dak/process_new.py

   1 #!/usr/bin/env python
   2
   3 # Handles NEW and BYHAND packages
   4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 ################################################################################
  21
  22 # 23:12|<aj> I will not hush!
  23 # 23:12|<elmo> :>
  24 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  25 # 23:13|<aj> I shall not be silenced!
  26 # 23:13|<aj> The world shall know!
  27 # 23:13|<aj> The world *must* know!
  28 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  29 # 23:13|<aj> yay powerpuff girls!!
  30 # 23:13|<aj> buttercup's my favourite, who's yours?
  31 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  32 # 23:14|<aj> *AREN'T YOU*?!
  33 # 23:15|<aj> I will not be treated like this.
  34 # 23:15|<aj> I shall have my revenge.
  35 # 23:15|<aj> I SHALL!!!
  36
  37 ################################################################################
  38
  39 import copy, errno, os, readline, stat, sys, time
  40 import apt_pkg, apt_inst
  41 import examine_package
  42 from daklib import database
  43 from daklib import logging
  44 from daklib import queue
  45 from daklib import utils
  46
  47 # Globals
  48 Cnf = None
  49 Options = None
  50 Upload = None
  51 projectB = None
  52 Logger = None
  53
  54 Priorities = None
  55 Sections = None
  56
  57 reject_message = ""
  58
  59 ################################################################################
  60 ################################################################################
  61 ################################################################################
  62
  63 def reject (str, prefix="Rejected: "):
  64     global reject_message
  65     if str:
  66         reject_message += prefix + str + "\n"
  67
  68 def recheck():
  69     global reject_message
  70     files = Upload.pkg.files
  71     reject_message = ""
  72
  73     for f in files.keys():
  74         # The .orig.tar.gz can disappear out from under us is it's a
  75         # duplicate of one in the archive.
  76         if not files.has_key(f):
  77             continue
  78         # Check that the source still exists
  79         if files[f]["type"] == "deb":
  80             source_version = files[f]["source version"]
  81             source_package = files[f]["source package"]
  82             if not Upload.pkg.changes["architecture"].has_key("source") \
  83                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  84                 source_epochless_version = utils.re_no_epoch.sub('', source_version)
  85                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  86                 found = 0
  87                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
  88                     if Cnf.has_key("Dir::Queue::%s" % (q)):
  89                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
  90                             found = 1
  91                 if not found:
  92                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
  93
  94         # Version and file overwrite checks
  95         if files[f]["type"] == "deb":
  96             reject(Upload.check_binary_against_db(f))
  97         elif files[f]["type"] == "dsc":
  98             reject(Upload.check_source_against_db(f))
  99             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 100             reject(reject_msg, "")
 101
 102     if reject_message.find("Rejected") != -1:
 103         answer = "XXX"
 104         if Options["No-Action"] or Options["Automatic"]:
 105             answer = 'S'
 106
 107         print "REJECT\n" + reject_message,
 108         prompt = "[R]eject, Skip, Quit ?"
 109
 110         while prompt.find(answer) == -1:
 111             answer = utils.our_raw_input(prompt)
 112             m = queue.re_default_answer.match(prompt)
 113             if answer == "":
 114                 answer = m.group(1)
 115             answer = answer[:1].upper()
 116
 117         if answer == 'R':
 118             Upload.do_reject(0, reject_message)
 119             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 120             return 0
 121         elif answer == 'S':
 122             return 0
 123         elif answer == 'Q':
 124             end()
 125             sys.exit(0)
 126
 127     return 1
 128
 129 ################################################################################
 130
 131 def indiv_sg_compare (a, b):
 132     """Sort by source name, source, version, 'have source', and
 133        finally by filename."""
 134     # Sort by source version
 135     q = apt_pkg.VersionCompare(a["version"], b["version"])
 136     if q:
 137         return -q
 138
 139     # Sort by 'have source'
 140     a_has_source = a["architecture"].get("source")
 141     b_has_source = b["architecture"].get("source")
 142     if a_has_source and not b_has_source:
 143         return -1
 144     elif b_has_source and not a_has_source:
 145         return 1
 146
 147     return cmp(a["filename"], b["filename"])
 148
 149 ############################################################
 150
 151 def sg_compare (a, b):
 152     a = a[1]
 153     b = b[1]
 154     """Sort by have note, source already in database and time of oldest upload."""
 155     # Sort by have note
 156     a_note_state = a["note_state"]
 157     b_note_state = b["note_state"]
 158     if a_note_state < b_note_state:
 159         return -1
 160     elif a_note_state > b_note_state:
 161         return 1
 162     # Sort by source already in database (descending)
 163     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 164     if source_in_database:
 165         return -source_in_database
 166
 167     # Sort by time of oldest upload
 168     return cmp(a["oldest"], b["oldest"])
 169
 170 def sort_changes(changes_files):
 171     """Sort into source groups, then sort each source group by version,
 172     have source, filename.  Finally, sort the source groups by have
 173     note, time of oldest upload of each source upload."""
 174     if len(changes_files) == 1:
 175         return changes_files
 176
 177     sorted_list = []
 178     cache = {}
 179     # Read in all the .changes files
 180     for filename in changes_files:
 181         try:
 182             Upload.pkg.changes_file = filename
 183             Upload.init_vars()
 184             Upload.update_vars()
 185             cache[filename] = copy.copy(Upload.pkg.changes)
 186             cache[filename]["filename"] = filename
 187         except:
 188             sorted_list.append(filename)
 189             break
 190     # Divide the .changes into per-source groups
 191     per_source = {}
 192     for filename in cache.keys():
 193         source = cache[filename]["source"]
 194         if not per_source.has_key(source):
 195             per_source[source] = {}
 196             per_source[source]["list"] = []
 197         per_source[source]["list"].append(cache[filename])
 198     # Determine oldest time and have note status for each source group
 199     for source in per_source.keys():
 200         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 201         ql = q.getresult()
 202         per_source[source]["source_in_database"] = len(ql)>0
 203         source_list = per_source[source]["list"]
 204         first = source_list[0]
 205         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 206         have_note = 0
 207         for d in per_source[source]["list"]:
 208             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 209             if mtime < oldest:
 210                 oldest = mtime
 211             have_note += (d.has_key("process-new note"))
 212         per_source[source]["oldest"] = oldest
 213         if not have_note:
 214             per_source[source]["note_state"] = 0; # none
 215         elif have_note < len(source_list):
 216             per_source[source]["note_state"] = 1; # some
 217         else:
 218             per_source[source]["note_state"] = 2; # all
 219         per_source[source]["list"].sort(indiv_sg_compare)
 220     per_source_items = per_source.items()
 221     per_source_items.sort(sg_compare)
 222     for i in per_source_items:
 223         for j in i[1]["list"]:
 224             sorted_list.append(j["filename"])
 225     return sorted_list
 226
 227 ################################################################################
 228
 229 class Section_Completer:
 230     def __init__ (self):
 231         self.sections = []
 232         q = projectB.query("SELECT section FROM section")
 233         for i in q.getresult():
 234             self.sections.append(i[0])
 235
 236     def complete(self, text, state):
 237         if state == 0:
 238             self.matches = []
 239             n = len(text)
 240             for word in self.sections:
 241                 if word[:n] == text:
 242                     self.matches.append(word)
 243         try:
 244             return self.matches[state]
 245         except IndexError:
 246             return None
 247
 248 ############################################################
 249
 250 class Priority_Completer:
 251     def __init__ (self):
 252         self.priorities = []
 253         q = projectB.query("SELECT priority FROM priority")
 254         for i in q.getresult():
 255             self.priorities.append(i[0])
 256
 257     def complete(self, text, state):
 258         if state == 0:
 259             self.matches = []
 260             n = len(text)
 261             for word in self.priorities:
 262                 if word[:n] == text:
 263                     self.matches.append(word)
 264         try:
 265             return self.matches[state]
 266         except IndexError:
 267             return None
 268
 269 ################################################################################
 270
 271 def print_new (new, indexed, file=sys.stdout):
 272     queue.check_valid(new)
 273     broken = 0
 274     index = 0
 275     for pkg in new.keys():
 276         index += 1
 277         section = new[pkg]["section"]
 278         priority = new[pkg]["priority"]
 279         if new[pkg]["section id"] == -1:
 280             section += "[!]"
 281             broken = 1
 282         if new[pkg]["priority id"] == -1:
 283             priority += "[!]"
 284             broken = 1
 285         if indexed:
 286             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 287         else:
 288             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 289         line = line.strip()+'\n'
 290         file.write(line)
 291     note = Upload.pkg.changes.get("process-new note")
 292     if note:
 293         print "*"*75
 294         print note
 295         print "*"*75
 296     return broken, note
 297
 298 ################################################################################
 299
 300 def index_range (index):
 301     if index == 1:
 302         return "1"
 303     else:
 304         return "1-%s" % (index)
 305
 306 ################################################################################
 307 ################################################################################
 308
 309 def edit_new (new):
 310     # Write the current data to a temporary file
 311     temp_filename = utils.temp_filename()
 312     temp_file = utils.open_file(temp_filename, 'w')
 313     print_new (new, 0, temp_file)
 314     temp_file.close()
 315     # Spawn an editor on that file
 316     editor = os.environ.get("EDITOR","vi")
 317     result = os.system("%s %s" % (editor, temp_filename))
 318     if result != 0:
 319         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 320     # Read the edited data back in
 321     temp_file = utils.open_file(temp_filename)
 322     lines = temp_file.readlines()
 323     temp_file.close()
 324     os.unlink(temp_filename)
 325     # Parse the new data
 326     for line in lines:
 327         line = line.strip()
 328         if line == "":
 329             continue
 330         s = line.split()
 331         # Pad the list if necessary
 332         s[len(s):3] = [None] * (3-len(s))
 333         (pkg, priority, section) = s[:3]
 334         if not new.has_key(pkg):
 335             utils.warn("Ignoring unknown package '%s'" % (pkg))
 336         else:
 337             # Strip off any invalid markers, print_new will readd them.
 338             if section.endswith("[!]"):
 339                 section = section[:-3]
 340             if priority.endswith("[!]"):
 341                 priority = priority[:-3]
 342             for f in new[pkg]["files"]:
 343                 Upload.pkg.files[f]["section"] = section
 344                 Upload.pkg.files[f]["priority"] = priority
 345             new[pkg]["section"] = section
 346             new[pkg]["priority"] = priority
 347
 348 ################################################################################
 349
 350 def edit_index (new, index):
 351     priority = new[index]["priority"]
 352     section = new[index]["section"]
 353     ftype = new[index]["type"]
 354     done = 0
 355     while not done:
 356         print "\t".join([index, priority, section])
 357
 358         answer = "XXX"
 359         if ftype != "dsc":
 360             prompt = "[B]oth, Priority, Section, Done ? "
 361         else:
 362             prompt = "[S]ection, Done ? "
 363         edit_priority = edit_section = 0
 364
 365         while prompt.find(answer) == -1:
 366             answer = utils.our_raw_input(prompt)
 367             m = queue.re_default_answer.match(prompt)
 368             if answer == "":
 369                 answer = m.group(1)
 370             answer = answer[:1].upper()
 371
 372         if answer == 'P':
 373             edit_priority = 1
 374         elif answer == 'S':
 375             edit_section = 1
 376         elif answer == 'B':
 377             edit_priority = edit_section = 1
 378         elif answer == 'D':
 379             done = 1
 380
 381         # Edit the priority
 382         if edit_priority:
 383             readline.set_completer(Priorities.complete)
 384             got_priority = 0
 385             while not got_priority:
 386                 new_priority = utils.our_raw_input("New priority: ").strip()
 387                 if new_priority not in Priorities.priorities:
 388                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 389                 else:
 390                     got_priority = 1
 391                     priority = new_priority
 392
 393         # Edit the section
 394         if edit_section:
 395             readline.set_completer(Sections.complete)
 396             got_section = 0
 397             while not got_section:
 398                 new_section = utils.our_raw_input("New section: ").strip()
 399                 if new_section not in Sections.sections:
 400                     print "E: '%s' is not a valid section, try again." % (new_section)
 401                 else:
 402                     got_section = 1
 403                     section = new_section
 404
 405         # Reset the readline completer
 406         readline.set_completer(None)
 407
 408     for f in new[index]["files"]:
 409         Upload.pkg.files[f]["section"] = section
 410         Upload.pkg.files[f]["priority"] = priority
 411     new[index]["priority"] = priority
 412     new[index]["section"] = section
 413     return new
 414
 415 ################################################################################
 416
 417 def edit_overrides (new):
 418     print
 419     done = 0
 420     while not done:
 421         print_new (new, 1)
 422         new_index = {}
 423         index = 0
 424         for i in new.keys():
 425             index += 1
 426             new_index[index] = i
 427
 428         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 429
 430         got_answer = 0
 431         while not got_answer:
 432             answer = utils.our_raw_input(prompt)
 433             if not answer.isdigit():
 434                 answer = answer[:1].upper()
 435             if answer == "E" or answer == "D":
 436                 got_answer = 1
 437             elif queue.re_isanum.match (answer):
 438                 answer = int(answer)
 439                 if (answer < 1) or (answer > index):
 440                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 441                 else:
 442                     got_answer = 1
 443
 444         if answer == 'E':
 445             edit_new(new)
 446         elif answer == 'D':
 447             done = 1
 448         else:
 449             edit_index (new, new_index[answer])
 450
 451     return new
 452
 453 ################################################################################
 454
 455 def edit_note(note):
 456     # Write the current data to a temporary file
 457     temp_filename = utils.temp_filename()
 458     temp_file = utils.open_file(temp_filename, 'w')
 459     temp_file.write(note)
 460     temp_file.close()
 461     editor = os.environ.get("EDITOR","vi")
 462     answer = 'E'
 463     while answer == 'E':
 464         os.system("%s %s" % (editor, temp_filename))
 465         temp_file = utils.open_file(temp_filename)
 466         note = temp_file.read().rstrip()
 467         temp_file.close()
 468         print "Note:"
 469         print utils.prefix_multi_line_string(note,"  ")
 470         prompt = "[D]one, Edit, Abandon, Quit ?"
 471         answer = "XXX"
 472         while prompt.find(answer) == -1:
 473             answer = utils.our_raw_input(prompt)
 474             m = queue.re_default_answer.search(prompt)
 475             if answer == "":
 476                 answer = m.group(1)
 477             answer = answer[:1].upper()
 478     os.unlink(temp_filename)
 479     if answer == 'A':
 480         return
 481     elif answer == 'Q':
 482         end()
 483         sys.exit(0)
 484     Upload.pkg.changes["process-new note"] = note
 485     Upload.dump_vars(Cnf["Dir::Queue::New"])
 486
 487 ################################################################################
 488
 489 def check_pkg ():
 490     try:
 491         less_fd = os.popen("less -R -", 'w', 0)
 492         stdout_fd = sys.stdout
 493         try:
 494             sys.stdout = less_fd
 495             examine_package.display_changes(Upload.pkg.changes_file)
 496             files = Upload.pkg.files
 497             for f in files.keys():
 498                 if files[f].has_key("new"):
 499                     ftype = files[f]["type"]
 500                     if ftype == "deb":
 501                         examine_package.check_deb(f)
 502                     elif ftype == "dsc":
 503                         examine_package.check_dsc(f)
 504         finally:
 505             sys.stdout = stdout_fd
 506     except IOError, e:
 507         if e.errno == errno.EPIPE:
 508             utils.warn("[examine_package] Caught EPIPE; skipping.")
 509             pass
 510         else:
 511             raise
 512     except KeyboardInterrupt:
 513         utils.warn("[examine_package] Caught C-c; skipping.")
 514         pass
 515
 516 ################################################################################
 517
 518 ## FIXME: horribly Debian specific
 519
 520 def do_bxa_notification():
 521     files = Upload.pkg.files
 522     summary = ""
 523     for f in files.keys():
 524         if files[f]["type"] == "deb":
 525             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 526             summary += "\n"
 527             summary += "Package: %s\n" % (control.Find("Package"))
 528             summary += "Description: %s\n" % (control.Find("Description"))
 529     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 530     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 531     utils.send_mail(bxa_mail)
 532
 533 ################################################################################
 534
 535 def add_overrides (new):
 536     changes = Upload.pkg.changes
 537     files = Upload.pkg.files
 538
 539     projectB.query("BEGIN WORK")
 540     for suite in changes["suite"].keys():
 541         suite_id = database.get_suite_id(suite)
 542         for pkg in new.keys():
 543             component_id = database.get_component_id(new[pkg]["component"])
 544             type_id = database.get_override_type_id(new[pkg]["type"])
 545             priority_id = new[pkg]["priority id"]
 546             section_id = new[pkg]["section id"]
 547             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 548             for f in new[pkg]["files"]:
 549                 if files[f].has_key("new"):
 550                     del files[f]["new"]
 551             del new[pkg]
 552
 553     projectB.query("COMMIT WORK")
 554
 555     if Cnf.FindB("Dinstall::BXANotify"):
 556         do_bxa_notification()
 557
 558 ################################################################################
 559
 560 def prod_maintainer ():
 561     # Here we prepare an editor and get them ready to prod...
 562     temp_filename = utils.temp_filename()
 563     editor = os.environ.get("EDITOR","vi")
 564     answer = 'E'
 565     while answer == 'E':
 566         os.system("%s %s" % (editor, temp_filename))
 567         f = utils.open_file(temp_filename)
 568         prod_message = "".join(f.readlines())
 569         f.close()
 570         print "Prod message:"
 571         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 572         prompt = "[P]rod, Edit, Abandon, Quit ?"
 573         answer = "XXX"
 574         while prompt.find(answer) == -1:
 575             answer = utils.our_raw_input(prompt)
 576             m = queue.re_default_answer.search(prompt)
 577             if answer == "":
 578                 answer = m.group(1)
 579             answer = answer[:1].upper()
 580         os.unlink(temp_filename)
 581         if answer == 'A':
 582             return
 583         elif answer == 'Q':
 584             end()
 585             sys.exit(0)
 586     # Otherwise, do the proding...
 587     user_email_address = utils.whoami() + " <%s>" % (
 588         Cnf["Dinstall::MyAdminAddress"])
 589
 590     Subst = Upload.Subst
 591
 592     Subst["__FROM_ADDRESS__"] = user_email_address
 593     Subst["__PROD_MESSAGE__"] = prod_message
 594     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 595
 596     prod_mail_message = utils.TemplateSubst(
 597         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 598
 599     # Send the prod mail if appropriate
 600     if not Cnf["Dinstall::Options::No-Mail"]:
 601         utils.send_mail(prod_mail_message)
 602
 603     print "Sent proding message"
 604
 605 ################################################################################
 606
 607 def do_new():
 608     print "NEW\n"
 609     files = Upload.pkg.files
 610     changes = Upload.pkg.changes
 611
 612     # Make a copy of distribution we can happily trample on
 613     changes["suite"] = copy.copy(changes["distribution"])
 614
 615     # Fix up the list of target suites
 616     for suite in changes["suite"].keys():
 617         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 618         if override:
 619             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 620               database.get_suite_id(override) == -1)
 621             if olderr or newerr:
 622                 (oinv, newinv) = ("", "")
 623                 if olderr: oinv = "invalid "
 624                 if newerr: ninv = "invalid "
 625                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 626                         oinv, suite, ninv, override)
 627             del changes["suite"][suite]
 628             changes["suite"][override] = 1
 629     # Validate suites
 630     for suite in changes["suite"].keys():
 631         suite_id = database.get_suite_id(suite)
 632         if suite_id == -1:
 633             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 634
 635     # The main NEW processing loop
 636     done = 0
 637     while not done:
 638         # Find out what's new
 639         new = queue.determine_new(changes, files, projectB)
 640
 641         if not new:
 642             break
 643
 644         answer = "XXX"
 645         if Options["No-Action"] or Options["Automatic"]:
 646             answer = 'S'
 647
 648         (broken, note) = print_new(new, 0)
 649         prompt = ""
 650
 651         if not broken and not note:
 652             prompt = "Add overrides, "
 653         if broken:
 654             print "W: [!] marked entries must be fixed before package can be processed."
 655         if note:
 656             print "W: note must be removed before package can be processed."
 657             prompt += "Remove note, "
 658
 659         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 660
 661         while prompt.find(answer) == -1:
 662             answer = utils.our_raw_input(prompt)
 663             m = queue.re_default_answer.search(prompt)
 664             if answer == "":
 665                 answer = m.group(1)
 666             answer = answer[:1].upper()
 667
 668         if answer == 'A':
 669             done = add_overrides (new)
 670         elif answer == 'C':
 671             check_pkg()
 672         elif answer == 'E':
 673             new = edit_overrides (new)
 674         elif answer == 'M':
 675             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 676             if not aborted:
 677                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 678                 done = 1
 679         elif answer == 'N':
 680             edit_note(changes.get("process-new note", ""))
 681         elif answer == 'P':
 682             prod_maintainer()
 683         elif answer == 'R':
 684             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 685             if confirm == "y":
 686                 del changes["process-new note"]
 687         elif answer == 'S':
 688             done = 1
 689         elif answer == 'Q':
 690             end()
 691             sys.exit(0)
 692
 693 ################################################################################
 694 ################################################################################
 695 ################################################################################
 696
 697 def usage (exit_code=0):
 698     print """Usage: dak process-new [OPTION]... [CHANGES]...
 699   -a, --automatic           automatic run
 700   -h, --help                show this help and exit.
 701   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 702   -m, --manual-reject=MSG   manual reject with `msg'
 703   -n, --no-action           don't do anything
 704   -V, --version             display the version number and exit"""
 705     sys.exit(exit_code)
 706
 707 ################################################################################
 708
 709 def init():
 710     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 711
 712     Cnf = utils.get_conf()
 713
 714     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 715                  ('h',"help","Process-New::Options::Help"),
 716                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 717                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 718                  ('n',"no-action","Process-New::Options::No-Action")]
 719
 720     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 721         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 722             Cnf["Process-New::Options::%s" % (i)] = ""
 723
 724     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 725     Options = Cnf.SubTree("Process-New::Options")
 726
 727     if Options["Help"]:
 728         usage()
 729
 730     Upload = queue.Upload(Cnf)
 731
 732     if not Options["No-Action"]:
 733         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 734
 735     projectB = Upload.projectB
 736
 737     Sections = Section_Completer()
 738     Priorities = Priority_Completer()
 739     readline.parse_and_bind("tab: complete")
 740
 741     return changes_files
 742
 743 ################################################################################
 744
 745 def do_byhand():
 746     done = 0
 747     while not done:
 748         files = Upload.pkg.files
 749         will_install = 1
 750         byhand = []
 751
 752         for f in files.keys():
 753             if files[f]["type"] == "byhand":
 754                 if os.path.exists(f):
 755                     print "W: %s still present; please process byhand components and try again." % (f)
 756                     will_install = 0
 757                 else:
 758                     byhand.append(f)
 759
 760         answer = "XXXX"
 761         if Options["No-Action"]:
 762             answer = "S"
 763         if will_install:
 764             if Options["Automatic"] and not Options["No-Action"]:
 765                 answer = 'A'
 766             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 767         else:
 768             prompt = "Manual reject, [S]kip, Quit ?"
 769
 770         while prompt.find(answer) == -1:
 771             answer = utils.our_raw_input(prompt)
 772             m = queue.re_default_answer.search(prompt)
 773             if answer == "":
 774                 answer = m.group(1)
 775             answer = answer[:1].upper()
 776
 777         if answer == 'A':
 778             done = 1
 779             for f in byhand:
 780                 del files[f]
 781         elif answer == 'M':
 782             Upload.do_reject(1, Options["Manual-Reject"])
 783             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 784             done = 1
 785         elif answer == 'S':
 786             done = 1
 787         elif answer == 'Q':
 788             end()
 789             sys.exit(0)
 790
 791 ################################################################################
 792
 793 def get_accept_lock():
 794     retry = 0
 795     while retry < 10:
 796         try:
 797             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 798             retry = 10
 799         except OSError, e:
 800             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 801                 retry += 1
 802                 if (retry >= 10):
 803                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 804                 else:
 805                     print("Unable to get accepted lock (try %d of 10)" % retry)
 806                 time.sleep(60)
 807             else:
 808                 raise
 809
 810 def move_to_dir (dest, perms=0660, changesperms=0664):
 811     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 812     file_keys = Upload.pkg.files.keys()
 813     for f in file_keys:
 814         utils.move (f, dest, perms=perms)
 815
 816 def is_source_in_queue_dir(qdir):
 817     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 818                 and x.endswith(".changes") ]
 819     for entry in entries:
 820         # read the .dak
 821         u = queue.Upload(Cnf)
 822         u.pkg.changes_file = entry
 823         u.update_vars()
 824         if not Upload.pkg.changes["architecture"].has_key("source"):
 825             # another binary upload, ignore
 826             continue
 827         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 828             # another version, ignore
 829             continue
 830         # found it!
 831         return True
 832     return False
 833
 834 def move_to_holding(suite, queue_dir):
 835     print "Moving to %s holding area." % (suite.upper(),)
 836     if Options["No-Action"]:
 837         return
 838     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 839     Upload.dump_vars(queue_dir)
 840     move_to_dir(queue_dir)
 841     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 842
 843 def do_accept_stableupdate(suite, q):
 844     (summary, short_summary) = Upload.build_summaries()
 845     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 846     if not Upload.pkg.changes["architecture"].has_key("source"):
 847         # It is not a sourceful upload.  So its source may be either in p-u
 848         # holding, in new, in accepted or already installed.
 849         if is_source_in_queue_dir(queue_dir):
 850             # It's in p-u holding, so move it there.
 851             move_to_holding(suite, queue_dir)
 852         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 853             # It's in NEW.  We expect the source to land in p-u holding
 854             # pretty soon.
 855             move_to_holding(suite, queue_dir)
 856         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 857             # The source is in accepted, the binary cleared NEW: accept it.
 858             Upload.accept(summary, short_summary)
 859             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 860         elif Upload.source_exists(Upload.pkg.changes["source"],
 861                 Upload.pkg.changes["version"]):
 862             # dak tells us that there is source available.  At time of
 863             # writing this means that it is installed, so put it into
 864             # accepted.
 865             Upload.accept(summary, short_summary)
 866             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 867     else:
 868         # We are handling a sourceful upload.  Move to accepted if currently
 869         # in p-u holding and to p-u holding otherwise.
 870         if is_source_in_queue_dir(queue_dir):
 871             Upload.accept(summary, short_summary)
 872             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 873         else:
 874             move_to_holding(suite, queue_dir)
 875
 876 def do_accept():
 877     print "ACCEPT"
 878     if not Options["No-Action"]:
 879         get_accept_lock()
 880         (summary, short_summary) = Upload.build_summaries()
 881     try:
 882         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 883             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 884             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 885             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 886             # Check for override disparities
 887             Upload.Subst["__SUMMARY__"] = summary
 888         else:
 889             # Stable updates need to be copied to proposed-updates holding
 890             # area instead of accepted.  Sourceful uploads need to go
 891             # to it directly, binaries only if the source has not yet been
 892             # accepted into p-u.
 893             for suite, q in [("proposed-updates", "ProposedUpdates"),
 894                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 895                 if not Upload.pkg.changes["distribution"].has_key(suite):
 896                     continue
 897                 return do_accept_stableupdate(suite, q)
 898             # Just a normal upload, accept it...
 899             Upload.accept(summary, short_summary)
 900             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 901     finally:
 902         if not Options["No-Action"]:
 903             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 904
 905 def check_status(files):
 906     new = byhand = 0
 907     for f in files.keys():
 908         if files[f]["type"] == "byhand":
 909             byhand = 1
 910         elif files[f].has_key("new"):
 911             new = 1
 912     return (new, byhand)
 913
 914 def do_pkg(changes_file):
 915     Upload.pkg.changes_file = changes_file
 916     Upload.init_vars()
 917     Upload.update_vars()
 918     Upload.update_subst()
 919     files = Upload.pkg.files
 920
 921     if not recheck():
 922         return
 923
 924     (new, byhand) = check_status(files)
 925     if new or byhand:
 926         if new:
 927             do_new()
 928         if byhand:
 929             do_byhand()
 930         (new, byhand) = check_status(files)
 931
 932     if not new and not byhand:
 933         do_accept()
 934
 935 ################################################################################
 936
 937 def end():
 938     accept_count = Upload.accept_count
 939     accept_bytes = Upload.accept_bytes
 940
 941     if accept_count:
 942         sets = "set"
 943         if accept_count > 1:
 944             sets = "sets"
 945         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 946         Logger.log(["total",accept_count,accept_bytes])
 947
 948     if not Options["No-Action"]:
 949         Logger.close()
 950
 951 ################################################################################
 952
 953 def do_comments(dir, opref, npref, line, fn):
 954     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 955         lines = open("%s/%s" % (dir, comm)).readlines()
 956         if len(lines) == 0 or lines[0] != line + "\n": continue
 957         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 958                                 and x.endswith(".changes") ]
 959         changes_files = sort_changes(changes_files)
 960         for f in changes_files:
 961             f = utils.validate_changes_file_arg(f, 0)
 962             if not f: continue
 963             print "\n" + f
 964             fn(f, "".join(lines[1:]))
 965
 966         if opref != npref and not Options["No-Action"]:
 967             newcomm = npref + comm[len(opref):]
 968             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 969
 970 ################################################################################
 971
 972 def comment_accept(changes_file, comments):
 973     Upload.pkg.changes_file = changes_file
 974     Upload.init_vars()
 975     Upload.update_vars()
 976     Upload.update_subst()
 977     files = Upload.pkg.files
 978
 979     if not recheck():
 980         return # dak wants to REJECT, crap
 981
 982     (new, byhand) = check_status(files)
 983     if not new and not byhand:
 984         do_accept()
 985
 986 ################################################################################
 987
 988 def comment_reject(changes_file, comments):
 989     Upload.pkg.changes_file = changes_file
 990     Upload.init_vars()
 991     Upload.update_vars()
 992     Upload.update_subst()
 993
 994     if not recheck():
 995         pass # dak has its own reasons to reject as well, which is fine
 996
 997     reject(comments)
 998     print "REJECT\n" + reject_message,
 999     if not Options["No-Action"]:
1000         Upload.do_reject(0, reject_message)
1001         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1002
1003 ################################################################################
1004
1005 def main():
1006     changes_files = init()
1007     if len(changes_files) > 50:
1008         sys.stderr.write("Sorting changes...\n")
1009     changes_files = sort_changes(changes_files)
1010
1011     # Kill me now? **FIXME**
1012     Cnf["Dinstall::Options::No-Mail"] = ""
1013     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1014     if Cnf.has_key("Dinstall::Bcc"):
1015         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1016     else:
1017         Upload.Subst["__BCC__"] = bcc
1018
1019     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1020     if commentsdir:
1021         if changes_files != []:
1022             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1023             sys.exit(1)
1024         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1025         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1026     else:
1027         for changes_file in changes_files:
1028             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1029             if not changes_file:
1030                 continue
1031             print "\n" + changes_file
1032             do_pkg (changes_file)
1033
1034     end()
1035
1036 ################################################################################
1037
1038 if __name__ == '__main__':
1039     main()