dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 from __future__ import with_statement
  46
  47 import copy
  48 import errno
  49 import os
  50 import readline
  51 import stat
  52 import sys
  53 import time
  54 import contextlib
  55 import pwd
  56 import apt_pkg, apt_inst
  57 import examine_package
  58
  59 from daklib.dbconn import *
  60 from daklib.queue import *
  61 from daklib import daklog
  62 from daklib import utils
  63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  65 from daklib.summarystats import SummaryStats
  66 from daklib.config import Config
  67
  68 # Globals
  69 Options = None
  70 Logger = None
  71
  72 Priorities = None
  73 Sections = None
  74
  75 ################################################################################
  76 ################################################################################
  77 ################################################################################
  78
  79 def recheck(upload, session):
  80 # STU: I'm not sure, but I don't thin kthis is necessary any longer:    upload.recheck(session)
  81     if len(upload.rejects) > 0:
  82         answer = "XXX"
  83         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
  84             answer = 'S'
  85
  86         print "REJECT\n%s" % '\n'.join(upload.rejects)
  87         prompt = "[R]eject, Skip, Quit ?"
  88
  89         while prompt.find(answer) == -1:
  90             answer = utils.our_raw_input(prompt)
  91             m = re_default_answer.match(prompt)
  92             if answer == "":
  93                 answer = m.group(1)
  94             answer = answer[:1].upper()
  95
  96         if answer == 'R':
  97             upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
  98             os.unlink(upload.pkg.changes_file[:-8]+".dak")
  99             return 0
 100         elif answer == 'S':
 101             return 0
 102         elif answer == 'Q':
 103             end()
 104             sys.exit(0)
 105
 106     return 1
 107
 108 ################################################################################
 109
 110 def indiv_sg_compare (a, b):
 111     """Sort by source name, source, version, 'have source', and
 112        finally by filename."""
 113     # Sort by source version
 114     q = apt_pkg.VersionCompare(a["version"], b["version"])
 115     if q:
 116         return -q
 117
 118     # Sort by 'have source'
 119     a_has_source = a["architecture"].get("source")
 120     b_has_source = b["architecture"].get("source")
 121     if a_has_source and not b_has_source:
 122         return -1
 123     elif b_has_source and not a_has_source:
 124         return 1
 125
 126     return cmp(a["filename"], b["filename"])
 127
 128 ############################################################
 129
 130 def sg_compare (a, b):
 131     a = a[1]
 132     b = b[1]
 133     """Sort by have note, source already in database and time of oldest upload."""
 134     # Sort by have note
 135     a_note_state = a["note_state"]
 136     b_note_state = b["note_state"]
 137     if a_note_state < b_note_state:
 138         return -1
 139     elif a_note_state > b_note_state:
 140         return 1
 141     # Sort by source already in database (descending)
 142     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 143     if source_in_database:
 144         return -source_in_database
 145
 146     # Sort by time of oldest upload
 147     return cmp(a["oldest"], b["oldest"])
 148
 149 def sort_changes(changes_files, session):
 150     """Sort into source groups, then sort each source group by version,
 151     have source, filename.  Finally, sort the source groups by have
 152     note, time of oldest upload of each source upload."""
 153     if len(changes_files) == 1:
 154         return changes_files
 155
 156     sorted_list = []
 157     cache = {}
 158     # Read in all the .changes files
 159     for filename in changes_files:
 160         u = Upload()
 161         try:
 162             u.pkg.changes_file = filename
 163             u.load_changes(filename)
 164             u.update_subst()
 165             cache[filename] = copy.copy(u.pkg.changes)
 166             cache[filename]["filename"] = filename
 167         except:
 168             sorted_list.append(filename)
 169             break
 170     # Divide the .changes into per-source groups
 171     per_source = {}
 172     for filename in cache.keys():
 173         source = cache[filename]["source"]
 174         if not per_source.has_key(source):
 175             per_source[source] = {}
 176             per_source[source]["list"] = []
 177         per_source[source]["list"].append(cache[filename])
 178     # Determine oldest time and have note status for each source group
 179     for source in per_source.keys():
 180         q = session.query(DBSource).filter_by(source = source).all()
 181         per_source[source]["source_in_database"] = len(q)>0
 182         source_list = per_source[source]["list"]
 183         first = source_list[0]
 184         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 185         have_note = 0
 186         for d in per_source[source]["list"]:
 187             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 188             if mtime < oldest:
 189                 oldest = mtime
 190             have_note += has_new_comment(d["source"], d["version"], session)
 191         per_source[source]["oldest"] = oldest
 192         if not have_note:
 193             per_source[source]["note_state"] = 0; # none
 194         elif have_note < len(source_list):
 195             per_source[source]["note_state"] = 1; # some
 196         else:
 197             per_source[source]["note_state"] = 2; # all
 198         per_source[source]["list"].sort(indiv_sg_compare)
 199     per_source_items = per_source.items()
 200     per_source_items.sort(sg_compare)
 201     for i in per_source_items:
 202         for j in i[1]["list"]:
 203             sorted_list.append(j["filename"])
 204     return sorted_list
 205
 206 ################################################################################
 207
 208 class Section_Completer:
 209     def __init__ (self, session):
 210         self.sections = []
 211         self.matches = []
 212         for s, in session.query(Section.section):
 213             self.sections.append(s)
 214
 215     def complete(self, text, state):
 216         if state == 0:
 217             self.matches = []
 218             n = len(text)
 219             for word in self.sections:
 220                 if word[:n] == text:
 221                     self.matches.append(word)
 222         try:
 223             return self.matches[state]
 224         except IndexError:
 225             return None
 226
 227 ############################################################
 228
 229 class Priority_Completer:
 230     def __init__ (self, session):
 231         self.priorities = []
 232         self.matches = []
 233         for p, in session.query(Priority.priority):
 234             self.priorities.append(p)
 235
 236     def complete(self, text, state):
 237         if state == 0:
 238             self.matches = []
 239             n = len(text)
 240             for word in self.priorities:
 241                 if word[:n] == text:
 242                     self.matches.append(word)
 243         try:
 244             return self.matches[state]
 245         except IndexError:
 246             return None
 247
 248 ################################################################################
 249
 250 def print_new (new, upload, indexed, file=sys.stdout):
 251     check_valid(new)
 252     broken = False
 253     index = 0
 254     for pkg in new.keys():
 255         index += 1
 256         section = new[pkg]["section"]
 257         priority = new[pkg]["priority"]
 258         if new[pkg]["section id"] == -1:
 259             section += "[!]"
 260             broken = True
 261         if new[pkg]["priority id"] == -1:
 262             priority += "[!]"
 263             broken = True
 264         if indexed:
 265             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 266         else:
 267             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 268         line = line.strip()+'\n'
 269         file.write(line)
 270     notes = get_new_comments(upload.pkg.changes.get("source"))
 271     for note in notes:
 272         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 273               % (note.author, note.version, note.notedate, note.comment)
 274         print "-" * 72
 275     return broken, len(notes) > 0
 276
 277 ################################################################################
 278
 279 def index_range (index):
 280     if index == 1:
 281         return "1"
 282     else:
 283         return "1-%s" % (index)
 284
 285 ################################################################################
 286 ################################################################################
 287
 288 def edit_new (new, upload):
 289     # Write the current data to a temporary file
 290     (fd, temp_filename) = utils.temp_filename()
 291     temp_file = os.fdopen(fd, 'w')
 292     print_new (new, upload, indexed=0, file=temp_file)
 293     temp_file.close()
 294     # Spawn an editor on that file
 295     editor = os.environ.get("EDITOR","vi")
 296     result = os.system("%s %s" % (editor, temp_filename))
 297     if result != 0:
 298         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 299     # Read the edited data back in
 300     temp_file = utils.open_file(temp_filename)
 301     lines = temp_file.readlines()
 302     temp_file.close()
 303     os.unlink(temp_filename)
 304     # Parse the new data
 305     for line in lines:
 306         line = line.strip()
 307         if line == "":
 308             continue
 309         s = line.split()
 310         # Pad the list if necessary
 311         s[len(s):3] = [None] * (3-len(s))
 312         (pkg, priority, section) = s[:3]
 313         if not new.has_key(pkg):
 314             utils.warn("Ignoring unknown package '%s'" % (pkg))
 315         else:
 316             # Strip off any invalid markers, print_new will readd them.
 317             if section.endswith("[!]"):
 318                 section = section[:-3]
 319             if priority.endswith("[!]"):
 320                 priority = priority[:-3]
 321             for f in new[pkg]["files"]:
 322                 upload.pkg.files[f]["section"] = section
 323                 upload.pkg.files[f]["priority"] = priority
 324             new[pkg]["section"] = section
 325             new[pkg]["priority"] = priority
 326
 327 ################################################################################
 328
 329 def edit_index (new, upload, index):
 330     priority = new[index]["priority"]
 331     section = new[index]["section"]
 332     ftype = new[index]["type"]
 333     done = 0
 334     while not done:
 335         print "\t".join([index, priority, section])
 336
 337         answer = "XXX"
 338         if ftype != "dsc":
 339             prompt = "[B]oth, Priority, Section, Done ? "
 340         else:
 341             prompt = "[S]ection, Done ? "
 342         edit_priority = edit_section = 0
 343
 344         while prompt.find(answer) == -1:
 345             answer = utils.our_raw_input(prompt)
 346             m = re_default_answer.match(prompt)
 347             if answer == "":
 348                 answer = m.group(1)
 349             answer = answer[:1].upper()
 350
 351         if answer == 'P':
 352             edit_priority = 1
 353         elif answer == 'S':
 354             edit_section = 1
 355         elif answer == 'B':
 356             edit_priority = edit_section = 1
 357         elif answer == 'D':
 358             done = 1
 359
 360         # Edit the priority
 361         if edit_priority:
 362             readline.set_completer(Priorities.complete)
 363             got_priority = 0
 364             while not got_priority:
 365                 new_priority = utils.our_raw_input("New priority: ").strip()
 366                 if new_priority not in Priorities.priorities:
 367                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 368                 else:
 369                     got_priority = 1
 370                     priority = new_priority
 371
 372         # Edit the section
 373         if edit_section:
 374             readline.set_completer(Sections.complete)
 375             got_section = 0
 376             while not got_section:
 377                 new_section = utils.our_raw_input("New section: ").strip()
 378                 if new_section not in Sections.sections:
 379                     print "E: '%s' is not a valid section, try again." % (new_section)
 380                 else:
 381                     got_section = 1
 382                     section = new_section
 383
 384         # Reset the readline completer
 385         readline.set_completer(None)
 386
 387     for f in new[index]["files"]:
 388         upload.pkg.files[f]["section"] = section
 389         upload.pkg.files[f]["priority"] = priority
 390     new[index]["priority"] = priority
 391     new[index]["section"] = section
 392     return new
 393
 394 ################################################################################
 395
 396 def edit_overrides (new, upload, session):
 397     print
 398     done = 0
 399     while not done:
 400         print_new (new, upload, indexed=1)
 401         new_index = {}
 402         index = 0
 403         for i in new.keys():
 404             index += 1
 405             new_index[index] = i
 406
 407         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 408
 409         got_answer = 0
 410         while not got_answer:
 411             answer = utils.our_raw_input(prompt)
 412             if not answer.isdigit():
 413                 answer = answer[:1].upper()
 414             if answer == "E" or answer == "D":
 415                 got_answer = 1
 416             elif re_isanum.match (answer):
 417                 answer = int(answer)
 418                 if (answer < 1) or (answer > index):
 419                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 420                 else:
 421                     got_answer = 1
 422
 423         if answer == 'E':
 424             edit_new(new, upload)
 425         elif answer == 'D':
 426             done = 1
 427         else:
 428             edit_index (new, upload, new_index[answer])
 429
 430     return new
 431
 432 ################################################################################
 433
 434 def edit_note(note, upload, session):
 435     # Write the current data to a temporary file
 436     (fd, temp_filename) = utils.temp_filename()
 437     editor = os.environ.get("EDITOR","vi")
 438     answer = 'E'
 439     while answer == 'E':
 440         os.system("%s %s" % (editor, temp_filename))
 441         temp_file = utils.open_file(temp_filename)
 442         newnote = temp_file.read().rstrip()
 443         temp_file.close()
 444         print "New Note:"
 445         print utils.prefix_multi_line_string(newnote,"  ")
 446         prompt = "[D]one, Edit, Abandon, Quit ?"
 447         answer = "XXX"
 448         while prompt.find(answer) == -1:
 449             answer = utils.our_raw_input(prompt)
 450             m = re_default_answer.search(prompt)
 451             if answer == "":
 452                 answer = m.group(1)
 453             answer = answer[:1].upper()
 454     os.unlink(temp_filename)
 455     if answer == 'A':
 456         return
 457     elif answer == 'Q':
 458         end()
 459         sys.exit(0)
 460
 461     comment = NewComment()
 462     comment.package = upload.pkg.changes["source"]
 463     comment.version = upload.pkg.changes["version"]
 464     comment.comment = newnote
 465     comment.author  = utils.whoami()
 466     comment.trainee = bool(Options["Trainee"])
 467     session.add(comment)
 468     session.commit()
 469
 470 ################################################################################
 471
 472 def check_pkg (upload):
 473     try:
 474         less_fd = os.popen("less -R -", 'w', 0)
 475         stdout_fd = sys.stdout
 476         try:
 477             sys.stdout = less_fd
 478             changes = utils.parse_changes (upload.pkg.changes_file)
 479             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
 480             files = upload.pkg.files
 481             for f in files.keys():
 482                 if files[f].has_key("new"):
 483                     ftype = files[f]["type"]
 484                     if ftype == "deb":
 485                         examine_package.check_deb(changes['distribution'], f)
 486                     elif ftype == "dsc":
 487                         examine_package.check_dsc(changes['distribution'], f)
 488         finally:
 489             examine_package.output_package_relations()
 490             sys.stdout = stdout_fd
 491     except IOError, e:
 492         if e.errno == errno.EPIPE:
 493             utils.warn("[examine_package] Caught EPIPE; skipping.")
 494             pass
 495         else:
 496             raise
 497     except KeyboardInterrupt:
 498         utils.warn("[examine_package] Caught C-c; skipping.")
 499         pass
 500
 501 ################################################################################
 502
 503 ## FIXME: horribly Debian specific
 504
 505 def do_bxa_notification(upload):
 506     files = upload.pkg.files
 507     summary = ""
 508     for f in files.keys():
 509         if files[f]["type"] == "deb":
 510             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 511             summary += "\n"
 512             summary += "Package: %s\n" % (control.Find("Package"))
 513             summary += "Description: %s\n" % (control.Find("Description"))
 514     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 515     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
 516     utils.send_mail(bxa_mail)
 517
 518 ################################################################################
 519
 520 def add_overrides (new, upload, session):
 521     changes = upload.pkg.changes
 522     files = upload.pkg.files
 523     srcpkg = changes.get("source")
 524
 525     for suite in changes["suite"].keys():
 526         suite_id = get_suite(suite).suite_id
 527         for pkg in new.keys():
 528             component_id = get_component(new[pkg]["component"]).component_id
 529             type_id = get_override_type(new[pkg]["type"]).overridetype_id
 530             priority_id = new[pkg]["priority id"]
 531             section_id = new[pkg]["section id"]
 532             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 533             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
 534                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
 535             for f in new[pkg]["files"]:
 536                 if files[f].has_key("new"):
 537                     del files[f]["new"]
 538             del new[pkg]
 539
 540     session.commit()
 541
 542     if Config().FindB("Dinstall::BXANotify"):
 543         do_bxa_notification(upload)
 544
 545 ################################################################################
 546
 547 def prod_maintainer (note, upload):
 548     cnf = Config()
 549     # Here we prepare an editor and get them ready to prod...
 550     (fd, temp_filename) = utils.temp_filename()
 551     temp_file = os.fdopen(fd, 'w')
 552     if len(note) > 0:
 553         for line in note:
 554             temp_file.write(line)
 555     temp_file.close()
 556     editor = os.environ.get("EDITOR","vi")
 557     answer = 'E'
 558     while answer == 'E':
 559         os.system("%s %s" % (editor, temp_filename))
 560         temp_fh = utils.open_file(temp_filename)
 561         prod_message = "".join(temp_fh.readlines())
 562         temp_fh.close()
 563         print "Prod message:"
 564         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 565         prompt = "[P]rod, Edit, Abandon, Quit ?"
 566         answer = "XXX"
 567         while prompt.find(answer) == -1:
 568             answer = utils.our_raw_input(prompt)
 569             m = re_default_answer.search(prompt)
 570             if answer == "":
 571                 answer = m.group(1)
 572             answer = answer[:1].upper()
 573     os.unlink(temp_filename)
 574     if answer == 'A':
 575         return
 576     elif answer == 'Q':
 577         end()
 578         sys.exit(0)
 579     # Otherwise, do the proding...
 580     user_email_address = utils.whoami() + " <%s>" % (
 581         cnf["Dinstall::MyAdminAddress"])
 582
 583     Subst = upload.Subst
 584
 585     Subst["__FROM_ADDRESS__"] = user_email_address
 586     Subst["__PROD_MESSAGE__"] = prod_message
 587     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
 588
 589     prod_mail_message = utils.TemplateSubst(
 590         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 591
 592     # Send the prod mail if appropriate
 593     if not cnf["Dinstall::Options::No-Mail"]:
 594         utils.send_mail(prod_mail_message)
 595
 596     print "Sent proding message"
 597
 598 ################################################################################
 599
 600 def do_new(upload, session):
 601     print "NEW\n"
 602     files = upload.pkg.files
 603     upload.check_files(not Options["No-Action"])
 604     changes = upload.pkg.changes
 605     cnf = Config()
 606
 607     # Make a copy of distribution we can happily trample on
 608     changes["suite"] = copy.copy(changes["distribution"])
 609
 610     # Fix up the list of target suites
 611     for suite in changes["suite"].keys():
 612         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
 613         if override:
 614             (olderr, newerr) = (get_suite(suite, session) == None,
 615                                 get_suite(override, session) == None)
 616             if olderr or newerr:
 617                 (oinv, newinv) = ("", "")
 618                 if olderr: oinv = "invalid "
 619                 if newerr: ninv = "invalid "
 620                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 621                         oinv, suite, ninv, override)
 622             del changes["suite"][suite]
 623             changes["suite"][override] = 1
 624     # Validate suites
 625     for suite in changes["suite"].keys():
 626         if get_suite(suite, session) is None:
 627             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 628
 629     # The main NEW processing loop
 630     done = 0
 631     while not done:
 632         # Find out what's new
 633         new = determine_new(changes, files)
 634
 635         if not new:
 636             break
 637
 638         answer = "XXX"
 639         if Options["No-Action"] or Options["Automatic"]:
 640             answer = 'S'
 641
 642         (broken, note) = print_new(new, upload, indexed=0)
 643         prompt = ""
 644
 645         if not broken and not note:
 646             prompt = "Add overrides, "
 647         if broken:
 648             print "W: [!] marked entries must be fixed before package can be processed."
 649         if note:
 650             print "W: note must be removed before package can be processed."
 651             prompt += "RemOve all notes, Remove note, "
 652
 653         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 654
 655         while prompt.find(answer) == -1:
 656             answer = utils.our_raw_input(prompt)
 657             m = re_default_answer.search(prompt)
 658             if answer == "":
 659                 answer = m.group(1)
 660             answer = answer[:1].upper()
 661
 662         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 663             utils.warn("Trainees can't do that")
 664             continue
 665
 666         if answer == 'A' and not Options["Trainee"]:
 667             try:
 668                 check_daily_lock()
 669                 done = add_overrides (new, upload, session)
 670                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
 671             except CantGetLockError:
 672                 print "Hello? Operator! Give me the number for 911!"
 673                 print "Dinstall in the locked area, cant process packages, come back later"
 674         elif answer == 'C':
 675             check_pkg(upload)
 676         elif answer == 'E' and not Options["Trainee"]:
 677             new = edit_overrides (new, upload, session)
 678         elif answer == 'M' and not Options["Trainee"]:
 679             upload.pkg.remove_known_changes()
 680             aborted = upload.do_reject(manual=1,
 681                                        reject_message=Options["Manual-Reject"],
 682                                        note=get_new_comments(changes.get("source", ""), session=session))
 683             if not aborted:
 684                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
 685                 os.unlink(upload.pkg.changes_file[:-8]+".dak")
 686                 done = 1
 687         elif answer == 'N':
 688             edit_note(get_new_comments(changes.get("source", ""), session=session),
 689                       upload, session)
 690         elif answer == 'P' and not Options["Trainee"]:
 691             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
 692                             upload)
 693             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
 694         elif answer == 'R' and not Options["Trainee"]:
 695             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 696             if confirm == "y":
 697                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
 698                     session.delete(c)
 699                 session.commit()
 700         elif answer == 'O' and not Options["Trainee"]:
 701             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 702             if confirm == "y":
 703                 for c in get_new_comments(changes.get("source", ""), session=session):
 704                     session.delete(c)
 705                 session.commit()
 706
 707         elif answer == 'S':
 708             done = 1
 709         elif answer == 'Q':
 710             end()
 711             sys.exit(0)
 712
 713 ################################################################################
 714 ################################################################################
 715 ################################################################################
 716
 717 def usage (exit_code=0):
 718     print """Usage: dak process-new [OPTION]... [CHANGES]...
 719   -a, --automatic           automatic run
 720   -h, --help                show this help and exit.
 721   -m, --manual-reject=MSG   manual reject with `msg'
 722   -n, --no-action           don't do anything
 723   -t, --trainee             FTP Trainee mode
 724   -V, --version             display the version number and exit"""
 725     sys.exit(exit_code)
 726
 727 ################################################################################
 728
 729 def do_byhand(upload, session):
 730     done = 0
 731     while not done:
 732         files = upload.pkg.files
 733         will_install = 1
 734         byhand = []
 735
 736         for f in files.keys():
 737             if files[f]["type"] == "byhand":
 738                 if os.path.exists(f):
 739                     print "W: %s still present; please process byhand components and try again." % (f)
 740                     will_install = 0
 741                 else:
 742                     byhand.append(f)
 743
 744         answer = "XXXX"
 745         if Options["No-Action"]:
 746             answer = "S"
 747         if will_install:
 748             if Options["Automatic"] and not Options["No-Action"]:
 749                 answer = 'A'
 750             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 751         else:
 752             prompt = "Manual reject, [S]kip, Quit ?"
 753
 754         while prompt.find(answer) == -1:
 755             answer = utils.our_raw_input(prompt)
 756             m = re_default_answer.search(prompt)
 757             if answer == "":
 758                 answer = m.group(1)
 759             answer = answer[:1].upper()
 760
 761         if answer == 'A':
 762             try:
 763                 check_daily_lock()
 764                 done = 1
 765                 for f in byhand:
 766                     del files[f]
 767                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
 768             except CantGetLockError:
 769                 print "Hello? Operator! Give me the number for 911!"
 770                 print "Dinstall in the locked area, cant process packages, come back later"
 771         elif answer == 'M':
 772             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
 773             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
 774             os.unlink(upload.pkg.changes_file[:-8]+".dak")
 775             done = 1
 776         elif answer == 'S':
 777             done = 1
 778         elif answer == 'Q':
 779             end()
 780             sys.exit(0)
 781
 782 ################################################################################
 783
 784 def check_daily_lock():
 785     """
 786     Raises CantGetLockError if the dinstall daily.lock exists.
 787     """
 788
 789     cnf = Config()
 790     try:
 791         os.open(cnf["Process-New::DinstallLockFile"],
 792                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 793     except OSError, e:
 794         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 795             raise CantGetLockError
 796
 797     os.unlink(cnf["Process-New::DinstallLockFile"])
 798
 799
 800 @contextlib.contextmanager
 801 def lock_package(package):
 802     """
 803     Lock C{package} so that noone else jumps in processing it.
 804
 805     @type package: string
 806     @param package: source package name to lock
 807     """
 808
 809     path = os.path.join(Config()["Process-New::LockDir"], package)
 810     try:
 811         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 812     except OSError, e:
 813         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 814             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 815             raise AlreadyLockedError, user
 816
 817     try:
 818         yield fd
 819     finally:
 820         os.unlink(path)
 821
 822 def move_file_to_queue(to_q, f, session):
 823     """mark a file as being in the unchecked queue"""
 824     # update the queue_file entry for the existing queue
 825     qf = session.query(QueueFile).filter_by(queueid=to_q.queueid,
 826                                             filename=f.filename)
 827     qf.queue = to_q
 828
 829     # update the changes_pending_files row
 830     f.queue = to_q
 831
 832 def changes_to_unchecked(changes, session):
 833     """move a changes file to unchecked"""
 834     unchecked = get_policy_queue('unchecked', session );
 835     changes.in_queue = unchecked
 836
 837     for f in changes.pkg.files:
 838         move_file_to_queue(unchecked, f)
 839
 840     # actually move files
 841     changes.move_to_queue(unchecked)
 842
 843 def _accept(upload):
 844     if Options["No-Action"]:
 845         return
 846     (summary, short_summary) = upload.build_summaries()
 847 #    upload.accept(summary, short_summary, targetqueue)
 848 #    os.unlink(upload.pkg.changes_file[:-8]+".dak")
 849     changes_to_unchecked(upload)
 850
 851 def do_accept(upload):
 852     print "ACCEPT"
 853     cnf = Config()
 854     if not Options["No-Action"]:
 855         (summary, short_summary) = upload.build_summaries()
 856
 857         if cnf.FindB("Dinstall::SecurityQueueHandling"):
 858             upload.dump_vars(cnf["Dir::Queue::Embargoed"])
 859             upload.move_to_queue(get_policy_queue('embargoed'))
 860             upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
 861             # Check for override disparities
 862             upload.Subst["__SUMMARY__"] = summary
 863         else:
 864             # Just a normal upload, accept it...
 865             _accept(upload)
 866
 867 def do_pkg(changes_file, session):
 868     new_queue = get_policy_queue('new', session );
 869     u = Upload()
 870     u.pkg.changes_file = changes_file
 871     u.load_changes(changes_file)
 872     u.pkg.directory = new_queue.path
 873     u.logger = Logger
 874     origchanges = os.path.abspath(u.pkg.changes_file)
 875
 876     cnf = Config()
 877     bcc = "X-DAK: dak process-new"
 878     if cnf.has_key("Dinstall::Bcc"):
 879         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 880     else:
 881         u.Subst["__BCC__"] = bcc
 882
 883     files = u.pkg.files
 884
 885     try:
 886         with lock_package(u.pkg.changes["source"]):
 887             if not recheck(u, session):
 888                 return
 889
 890             do_new(u,session)
 891
 892 #             (new, byhand) = check_status(files)
 893 #             if new or byhand:
 894 #                 if new:
 895 #                     do_new(u, session)
 896 #                 if byhand:
 897 #                     do_byhand(u, session)
 898 #                 (new, byhand) = check_status(files)
 899
 900 #             if not new and not byhand:
 901 #                 try:
 902 #                     check_daily_lock()
 903 #                     do_accept(u)
 904 #                 except CantGetLockError:
 905 #                     print "Hello? Operator! Give me the number for 911!"
 906 #                     print "Dinstall in the locked area, cant process packages, come back later"
 907     except AlreadyLockedError, e:
 908         print "Seems to be locked by %s already, skipping..." % (e)
 909
 910 ################################################################################
 911
 912 def end():
 913     accept_count = SummaryStats().accept_count
 914     accept_bytes = SummaryStats().accept_bytes
 915
 916     if accept_count:
 917         sets = "set"
 918         if accept_count > 1:
 919             sets = "sets"
 920         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 921         Logger.log(["total",accept_count,accept_bytes])
 922
 923     if not Options["No-Action"] and not Options["Trainee"]:
 924         Logger.close()
 925
 926 ################################################################################
 927
 928 def main():
 929     global Options, Logger, Sections, Priorities
 930
 931     cnf = Config()
 932     session = DBConn().session()
 933
 934     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 935                  ('h',"help","Process-New::Options::Help"),
 936                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 937                  ('t',"trainee","Process-New::Options::Trainee"),
 938                  ('n',"no-action","Process-New::Options::No-Action")]
 939
 940     for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
 941         if not cnf.has_key("Process-New::Options::%s" % (i)):
 942             cnf["Process-New::Options::%s" % (i)] = ""
 943
 944     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
 945     if len(changes_files) == 0:
 946         new_queue = get_policy_queue('new', session );
 947         changes_files = utils.get_changes_files(new_queue.path)
 948
 949     Options = cnf.SubTree("Process-New::Options")
 950
 951     if Options["Help"]:
 952         usage()
 953
 954     if not Options["No-Action"]:
 955         try:
 956             Logger = daklog.Logger(cnf, "process-new")
 957         except CantOpenError, e:
 958             Options["Trainee"] = "True"
 959
 960     Sections = Section_Completer(session)
 961     Priorities = Priority_Completer(session)
 962     readline.parse_and_bind("tab: complete")
 963
 964     if len(changes_files) > 1:
 965         sys.stderr.write("Sorting changes...\n")
 966     changes_files = sort_changes(changes_files, session)
 967
 968     # Kill me now? **FIXME**
 969     cnf["Dinstall::Options::No-Mail"] = ""
 970
 971     for changes_file in changes_files:
 972         changes_file = utils.validate_changes_file_arg(changes_file, 0)
 973         if not changes_file:
 974             continue
 975         print "\n" + changes_file
 976
 977         do_pkg (changes_file, session)
 978
 979     end()
 980
 981 ################################################################################
 982
 983 if __name__ == '__main__':
 984     main()