dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 import copy
  46 import errno
  47 import os
  48 import readline
  49 import stat
  50 import sys
  51 import time
  52 import contextlib
  53 import pwd
  54 import apt_pkg, apt_inst
  55 import examine_package
  56 import subprocess
  57 import daklib.daksubprocess
  58 from sqlalchemy import or_
  59
  60 from daklib.dbconn import *
  61 from daklib.queue import *
  62 from daklib import daklog
  63 from daklib import utils
  64 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
  65 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  66 from daklib.summarystats import SummaryStats
  67 from daklib.config import Config
  68 from daklib.policy import UploadCopy, PolicyQueueUploadHandler
  69
  70 # Globals
  71 Options = None
  72 Logger = None
  73
  74 Priorities = None
  75 Sections = None
  76
  77 ################################################################################
  78 ################################################################################
  79 ################################################################################
  80
  81 class Section_Completer:
  82     def __init__ (self, session):
  83         self.sections = []
  84         self.matches = []
  85         for s, in session.query(Section.section):
  86             self.sections.append(s)
  87
  88     def complete(self, text, state):
  89         if state == 0:
  90             self.matches = []
  91             n = len(text)
  92             for word in self.sections:
  93                 if word[:n] == text:
  94                     self.matches.append(word)
  95         try:
  96             return self.matches[state]
  97         except IndexError:
  98             return None
  99
 100 ############################################################
 101
 102 class Priority_Completer:
 103     def __init__ (self, session):
 104         self.priorities = []
 105         self.matches = []
 106         for p, in session.query(Priority.priority):
 107             self.priorities.append(p)
 108
 109     def complete(self, text, state):
 110         if state == 0:
 111             self.matches = []
 112             n = len(text)
 113             for word in self.priorities:
 114                 if word[:n] == text:
 115                     self.matches.append(word)
 116         try:
 117             return self.matches[state]
 118         except IndexError:
 119             return None
 120
 121 ################################################################################
 122
 123 def takenover_binaries(upload, missing, session):
 124     rows = []
 125     binaries = set([x.package for x in upload.binaries])
 126     for m in missing:
 127         if m['type'] != 'dsc':
 128             binaries.discard(m['package'])
 129     if binaries:
 130         source = upload.binaries[0].source.source
 131         suite = upload.target_suite.overridesuite or \
 132                     upload.target_suite.suite_name
 133         suites = [s[0] for s in session.query(Suite.suite_name).filter \
 134                                     (or_(Suite.suite_name == suite,
 135                                      Suite.overridesuite == suite)).all()]
 136         rows = session.query(DBSource.source, DBBinary.package).distinct(). \
 137                              filter(DBBinary.package.in_(binaries)). \
 138                              join(DBBinary.source). \
 139                              filter(DBSource.source != source). \
 140                              join(DBBinary.suites). \
 141                              filter(Suite.suite_name.in_(suites)). \
 142                              order_by(DBSource.source, DBBinary.package).all()
 143     return rows
 144
 145 ################################################################################
 146
 147 def print_new (upload, missing, indexed, session, file=sys.stdout):
 148     check_valid(missing, session)
 149     index = 0
 150     for m in missing:
 151         index += 1
 152         if m['type'] != 'deb':
 153             package = '{0}:{1}'.format(m['type'], m['package'])
 154         else:
 155             package = m['package']
 156         section = m['section']
 157         priority = m['priority']
 158         included = "" if m['included'] else "NOT UPLOADED"
 159         if indexed:
 160             line = "(%s): %-20s %-20s %-20s %s" % (index, package, priority, section, included)
 161         else:
 162             line = "%-20s %-20s %-20s %s" % (package, priority, section, included)
 163         line = line.strip()
 164         if not m['valid']:
 165             line = line + ' [!]'
 166         print >>file, line
 167     takenover = takenover_binaries(upload, missing, session)
 168     if takenover:
 169         print '\n\nBINARIES TAKEN OVER\n'
 170         for t in takenover:
 171             print '%s: %s' % (t[0], t[1])
 172     notes = get_new_comments(upload.policy_queue, upload.changes.source)
 173     for note in notes:
 174         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 175               % (note.author, note.version, note.notedate, note.comment)
 176         print "-" * 72
 177     return len(notes) > 0
 178
 179 ################################################################################
 180
 181 def index_range (index):
 182     if index == 1:
 183         return "1"
 184     else:
 185         return "1-%s" % (index)
 186
 187 ################################################################################
 188 ################################################################################
 189
 190 def edit_new (overrides, upload, session):
 191     # Write the current data to a temporary file
 192     (fd, temp_filename) = utils.temp_filename()
 193     temp_file = os.fdopen(fd, 'w')
 194     print_new (upload, overrides, indexed=0, session=session, file=temp_file)
 195     temp_file.close()
 196     # Spawn an editor on that file
 197     editor = os.environ.get("EDITOR","vi")
 198     result = os.system("%s %s" % (editor, temp_filename))
 199     if result != 0:
 200         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 201     # Read the edited data back in
 202     temp_file = utils.open_file(temp_filename)
 203     lines = temp_file.readlines()
 204     temp_file.close()
 205     os.unlink(temp_filename)
 206
 207     overrides_map = dict([ ((o['type'], o['package']), o) for o in overrides ])
 208     new_overrides = []
 209     # Parse the new data
 210     for line in lines:
 211         line = line.strip()
 212         if line == "" or line[0] == '#':
 213             continue
 214         s = line.split()
 215         # Pad the list if necessary
 216         s[len(s):3] = [None] * (3-len(s))
 217         (pkg, priority, section) = s[:3]
 218         if pkg.find(':') != -1:
 219             type, pkg = pkg.split(':', 1)
 220         else:
 221             type = 'deb'
 222         o = overrides_map.get((type, pkg), None)
 223         if o is None:
 224             utils.warn("Ignoring unknown package '%s'" % (pkg))
 225         else:
 226             if section.find('/') != -1:
 227                 component = section.split('/', 1)[0]
 228             else:
 229                 component = 'main'
 230             new_overrides.append(dict(
 231                     package=pkg,
 232                     type=type,
 233                     section=section,
 234                     component=component,
 235                     priority=priority,
 236                     included=o['included'],
 237                     ))
 238     return new_overrides
 239
 240 ################################################################################
 241
 242 def edit_index (new, upload, index):
 243     package = new[index]['package']
 244     priority = new[index]["priority"]
 245     section = new[index]["section"]
 246     ftype = new[index]["type"]
 247     done = 0
 248     while not done:
 249         print "\t".join([package, priority, section])
 250
 251         answer = "XXX"
 252         if ftype != "dsc":
 253             prompt = "[B]oth, Priority, Section, Done ? "
 254         else:
 255             prompt = "[S]ection, Done ? "
 256         edit_priority = edit_section = 0
 257
 258         while prompt.find(answer) == -1:
 259             answer = utils.our_raw_input(prompt)
 260             m = re_default_answer.match(prompt)
 261             if answer == "":
 262                 answer = m.group(1)
 263             answer = answer[:1].upper()
 264
 265         if answer == 'P':
 266             edit_priority = 1
 267         elif answer == 'S':
 268             edit_section = 1
 269         elif answer == 'B':
 270             edit_priority = edit_section = 1
 271         elif answer == 'D':
 272             done = 1
 273
 274         # Edit the priority
 275         if edit_priority:
 276             readline.set_completer(Priorities.complete)
 277             got_priority = 0
 278             while not got_priority:
 279                 new_priority = utils.our_raw_input("New priority: ").strip()
 280                 if new_priority not in Priorities.priorities:
 281                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 282                 else:
 283                     got_priority = 1
 284                     priority = new_priority
 285
 286         # Edit the section
 287         if edit_section:
 288             readline.set_completer(Sections.complete)
 289             got_section = 0
 290             while not got_section:
 291                 new_section = utils.our_raw_input("New section: ").strip()
 292                 if new_section not in Sections.sections:
 293                     print "E: '%s' is not a valid section, try again." % (new_section)
 294                 else:
 295                     got_section = 1
 296                     section = new_section
 297
 298         # Reset the readline completer
 299         readline.set_completer(None)
 300
 301     new[index]["priority"] = priority
 302     new[index]["section"] = section
 303     if section.find('/') != -1:
 304         component = section.split('/', 1)[0]
 305     else:
 306         component = 'main'
 307     new[index]['component'] = component
 308
 309     return new
 310
 311 ################################################################################
 312
 313 def edit_overrides (new, upload, session):
 314     print
 315     done = 0
 316     while not done:
 317         print_new (upload, new, indexed=1, session=session)
 318         prompt = "edit override <n>, Editor, Done ? "
 319
 320         got_answer = 0
 321         while not got_answer:
 322             answer = utils.our_raw_input(prompt)
 323             if not answer.isdigit():
 324                 answer = answer[:1].upper()
 325             if answer == "E" or answer == "D":
 326                 got_answer = 1
 327             elif re_isanum.match (answer):
 328                 answer = int(answer)
 329                 if answer < 1 or answer > len(new):
 330                     print "{0} is not a valid index.  Please retry.".format(answer)
 331                 else:
 332                     got_answer = 1
 333
 334         if answer == 'E':
 335             new = edit_new(new, upload, session)
 336         elif answer == 'D':
 337             done = 1
 338         else:
 339             edit_index (new, upload, answer - 1)
 340
 341     return new
 342
 343
 344 ################################################################################
 345
 346 def check_pkg (upload, upload_copy, session):
 347     missing = []
 348     save_stdout = sys.stdout
 349     changes = os.path.join(upload_copy.directory, upload.changes.changesname)
 350     suite_name = upload.target_suite.suite_name
 351     handler = PolicyQueueUploadHandler(upload, session)
 352     missing = [(m['type'], m["package"]) for m in handler.missing_overrides(hints=missing)]
 353
 354     less_cmd = ("less", "-R", "-")
 355     less_process = daklib.daksubprocess.Popen(less_cmd, bufsize=0, stdin=subprocess.PIPE)
 356     try:
 357         sys.stdout = less_process.stdin
 358         print examine_package.display_changes(suite_name, changes)
 359
 360         source = upload.source
 361         if source is not None:
 362             source_file = os.path.join(upload_copy.directory, os.path.basename(source.poolfile.filename))
 363             print examine_package.check_dsc(suite_name, source_file)
 364
 365         for binary in upload.binaries:
 366             binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename))
 367             examined = examine_package.check_deb(suite_name, binary_file)
 368             # We always need to call check_deb to display package relations for every binary,
 369             # but we print its output only if new overrides are being added.
 370             if ("deb", binary.package) in missing:
 371                 print examined
 372
 373         print examine_package.output_package_relations()
 374         less_process.stdin.close()
 375     except IOError as e:
 376         if e.errno == errno.EPIPE:
 377             utils.warn("[examine_package] Caught EPIPE; skipping.")
 378         else:
 379             raise
 380     except KeyboardInterrupt:
 381         utils.warn("[examine_package] Caught C-c; skipping.")
 382     finally:
 383         less_process.wait()
 384         sys.stdout = save_stdout
 385
 386 ################################################################################
 387
 388 ## FIXME: horribly Debian specific
 389
 390 def do_bxa_notification(new, upload, session):
 391     cnf = Config()
 392
 393     new = set([ o['package'] for o in new if o['type'] == 'deb' ])
 394     if len(new) == 0:
 395         return
 396
 397     key = session.query(MetadataKey).filter_by(key='Description').one()
 398     summary = ""
 399     for binary in upload.binaries:
 400         if binary.package not in new:
 401             continue
 402         description = session.query(BinaryMetadata).filter_by(binary=binary, key=key).one().value
 403         summary += "\n"
 404         summary += "Package: {0}\n".format(binary.package)
 405         summary += "Description: {0}\n".format(description)
 406
 407     subst = {
 408         '__DISTRO__': cnf['Dinstall::MyDistribution'],
 409         '__BCC__': 'X-DAK: dak process-new',
 410         '__BINARY_DESCRIPTIONS__': summary,
 411         }
 412
 413     bxa_mail = utils.TemplateSubst(subst,os.path.join(cnf["Dir::Templates"], "process-new.bxa_notification"))
 414     utils.send_mail(bxa_mail)
 415
 416 ################################################################################
 417
 418 def add_overrides (new_overrides, suite, session):
 419     if suite.overridesuite is not None:
 420         suite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 421
 422     for override in new_overrides:
 423         package = override['package']
 424         priority = session.query(Priority).filter_by(priority=override['priority']).first()
 425         section = session.query(Section).filter_by(section=override['section']).first()
 426         component = get_mapped_component(override['component'], session)
 427         overridetype = session.query(OverrideType).filter_by(overridetype=override['type']).one()
 428
 429         if priority is None:
 430             raise Exception('Invalid priority {0} for package {1}'.format(priority, package))
 431         if section is None:
 432             raise Exception('Invalid section {0} for package {1}'.format(section, package))
 433         if component is None:
 434             raise Exception('Invalid component {0} for package {1}'.format(component, package))
 435
 436         o = Override(package=package, suite=suite, component=component, priority=priority, section=section, overridetype=overridetype)
 437         session.add(o)
 438
 439     session.commit()
 440
 441 ################################################################################
 442
 443 def run_user_inspect_command(upload, upload_copy):
 444     command = os.environ.get('DAK_INSPECT_UPLOAD')
 445     if command is None:
 446         return
 447
 448     directory = upload_copy.directory
 449     if upload.source:
 450         dsc = os.path.basename(upload.source.poolfile.filename)
 451     else:
 452         dsc = ''
 453     changes = upload.changes.changesname
 454
 455     shell_command = command.format(
 456             directory=directory,
 457             dsc=dsc,
 458             changes=changes,
 459             )
 460
 461     daklib.daksubprocess.check_call(shell_command, shell=True)
 462
 463 ################################################################################
 464
 465 def get_reject_reason(reason=''):
 466     """get reason for rejection
 467
 468     @rtype:  str
 469     @return: string giving the reason for the rejection or C{None} if the
 470              rejection should be cancelled
 471     """
 472     answer = 'E'
 473     if Options['Automatic']:
 474         answer = 'R'
 475
 476     while answer == 'E':
 477         reason = utils.call_editor(reason)
 478         print "Reject message:"
 479         print utils.prefix_multi_line_string(reason, "  ", include_blank_lines=1)
 480         prompt = "[R]eject, Edit, Abandon, Quit ?"
 481         answer = "XXX"
 482         while prompt.find(answer) == -1:
 483             answer = utils.our_raw_input(prompt)
 484             m = re_default_answer.search(prompt)
 485             if answer == "":
 486                 answer = m.group(1)
 487             answer = answer[:1].upper()
 488
 489     if answer == 'Q':
 490         sys.exit(0)
 491
 492     if answer == 'R':
 493         return reason
 494     return None
 495
 496 ################################################################################
 497
 498 def do_new(upload, upload_copy, handler, session):
 499     cnf = Config()
 500
 501     run_user_inspect_command(upload, upload_copy)
 502
 503     # The main NEW processing loop
 504     done = False
 505     missing = []
 506     while not done:
 507         queuedir = upload.policy_queue.path
 508         byhand = upload.byhand
 509
 510         missing = handler.missing_overrides(hints=missing)
 511         broken = not check_valid(missing, session)
 512
 513         changesname = os.path.basename(upload.changes.changesname)
 514
 515         print
 516         print changesname
 517         print "-" * len(changesname)
 518         print
 519         print "   Target:     {0}".format(upload.target_suite.suite_name)
 520         print "   Changed-By: {0}".format(upload.changes.changedby)
 521         print
 522
 523         #if len(byhand) == 0 and len(missing) == 0:
 524         #    break
 525
 526         if missing:
 527             print "NEW\n"
 528
 529         answer = "XXX"
 530         if Options["No-Action"] or Options["Automatic"]:
 531             answer = 'S'
 532
 533         note = print_new(upload, missing, indexed=0, session=session)
 534         prompt = ""
 535
 536         has_unprocessed_byhand = False
 537         for f in byhand:
 538             path = os.path.join(queuedir, f.filename)
 539             if not f.processed and os.path.exists(path):
 540                 print "W: {0} still present; please process byhand components and try again".format(f.filename)
 541                 has_unprocessed_byhand = True
 542
 543         if not has_unprocessed_byhand and not broken and not note:
 544             if len(missing) == 0:
 545                 prompt = "Accept, "
 546                 answer = 'A'
 547             else:
 548                 prompt = "Add overrides, "
 549         if broken:
 550             print "W: [!] marked entries must be fixed before package can be processed."
 551         if note:
 552             print "W: note must be removed before package can be processed."
 553             prompt += "RemOve all notes, Remove note, "
 554
 555         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 556
 557         while prompt.find(answer) == -1:
 558             answer = utils.our_raw_input(prompt)
 559             m = re_default_answer.search(prompt)
 560             if answer == "":
 561                 answer = m.group(1)
 562             answer = answer[:1].upper()
 563
 564         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 565             utils.warn("Trainees can't do that")
 566             continue
 567
 568         if answer == 'A' and not Options["Trainee"]:
 569             add_overrides(missing, upload.target_suite, session)
 570             if Config().find_b("Dinstall::BXANotify"):
 571                 do_bxa_notification(missing, upload, session)
 572             handler.accept()
 573             done = True
 574             Logger.log(["NEW ACCEPT", upload.changes.changesname])
 575         elif answer == 'C':
 576             check_pkg(upload, upload_copy, session)
 577         elif answer == 'E' and not Options["Trainee"]:
 578             missing = edit_overrides (missing, upload, session)
 579         elif answer == 'M' and not Options["Trainee"]:
 580             reason = Options.get('Manual-Reject', '') + "\n"
 581             reason = reason + "\n\n=====\n\n".join([n.comment for n in get_new_comments(upload.policy_queue, upload.changes.source, session=session)])
 582             reason = get_reject_reason(reason)
 583             if reason is not None:
 584                 Logger.log(["NEW REJECT", upload.changes.changesname])
 585                 handler.reject(reason)
 586                 done = True
 587         elif answer == 'N':
 588             if edit_note(get_new_comments(upload.policy_queue, upload.changes.source, session=session),
 589                          upload, session, bool(Options["Trainee"])) == 0:
 590                 end()
 591                 sys.exit(0)
 592         elif answer == 'P' and not Options["Trainee"]:
 593             if prod_maintainer(get_new_comments(upload.policy_queue, upload.changes.source, session=session),
 594                                upload) == 0:
 595                 end()
 596                 sys.exit(0)
 597             Logger.log(["NEW PROD", upload.changes.changesname])
 598         elif answer == 'R' and not Options["Trainee"]:
 599             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 600             if confirm == "y":
 601                 for c in get_new_comments(upload.policy_queue, upload.changes.source, upload.changes.version, session=session):
 602                     session.delete(c)
 603                 session.commit()
 604         elif answer == 'O' and not Options["Trainee"]:
 605             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 606             if confirm == "y":
 607                 for c in get_new_comments(upload.policy_queue, upload.changes.source, session=session):
 608                     session.delete(c)
 609                 session.commit()
 610
 611         elif answer == 'S':
 612             done = True
 613         elif answer == 'Q':
 614             end()
 615             sys.exit(0)
 616
 617         if handler.get_action():
 618             print "PENDING %s\n" % handler.get_action()
 619
 620 ################################################################################
 621 ################################################################################
 622 ################################################################################
 623
 624 def usage (exit_code=0):
 625     print """Usage: dak process-new [OPTION]... [CHANGES]...
 626   -a, --automatic           automatic run
 627   -b, --no-binaries         do not sort binary-NEW packages first
 628   -c, --comments            show NEW comments
 629   -h, --help                show this help and exit.
 630   -m, --manual-reject=MSG   manual reject with `msg'
 631   -n, --no-action           don't do anything
 632   -q, --queue=QUEUE         operate on a different queue
 633   -t, --trainee             FTP Trainee mode
 634   -V, --version             display the version number and exit
 635
 636 ENVIRONMENT VARIABLES
 637
 638   DAK_INSPECT_UPLOAD: shell command to run to inspect a package
 639       The command is automatically run in a shell when an upload
 640       is checked.  The following substitutions are available:
 641
 642         {directory}: directory the upload is contained in
 643         {dsc}:       name of the included dsc or the empty string
 644         {changes}:   name of the changes file
 645
 646       Note that Python's 'format' method is used to format the command.
 647
 648       Example: run mc in a tmux session to inspect the upload
 649
 650       export DAK_INSPECT_UPLOAD='tmux new-session -d -s process-new 2>/dev/null; tmux new-window -n "{changes}" -t process-new:0 -k "cd {directory}; mc"'
 651
 652       and run
 653
 654       tmux attach -t process-new
 655
 656       in a separate terminal session.
 657 """
 658     sys.exit(exit_code)
 659
 660 ################################################################################
 661
 662 @contextlib.contextmanager
 663 def lock_package(package):
 664     """
 665     Lock C{package} so that noone else jumps in processing it.
 666
 667     @type package: string
 668     @param package: source package name to lock
 669     """
 670
 671     cnf = Config()
 672
 673     path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package)
 674
 675     try:
 676         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 677     except OSError as e:
 678         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 679             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 680             raise AlreadyLockedError(user)
 681
 682     try:
 683         yield fd
 684     finally:
 685         os.unlink(path)
 686
 687 def do_pkg(upload, session):
 688     # Try to get an included dsc
 689     dsc = upload.source
 690
 691     cnf = Config()
 692     group = cnf.get('Dinstall::UnprivGroup') or None
 693
 694     #bcc = "X-DAK: dak process-new"
 695     #if cnf.has_key("Dinstall::Bcc"):
 696     #    u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 697     #else:
 698     #    u.Subst["__BCC__"] = bcc
 699
 700     try:
 701       with lock_package(upload.changes.source):
 702        with UploadCopy(upload, group=group) as upload_copy:
 703         handler = PolicyQueueUploadHandler(upload, session)
 704         if handler.get_action() is not None:
 705             print "PENDING %s\n" % handler.get_action()
 706             return
 707
 708         do_new(upload, upload_copy, handler, session)
 709     except AlreadyLockedError as e:
 710         print "Seems to be locked by %s already, skipping..." % (e)
 711
 712 def show_new_comments(uploads, session):
 713     sources = [ upload.changes.source for upload in uploads ]
 714     if len(sources) == 0:
 715         return
 716
 717     query = """SELECT package, version, comment, author
 718                FROM new_comments
 719                WHERE package IN :sources
 720                ORDER BY package, version"""
 721
 722     r = session.execute(query, params=dict(sources=tuple(sources)))
 723
 724     for i in r:
 725         print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3])
 726
 727     session.rollback()
 728
 729 ################################################################################
 730
 731 def sort_uploads(new_queue, uploads, session, nobinaries=False):
 732     sources = {}
 733     sorteduploads = []
 734     suitesrc = [s.source for s in session.query(DBSource.source). \
 735       filter(DBSource.suites.any(Suite.suite_name.in_(['unstable', 'experimental'])))]
 736     comments = [p.package for p in session.query(NewComment.package). \
 737       filter_by(trainee=False, policy_queue=new_queue).distinct()]
 738     for upload in uploads:
 739         source = upload.changes.source
 740         if not source in sources:
 741             sources[source] = []
 742         sources[source].append({'upload': upload,
 743                                 'date': upload.changes.created,
 744                                 'stack': 1,
 745                                 'binary': True if source in suitesrc else False,
 746                                 'comments': True if source in comments else False})
 747     for src in sources:
 748         if len(sources[src]) > 1:
 749             changes = sources[src]
 750             firstseen = sorted(changes, key=lambda k: (k['date']))[0]['date']
 751             changes.sort(key=lambda item:item['date'])
 752             for i in range (0, len(changes)):
 753                 changes[i]['date'] = firstseen
 754                 changes[i]['stack'] = i + 1
 755         sorteduploads += sources[src]
 756     if nobinaries:
 757         sorteduploads = [u["upload"] for u in sorted(sorteduploads,
 758                          key=lambda k: (k["comments"], k["binary"],
 759                          k["date"], -k["stack"]))]
 760     else:
 761         sorteduploads = [u["upload"] for u in sorted(sorteduploads,
 762                          key=lambda k: (k["comments"], -k["binary"],
 763                          k["date"], -k["stack"]))]
 764     return sorteduploads
 765
 766 ################################################################################
 767
 768 def end():
 769     accept_count = SummaryStats().accept_count
 770     accept_bytes = SummaryStats().accept_bytes
 771
 772     if accept_count:
 773         sets = "set"
 774         if accept_count > 1:
 775             sets = "sets"
 776         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 777         Logger.log(["total",accept_count,accept_bytes])
 778
 779     if not Options["No-Action"] and not Options["Trainee"]:
 780         Logger.close()
 781
 782 ################################################################################
 783
 784 def main():
 785     global Options, Logger, Sections, Priorities
 786
 787     cnf = Config()
 788     session = DBConn().session()
 789
 790     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 791                  ('b',"no-binaries","Process-New::Options::No-Binaries"),
 792                  ('c',"comments","Process-New::Options::Comments"),
 793                  ('h',"help","Process-New::Options::Help"),
 794                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 795                  ('t',"trainee","Process-New::Options::Trainee"),
 796                  ('q','queue','Process-New::Options::Queue', 'HasArg'),
 797                  ('n',"no-action","Process-New::Options::No-Action")]
 798
 799     changes_files = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
 800
 801     for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:
 802         if not cnf.has_key("Process-New::Options::%s" % (i)):
 803             cnf["Process-New::Options::%s" % (i)] = ""
 804
 805     queue_name = cnf.get('Process-New::Options::Queue', 'new')
 806     new_queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
 807     if len(changes_files) == 0:
 808         uploads = new_queue.uploads
 809     else:
 810         uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \
 811             .join(DBChange).filter(DBChange.changesname.in_(changes_files)).all()
 812
 813     Options = cnf.subtree("Process-New::Options")
 814
 815     if Options["Help"]:
 816         usage()
 817
 818     if not Options["No-Action"]:
 819         try:
 820             Logger = daklog.Logger("process-new")
 821         except CantOpenError as e:
 822             Options["Trainee"] = "True"
 823
 824     Sections = Section_Completer(session)
 825     Priorities = Priority_Completer(session)
 826     readline.parse_and_bind("tab: complete")
 827
 828     if len(uploads) > 1:
 829         sys.stderr.write("Sorting changes...\n")
 830         uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"])
 831
 832     if Options["Comments"]:
 833         show_new_comments(uploads, session)
 834     else:
 835         for upload in uploads:
 836             do_pkg (upload, session)
 837
 838     end()
 839
 840 ################################################################################
 841
 842 if __name__ == '__main__':
 843     main()