dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 import copy
  46 import errno
  47 import os
  48 import readline
  49 import stat
  50 import sys
  51 import time
  52 import contextlib
  53 import pwd
  54 import apt_pkg, apt_inst
  55 import examine_package
  56 import subprocess
  57
  58 from daklib.dbconn import *
  59 from daklib.queue import *
  60 from daklib import daklog
  61 from daklib import utils
  62 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
  63 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  64 from daklib.summarystats import SummaryStats
  65 from daklib.config import Config
  66 from daklib.policy import UploadCopy, PolicyQueueUploadHandler
  67 from sqlalchemy.sql import not_
  68
  69 # Globals
  70 Options = None
  71 Logger = None
  72
  73 Priorities = None
  74 Sections = None
  75
  76 ################################################################################
  77 ################################################################################
  78 ################################################################################
  79
  80 class Section_Completer:
  81     def __init__ (self, session):
  82         self.sections = []
  83         self.matches = []
  84         for s, in session.query(Section.section):
  85             self.sections.append(s)
  86
  87     def complete(self, text, state):
  88         if state == 0:
  89             self.matches = []
  90             n = len(text)
  91             for word in self.sections:
  92                 if word[:n] == text:
  93                     self.matches.append(word)
  94         try:
  95             return self.matches[state]
  96         except IndexError:
  97             return None
  98
  99 ############################################################
 100
 101 class Priority_Completer:
 102     def __init__ (self, session):
 103         self.priorities = []
 104         self.matches = []
 105         for p, in session.query(Priority.priority):
 106             self.priorities.append(p)
 107
 108     def complete(self, text, state):
 109         if state == 0:
 110             self.matches = []
 111             n = len(text)
 112             for word in self.priorities:
 113                 if word[:n] == text:
 114                     self.matches.append(word)
 115         try:
 116             return self.matches[state]
 117         except IndexError:
 118             return None
 119
 120 ################################################################################
 121
 122 def claimed_overrides(upload, missing, session):
 123     source = [upload.source.source]
 124     binaries = set([x.package for x in upload.binaries])
 125     suites = ('unstable','experimental')
 126     for m in missing:
 127         if m['type'] != 'dsc':
 128             binaries.remove(m['package'])
 129     return session.query(DBBinary).filter(DBBinary.package.in_(binaries)). \
 130                          join(DBBinary.source). \
 131                          filter(not_(DBSource.source.in_(source))). \
 132                          join(DBBinary.suites). \
 133                          filter(Suite.suite_name.in_(suites))
 134
 135 ################################################################################
 136
 137 def print_new (upload, missing, indexed, session, file=sys.stdout):
 138     check_valid(missing, session)
 139     index = 0
 140     for m in missing:
 141         index += 1
 142         if m['type'] != 'deb':
 143             package = '{0}:{1}'.format(m['type'], m['package'])
 144         else:
 145             package = m['package']
 146         section = m['section']
 147         priority = m['priority']
 148         if indexed:
 149             line = "(%s): %-20s %-20s %-20s" % (index, package, priority, section)
 150         else:
 151             line = "%-20s %-20s %-20s" % (package, priority, section)
 152         line = line.strip()
 153         if not m['valid']:
 154             line = line + ' [!]'
 155         print >>file, line
 156     claimed = claimed_overrides(upload, missing, session)
 157     if claimed.count():
 158         print '\nCLAIMED OVERRIDES'
 159         for c in claimed:
 160             print '%s:\t%s' % (c.source.source, c.package)
 161     notes = get_new_comments(upload.policy_queue, upload.changes.source)
 162     for note in notes:
 163         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 164               % (note.author, note.version, note.notedate, note.comment)
 165         print "-" * 72
 166     return len(notes) > 0
 167
 168 ################################################################################
 169
 170 def index_range (index):
 171     if index == 1:
 172         return "1"
 173     else:
 174         return "1-%s" % (index)
 175
 176 ################################################################################
 177 ################################################################################
 178
 179 def edit_new (overrides, upload, session):
 180     # Write the current data to a temporary file
 181     (fd, temp_filename) = utils.temp_filename()
 182     temp_file = os.fdopen(fd, 'w')
 183     print_new (upload, overrides, indexed=0, session=session, file=temp_file)
 184     temp_file.close()
 185     # Spawn an editor on that file
 186     editor = os.environ.get("EDITOR","vi")
 187     result = os.system("%s %s" % (editor, temp_filename))
 188     if result != 0:
 189         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 190     # Read the edited data back in
 191     temp_file = utils.open_file(temp_filename)
 192     lines = temp_file.readlines()
 193     temp_file.close()
 194     os.unlink(temp_filename)
 195
 196     overrides_map = dict([ ((o['type'], o['package']), o) for o in overrides ])
 197     new_overrides = []
 198     # Parse the new data
 199     for line in lines:
 200         line = line.strip()
 201         if line == "" or line[0] == '#':
 202             continue
 203         s = line.split()
 204         # Pad the list if necessary
 205         s[len(s):3] = [None] * (3-len(s))
 206         (pkg, priority, section) = s[:3]
 207         if pkg.find(':') != -1:
 208             type, pkg = pkg.split(':', 1)
 209         else:
 210             type = 'deb'
 211         if (type, pkg) not in overrides_map:
 212             utils.warn("Ignoring unknown package '%s'" % (pkg))
 213         else:
 214             if section.find('/') != -1:
 215                 component = section.split('/', 1)[0]
 216             else:
 217                 component = 'main'
 218             new_overrides.append(dict(
 219                     package=pkg,
 220                     type=type,
 221                     section=section,
 222                     component=component,
 223                     priority=priority,
 224                     ))
 225     return new_overrides
 226
 227 ################################################################################
 228
 229 def edit_index (new, upload, index):
 230     package = new[index]['package']
 231     priority = new[index]["priority"]
 232     section = new[index]["section"]
 233     ftype = new[index]["type"]
 234     done = 0
 235     while not done:
 236         print "\t".join([package, priority, section])
 237
 238         answer = "XXX"
 239         if ftype != "dsc":
 240             prompt = "[B]oth, Priority, Section, Done ? "
 241         else:
 242             prompt = "[S]ection, Done ? "
 243         edit_priority = edit_section = 0
 244
 245         while prompt.find(answer) == -1:
 246             answer = utils.our_raw_input(prompt)
 247             m = re_default_answer.match(prompt)
 248             if answer == "":
 249                 answer = m.group(1)
 250             answer = answer[:1].upper()
 251
 252         if answer == 'P':
 253             edit_priority = 1
 254         elif answer == 'S':
 255             edit_section = 1
 256         elif answer == 'B':
 257             edit_priority = edit_section = 1
 258         elif answer == 'D':
 259             done = 1
 260
 261         # Edit the priority
 262         if edit_priority:
 263             readline.set_completer(Priorities.complete)
 264             got_priority = 0
 265             while not got_priority:
 266                 new_priority = utils.our_raw_input("New priority: ").strip()
 267                 if new_priority not in Priorities.priorities:
 268                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 269                 else:
 270                     got_priority = 1
 271                     priority = new_priority
 272
 273         # Edit the section
 274         if edit_section:
 275             readline.set_completer(Sections.complete)
 276             got_section = 0
 277             while not got_section:
 278                 new_section = utils.our_raw_input("New section: ").strip()
 279                 if new_section not in Sections.sections:
 280                     print "E: '%s' is not a valid section, try again." % (new_section)
 281                 else:
 282                     got_section = 1
 283                     section = new_section
 284
 285         # Reset the readline completer
 286         readline.set_completer(None)
 287
 288     new[index]["priority"] = priority
 289     new[index]["section"] = section
 290     if section.find('/') != -1:
 291         component = section.split('/', 1)[0]
 292     else:
 293         component = 'main'
 294     new[index]['component'] = component
 295
 296     return new
 297
 298 ################################################################################
 299
 300 def edit_overrides (new, upload, session):
 301     print
 302     done = 0
 303     while not done:
 304         print_new (upload, new, indexed=1, session=session)
 305         prompt = "edit override <n>, Editor, Done ? "
 306
 307         got_answer = 0
 308         while not got_answer:
 309             answer = utils.our_raw_input(prompt)
 310             if not answer.isdigit():
 311                 answer = answer[:1].upper()
 312             if answer == "E" or answer == "D":
 313                 got_answer = 1
 314             elif re_isanum.match (answer):
 315                 answer = int(answer)
 316                 if answer < 1 or answer > len(new):
 317                     print "{0} is not a valid index.  Please retry.".format(answer)
 318                 else:
 319                     got_answer = 1
 320
 321         if answer == 'E':
 322             new = edit_new(new, upload, session)
 323         elif answer == 'D':
 324             done = 1
 325         else:
 326             edit_index (new, upload, answer - 1)
 327
 328     return new
 329
 330
 331 ################################################################################
 332
 333 def check_pkg (upload, upload_copy, session):
 334     missing = []
 335     save_stdout = sys.stdout
 336     changes = os.path.join(upload_copy.directory, upload.changes.changesname)
 337     suite_name = upload.target_suite.suite_name
 338     handler = PolicyQueueUploadHandler(upload, session)
 339     missing = [(m['type'], m["package"]) for m in handler.missing_overrides(hints=missing)]
 340     try:
 341         sys.stdout = os.popen("less -R -", 'w', 0)
 342         print examine_package.display_changes(suite_name, changes)
 343
 344         source = upload.source
 345         if source is not None:
 346             source_file = os.path.join(upload_copy.directory, os.path.basename(source.poolfile.filename))
 347             print examine_package.check_dsc(suite_name, source_file)
 348
 349         for binary in upload.binaries:
 350             binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename))
 351             examined = examine_package.check_deb(suite_name, binary_file)
 352             # We always need to call check_deb to display package relations for every binary,
 353             # but we print its output only if new overrides are being added.
 354             if ("deb", binary.package) in missing:
 355                 print examined
 356
 357         print examine_package.output_package_relations()
 358     except IOError as e:
 359         if e.errno == errno.EPIPE:
 360             utils.warn("[examine_package] Caught EPIPE; skipping.")
 361         else:
 362             raise
 363     except KeyboardInterrupt:
 364         utils.warn("[examine_package] Caught C-c; skipping.")
 365     finally:
 366         sys.stdout = save_stdout
 367
 368 ################################################################################
 369
 370 ## FIXME: horribly Debian specific
 371
 372 def do_bxa_notification(new, upload, session):
 373     cnf = Config()
 374
 375     new = set([ o['package'] for o in new if o['type'] == 'deb' ])
 376     if len(new) == 0:
 377         return
 378
 379     key = session.query(MetadataKey).filter_by(key='Description').one()
 380     summary = ""
 381     for binary in upload.binaries:
 382         if binary.package not in new:
 383             continue
 384         description = session.query(BinaryMetadata).filter_by(binary=binary, key=key).one().value
 385         summary += "\n"
 386         summary += "Package: {0}\n".format(binary.package)
 387         summary += "Description: {0}\n".format(description)
 388
 389     subst = {
 390         '__DISTRO__': cnf['Dinstall::MyDistribution'],
 391         '__BCC__': 'X-DAK: dak process-new',
 392         '__BINARY_DESCRIPTIONS__': summary,
 393         }
 394
 395     bxa_mail = utils.TemplateSubst(subst,os.path.join(cnf["Dir::Templates"], "process-new.bxa_notification"))
 396     utils.send_mail(bxa_mail)
 397
 398 ################################################################################
 399
 400 def add_overrides (new_overrides, suite, session):
 401     if suite.overridesuite is not None:
 402         suite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 403
 404     for override in new_overrides:
 405         package = override['package']
 406         priority = session.query(Priority).filter_by(priority=override['priority']).first()
 407         section = session.query(Section).filter_by(section=override['section']).first()
 408         component = get_mapped_component(override['component'], session)
 409         overridetype = session.query(OverrideType).filter_by(overridetype=override['type']).one()
 410
 411         if priority is None:
 412             raise Exception('Invalid priority {0} for package {1}'.format(priority, package))
 413         if section is None:
 414             raise Exception('Invalid section {0} for package {1}'.format(section, package))
 415         if component is None:
 416             raise Exception('Invalid component {0} for package {1}'.format(component, package))
 417
 418         o = Override(package=package, suite=suite, component=component, priority=priority, section=section, overridetype=overridetype)
 419         session.add(o)
 420
 421     session.commit()
 422
 423 ################################################################################
 424
 425 def run_user_inspect_command(upload, upload_copy):
 426     command = os.environ.get('DAK_INSPECT_UPLOAD')
 427     if command is None:
 428         return
 429
 430     directory = upload_copy.directory
 431     if upload.source:
 432         dsc = os.path.basename(upload.source.poolfile.filename)
 433     else:
 434         dsc = ''
 435     changes = upload.changes.changesname
 436
 437     shell_command = command.format(
 438             directory=directory,
 439             dsc=dsc,
 440             changes=changes,
 441             )
 442
 443     subprocess.check_call(shell_command, shell=True)
 444
 445 ################################################################################
 446
 447 def get_reject_reason(reason=''):
 448     """get reason for rejection
 449
 450     @rtype:  str
 451     @return: string giving the reason for the rejection or C{None} if the
 452              rejection should be cancelled
 453     """
 454     answer = 'E'
 455     if Options['Automatic']:
 456         answer = 'R'
 457
 458     while answer == 'E':
 459         reason = utils.call_editor(reason)
 460         print "Reject message:"
 461         print utils.prefix_multi_line_string(reason, "  ", include_blank_lines=1)
 462         prompt = "[R]eject, Edit, Abandon, Quit ?"
 463         answer = "XXX"
 464         while prompt.find(answer) == -1:
 465             answer = utils.our_raw_input(prompt)
 466             m = re_default_answer.search(prompt)
 467             if answer == "":
 468                 answer = m.group(1)
 469             answer = answer[:1].upper()
 470
 471     if answer == 'Q':
 472         sys.exit(0)
 473
 474     if answer == 'R':
 475         return reason
 476     return None
 477
 478 ################################################################################
 479
 480 def do_new(upload, upload_copy, handler, session):
 481     cnf = Config()
 482
 483     run_user_inspect_command(upload, upload_copy)
 484
 485     # The main NEW processing loop
 486     done = False
 487     missing = []
 488     while not done:
 489         queuedir = upload.policy_queue.path
 490         byhand = upload.byhand
 491
 492         missing = handler.missing_overrides(hints=missing)
 493         broken = not check_valid(missing, session)
 494
 495         changesname = os.path.basename(upload.changes.changesname)
 496
 497         print
 498         print changesname
 499         print "-" * len(changesname)
 500         print
 501         print "   Target:     {0}".format(upload.target_suite.suite_name)
 502         print "   Changed-By: {0}".format(upload.changes.changedby)
 503         print
 504
 505         #if len(byhand) == 0 and len(missing) == 0:
 506         #    break
 507
 508         if missing:
 509             print "NEW\n"
 510
 511         answer = "XXX"
 512         if Options["No-Action"] or Options["Automatic"]:
 513             answer = 'S'
 514
 515         note = print_new(upload, missing, indexed=0, session=session)
 516         prompt = ""
 517
 518         has_unprocessed_byhand = False
 519         for f in byhand:
 520             path = os.path.join(queuedir, f.filename)
 521             if not f.processed and os.path.exists(path):
 522                 print "W: {0} still present; please process byhand components and try again".format(f.filename)
 523                 has_unprocessed_byhand = True
 524
 525         if not has_unprocessed_byhand and not broken and not note:
 526             if len(missing) == 0:
 527                 prompt = "Accept, "
 528                 answer = 'A'
 529             else:
 530                 prompt = "Add overrides, "
 531         if broken:
 532             print "W: [!] marked entries must be fixed before package can be processed."
 533         if note:
 534             print "W: note must be removed before package can be processed."
 535             prompt += "RemOve all notes, Remove note, "
 536
 537         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 538
 539         while prompt.find(answer) == -1:
 540             answer = utils.our_raw_input(prompt)
 541             m = re_default_answer.search(prompt)
 542             if answer == "":
 543                 answer = m.group(1)
 544             answer = answer[:1].upper()
 545
 546         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 547             utils.warn("Trainees can't do that")
 548             continue
 549
 550         if answer == 'A' and not Options["Trainee"]:
 551             add_overrides(missing, upload.target_suite, session)
 552             if Config().find_b("Dinstall::BXANotify"):
 553                 do_bxa_notification(missing, upload, session)
 554             handler.accept()
 555             done = True
 556             Logger.log(["NEW ACCEPT", upload.changes.changesname])
 557         elif answer == 'C':
 558             check_pkg(upload, upload_copy, session)
 559         elif answer == 'E' and not Options["Trainee"]:
 560             missing = edit_overrides (missing, upload, session)
 561         elif answer == 'M' and not Options["Trainee"]:
 562             reason = Options.get('Manual-Reject', '') + "\n"
 563             reason = reason + "\n\n=====\n\n".join([n.comment for n in get_new_comments(upload.policy_queue, upload.changes.source, session=session)])
 564             reason = get_reject_reason(reason)
 565             if reason is not None:
 566                 Logger.log(["NEW REJECT", upload.changes.changesname])
 567                 handler.reject(reason)
 568                 done = True
 569         elif answer == 'N':
 570             if edit_note(get_new_comments(upload.policy_queue, upload.changes.source, session=session),
 571                          upload, session, bool(Options["Trainee"])) == 0:
 572                 end()
 573                 sys.exit(0)
 574         elif answer == 'P' and not Options["Trainee"]:
 575             if prod_maintainer(get_new_comments(upload.policy_queue, upload.changes.source, session=session),
 576                                upload) == 0:
 577                 end()
 578                 sys.exit(0)
 579             Logger.log(["NEW PROD", upload.changes.changesname])
 580         elif answer == 'R' and not Options["Trainee"]:
 581             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 582             if confirm == "y":
 583                 for c in get_new_comments(upload.policy_queue, upload.changes.source, upload.changes.version, session=session):
 584                     session.delete(c)
 585                 session.commit()
 586         elif answer == 'O' and not Options["Trainee"]:
 587             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 588             if confirm == "y":
 589                 for c in get_new_comments(upload.policy_queue, upload.changes.source, session=session):
 590                     session.delete(c)
 591                 session.commit()
 592
 593         elif answer == 'S':
 594             done = True
 595         elif answer == 'Q':
 596             end()
 597             sys.exit(0)
 598
 599         if handler.get_action():
 600             print "PENDING %s\n" % handler.get_action()
 601
 602 ################################################################################
 603 ################################################################################
 604 ################################################################################
 605
 606 def usage (exit_code=0):
 607     print """Usage: dak process-new [OPTION]... [CHANGES]...
 608   -a, --automatic           automatic run
 609   -b, --no-binaries         do not sort binary-NEW packages first
 610   -c, --comments            show NEW comments
 611   -h, --help                show this help and exit.
 612   -m, --manual-reject=MSG   manual reject with `msg'
 613   -n, --no-action           don't do anything
 614   -q, --queue=QUEUE         operate on a different queue
 615   -t, --trainee             FTP Trainee mode
 616   -V, --version             display the version number and exit
 617
 618 ENVIRONMENT VARIABLES
 619
 620   DAK_INSPECT_UPLOAD: shell command to run to inspect a package
 621       The command is automatically run in a shell when an upload
 622       is checked.  The following substitutions are available:
 623
 624         {directory}: directory the upload is contained in
 625         {dsc}:       name of the included dsc or the empty string
 626         {changes}:   name of the changes file
 627
 628       Note that Python's 'format' method is used to format the command.
 629
 630       Example: run mc in a tmux session to inspect the upload
 631
 632       export DAK_INSPECT_UPLOAD='tmux new-session -d -s process-new 2>/dev/null; tmux new-window -n "{changes}" -t process-new:0 -k "cd {directory}; mc"'
 633
 634       and run
 635
 636       tmux attach -t process-new
 637
 638       in a separate terminal session.
 639 """
 640     sys.exit(exit_code)
 641
 642 ################################################################################
 643
 644 @contextlib.contextmanager
 645 def lock_package(package):
 646     """
 647     Lock C{package} so that noone else jumps in processing it.
 648
 649     @type package: string
 650     @param package: source package name to lock
 651     """
 652
 653     cnf = Config()
 654
 655     path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package)
 656
 657     try:
 658         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 659     except OSError as e:
 660         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 661             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 662             raise AlreadyLockedError(user)
 663
 664     try:
 665         yield fd
 666     finally:
 667         os.unlink(path)
 668
 669 def do_pkg(upload, session):
 670     # Try to get an included dsc
 671     dsc = upload.source
 672
 673     cnf = Config()
 674     group = cnf.get('Dinstall::UnprivGroup') or None
 675
 676     #bcc = "X-DAK: dak process-new"
 677     #if cnf.has_key("Dinstall::Bcc"):
 678     #    u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 679     #else:
 680     #    u.Subst["__BCC__"] = bcc
 681
 682     try:
 683       with lock_package(upload.changes.source):
 684        with UploadCopy(upload, group=group) as upload_copy:
 685         handler = PolicyQueueUploadHandler(upload, session)
 686         if handler.get_action() is not None:
 687             print "PENDING %s\n" % handler.get_action()
 688             return
 689
 690         do_new(upload, upload_copy, handler, session)
 691     except AlreadyLockedError as e:
 692         print "Seems to be locked by %s already, skipping..." % (e)
 693
 694 def show_new_comments(uploads, session):
 695     sources = [ upload.changes.source for upload in uploads ]
 696     if len(sources) == 0:
 697         return
 698
 699     query = """SELECT package, version, comment, author
 700                FROM new_comments
 701                WHERE package IN :sources
 702                ORDER BY package, version"""
 703
 704     r = session.execute(query, params=dict(sources=tuple(sources)))
 705
 706     for i in r:
 707         print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3])
 708
 709     session.rollback()
 710
 711 ################################################################################
 712
 713 def sort_uploads(new_queue, uploads, session, nobinaries=False):
 714     sources = {}
 715     sorteduploads = []
 716     suitesrc = [s.source for s in session.query(DBSource.source). \
 717       filter(DBSource.suites.any(Suite.suite_name.in_(['unstable', 'experimental'])))]
 718     comments = [p.package for p in session.query(NewComment.package). \
 719       filter_by(trainee=False, policy_queue=new_queue).distinct()]
 720     for upload in uploads:
 721         source = upload.changes.source
 722         if not source in sources:
 723             sources[source] = []
 724         sources[source].append({'upload': upload,
 725                                 'date': upload.changes.created,
 726                                 'stack': 1,
 727                                 'binary': True if source in suitesrc else False,
 728                                 'comments': True if source in comments else False})
 729     for src in sources:
 730         if len(sources[src]) > 1:
 731             changes = sources[src]
 732             firstseen = sorted(changes, key=lambda k: (k['date']))[0]['date']
 733             changes.sort(key=lambda item:item['date'])
 734             for i in range (0, len(changes)):
 735                 changes[i]['date'] = firstseen
 736                 changes[i]['stack'] = i + 1
 737         sorteduploads += sources[src]
 738     if nobinaries:
 739         sorteduploads = [u["upload"] for u in sorted(sorteduploads,
 740                          key=lambda k: (k["comments"], k["binary"],
 741                          k["date"], -k["stack"]))]
 742     else:
 743         sorteduploads = [u["upload"] for u in sorted(sorteduploads,
 744                          key=lambda k: (k["comments"], -k["binary"],
 745                          k["date"], -k["stack"]))]
 746     return sorteduploads
 747
 748 ################################################################################
 749
 750 def end():
 751     accept_count = SummaryStats().accept_count
 752     accept_bytes = SummaryStats().accept_bytes
 753
 754     if accept_count:
 755         sets = "set"
 756         if accept_count > 1:
 757             sets = "sets"
 758         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 759         Logger.log(["total",accept_count,accept_bytes])
 760
 761     if not Options["No-Action"] and not Options["Trainee"]:
 762         Logger.close()
 763
 764 ################################################################################
 765
 766 def main():
 767     global Options, Logger, Sections, Priorities
 768
 769     cnf = Config()
 770     session = DBConn().session()
 771
 772     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 773                  ('b',"no-binaries","Process-New::Options::No-Binaries"),
 774                  ('c',"comments","Process-New::Options::Comments"),
 775                  ('h',"help","Process-New::Options::Help"),
 776                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 777                  ('t',"trainee","Process-New::Options::Trainee"),
 778                  ('q','queue','Process-New::Options::Queue', 'HasArg'),
 779                  ('n',"no-action","Process-New::Options::No-Action")]
 780
 781     changes_files = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
 782
 783     for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:
 784         if not cnf.has_key("Process-New::Options::%s" % (i)):
 785             cnf["Process-New::Options::%s" % (i)] = ""
 786
 787     queue_name = cnf.get('Process-New::Options::Queue', 'new')
 788     new_queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
 789     if len(changes_files) == 0:
 790         uploads = new_queue.uploads
 791     else:
 792         uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \
 793             .join(DBChange).filter(DBChange.changesname.in_(changes_files)).all()
 794
 795     Options = cnf.subtree("Process-New::Options")
 796
 797     if Options["Help"]:
 798         usage()
 799
 800     if not Options["No-Action"]:
 801         try:
 802             Logger = daklog.Logger("process-new")
 803         except CantOpenError as e:
 804             Options["Trainee"] = "True"
 805
 806     Sections = Section_Completer(session)
 807     Priorities = Priority_Completer(session)
 808     readline.parse_and_bind("tab: complete")
 809
 810     if len(uploads) > 1:
 811         sys.stderr.write("Sorting changes...\n")
 812         uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"])
 813
 814     if Options["Comments"]:
 815         show_new_comments(uploads, session)
 816     else:
 817         for upload in uploads:
 818             do_pkg (upload, session)
 819
 820     end()
 821
 822 ################################################################################
 823
 824 if __name__ == '__main__':
 825     main()