dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 from __future__ import with_statement
  46
  47 import copy
  48 import errno
  49 import os
  50 import readline
  51 import stat
  52 import sys
  53 import time
  54 import contextlib
  55 import pwd
  56 import apt_pkg, apt_inst
  57 import examine_package
  58
  59 from daklib.dbconn import *
  60 from daklib.queue import *
  61 from daklib import daklog
  62 from daklib import utils
  63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  65 from daklib.summarystats import SummaryStats
  66 from daklib.config import Config
  67
  68 # Globals
  69 Options = None
  70 Logger = None
  71
  72 Priorities = None
  73 Sections = None
  74
  75 ################################################################################
  76 ################################################################################
  77 ################################################################################
  78
  79 def recheck(upload, session):
  80     upload.recheck()
  81     if len(upload.rejects) > 0:
  82         answer = "XXX"
  83         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
  84             answer = 'S'
  85
  86         print "REJECT\n%s" % '\n'.join(upload.rejects)
  87         prompt = "[R]eject, Skip, Quit ?"
  88
  89         while prompt.find(answer) == -1:
  90             answer = utils.our_raw_input(prompt)
  91             m = re_default_answer.match(prompt)
  92             if answer == "":
  93                 answer = m.group(1)
  94             answer = answer[:1].upper()
  95
  96         if answer == 'R':
  97             upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
  98             return 0
  99         elif answer == 'S':
 100             return 0
 101         elif answer == 'Q':
 102             end()
 103             sys.exit(0)
 104
 105     return 1
 106
 107 ################################################################################
 108
 109 def indiv_sg_compare (a, b):
 110     """Sort by source name, source, version, 'have source', and
 111        finally by filename."""
 112     # Sort by source version
 113     q = apt_pkg.VersionCompare(a["version"], b["version"])
 114     if q:
 115         return -q
 116
 117     # Sort by 'have source'
 118     a_has_source = a["architecture"].get("source")
 119     b_has_source = b["architecture"].get("source")
 120     if a_has_source and not b_has_source:
 121         return -1
 122     elif b_has_source and not a_has_source:
 123         return 1
 124
 125     return cmp(a["filename"], b["filename"])
 126
 127 ############################################################
 128
 129 def sg_compare (a, b):
 130     a = a[1]
 131     b = b[1]
 132     """Sort by have note, source already in database and time of oldest upload."""
 133     # Sort by have note
 134     a_note_state = a["note_state"]
 135     b_note_state = b["note_state"]
 136     if a_note_state < b_note_state:
 137         return -1
 138     elif a_note_state > b_note_state:
 139         return 1
 140     # Sort by source already in database (descending)
 141     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 142     if source_in_database:
 143         return -source_in_database
 144
 145     # Sort by time of oldest upload
 146     return cmp(a["oldest"], b["oldest"])
 147
 148 def sort_changes(changes_files, session):
 149     """Sort into source groups, then sort each source group by version,
 150     have source, filename.  Finally, sort the source groups by have
 151     note, time of oldest upload of each source upload."""
 152     if len(changes_files) == 1:
 153         return changes_files
 154
 155     sorted_list = []
 156     cache = {}
 157     # Read in all the .changes files
 158     for filename in changes_files:
 159         u = Upload()
 160         try:
 161             u.pkg.load_dot_dak(filename)
 162             u.update_subst()
 163             cache[filename] = copy.copy(u.pkg.changes)
 164             cache[filename]["filename"] = filename
 165         except:
 166             sorted_list.append(filename)
 167             break
 168     # Divide the .changes into per-source groups
 169     per_source = {}
 170     for filename in cache.keys():
 171         source = cache[filename]["source"]
 172         if not per_source.has_key(source):
 173             per_source[source] = {}
 174             per_source[source]["list"] = []
 175         per_source[source]["list"].append(cache[filename])
 176     # Determine oldest time and have note status for each source group
 177     for source in per_source.keys():
 178         q = session.query(DBSource).filter_by(source = source).all()
 179         per_source[source]["source_in_database"] = len(q)>0
 180         source_list = per_source[source]["list"]
 181         first = source_list[0]
 182         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 183         have_note = 0
 184         for d in per_source[source]["list"]:
 185             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 186             if mtime < oldest:
 187                 oldest = mtime
 188             have_note += has_new_comment(d["source"], d["version"], session)
 189         per_source[source]["oldest"] = oldest
 190         if not have_note:
 191             per_source[source]["note_state"] = 0; # none
 192         elif have_note < len(source_list):
 193             per_source[source]["note_state"] = 1; # some
 194         else:
 195             per_source[source]["note_state"] = 2; # all
 196         per_source[source]["list"].sort(indiv_sg_compare)
 197     per_source_items = per_source.items()
 198     per_source_items.sort(sg_compare)
 199     for i in per_source_items:
 200         for j in i[1]["list"]:
 201             sorted_list.append(j["filename"])
 202     return sorted_list
 203
 204 ################################################################################
 205
 206 class Section_Completer:
 207     def __init__ (self, session):
 208         self.sections = []
 209         self.matches = []
 210         for s, in session.query(Section.section):
 211             self.sections.append(s)
 212
 213     def complete(self, text, state):
 214         if state == 0:
 215             self.matches = []
 216             n = len(text)
 217             for word in self.sections:
 218                 if word[:n] == text:
 219                     self.matches.append(word)
 220         try:
 221             return self.matches[state]
 222         except IndexError:
 223             return None
 224
 225 ############################################################
 226
 227 class Priority_Completer:
 228     def __init__ (self, session):
 229         self.priorities = []
 230         self.matches = []
 231         for p, in session.query(Priority.priority):
 232             self.priorities.append(p)
 233
 234     def complete(self, text, state):
 235         if state == 0:
 236             self.matches = []
 237             n = len(text)
 238             for word in self.priorities:
 239                 if word[:n] == text:
 240                     self.matches.append(word)
 241         try:
 242             return self.matches[state]
 243         except IndexError:
 244             return None
 245
 246 ################################################################################
 247
 248 def print_new (new, upload, indexed, file=sys.stdout):
 249     check_valid(new)
 250     broken = False
 251     index = 0
 252     for pkg in new.keys():
 253         index += 1
 254         section = new[pkg]["section"]
 255         priority = new[pkg]["priority"]
 256         if new[pkg]["section id"] == -1:
 257             section += "[!]"
 258             broken = True
 259         if new[pkg]["priority id"] == -1:
 260             priority += "[!]"
 261             broken = True
 262         if indexed:
 263             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 264         else:
 265             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 266         line = line.strip()+'\n'
 267         file.write(line)
 268     notes = get_new_comments(upload.pkg.changes.get("source"))
 269     for note in notes:
 270         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 271               % (note.author, note.version, note.notedate, note.comment)
 272         print "-" * 72
 273     return broken, len(notes) > 0
 274
 275 ################################################################################
 276
 277 def index_range (index):
 278     if index == 1:
 279         return "1"
 280     else:
 281         return "1-%s" % (index)
 282
 283 ################################################################################
 284 ################################################################################
 285
 286 def edit_new (new, upload):
 287     # Write the current data to a temporary file
 288     (fd, temp_filename) = utils.temp_filename()
 289     temp_file = os.fdopen(fd, 'w')
 290     print_new (new, upload, indexed=0, file=temp_file)
 291     temp_file.close()
 292     # Spawn an editor on that file
 293     editor = os.environ.get("EDITOR","vi")
 294     result = os.system("%s %s" % (editor, temp_filename))
 295     if result != 0:
 296         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 297     # Read the edited data back in
 298     temp_file = utils.open_file(temp_filename)
 299     lines = temp_file.readlines()
 300     temp_file.close()
 301     os.unlink(temp_filename)
 302     # Parse the new data
 303     for line in lines:
 304         line = line.strip()
 305         if line == "":
 306             continue
 307         s = line.split()
 308         # Pad the list if necessary
 309         s[len(s):3] = [None] * (3-len(s))
 310         (pkg, priority, section) = s[:3]
 311         if not new.has_key(pkg):
 312             utils.warn("Ignoring unknown package '%s'" % (pkg))
 313         else:
 314             # Strip off any invalid markers, print_new will readd them.
 315             if section.endswith("[!]"):
 316                 section = section[:-3]
 317             if priority.endswith("[!]"):
 318                 priority = priority[:-3]
 319             for f in new[pkg]["files"]:
 320                 upload.pkg.files[f]["section"] = section
 321                 upload.pkg.files[f]["priority"] = priority
 322             new[pkg]["section"] = section
 323             new[pkg]["priority"] = priority
 324
 325 ################################################################################
 326
 327 def edit_index (new, upload, index):
 328     priority = new[index]["priority"]
 329     section = new[index]["section"]
 330     ftype = new[index]["type"]
 331     done = 0
 332     while not done:
 333         print "\t".join([index, priority, section])
 334
 335         answer = "XXX"
 336         if ftype != "dsc":
 337             prompt = "[B]oth, Priority, Section, Done ? "
 338         else:
 339             prompt = "[S]ection, Done ? "
 340         edit_priority = edit_section = 0
 341
 342         while prompt.find(answer) == -1:
 343             answer = utils.our_raw_input(prompt)
 344             m = re_default_answer.match(prompt)
 345             if answer == "":
 346                 answer = m.group(1)
 347             answer = answer[:1].upper()
 348
 349         if answer == 'P':
 350             edit_priority = 1
 351         elif answer == 'S':
 352             edit_section = 1
 353         elif answer == 'B':
 354             edit_priority = edit_section = 1
 355         elif answer == 'D':
 356             done = 1
 357
 358         # Edit the priority
 359         if edit_priority:
 360             readline.set_completer(Priorities.complete)
 361             got_priority = 0
 362             while not got_priority:
 363                 new_priority = utils.our_raw_input("New priority: ").strip()
 364                 if new_priority not in Priorities.priorities:
 365                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 366                 else:
 367                     got_priority = 1
 368                     priority = new_priority
 369
 370         # Edit the section
 371         if edit_section:
 372             readline.set_completer(Sections.complete)
 373             got_section = 0
 374             while not got_section:
 375                 new_section = utils.our_raw_input("New section: ").strip()
 376                 if new_section not in Sections.sections:
 377                     print "E: '%s' is not a valid section, try again." % (new_section)
 378                 else:
 379                     got_section = 1
 380                     section = new_section
 381
 382         # Reset the readline completer
 383         readline.set_completer(None)
 384
 385     for f in new[index]["files"]:
 386         upload.pkg.files[f]["section"] = section
 387         upload.pkg.files[f]["priority"] = priority
 388     new[index]["priority"] = priority
 389     new[index]["section"] = section
 390     return new
 391
 392 ################################################################################
 393
 394 def edit_overrides (new, upload, session):
 395     print
 396     done = 0
 397     while not done:
 398         print_new (new, upload, indexed=1)
 399         new_index = {}
 400         index = 0
 401         for i in new.keys():
 402             index += 1
 403             new_index[index] = i
 404
 405         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 406
 407         got_answer = 0
 408         while not got_answer:
 409             answer = utils.our_raw_input(prompt)
 410             if not answer.isdigit():
 411                 answer = answer[:1].upper()
 412             if answer == "E" or answer == "D":
 413                 got_answer = 1
 414             elif re_isanum.match (answer):
 415                 answer = int(answer)
 416                 if (answer < 1) or (answer > index):
 417                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 418                 else:
 419                     got_answer = 1
 420
 421         if answer == 'E':
 422             edit_new(new, upload)
 423         elif answer == 'D':
 424             done = 1
 425         else:
 426             edit_index (new, upload, new_index[answer])
 427
 428     return new
 429
 430 ################################################################################
 431
 432 def edit_note(note, upload, session):
 433     # Write the current data to a temporary file
 434     (fd, temp_filename) = utils.temp_filename()
 435     editor = os.environ.get("EDITOR","vi")
 436     answer = 'E'
 437     while answer == 'E':
 438         os.system("%s %s" % (editor, temp_filename))
 439         temp_file = utils.open_file(temp_filename)
 440         newnote = temp_file.read().rstrip()
 441         temp_file.close()
 442         print "New Note:"
 443         print utils.prefix_multi_line_string(newnote,"  ")
 444         prompt = "[D]one, Edit, Abandon, Quit ?"
 445         answer = "XXX"
 446         while prompt.find(answer) == -1:
 447             answer = utils.our_raw_input(prompt)
 448             m = re_default_answer.search(prompt)
 449             if answer == "":
 450                 answer = m.group(1)
 451             answer = answer[:1].upper()
 452     os.unlink(temp_filename)
 453     if answer == 'A':
 454         return
 455     elif answer == 'Q':
 456         end()
 457         sys.exit(0)
 458
 459     comment = NewComment()
 460     comment.package = upload.pkg.changes["source"]
 461     comment.version = upload.pkg.changes["version"]
 462     comment.comment = newnote
 463     comment.author  = utils.whoami()
 464     comment.trainee = bool(Options["Trainee"])
 465     session.add(comment)
 466     session.commit()
 467
 468 ################################################################################
 469
 470 def check_pkg (upload):
 471     try:
 472         less_fd = os.popen("less -R -", 'w', 0)
 473         stdout_fd = sys.stdout
 474         try:
 475             sys.stdout = less_fd
 476             changes = utils.parse_changes (upload.pkg.changes_file)
 477             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
 478             files = upload.pkg.files
 479             for f in files.keys():
 480                 if files[f].has_key("new"):
 481                     ftype = files[f]["type"]
 482                     if ftype == "deb":
 483                         examine_package.check_deb(changes['distribution'], f)
 484                     elif ftype == "dsc":
 485                         examine_package.check_dsc(changes['distribution'], f)
 486         finally:
 487             examine_package.output_package_relations()
 488             sys.stdout = stdout_fd
 489     except IOError, e:
 490         if e.errno == errno.EPIPE:
 491             utils.warn("[examine_package] Caught EPIPE; skipping.")
 492             pass
 493         else:
 494             raise
 495     except KeyboardInterrupt:
 496         utils.warn("[examine_package] Caught C-c; skipping.")
 497         pass
 498
 499 ################################################################################
 500
 501 ## FIXME: horribly Debian specific
 502
 503 def do_bxa_notification(upload):
 504     files = upload.pkg.files
 505     summary = ""
 506     for f in files.keys():
 507         if files[f]["type"] == "deb":
 508             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 509             summary += "\n"
 510             summary += "Package: %s\n" % (control.Find("Package"))
 511             summary += "Description: %s\n" % (control.Find("Description"))
 512     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 513     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
 514     utils.send_mail(bxa_mail)
 515
 516 ################################################################################
 517
 518 def add_overrides (new, upload, session):
 519     changes = upload.pkg.changes
 520     files = upload.pkg.files
 521     srcpkg = changes.get("source")
 522
 523     for suite in changes["suite"].keys():
 524         suite_id = get_suite(suite).suite_id
 525         for pkg in new.keys():
 526             component_id = get_component(new[pkg]["component"]).component_id
 527             type_id = get_override_type(new[pkg]["type"]).overridetype_id
 528             priority_id = new[pkg]["priority id"]
 529             section_id = new[pkg]["section id"]
 530             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 531             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
 532                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
 533             for f in new[pkg]["files"]:
 534                 if files[f].has_key("new"):
 535                     del files[f]["new"]
 536             del new[pkg]
 537
 538     session.commit()
 539
 540     if Config().FindB("Dinstall::BXANotify"):
 541         do_bxa_notification(upload)
 542
 543 ################################################################################
 544
 545 def prod_maintainer (note, upload):
 546     cnf = Config()
 547     # Here we prepare an editor and get them ready to prod...
 548     (fd, temp_filename) = utils.temp_filename()
 549     temp_file = os.fdopen(fd, 'w')
 550     if len(note) > 0:
 551         for line in note:
 552             temp_file.write(line)
 553     temp_file.close()
 554     editor = os.environ.get("EDITOR","vi")
 555     answer = 'E'
 556     while answer == 'E':
 557         os.system("%s %s" % (editor, temp_filename))
 558         temp_fh = utils.open_file(temp_filename)
 559         prod_message = "".join(temp_fh.readlines())
 560         temp_fh.close()
 561         print "Prod message:"
 562         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 563         prompt = "[P]rod, Edit, Abandon, Quit ?"
 564         answer = "XXX"
 565         while prompt.find(answer) == -1:
 566             answer = utils.our_raw_input(prompt)
 567             m = re_default_answer.search(prompt)
 568             if answer == "":
 569                 answer = m.group(1)
 570             answer = answer[:1].upper()
 571     os.unlink(temp_filename)
 572     if answer == 'A':
 573         return
 574     elif answer == 'Q':
 575         end()
 576         sys.exit(0)
 577     # Otherwise, do the proding...
 578     user_email_address = utils.whoami() + " <%s>" % (
 579         cnf["Dinstall::MyAdminAddress"])
 580
 581     Subst = upload.Subst
 582
 583     Subst["__FROM_ADDRESS__"] = user_email_address
 584     Subst["__PROD_MESSAGE__"] = prod_message
 585     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
 586
 587     prod_mail_message = utils.TemplateSubst(
 588         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 589
 590     # Send the prod mail if appropriate
 591     if not cnf["Dinstall::Options::No-Mail"]:
 592         utils.send_mail(prod_mail_message)
 593
 594     print "Sent proding message"
 595
 596 ################################################################################
 597
 598 def do_new(upload, session):
 599     print "NEW\n"
 600     files = upload.pkg.files
 601     changes = upload.pkg.changes
 602     cnf = Config()
 603
 604     # Make a copy of distribution we can happily trample on
 605     changes["suite"] = copy.copy(changes["distribution"])
 606
 607     # Fix up the list of target suites
 608     for suite in changes["suite"].keys():
 609         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
 610         if override:
 611             (olderr, newerr) = (get_suite(suite, session) == None,
 612                                 get_suite(override, session) == None)
 613             if olderr or newerr:
 614                 (oinv, newinv) = ("", "")
 615                 if olderr: oinv = "invalid "
 616                 if newerr: ninv = "invalid "
 617                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 618                         oinv, suite, ninv, override)
 619             del changes["suite"][suite]
 620             changes["suite"][override] = 1
 621     # Validate suites
 622     for suite in changes["suite"].keys():
 623         if get_suite(suite, session) is None:
 624             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 625
 626     # The main NEW processing loop
 627     done = 0
 628     while not done:
 629         # Find out what's new
 630         new = determine_new(changes, files)
 631
 632         if not new:
 633             break
 634
 635         answer = "XXX"
 636         if Options["No-Action"] or Options["Automatic"]:
 637             answer = 'S'
 638
 639         (broken, note) = print_new(new, upload, indexed=0)
 640         prompt = ""
 641
 642         if not broken and not note:
 643             prompt = "Add overrides, "
 644         if broken:
 645             print "W: [!] marked entries must be fixed before package can be processed."
 646         if note:
 647             print "W: note must be removed before package can be processed."
 648             prompt += "RemOve all notes, Remove note, "
 649
 650         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 651
 652         while prompt.find(answer) == -1:
 653             answer = utils.our_raw_input(prompt)
 654             m = re_default_answer.search(prompt)
 655             if answer == "":
 656                 answer = m.group(1)
 657             answer = answer[:1].upper()
 658
 659         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 660             utils.warn("Trainees can't do that")
 661             continue
 662
 663         if answer == 'A' and not Options["Trainee"]:
 664             try:
 665                 check_daily_lock()
 666                 done = add_overrides (new, upload, session)
 667                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
 668             except CantGetLockError:
 669                 print "Hello? Operator! Give me the number for 911!"
 670                 print "Dinstall in the locked area, cant process packages, come back later"
 671         elif answer == 'C':
 672             check_pkg(upload)
 673         elif answer == 'E' and not Options["Trainee"]:
 674             new = edit_overrides (new, upload, session)
 675         elif answer == 'M' and not Options["Trainee"]:
 676             upload.pkg.remove_known_changes()
 677             aborted = upload.do_reject(manual=1,
 678                                        reject_message=Options["Manual-Reject"],
 679                                        note=get_new_comments(changes.get("source", ""), session=session))
 680             if not aborted:
 681                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
 682                 done = 1
 683         elif answer == 'N':
 684             edit_note(get_new_comments(changes.get("source", ""), session=session),
 685                       upload, session)
 686         elif answer == 'P' and not Options["Trainee"]:
 687             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
 688                             upload)
 689             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
 690         elif answer == 'R' and not Options["Trainee"]:
 691             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 692             if confirm == "y":
 693                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
 694                     session.delete(c)
 695                 session.commit()
 696         elif answer == 'O' and not Options["Trainee"]:
 697             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 698             if confirm == "y":
 699                 for c in get_new_comments(changes.get("source", ""), session=session):
 700                     session.delete(c)
 701                 session.commit()
 702
 703         elif answer == 'S':
 704             done = 1
 705         elif answer == 'Q':
 706             end()
 707             sys.exit(0)
 708
 709 ################################################################################
 710 ################################################################################
 711 ################################################################################
 712
 713 def usage (exit_code=0):
 714     print """Usage: dak process-new [OPTION]... [CHANGES]...
 715   -a, --automatic           automatic run
 716   -h, --help                show this help and exit.
 717   -m, --manual-reject=MSG   manual reject with `msg'
 718   -n, --no-action           don't do anything
 719   -t, --trainee             FTP Trainee mode
 720   -V, --version             display the version number and exit"""
 721     sys.exit(exit_code)
 722
 723 ################################################################################
 724
 725 def do_byhand(upload, session):
 726     done = 0
 727     while not done:
 728         files = upload.pkg.files
 729         will_install = 1
 730         byhand = []
 731
 732         for f in files.keys():
 733             if files[f]["type"] == "byhand":
 734                 if os.path.exists(f):
 735                     print "W: %s still present; please process byhand components and try again." % (f)
 736                     will_install = 0
 737                 else:
 738                     byhand.append(f)
 739
 740         answer = "XXXX"
 741         if Options["No-Action"]:
 742             answer = "S"
 743         if will_install:
 744             if Options["Automatic"] and not Options["No-Action"]:
 745                 answer = 'A'
 746             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 747         else:
 748             prompt = "Manual reject, [S]kip, Quit ?"
 749
 750         while prompt.find(answer) == -1:
 751             answer = utils.our_raw_input(prompt)
 752             m = re_default_answer.search(prompt)
 753             if answer == "":
 754                 answer = m.group(1)
 755             answer = answer[:1].upper()
 756
 757         if answer == 'A':
 758             try:
 759                 check_daily_lock()
 760                 done = 1
 761                 for f in byhand:
 762                     del files[f]
 763                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
 764             except CantGetLockError:
 765                 print "Hello? Operator! Give me the number for 911!"
 766                 print "Dinstall in the locked area, cant process packages, come back later"
 767         elif answer == 'M':
 768             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
 769             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
 770             done = 1
 771         elif answer == 'S':
 772             done = 1
 773         elif answer == 'Q':
 774             end()
 775             sys.exit(0)
 776
 777 ################################################################################
 778
 779 def check_daily_lock():
 780     """
 781     Raises CantGetLockError if the dinstall daily.lock exists.
 782     """
 783
 784     cnf = Config()
 785     try:
 786         os.open(cnf["Process-New::DinstallLockFile"],
 787                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 788     except OSError, e:
 789         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 790             raise CantGetLockError
 791
 792     os.unlink(cnf["Process-New::DinstallLockFile"])
 793
 794
 795 @contextlib.contextmanager
 796 def lock_package(package):
 797     """
 798     Lock C{package} so that noone else jumps in processing it.
 799
 800     @type package: string
 801     @param package: source package name to lock
 802     """
 803
 804     path = os.path.join(Config()["Process-New::LockDir"], package)
 805     try:
 806         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 807     except OSError, e:
 808         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 809             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 810             raise AlreadyLockedError, user
 811
 812     try:
 813         yield fd
 814     finally:
 815         os.unlink(path)
 816
 817 def _accept(upload):
 818     if Options["No-Action"]:
 819         return
 820     (summary, short_summary) = upload.build_summaries()
 821     upload.accept(summary, short_summary, targetqueue)
 822
 823 def do_accept(upload):
 824     print "ACCEPT"
 825     cnf = Config()
 826     if not Options["No-Action"]:
 827         (summary, short_summary) = upload.build_summaries()
 828
 829         if cnf.FindB("Dinstall::SecurityQueueHandling"):
 830             upload.dump_vars(cnf["Dir::Queue::Embargoed"])
 831             upload.move_to_queue(get_policy_queue('embargoed'))
 832             upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
 833             # Check for override disparities
 834             upload.Subst["__SUMMARY__"] = summary
 835         else:
 836             # Just a normal upload, accept it...
 837             _accept(upload)
 838
 839 def do_pkg(changes_file, session):
 840     u = Upload()
 841     u.pkg.load_dot_dak(changes_file)
 842     u.update_subst()
 843
 844     cnf = Config()
 845     bcc = "X-DAK: dak process-new"
 846     if cnf.has_key("Dinstall::Bcc"):
 847         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 848     else:
 849         u.Subst["__BCC__"] = bcc
 850
 851     files = u.pkg.files
 852
 853     try:
 854         with lock_package(u.pkg.changes["source"]):
 855             if not recheck(u, session):
 856                 return
 857
 858             (new, byhand) = check_status(files)
 859             if new or byhand:
 860                 if new:
 861                     do_new(u, session)
 862                 if byhand:
 863                     do_byhand(u, session)
 864                 (new, byhand) = check_status(files)
 865
 866             if not new and not byhand:
 867                 try:
 868                     check_daily_lock()
 869                     do_accept(u)
 870                 except CantGetLockError:
 871                     print "Hello? Operator! Give me the number for 911!"
 872                     print "Dinstall in the locked area, can't process packages, come back later"
 873     except AlreadyLockedError, e:
 874         print "Seems to be locked by %s already, skipping..." % (e)
 875
 876 ################################################################################
 877
 878 def end():
 879     accept_count = SummaryStats().accept_count
 880     accept_bytes = SummaryStats().accept_bytes
 881
 882     if accept_count:
 883         sets = "set"
 884         if accept_count > 1:
 885             sets = "sets"
 886         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 887         Logger.log(["total",accept_count,accept_bytes])
 888
 889     if not Options["No-Action"] and not Options["Trainee"]:
 890         Logger.close()
 891
 892 ################################################################################
 893
 894 def main():
 895     global Options, Logger, Sections, Priorities
 896
 897     cnf = Config()
 898     session = DBConn().session()
 899
 900     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 901                  ('h',"help","Process-New::Options::Help"),
 902                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 903                  ('t',"trainee","Process-New::Options::Trainee"),
 904                  ('n',"no-action","Process-New::Options::No-Action")]
 905
 906     for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
 907         if not cnf.has_key("Process-New::Options::%s" % (i)):
 908             cnf["Process-New::Options::%s" % (i)] = ""
 909
 910     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
 911     if len(changes_files) == 0:
 912         changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
 913
 914     Options = cnf.SubTree("Process-New::Options")
 915
 916     if Options["Help"]:
 917         usage()
 918
 919     if not Options["No-Action"]:
 920         try:
 921             Logger = daklog.Logger(cnf, "process-new")
 922         except CantOpenError, e:
 923             Options["Trainee"] = "True"
 924
 925     Sections = Section_Completer(session)
 926     Priorities = Priority_Completer(session)
 927     readline.parse_and_bind("tab: complete")
 928
 929     if len(changes_files) > 1:
 930         sys.stderr.write("Sorting changes...\n")
 931     changes_files = sort_changes(changes_files, session)
 932
 933     # Kill me now? **FIXME**
 934     cnf["Dinstall::Options::No-Mail"] = ""
 935
 936     for changes_file in changes_files:
 937         changes_file = utils.validate_changes_file_arg(changes_file, 0)
 938         if not changes_file:
 939             continue
 940         print "\n" + changes_file
 941
 942         do_pkg (changes_file, session)
 943
 944     end()
 945
 946 ################################################################################
 947
 948 if __name__ == '__main__':
 949     main()