dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 from __future__ import with_statement
  46
  47 import copy
  48 import errno
  49 import os
  50 import readline
  51 import stat
  52 import sys
  53 import time
  54 import contextlib
  55 import pwd
  56 import apt_pkg, apt_inst
  57 import examine_package
  58
  59 from daklib.dbconn import *
  60 from daklib.queue import *
  61 from daklib import daklog
  62 from daklib import utils
  63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
  64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  65 from daklib.summarystats import SummaryStats
  66 from daklib.config import Config
  67
  68 # Globals
  69 Options = None
  70 Logger = None
  71
  72 Priorities = None
  73 Sections = None
  74
  75 ################################################################################
  76 ################################################################################
  77 ################################################################################
  78
  79 def recheck(upload, session):
  80 # STU: I'm not sure, but I don't thin kthis is necessary any longer:    upload.recheck(session)
  81     if len(upload.rejects) > 0:
  82         answer = "XXX"
  83         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
  84             answer = 'S'
  85
  86         print "REJECT\n%s" % '\n'.join(upload.rejects)
  87         prompt = "[R]eject, Skip, Quit ?"
  88
  89         while prompt.find(answer) == -1:
  90             answer = utils.our_raw_input(prompt)
  91             m = re_default_answer.match(prompt)
  92             if answer == "":
  93                 answer = m.group(1)
  94             answer = answer[:1].upper()
  95
  96         if answer == 'R':
  97             upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
  98             return 0
  99         elif answer == 'S':
 100             return 0
 101         elif answer == 'Q':
 102             end()
 103             sys.exit(0)
 104
 105     return 1
 106
 107 ################################################################################
 108
 109 def indiv_sg_compare (a, b):
 110     """Sort by source name, source, version, 'have source', and
 111        finally by filename."""
 112     # Sort by source version
 113     q = apt_pkg.VersionCompare(a["version"], b["version"])
 114     if q:
 115         return -q
 116
 117     # Sort by 'have source'
 118     a_has_source = a["architecture"].get("source")
 119     b_has_source = b["architecture"].get("source")
 120     if a_has_source and not b_has_source:
 121         return -1
 122     elif b_has_source and not a_has_source:
 123         return 1
 124
 125     return cmp(a["filename"], b["filename"])
 126
 127 ############################################################
 128
 129 def sg_compare (a, b):
 130     a = a[1]
 131     b = b[1]
 132     """Sort by have note, source already in database and time of oldest upload."""
 133     # Sort by have note
 134     a_note_state = a["note_state"]
 135     b_note_state = b["note_state"]
 136     if a_note_state < b_note_state:
 137         return -1
 138     elif a_note_state > b_note_state:
 139         return 1
 140     # Sort by source already in database (descending)
 141     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 142     if source_in_database:
 143         return -source_in_database
 144
 145     # Sort by time of oldest upload
 146     return cmp(a["oldest"], b["oldest"])
 147
 148 def sort_changes(changes_files, session):
 149     """Sort into source groups, then sort each source group by version,
 150     have source, filename.  Finally, sort the source groups by have
 151     note, time of oldest upload of each source upload."""
 152     if len(changes_files) == 1:
 153         return changes_files
 154
 155     sorted_list = []
 156     cache = {}
 157     # Read in all the .changes files
 158     for filename in changes_files:
 159         u = Upload()
 160         try:
 161             u.pkg.changes_file = filename
 162             u.load_changes(filename)
 163             u.update_subst()
 164             cache[filename] = copy.copy(u.pkg.changes)
 165             cache[filename]["filename"] = filename
 166         except:
 167             sorted_list.append(filename)
 168             break
 169     # Divide the .changes into per-source groups
 170     per_source = {}
 171     for filename in cache.keys():
 172         source = cache[filename]["source"]
 173         if not per_source.has_key(source):
 174             per_source[source] = {}
 175             per_source[source]["list"] = []
 176         per_source[source]["list"].append(cache[filename])
 177     # Determine oldest time and have note status for each source group
 178     for source in per_source.keys():
 179         q = session.query(DBSource).filter_by(source = source).all()
 180         per_source[source]["source_in_database"] = len(q)>0
 181         source_list = per_source[source]["list"]
 182         first = source_list[0]
 183         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 184         have_note = 0
 185         for d in per_source[source]["list"]:
 186             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 187             if mtime < oldest:
 188                 oldest = mtime
 189             have_note += has_new_comment(d["source"], d["version"], session)
 190         per_source[source]["oldest"] = oldest
 191         if not have_note:
 192             per_source[source]["note_state"] = 0; # none
 193         elif have_note < len(source_list):
 194             per_source[source]["note_state"] = 1; # some
 195         else:
 196             per_source[source]["note_state"] = 2; # all
 197         per_source[source]["list"].sort(indiv_sg_compare)
 198     per_source_items = per_source.items()
 199     per_source_items.sort(sg_compare)
 200     for i in per_source_items:
 201         for j in i[1]["list"]:
 202             sorted_list.append(j["filename"])
 203     return sorted_list
 204
 205 ################################################################################
 206
 207 class Section_Completer:
 208     def __init__ (self, session):
 209         self.sections = []
 210         self.matches = []
 211         for s, in session.query(Section.section):
 212             self.sections.append(s)
 213
 214     def complete(self, text, state):
 215         if state == 0:
 216             self.matches = []
 217             n = len(text)
 218             for word in self.sections:
 219                 if word[:n] == text:
 220                     self.matches.append(word)
 221         try:
 222             return self.matches[state]
 223         except IndexError:
 224             return None
 225
 226 ############################################################
 227
 228 class Priority_Completer:
 229     def __init__ (self, session):
 230         self.priorities = []
 231         self.matches = []
 232         for p, in session.query(Priority.priority):
 233             self.priorities.append(p)
 234
 235     def complete(self, text, state):
 236         if state == 0:
 237             self.matches = []
 238             n = len(text)
 239             for word in self.priorities:
 240                 if word[:n] == text:
 241                     self.matches.append(word)
 242         try:
 243             return self.matches[state]
 244         except IndexError:
 245             return None
 246
 247 ################################################################################
 248
 249 def print_new (new, upload, indexed, file=sys.stdout):
 250     check_valid(new)
 251     broken = False
 252     index = 0
 253     for pkg in new.keys():
 254         index += 1
 255         section = new[pkg]["section"]
 256         priority = new[pkg]["priority"]
 257         if new[pkg]["section id"] == -1:
 258             section += "[!]"
 259             broken = True
 260         if new[pkg]["priority id"] == -1:
 261             priority += "[!]"
 262             broken = True
 263         if indexed:
 264             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 265         else:
 266             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 267         line = line.strip()+'\n'
 268         file.write(line)
 269     notes = get_new_comments(upload.pkg.changes.get("source"))
 270     for note in notes:
 271         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 272               % (note.author, note.version, note.notedate, note.comment)
 273         print "-" * 72
 274     return broken, len(notes) > 0
 275
 276 ################################################################################
 277
 278 def index_range (index):
 279     if index == 1:
 280         return "1"
 281     else:
 282         return "1-%s" % (index)
 283
 284 ################################################################################
 285 ################################################################################
 286
 287 def edit_new (new, upload):
 288     # Write the current data to a temporary file
 289     (fd, temp_filename) = utils.temp_filename()
 290     temp_file = os.fdopen(fd, 'w')
 291     print_new (new, upload, indexed=0, file=temp_file)
 292     temp_file.close()
 293     # Spawn an editor on that file
 294     editor = os.environ.get("EDITOR","vi")
 295     result = os.system("%s %s" % (editor, temp_filename))
 296     if result != 0:
 297         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 298     # Read the edited data back in
 299     temp_file = utils.open_file(temp_filename)
 300     lines = temp_file.readlines()
 301     temp_file.close()
 302     os.unlink(temp_filename)
 303     # Parse the new data
 304     for line in lines:
 305         line = line.strip()
 306         if line == "":
 307             continue
 308         s = line.split()
 309         # Pad the list if necessary
 310         s[len(s):3] = [None] * (3-len(s))
 311         (pkg, priority, section) = s[:3]
 312         if not new.has_key(pkg):
 313             utils.warn("Ignoring unknown package '%s'" % (pkg))
 314         else:
 315             # Strip off any invalid markers, print_new will readd them.
 316             if section.endswith("[!]"):
 317                 section = section[:-3]
 318             if priority.endswith("[!]"):
 319                 priority = priority[:-3]
 320             for f in new[pkg]["files"]:
 321                 upload.pkg.files[f]["section"] = section
 322                 upload.pkg.files[f]["priority"] = priority
 323             new[pkg]["section"] = section
 324             new[pkg]["priority"] = priority
 325
 326 ################################################################################
 327
 328 def edit_index (new, upload, index):
 329     priority = new[index]["priority"]
 330     section = new[index]["section"]
 331     ftype = new[index]["type"]
 332     done = 0
 333     while not done:
 334         print "\t".join([index, priority, section])
 335
 336         answer = "XXX"
 337         if ftype != "dsc":
 338             prompt = "[B]oth, Priority, Section, Done ? "
 339         else:
 340             prompt = "[S]ection, Done ? "
 341         edit_priority = edit_section = 0
 342
 343         while prompt.find(answer) == -1:
 344             answer = utils.our_raw_input(prompt)
 345             m = re_default_answer.match(prompt)
 346             if answer == "":
 347                 answer = m.group(1)
 348             answer = answer[:1].upper()
 349
 350         if answer == 'P':
 351             edit_priority = 1
 352         elif answer == 'S':
 353             edit_section = 1
 354         elif answer == 'B':
 355             edit_priority = edit_section = 1
 356         elif answer == 'D':
 357             done = 1
 358
 359         # Edit the priority
 360         if edit_priority:
 361             readline.set_completer(Priorities.complete)
 362             got_priority = 0
 363             while not got_priority:
 364                 new_priority = utils.our_raw_input("New priority: ").strip()
 365                 if new_priority not in Priorities.priorities:
 366                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 367                 else:
 368                     got_priority = 1
 369                     priority = new_priority
 370
 371         # Edit the section
 372         if edit_section:
 373             readline.set_completer(Sections.complete)
 374             got_section = 0
 375             while not got_section:
 376                 new_section = utils.our_raw_input("New section: ").strip()
 377                 if new_section not in Sections.sections:
 378                     print "E: '%s' is not a valid section, try again." % (new_section)
 379                 else:
 380                     got_section = 1
 381                     section = new_section
 382
 383         # Reset the readline completer
 384         readline.set_completer(None)
 385
 386     for f in new[index]["files"]:
 387         upload.pkg.files[f]["section"] = section
 388         upload.pkg.files[f]["priority"] = priority
 389     new[index]["priority"] = priority
 390     new[index]["section"] = section
 391     return new
 392
 393 ################################################################################
 394
 395 def edit_overrides (new, upload, session):
 396     print
 397     done = 0
 398     while not done:
 399         print_new (new, upload, indexed=1)
 400         new_index = {}
 401         index = 0
 402         for i in new.keys():
 403             index += 1
 404             new_index[index] = i
 405
 406         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 407
 408         got_answer = 0
 409         while not got_answer:
 410             answer = utils.our_raw_input(prompt)
 411             if not answer.isdigit():
 412                 answer = answer[:1].upper()
 413             if answer == "E" or answer == "D":
 414                 got_answer = 1
 415             elif re_isanum.match (answer):
 416                 answer = int(answer)
 417                 if (answer < 1) or (answer > index):
 418                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 419                 else:
 420                     got_answer = 1
 421
 422         if answer == 'E':
 423             edit_new(new, upload)
 424         elif answer == 'D':
 425             done = 1
 426         else:
 427             edit_index (new, upload, new_index[answer])
 428
 429     return new
 430
 431 ################################################################################
 432
 433 def edit_note(note, upload, session):
 434     # Write the current data to a temporary file
 435     (fd, temp_filename) = utils.temp_filename()
 436     editor = os.environ.get("EDITOR","vi")
 437     answer = 'E'
 438     while answer == 'E':
 439         os.system("%s %s" % (editor, temp_filename))
 440         temp_file = utils.open_file(temp_filename)
 441         newnote = temp_file.read().rstrip()
 442         temp_file.close()
 443         print "New Note:"
 444         print utils.prefix_multi_line_string(newnote,"  ")
 445         prompt = "[D]one, Edit, Abandon, Quit ?"
 446         answer = "XXX"
 447         while prompt.find(answer) == -1:
 448             answer = utils.our_raw_input(prompt)
 449             m = re_default_answer.search(prompt)
 450             if answer == "":
 451                 answer = m.group(1)
 452             answer = answer[:1].upper()
 453     os.unlink(temp_filename)
 454     if answer == 'A':
 455         return
 456     elif answer == 'Q':
 457         end()
 458         sys.exit(0)
 459
 460     comment = NewComment()
 461     comment.package = upload.pkg.changes["source"]
 462     comment.version = upload.pkg.changes["version"]
 463     comment.comment = newnote
 464     comment.author  = utils.whoami()
 465     comment.trainee = bool(Options["Trainee"])
 466     session.add(comment)
 467     session.commit()
 468
 469 ################################################################################
 470
 471 def check_pkg (upload):
 472     try:
 473         less_fd = os.popen("less -R -", 'w', 0)
 474         stdout_fd = sys.stdout
 475         try:
 476             sys.stdout = less_fd
 477             changes = utils.parse_changes (upload.pkg.changes_file)
 478             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
 479             files = upload.pkg.files
 480             for f in files.keys():
 481                 if files[f].has_key("new"):
 482                     ftype = files[f]["type"]
 483                     if ftype == "deb":
 484                         examine_package.check_deb(changes['distribution'], f)
 485                     elif ftype == "dsc":
 486                         examine_package.check_dsc(changes['distribution'], f)
 487         finally:
 488             examine_package.output_package_relations()
 489             sys.stdout = stdout_fd
 490     except IOError, e:
 491         if e.errno == errno.EPIPE:
 492             utils.warn("[examine_package] Caught EPIPE; skipping.")
 493             pass
 494         else:
 495             raise
 496     except KeyboardInterrupt:
 497         utils.warn("[examine_package] Caught C-c; skipping.")
 498         pass
 499
 500 ################################################################################
 501
 502 ## FIXME: horribly Debian specific
 503
 504 def do_bxa_notification(upload):
 505     files = upload.pkg.files
 506     summary = ""
 507     for f in files.keys():
 508         if files[f]["type"] == "deb":
 509             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 510             summary += "\n"
 511             summary += "Package: %s\n" % (control.Find("Package"))
 512             summary += "Description: %s\n" % (control.Find("Description"))
 513     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 514     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
 515     utils.send_mail(bxa_mail)
 516
 517 ################################################################################
 518
 519 def add_overrides (new, upload, session):
 520     changes = upload.pkg.changes
 521     files = upload.pkg.files
 522     srcpkg = changes.get("source")
 523
 524     for suite in changes["suite"].keys():
 525         suite_id = get_suite(suite).suite_id
 526         for pkg in new.keys():
 527             component_id = get_component(new[pkg]["component"]).component_id
 528             type_id = get_override_type(new[pkg]["type"]).overridetype_id
 529             priority_id = new[pkg]["priority id"]
 530             section_id = new[pkg]["section id"]
 531             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 532             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
 533                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
 534             for f in new[pkg]["files"]:
 535                 if files[f].has_key("new"):
 536                     del files[f]["new"]
 537             del new[pkg]
 538
 539     session.commit()
 540
 541     if Config().FindB("Dinstall::BXANotify"):
 542         do_bxa_notification(upload)
 543
 544 ################################################################################
 545
 546 def prod_maintainer (note, upload):
 547     cnf = Config()
 548     # Here we prepare an editor and get them ready to prod...
 549     (fd, temp_filename) = utils.temp_filename()
 550     temp_file = os.fdopen(fd, 'w')
 551     if len(note) > 0:
 552         for line in note:
 553             temp_file.write(line)
 554     temp_file.close()
 555     editor = os.environ.get("EDITOR","vi")
 556     answer = 'E'
 557     while answer == 'E':
 558         os.system("%s %s" % (editor, temp_filename))
 559         temp_fh = utils.open_file(temp_filename)
 560         prod_message = "".join(temp_fh.readlines())
 561         temp_fh.close()
 562         print "Prod message:"
 563         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 564         prompt = "[P]rod, Edit, Abandon, Quit ?"
 565         answer = "XXX"
 566         while prompt.find(answer) == -1:
 567             answer = utils.our_raw_input(prompt)
 568             m = re_default_answer.search(prompt)
 569             if answer == "":
 570                 answer = m.group(1)
 571             answer = answer[:1].upper()
 572     os.unlink(temp_filename)
 573     if answer == 'A':
 574         return
 575     elif answer == 'Q':
 576         end()
 577         sys.exit(0)
 578     # Otherwise, do the proding...
 579     user_email_address = utils.whoami() + " <%s>" % (
 580         cnf["Dinstall::MyAdminAddress"])
 581
 582     Subst = upload.Subst
 583
 584     Subst["__FROM_ADDRESS__"] = user_email_address
 585     Subst["__PROD_MESSAGE__"] = prod_message
 586     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
 587
 588     prod_mail_message = utils.TemplateSubst(
 589         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 590
 591     # Send the prod mail if appropriate
 592     if not cnf["Dinstall::Options::No-Mail"]:
 593         utils.send_mail(prod_mail_message)
 594
 595     print "Sent proding message"
 596
 597 ################################################################################
 598
 599 def do_new(upload, session):
 600     print "NEW\n"
 601     files = upload.pkg.files
 602     upload.check_files(not Options["No-Action"])
 603     changes = upload.pkg.changes
 604     cnf = Config()
 605
 606     # Make a copy of distribution we can happily trample on
 607     changes["suite"] = copy.copy(changes["distribution"])
 608
 609     # Fix up the list of target suites
 610     for suite in changes["suite"].keys():
 611         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
 612         if override:
 613             (olderr, newerr) = (get_suite(suite, session) == None,
 614                                 get_suite(override, session) == None)
 615             if olderr or newerr:
 616                 (oinv, newinv) = ("", "")
 617                 if olderr: oinv = "invalid "
 618                 if newerr: ninv = "invalid "
 619                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 620                         oinv, suite, ninv, override)
 621             del changes["suite"][suite]
 622             changes["suite"][override] = 1
 623     # Validate suites
 624     for suite in changes["suite"].keys():
 625         if get_suite(suite, session) is None:
 626             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 627
 628     # The main NEW processing loop
 629     done = 0
 630     while not done:
 631         # Find out what's new
 632         new = determine_new(changes, files)
 633
 634         if not new:
 635             break
 636
 637         answer = "XXX"
 638         if Options["No-Action"] or Options["Automatic"]:
 639             answer = 'S'
 640
 641         (broken, note) = print_new(new, upload, indexed=0)
 642         prompt = ""
 643
 644         if not broken and not note:
 645             prompt = "Add overrides, "
 646         if broken:
 647             print "W: [!] marked entries must be fixed before package can be processed."
 648         if note:
 649             print "W: note must be removed before package can be processed."
 650             prompt += "RemOve all notes, Remove note, "
 651
 652         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 653
 654         while prompt.find(answer) == -1:
 655             answer = utils.our_raw_input(prompt)
 656             m = re_default_answer.search(prompt)
 657             if answer == "":
 658                 answer = m.group(1)
 659             answer = answer[:1].upper()
 660
 661         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 662             utils.warn("Trainees can't do that")
 663             continue
 664
 665         if answer == 'A' and not Options["Trainee"]:
 666             try:
 667                 check_daily_lock()
 668                 done = add_overrides (new, upload, session)
 669                 do_accept(upload, session)
 670                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
 671             except CantGetLockError:
 672                 print "Hello? Operator! Give me the number for 911!"
 673                 print "Dinstall in the locked area, cant process packages, come back later"
 674         elif answer == 'C':
 675             check_pkg(upload)
 676         elif answer == 'E' and not Options["Trainee"]:
 677             new = edit_overrides (new, upload, session)
 678         elif answer == 'M' and not Options["Trainee"]:
 679             upload.pkg.remove_known_changes()
 680             aborted = upload.do_reject(manual=1,
 681                                        reject_message=Options["Manual-Reject"],
 682                                        note=get_new_comments(changes.get("source", ""), session=session))
 683             if not aborted:
 684                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
 685                 done = 1
 686         elif answer == 'N':
 687             edit_note(get_new_comments(changes.get("source", ""), session=session),
 688                       upload, session)
 689         elif answer == 'P' and not Options["Trainee"]:
 690             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
 691                             upload)
 692             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
 693         elif answer == 'R' and not Options["Trainee"]:
 694             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 695             if confirm == "y":
 696                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
 697                     session.delete(c)
 698                 session.commit()
 699         elif answer == 'O' and not Options["Trainee"]:
 700             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 701             if confirm == "y":
 702                 for c in get_new_comments(changes.get("source", ""), session=session):
 703                     session.delete(c)
 704                 session.commit()
 705
 706         elif answer == 'S':
 707             done = 1
 708         elif answer == 'Q':
 709             end()
 710             sys.exit(0)
 711
 712 ################################################################################
 713 ################################################################################
 714 ################################################################################
 715
 716 def usage (exit_code=0):
 717     print """Usage: dak process-new [OPTION]... [CHANGES]...
 718   -a, --automatic           automatic run
 719   -h, --help                show this help and exit.
 720   -m, --manual-reject=MSG   manual reject with `msg'
 721   -n, --no-action           don't do anything
 722   -t, --trainee             FTP Trainee mode
 723   -V, --version             display the version number and exit"""
 724     sys.exit(exit_code)
 725
 726 ################################################################################
 727
 728 def do_byhand(upload, session):
 729     done = 0
 730     while not done:
 731         files = upload.pkg.files
 732         will_install = 1
 733         byhand = []
 734
 735         for f in files.keys():
 736             if files[f]["type"] == "byhand":
 737                 if os.path.exists(f):
 738                     print "W: %s still present; please process byhand components and try again." % (f)
 739                     will_install = 0
 740                 else:
 741                     byhand.append(f)
 742
 743         answer = "XXXX"
 744         if Options["No-Action"]:
 745             answer = "S"
 746         if will_install:
 747             if Options["Automatic"] and not Options["No-Action"]:
 748                 answer = 'A'
 749             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 750         else:
 751             prompt = "Manual reject, [S]kip, Quit ?"
 752
 753         while prompt.find(answer) == -1:
 754             answer = utils.our_raw_input(prompt)
 755             m = re_default_answer.search(prompt)
 756             if answer == "":
 757                 answer = m.group(1)
 758             answer = answer[:1].upper()
 759
 760         if answer == 'A':
 761             try:
 762                 check_daily_lock()
 763                 done = 1
 764                 for f in byhand:
 765                     del files[f]
 766                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
 767             except CantGetLockError:
 768                 print "Hello? Operator! Give me the number for 911!"
 769                 print "Dinstall in the locked area, cant process packages, come back later"
 770         elif answer == 'M':
 771             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
 772             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
 773             done = 1
 774         elif answer == 'S':
 775             done = 1
 776         elif answer == 'Q':
 777             end()
 778             sys.exit(0)
 779
 780 ################################################################################
 781
 782 def check_daily_lock():
 783     """
 784     Raises CantGetLockError if the dinstall daily.lock exists.
 785     """
 786
 787     cnf = Config()
 788     try:
 789         os.open(cnf["Process-New::DinstallLockFile"],
 790                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 791     except OSError, e:
 792         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 793             raise CantGetLockError
 794
 795     os.unlink(cnf["Process-New::DinstallLockFile"])
 796
 797
 798 @contextlib.contextmanager
 799 def lock_package(package):
 800     """
 801     Lock C{package} so that noone else jumps in processing it.
 802
 803     @type package: string
 804     @param package: source package name to lock
 805     """
 806
 807     path = os.path.join(Config()["Process-New::LockDir"], package)
 808     try:
 809         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 810     except OSError, e:
 811         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 812             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 813             raise AlreadyLockedError, user
 814
 815     try:
 816         yield fd
 817     finally:
 818         os.unlink(path)
 819
 820 class clean_holding(object):
 821     def __init__(self,pkg):
 822         self.pkg = pkg
 823
 824     def __enter__(self):
 825         pass
 826
 827     def __exit__(self, type, value, traceback):
 828         h = Holding()
 829
 830         for f in self.pkg.files.keys():
 831             if os.path.exists(os.path.join(h.holding_dir, f)):
 832                 os.unlink(os.path.join(h.holding_dir, f))
 833
 834
 835
 836 def changes_to_newstage(upload, session):
 837     """move a changes file to newstage"""
 838     new = get_policy_queue('new', session );
 839     newstage = get_policy_queue('newstage', session );
 840
 841     chg = session.query(DBChange).filter_by(changesname=os.path.basename(upload.pkg.changes_file)).one()
 842     chg.approved_for = newstage.policy_queue_id
 843
 844     for f in chg.files:
 845         # update the changes_pending_files row
 846         f.queue = newstage
 847         utils.move(os.path.join(new.path, f.filename), newstage.path, perms=int(newstage.perms, 8))
 848
 849     utils.move(os.path.join(new.path, upload.pkg.changes_file), newstage.path, perms=int(newstage.perms, 8))
 850     chg.in_queue = newstage
 851     session.commit()
 852
 853 def _accept(upload, session):
 854     if Options["No-Action"]:
 855         return
 856     (summary, short_summary) = upload.build_summaries()
 857     # upload.accept(summary, short_summary, targetqueue)
 858
 859     changes_to_newstage(upload, session)
 860
 861 def do_accept(upload, session):
 862     print "ACCEPT"
 863     cnf = Config()
 864     if not Options["No-Action"]:
 865         (summary, short_summary) = upload.build_summaries()
 866
 867         if cnf.FindB("Dinstall::SecurityQueueHandling"):
 868             upload.dump_vars(cnf["Dir::Queue::Embargoed"])
 869             upload.move_to_queue(get_policy_queue('embargoed'))
 870             upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
 871             # Check for override disparities
 872             upload.Subst["__SUMMARY__"] = summary
 873         else:
 874             # Just a normal upload, accept it...
 875             _accept(upload, session)
 876
 877 def do_pkg(changes_file, session):
 878     new_queue = get_policy_queue('new', session );
 879     u = Upload()
 880     u.pkg.changes_file = changes_file
 881     (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
 882     u.load_changes(changes_file)
 883     u.pkg.directory = new_queue.path
 884     u.update_subst()
 885     u.logger = Logger
 886     origchanges = os.path.abspath(u.pkg.changes_file)
 887
 888     cnf = Config()
 889     bcc = "X-DAK: dak process-new"
 890     if cnf.has_key("Dinstall::Bcc"):
 891         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 892     else:
 893         u.Subst["__BCC__"] = bcc
 894
 895     files = u.pkg.files
 896     for deb_filename, f in files.items():
 897         if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
 898             u.binary_file_checks(deb_filename, session)
 899             u.check_binary_against_db(deb_filename, session)
 900         else:
 901             u.source_file_checks(deb_filename, session)
 902             u.check_source_against_db(deb_filename, session)
 903
 904         u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
 905
 906     try:
 907         with lock_package(u.pkg.changes["source"]):
 908             with clean_holding(u.pkg):
 909                 if not recheck(u, session):
 910                     return
 911
 912             # FIXME: This does need byhand checks added!
 913             new = determine_new(u.pkg.changes, files)
 914             if new:
 915                 do_new(u, session)
 916             else:
 917                 try:
 918                     check_daily_lock()
 919                     do_accept(u, session)
 920                 except CantGetLockError:
 921                     print "Hello? Operator! Give me the number for 911!"
 922                     print "Dinstall in the locked area, cant process packages, come back later"
 923 #             (new, byhand) = check_status(files)
 924 #             if new or byhand:
 925 #                 if new:
 926 #                     do_new(u, session)
 927 #                 if byhand:
 928 #                     do_byhand(u, session)
 929 #                 (new, byhand) = check_status(files)
 930
 931 #             if not new and not byhand:
 932 #                 try:
 933 #                     check_daily_lock()
 934 #                     do_accept(u)
 935 #                 except CantGetLockError:
 936 #                     print "Hello? Operator! Give me the number for 911!"
 937 #                     print "Dinstall in the locked area, cant process packages, come back later"
 938     except AlreadyLockedError, e:
 939         print "Seems to be locked by %s already, skipping..." % (e)
 940
 941 ################################################################################
 942
 943 def end():
 944     accept_count = SummaryStats().accept_count
 945     accept_bytes = SummaryStats().accept_bytes
 946
 947     if accept_count:
 948         sets = "set"
 949         if accept_count > 1:
 950             sets = "sets"
 951         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 952         Logger.log(["total",accept_count,accept_bytes])
 953
 954     if not Options["No-Action"] and not Options["Trainee"]:
 955         Logger.close()
 956
 957 ################################################################################
 958
 959 def main():
 960     global Options, Logger, Sections, Priorities
 961
 962     cnf = Config()
 963     session = DBConn().session()
 964
 965     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 966                  ('h',"help","Process-New::Options::Help"),
 967                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 968                  ('t',"trainee","Process-New::Options::Trainee"),
 969                  ('n',"no-action","Process-New::Options::No-Action")]
 970
 971     for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
 972         if not cnf.has_key("Process-New::Options::%s" % (i)):
 973             cnf["Process-New::Options::%s" % (i)] = ""
 974
 975     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
 976     if len(changes_files) == 0:
 977         new_queue = get_policy_queue('new', session );
 978         changes_files = utils.get_changes_files(new_queue.path)
 979
 980     Options = cnf.SubTree("Process-New::Options")
 981
 982     if Options["Help"]:
 983         usage()
 984
 985     if not Options["No-Action"]:
 986         try:
 987             Logger = daklog.Logger(cnf, "process-new")
 988         except CantOpenError, e:
 989             Options["Trainee"] = "True"
 990
 991     Sections = Section_Completer(session)
 992     Priorities = Priority_Completer(session)
 993     readline.parse_and_bind("tab: complete")
 994
 995     if len(changes_files) > 1:
 996         sys.stderr.write("Sorting changes...\n")
 997     changes_files = sort_changes(changes_files, session)
 998
 999     # Kill me now? **FIXME**
1000     cnf["Dinstall::Options::No-Mail"] = ""
1001
1002     for changes_file in changes_files:
1003         changes_file = utils.validate_changes_file_arg(changes_file, 0)
1004         if not changes_file:
1005             continue
1006         print "\n" + changes_file
1007
1008         do_pkg (changes_file, session)
1009
1010     end()
1011
1012 ################################################################################
1013
1014 if __name__ == '__main__':
1015     main()