dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 from __future__ import with_statement
  46
  47 import copy
  48 import errno
  49 import os
  50 import readline
  51 import stat
  52 import sys
  53 import time
  54 import contextlib
  55 import pwd
  56 import apt_pkg, apt_inst
  57 import examine_package
  58
  59 from daklib.dbconn import *
  60 from daklib.queue import *
  61 from daklib import daklog
  62 from daklib import utils
  63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  65 from daklib.summarystats import SummaryStats
  66 from daklib.config import Config
  67
  68 # Globals
  69 Options = None
  70 Logger = None
  71
  72 Priorities = None
  73 Sections = None
  74
  75 ################################################################################
  76 ################################################################################
  77 ################################################################################
  78
  79 def recheck(upload, session):
  80     files = upload.pkg.files
  81
  82     cnf = Config()
  83     for f in files.keys():
  84         # The .orig.tar.gz can disappear out from under us is it's a
  85         # duplicate of one in the archive.
  86         if not files.has_key(f):
  87             continue
  88         # Check that the source still exists
  89         if files[f]["type"] == "deb":
  90             source_version = files[f]["source version"]
  91             source_package = files[f]["source package"]
  92             if not upload.pkg.changes["architecture"].has_key("source") \
  93                and not upload.source_exists(source_package, source_version, upload.pkg.changes["distribution"].keys()):
  94                 source_epochless_version = re_no_epoch.sub('', source_version)
  95                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  96                 found = 0
  97                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
  98                     if cnf.has_key("Dir::Queue::%s" % (q)):
  99                         if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 100                             found = 1
 101                 if not found:
 102                     upload.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
 103
 104         # Version and file overwrite checks
 105         if files[f]["type"] == "deb":
 106             upload.check_binary_against_db(f, session)
 107         elif files[f]["type"] == "dsc":
 108             upload.check_source_against_db(f, session)
 109             upload.check_dsc_against_db(f, session)
 110
 111     if len(upload.rejects) > 0:
 112         answer = "XXX"
 113         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 114             answer = 'S'
 115
 116         print "REJECT\n" + upload.rejects.join("\n"),
 117         prompt = "[R]eject, Skip, Quit ?"
 118
 119         while prompt.find(answer) == -1:
 120             answer = utils.our_raw_input(prompt)
 121             m = re_default_answer.match(prompt)
 122             if answer == "":
 123                 answer = m.group(1)
 124             answer = answer[:1].upper()
 125
 126         if answer == 'R':
 127             upload.do_reject(manual=0, reject_message=upload.rejects.join("\n"))
 128             os.unlink(upload.pkg.changes_file[:-8]+".dak")
 129             return 0
 130         elif answer == 'S':
 131             return 0
 132         elif answer == 'Q':
 133             end()
 134             sys.exit(0)
 135
 136     return 1
 137
 138 ################################################################################
 139
 140 def indiv_sg_compare (a, b):
 141     """Sort by source name, source, version, 'have source', and
 142        finally by filename."""
 143     # Sort by source version
 144     q = apt_pkg.VersionCompare(a["version"], b["version"])
 145     if q:
 146         return -q
 147
 148     # Sort by 'have source'
 149     a_has_source = a["architecture"].get("source")
 150     b_has_source = b["architecture"].get("source")
 151     if a_has_source and not b_has_source:
 152         return -1
 153     elif b_has_source and not a_has_source:
 154         return 1
 155
 156     return cmp(a["filename"], b["filename"])
 157
 158 ############################################################
 159
 160 def sg_compare (a, b):
 161     a = a[1]
 162     b = b[1]
 163     """Sort by have note, source already in database and time of oldest upload."""
 164     # Sort by have note
 165     a_note_state = a["note_state"]
 166     b_note_state = b["note_state"]
 167     if a_note_state < b_note_state:
 168         return -1
 169     elif a_note_state > b_note_state:
 170         return 1
 171     # Sort by source already in database (descending)
 172     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 173     if source_in_database:
 174         return -source_in_database
 175
 176     # Sort by time of oldest upload
 177     return cmp(a["oldest"], b["oldest"])
 178
 179 def sort_changes(changes_files, session):
 180     """Sort into source groups, then sort each source group by version,
 181     have source, filename.  Finally, sort the source groups by have
 182     note, time of oldest upload of each source upload."""
 183     if len(changes_files) == 1:
 184         return changes_files
 185
 186     sorted_list = []
 187     cache = {}
 188     # Read in all the .changes files
 189     for filename in changes_files:
 190         u = Upload()
 191         try:
 192             u.pkg.load_dot_dak(filename)
 193             u.update_subst()
 194             cache[filename] = copy.copy(u.pkg.changes)
 195             cache[filename]["filename"] = filename
 196         except:
 197             sorted_list.append(filename)
 198             break
 199     # Divide the .changes into per-source groups
 200     per_source = {}
 201     for filename in cache.keys():
 202         source = cache[filename]["source"]
 203         if not per_source.has_key(source):
 204             per_source[source] = {}
 205             per_source[source]["list"] = []
 206         per_source[source]["list"].append(cache[filename])
 207     # Determine oldest time and have note status for each source group
 208     for source in per_source.keys():
 209         q = session.query(DBSource).filter_by(source = source).all()
 210         per_source[source]["source_in_database"] = len(q)>0
 211         source_list = per_source[source]["list"]
 212         first = source_list[0]
 213         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 214         have_note = 0
 215         for d in per_source[source]["list"]:
 216             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 217             if mtime < oldest:
 218                 oldest = mtime
 219             have_note += has_new_comment(d["source"], d["version"], session)
 220         per_source[source]["oldest"] = oldest
 221         if not have_note:
 222             per_source[source]["note_state"] = 0; # none
 223         elif have_note < len(source_list):
 224             per_source[source]["note_state"] = 1; # some
 225         else:
 226             per_source[source]["note_state"] = 2; # all
 227         per_source[source]["list"].sort(indiv_sg_compare)
 228     per_source_items = per_source.items()
 229     per_source_items.sort(sg_compare)
 230     for i in per_source_items:
 231         for j in i[1]["list"]:
 232             sorted_list.append(j["filename"])
 233     return sorted_list
 234
 235 ################################################################################
 236
 237 class Section_Completer:
 238     def __init__ (self, session):
 239         self.sections = []
 240         self.matches = []
 241         for s, in session.query(Section.section):
 242             self.sections.append(s)
 243
 244     def complete(self, text, state):
 245         if state == 0:
 246             self.matches = []
 247             n = len(text)
 248             for word in self.sections:
 249                 if word[:n] == text:
 250                     self.matches.append(word)
 251         try:
 252             return self.matches[state]
 253         except IndexError:
 254             return None
 255
 256 ############################################################
 257
 258 class Priority_Completer:
 259     def __init__ (self, session):
 260         self.priorities = []
 261         self.matches = []
 262         for p, in session.query(Priority.priority):
 263             self.priorities.append(p)
 264
 265     def complete(self, text, state):
 266         if state == 0:
 267             self.matches = []
 268             n = len(text)
 269             for word in self.priorities:
 270                 if word[:n] == text:
 271                     self.matches.append(word)
 272         try:
 273             return self.matches[state]
 274         except IndexError:
 275             return None
 276
 277 ################################################################################
 278
 279 def print_new (new, upload, indexed, file=sys.stdout):
 280     check_valid(new)
 281     broken = False
 282     index = 0
 283     for pkg in new.keys():
 284         index += 1
 285         section = new[pkg]["section"]
 286         priority = new[pkg]["priority"]
 287         if new[pkg]["section id"] == -1:
 288             section += "[!]"
 289             broken = True
 290         if new[pkg]["priority id"] == -1:
 291             priority += "[!]"
 292             broken = True
 293         if indexed:
 294             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 295         else:
 296             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 297         line = line.strip()+'\n'
 298         file.write(line)
 299     notes = get_new_comments(upload.pkg.changes.get("source"))
 300     for note in notes:
 301         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 302               % (note.author, note.version, note.notedate, note.comment)
 303         print "-" * 72
 304     return broken, len(notes) > 0
 305
 306 ################################################################################
 307
 308 def index_range (index):
 309     if index == 1:
 310         return "1"
 311     else:
 312         return "1-%s" % (index)
 313
 314 ################################################################################
 315 ################################################################################
 316
 317 def edit_new (new, upload):
 318     # Write the current data to a temporary file
 319     (fd, temp_filename) = utils.temp_filename()
 320     temp_file = os.fdopen(fd, 'w')
 321     print_new (new, upload, indexed=0, file=temp_file)
 322     temp_file.close()
 323     # Spawn an editor on that file
 324     editor = os.environ.get("EDITOR","vi")
 325     result = os.system("%s %s" % (editor, temp_filename))
 326     if result != 0:
 327         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 328     # Read the edited data back in
 329     temp_file = utils.open_file(temp_filename)
 330     lines = temp_file.readlines()
 331     temp_file.close()
 332     os.unlink(temp_filename)
 333     # Parse the new data
 334     for line in lines:
 335         line = line.strip()
 336         if line == "":
 337             continue
 338         s = line.split()
 339         # Pad the list if necessary
 340         s[len(s):3] = [None] * (3-len(s))
 341         (pkg, priority, section) = s[:3]
 342         if not new.has_key(pkg):
 343             utils.warn("Ignoring unknown package '%s'" % (pkg))
 344         else:
 345             # Strip off any invalid markers, print_new will readd them.
 346             if section.endswith("[!]"):
 347                 section = section[:-3]
 348             if priority.endswith("[!]"):
 349                 priority = priority[:-3]
 350             for f in new[pkg]["files"]:
 351                 upload.pkg.files[f]["section"] = section
 352                 upload.pkg.files[f]["priority"] = priority
 353             new[pkg]["section"] = section
 354             new[pkg]["priority"] = priority
 355
 356 ################################################################################
 357
 358 def edit_index (new, upload, index):
 359     priority = new[index]["priority"]
 360     section = new[index]["section"]
 361     ftype = new[index]["type"]
 362     done = 0
 363     while not done:
 364         print "\t".join([index, priority, section])
 365
 366         answer = "XXX"
 367         if ftype != "dsc":
 368             prompt = "[B]oth, Priority, Section, Done ? "
 369         else:
 370             prompt = "[S]ection, Done ? "
 371         edit_priority = edit_section = 0
 372
 373         while prompt.find(answer) == -1:
 374             answer = utils.our_raw_input(prompt)
 375             m = re_default_answer.match(prompt)
 376             if answer == "":
 377                 answer = m.group(1)
 378             answer = answer[:1].upper()
 379
 380         if answer == 'P':
 381             edit_priority = 1
 382         elif answer == 'S':
 383             edit_section = 1
 384         elif answer == 'B':
 385             edit_priority = edit_section = 1
 386         elif answer == 'D':
 387             done = 1
 388
 389         # Edit the priority
 390         if edit_priority:
 391             readline.set_completer(Priorities.complete)
 392             got_priority = 0
 393             while not got_priority:
 394                 new_priority = utils.our_raw_input("New priority: ").strip()
 395                 if new_priority not in Priorities.priorities:
 396                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 397                 else:
 398                     got_priority = 1
 399                     priority = new_priority
 400
 401         # Edit the section
 402         if edit_section:
 403             readline.set_completer(Sections.complete)
 404             got_section = 0
 405             while not got_section:
 406                 new_section = utils.our_raw_input("New section: ").strip()
 407                 if new_section not in Sections.sections:
 408                     print "E: '%s' is not a valid section, try again." % (new_section)
 409                 else:
 410                     got_section = 1
 411                     section = new_section
 412
 413         # Reset the readline completer
 414         readline.set_completer(None)
 415
 416     for f in new[index]["files"]:
 417         upload.pkg.files[f]["section"] = section
 418         upload.pkg.files[f]["priority"] = priority
 419     new[index]["priority"] = priority
 420     new[index]["section"] = section
 421     return new
 422
 423 ################################################################################
 424
 425 def edit_overrides (new, upload, session):
 426     print
 427     done = 0
 428     while not done:
 429         print_new (new, upload, indexed=1)
 430         new_index = {}
 431         index = 0
 432         for i in new.keys():
 433             index += 1
 434             new_index[index] = i
 435
 436         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 437
 438         got_answer = 0
 439         while not got_answer:
 440             answer = utils.our_raw_input(prompt)
 441             if not answer.isdigit():
 442                 answer = answer[:1].upper()
 443             if answer == "E" or answer == "D":
 444                 got_answer = 1
 445             elif re_isanum.match (answer):
 446                 answer = int(answer)
 447                 if (answer < 1) or (answer > index):
 448                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 449                 else:
 450                     got_answer = 1
 451
 452         if answer == 'E':
 453             edit_new(new, upload)
 454         elif answer == 'D':
 455             done = 1
 456         else:
 457             edit_index (new, upload, new_index[answer])
 458
 459     return new
 460
 461 ################################################################################
 462
 463 def edit_note(note, upload, session):
 464     # Write the current data to a temporary file
 465     (fd, temp_filename) = utils.temp_filename()
 466     editor = os.environ.get("EDITOR","vi")
 467     answer = 'E'
 468     while answer == 'E':
 469         os.system("%s %s" % (editor, temp_filename))
 470         temp_file = utils.open_file(temp_filename)
 471         newnote = temp_file.read().rstrip()
 472         temp_file.close()
 473         print "New Note:"
 474         print utils.prefix_multi_line_string(newnote,"  ")
 475         prompt = "[D]one, Edit, Abandon, Quit ?"
 476         answer = "XXX"
 477         while prompt.find(answer) == -1:
 478             answer = utils.our_raw_input(prompt)
 479             m = re_default_answer.search(prompt)
 480             if answer == "":
 481                 answer = m.group(1)
 482             answer = answer[:1].upper()
 483     os.unlink(temp_filename)
 484     if answer == 'A':
 485         return
 486     elif answer == 'Q':
 487         end()
 488         sys.exit(0)
 489
 490     comment = NewComment()
 491     comment.package = upload.pkg.changes["source"]
 492     comment.version = upload.pkg.changes["version"]
 493     comment.comment = newnote
 494     comment.author  = utils.whoami()
 495     comment.trainee = bool(Options["Trainee"])
 496     session.add(comment)
 497     session.commit()
 498
 499 ################################################################################
 500
 501 def check_pkg (upload):
 502     try:
 503         less_fd = os.popen("less -R -", 'w', 0)
 504         stdout_fd = sys.stdout
 505         try:
 506             sys.stdout = less_fd
 507             changes = utils.parse_changes (upload.pkg.changes_file)
 508             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
 509             files = upload.pkg.files
 510             for f in files.keys():
 511                 if files[f].has_key("new"):
 512                     ftype = files[f]["type"]
 513                     if ftype == "deb":
 514                         examine_package.check_deb(changes['distribution'], f)
 515                     elif ftype == "dsc":
 516                         examine_package.check_dsc(changes['distribution'], f)
 517         finally:
 518             examine_package.output_package_relations()
 519             sys.stdout = stdout_fd
 520     except IOError, e:
 521         if e.errno == errno.EPIPE:
 522             utils.warn("[examine_package] Caught EPIPE; skipping.")
 523             pass
 524         else:
 525             raise
 526     except KeyboardInterrupt:
 527         utils.warn("[examine_package] Caught C-c; skipping.")
 528         pass
 529
 530 ################################################################################
 531
 532 ## FIXME: horribly Debian specific
 533
 534 def do_bxa_notification(upload):
 535     files = upload.pkg.files
 536     summary = ""
 537     for f in files.keys():
 538         if files[f]["type"] == "deb":
 539             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 540             summary += "\n"
 541             summary += "Package: %s\n" % (control.Find("Package"))
 542             summary += "Description: %s\n" % (control.Find("Description"))
 543     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 544     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
 545     utils.send_mail(bxa_mail)
 546
 547 ################################################################################
 548
 549 def add_overrides (new, upload, session):
 550     changes = upload.pkg.changes
 551     files = upload.pkg.files
 552     srcpkg = changes.get("source")
 553
 554     for suite in changes["suite"].keys():
 555         suite_id = get_suite(suite).suite_id
 556         for pkg in new.keys():
 557             component_id = get_component(new[pkg]["component"]).component_id
 558             type_id = get_override_type(new[pkg]["type"]).overridetype_id
 559             priority_id = new[pkg]["priority id"]
 560             section_id = new[pkg]["section id"]
 561             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 562             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
 563                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
 564             for f in new[pkg]["files"]:
 565                 if files[f].has_key("new"):
 566                     del files[f]["new"]
 567             del new[pkg]
 568
 569     session.commit()
 570
 571     if Config().FindB("Dinstall::BXANotify"):
 572         do_bxa_notification(upload)
 573
 574 ################################################################################
 575
 576 def prod_maintainer (note, upload):
 577     cnf = Config()
 578     # Here we prepare an editor and get them ready to prod...
 579     (fd, temp_filename) = utils.temp_filename()
 580     temp_file = os.fdopen(fd, 'w')
 581     if len(note) > 0:
 582         for line in note:
 583             temp_file.write(line)
 584     temp_file.close()
 585     editor = os.environ.get("EDITOR","vi")
 586     answer = 'E'
 587     while answer == 'E':
 588         os.system("%s %s" % (editor, temp_filename))
 589         temp_fh = utils.open_file(temp_filename)
 590         prod_message = "".join(temp_fh.readlines())
 591         temp_fh.close()
 592         print "Prod message:"
 593         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 594         prompt = "[P]rod, Edit, Abandon, Quit ?"
 595         answer = "XXX"
 596         while prompt.find(answer) == -1:
 597             answer = utils.our_raw_input(prompt)
 598             m = re_default_answer.search(prompt)
 599             if answer == "":
 600                 answer = m.group(1)
 601             answer = answer[:1].upper()
 602     os.unlink(temp_filename)
 603     if answer == 'A':
 604         return
 605     elif answer == 'Q':
 606         end()
 607         sys.exit(0)
 608     # Otherwise, do the proding...
 609     user_email_address = utils.whoami() + " <%s>" % (
 610         cnf["Dinstall::MyAdminAddress"])
 611
 612     Subst = upload.Subst
 613
 614     Subst["__FROM_ADDRESS__"] = user_email_address
 615     Subst["__PROD_MESSAGE__"] = prod_message
 616     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
 617
 618     prod_mail_message = utils.TemplateSubst(
 619         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 620
 621     # Send the prod mail if appropriate
 622     if not cnf["Dinstall::Options::No-Mail"]:
 623         utils.send_mail(prod_mail_message)
 624
 625     print "Sent proding message"
 626
 627 ################################################################################
 628
 629 def do_new(upload, session):
 630     print "NEW\n"
 631     files = upload.pkg.files
 632     changes = upload.pkg.changes
 633     cnf = Config()
 634
 635     # Make a copy of distribution we can happily trample on
 636     changes["suite"] = copy.copy(changes["distribution"])
 637
 638     # Fix up the list of target suites
 639     for suite in changes["suite"].keys():
 640         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
 641         if override:
 642             (olderr, newerr) = (get_suite(suite, session) == None,
 643                                 get_suite(override, session) == None)
 644             if olderr or newerr:
 645                 (oinv, newinv) = ("", "")
 646                 if olderr: oinv = "invalid "
 647                 if newerr: ninv = "invalid "
 648                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 649                         oinv, suite, ninv, override)
 650             del changes["suite"][suite]
 651             changes["suite"][override] = 1
 652     # Validate suites
 653     for suite in changes["suite"].keys():
 654         if get_suite(suite, session) is None:
 655             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 656
 657     # The main NEW processing loop
 658     done = 0
 659     while not done:
 660         # Find out what's new
 661         new = determine_new(changes, files)
 662
 663         if not new:
 664             break
 665
 666         answer = "XXX"
 667         if Options["No-Action"] or Options["Automatic"]:
 668             answer = 'S'
 669
 670         (broken, note) = print_new(new, upload, indexed=0)
 671         prompt = ""
 672
 673         if not broken and not note:
 674             prompt = "Add overrides, "
 675         if broken:
 676             print "W: [!] marked entries must be fixed before package can be processed."
 677         if note:
 678             print "W: note must be removed before package can be processed."
 679             prompt += "RemOve all notes, Remove note, "
 680
 681         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 682
 683         while prompt.find(answer) == -1:
 684             answer = utils.our_raw_input(prompt)
 685             m = re_default_answer.search(prompt)
 686             if answer == "":
 687                 answer = m.group(1)
 688             answer = answer[:1].upper()
 689
 690         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 691             utils.warn("Trainees can't do that")
 692             continue
 693
 694         if answer == 'A' and not Options["Trainee"]:
 695             try:
 696                 check_daily_lock()
 697                 done = add_overrides (new, upload, session)
 698                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
 699             except CantGetLockError:
 700                 print "Hello? Operator! Give me the number for 911!"
 701                 print "Dinstall in the locked area, cant process packages, come back later"
 702         elif answer == 'C':
 703             check_pkg(upload)
 704         elif answer == 'E' and not Options["Trainee"]:
 705             new = edit_overrides (new, upload, session)
 706         elif answer == 'M' and not Options["Trainee"]:
 707             upload.pkg.remove_known_changes()
 708             aborted = upload.do_reject(manual=1,
 709                                        reject_message=Options["Manual-Reject"],
 710                                        note=get_new_comments(changes.get("source", ""), session=session))
 711             if not aborted:
 712                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
 713                 os.unlink(upload.pkg.changes_file[:-8]+".dak")
 714                 done = 1
 715         elif answer == 'N':
 716             edit_note(get_new_comments(changes.get("source", ""), session=session),
 717                       upload, session)
 718         elif answer == 'P' and not Options["Trainee"]:
 719             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
 720                             upload)
 721             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
 722         elif answer == 'R' and not Options["Trainee"]:
 723             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 724             if confirm == "y":
 725                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
 726                     session.delete(c)
 727                 session.commit()
 728         elif answer == 'O' and not Options["Trainee"]:
 729             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 730             if confirm == "y":
 731                 for c in get_new_comments(changes.get("source", ""), session=session):
 732                     session.delete(c)
 733                 session.commit()
 734
 735         elif answer == 'S':
 736             done = 1
 737         elif answer == 'Q':
 738             end()
 739             sys.exit(0)
 740
 741 ################################################################################
 742 ################################################################################
 743 ################################################################################
 744
 745 def usage (exit_code=0):
 746     print """Usage: dak process-new [OPTION]... [CHANGES]...
 747   -a, --automatic           automatic run
 748   -h, --help                show this help and exit.
 749   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 750   -m, --manual-reject=MSG   manual reject with `msg'
 751   -n, --no-action           don't do anything
 752   -t, --trainee             FTP Trainee mode
 753   -V, --version             display the version number and exit"""
 754     sys.exit(exit_code)
 755
 756 ################################################################################
 757
 758 def do_byhand(upload, session):
 759     done = 0
 760     while not done:
 761         files = upload.pkg.files
 762         will_install = 1
 763         byhand = []
 764
 765         for f in files.keys():
 766             if files[f]["type"] == "byhand":
 767                 if os.path.exists(f):
 768                     print "W: %s still present; please process byhand components and try again." % (f)
 769                     will_install = 0
 770                 else:
 771                     byhand.append(f)
 772
 773         answer = "XXXX"
 774         if Options["No-Action"]:
 775             answer = "S"
 776         if will_install:
 777             if Options["Automatic"] and not Options["No-Action"]:
 778                 answer = 'A'
 779             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 780         else:
 781             prompt = "Manual reject, [S]kip, Quit ?"
 782
 783         while prompt.find(answer) == -1:
 784             answer = utils.our_raw_input(prompt)
 785             m = re_default_answer.search(prompt)
 786             if answer == "":
 787                 answer = m.group(1)
 788             answer = answer[:1].upper()
 789
 790         if answer == 'A':
 791             try:
 792                 check_daily_lock()
 793                 done = 1
 794                 for f in byhand:
 795                     del files[f]
 796                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
 797             except CantGetLockError:
 798                 print "Hello? Operator! Give me the number for 911!"
 799                 print "Dinstall in the locked area, cant process packages, come back later"
 800         elif answer == 'M':
 801             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
 802             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
 803             os.unlink(upload.pkg.changes_file[:-8]+".dak")
 804             done = 1
 805         elif answer == 'S':
 806             done = 1
 807         elif answer == 'Q':
 808             end()
 809             sys.exit(0)
 810
 811 ################################################################################
 812
 813 def check_daily_lock():
 814     """
 815     Raises CantGetLockError if the dinstall daily.lock exists.
 816     """
 817
 818     cnf = Config()
 819     try:
 820         os.open(cnf["Process-New::DinstallLockFile"],
 821                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 822     except OSError, e:
 823         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 824             raise CantGetLockError
 825
 826     os.unlink(cnf["Process-New::DinstallLockFile"])
 827
 828
 829 @contextlib.contextmanager
 830 def lock_package(package):
 831     """
 832     Lock C{package} so that noone else jumps in processing it.
 833
 834     @type package: string
 835     @param package: source package name to lock
 836     """
 837
 838     path = os.path.join(Config()["Process-New::LockDir"], package)
 839     try:
 840         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 841     except OSError, e:
 842         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 843             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 844             raise AlreadyLockedError, user
 845
 846     try:
 847         yield fd
 848     finally:
 849         os.unlink(path)
 850
 851 # def move_to_dir (upload, dest, perms=0660, changesperms=0664):
 852 #     utils.move (upload.pkg.changes_file, dest, perms=changesperms)
 853 #     file_keys = upload.pkg.files.keys()
 854 #     for f in file_keys:
 855 #         utils.move (f, dest, perms=perms)
 856
 857 # def is_source_in_queue_dir(qdir):
 858 #     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 859 #                 and x.endswith(".changes") ]
 860 #     for entry in entries:
 861 #         # read the .dak
 862 #         u = queue.Upload(Cnf)
 863 #         u.pkg.changes_file = os.path.join(qdir, entry)
 864 #         u.update_vars()
 865 #         if not u.pkg.changes["architecture"].has_key("source"):
 866 #             # another binary upload, ignore
 867 #             continue
 868 #         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 869 #             # another version, ignore
 870 #             continue
 871 #         # found it!
 872 #         return True
 873 #     return False
 874
 875 # def move_to_holding(suite, queue_dir):
 876 #     print "Moving to %s holding area." % (suite.upper(),)
 877 #     if Options["No-Action"]:
 878 #       return
 879 #     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 880 #     Upload.dump_vars(queue_dir)
 881 #     move_to_dir(queue_dir, perms=0664)
 882 #     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 883
 884 def _accept(upload):
 885     if Options["No-Action"]:
 886         return
 887     (summary, short_summary) = upload.build_summaries()
 888     upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"])
 889     os.unlink(upload.pkg.changes_file[:-8]+".dak")
 890
 891 # def do_accept_stableupdate(upload,suite, q):
 892 #     cnf = Config()
 893 #     queue_dir = cnf["Dir::Queue::%s" % (q,)]
 894 #     if not upload.pkg.changes["architecture"].has_key("source"):
 895 #         # It is not a sourceful upload.  So its source may be either in p-u
 896 #         # holding, in new, in accepted or already installed.
 897 #         if is_source_in_queue_dir(queue_dir):
 898 #             # It's in p-u holding, so move it there.
 899 #             print "Binary-only upload, source in %s." % (q,)
 900 #             move_to_holding(suite, queue_dir)
 901 #         elif Upload.source_exists(Upload.pkg.changes["source"],
 902 #                 Upload.pkg.changes["version"]):
 903 #             # dak tells us that there is source available.  At time of
 904 #             # writing this means that it is installed, so put it into
 905 #             # accepted.
 906 #             print "Binary-only upload, source installed."
 907 #             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 908 #             _accept()
 909 #         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 910 #             # The source is in accepted, the binary cleared NEW: accept it.
 911 #             print "Binary-only upload, source in accepted."
 912 #             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 913 #             _accept()
 914 #         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 915 #             # It's in NEW.  We expect the source to land in p-u holding
 916 #             # pretty soon.
 917 #             print "Binary-only upload, source in new."
 918 #             move_to_holding(suite, queue_dir)
 919 #         elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
 920 #             # It's in newstage.  Accept into the holding area
 921 #             print "Binary-only upload, source in newstage."
 922 #             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 923 #             _accept()
 924 #         else:
 925 #             # No case applicable.  Bail out.  Return will cause the upload
 926 #             # to be skipped.
 927 #             print "ERROR"
 928 #             print "Stable update failed.  Source not found."
 929 #             return
 930 #     else:
 931 #         # We are handling a sourceful upload.  Move to accepted if currently
 932 #         # in p-u holding and to p-u holding otherwise.
 933 #         if is_source_in_queue_dir(queue_dir):
 934 #             print "Sourceful upload in %s, accepting." % (q,)
 935 #             _accept()
 936 #         else:
 937 #             move_to_holding(suite, queue_dir)
 938
 939 def do_accept(upload):
 940     print "ACCEPT"
 941     cnf = Config()
 942     if not Options["No-Action"]:
 943         (summary, short_summary) = upload.build_summaries()
 944 #     if cnf.FindB("Dinstall::SecurityQueueHandling"):
 945 #         upload.dump_vars(cnf["Dir::Queue::Embargoed"])
 946 #         move_to_dir(cnf["Dir::Queue::Embargoed"])
 947 #         upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
 948 #         # Check for override disparities
 949 #         upload.Subst["__SUMMARY__"] = summary
 950 #     else:
 951         # Stable updates need to be copied to proposed-updates holding
 952         # area instead of accepted.  Sourceful uploads need to go
 953         # to it directly, binaries only if the source has not yet been
 954         # accepted into p-u.
 955         for suite, q in [("proposed-updates", "ProposedUpdates"),
 956                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
 957             if not upload.pkg.changes["distribution"].has_key(suite):
 958                 continue
 959             utils.fubar("stable accept not supported yet")
 960 #            return do_accept_stableupdate(suite, q)
 961         # Just a normal upload, accept it...
 962         _accept(upload)
 963
 964 def check_status(files):
 965     new = byhand = 0
 966     for f in files.keys():
 967         if files[f]["type"] == "byhand":
 968             byhand = 1
 969         elif files[f].has_key("new"):
 970             new = 1
 971     return (new, byhand)
 972
 973 def do_pkg(changes_file, session):
 974     u = Upload()
 975     u.pkg.load_dot_dak(changes_file)
 976     u.update_subst()
 977
 978     cnf = Config()
 979     bcc = "X-DAK: dak process-new"
 980     if cnf.has_key("Dinstall::Bcc"):
 981         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 982     else:
 983         u.Subst["__BCC__"] = bcc
 984
 985     files = u.pkg.files
 986
 987     try:
 988         with lock_package(u.pkg.changes["source"]):
 989             if not recheck(u, session):
 990                 return
 991
 992             (new, byhand) = check_status(files)
 993             if new or byhand:
 994                 if new:
 995                     do_new(u, session)
 996                 if byhand:
 997                     do_byhand(u, session)
 998                 (new, byhand) = check_status(files)
 999
1000             if not new and not byhand:
1001                 try:
1002                     check_daily_lock()
1003                     do_accept(u)
1004                 except CantGetLockError:
1005                     print "Hello? Operator! Give me the number for 911!"
1006                     print "Dinstall in the locked area, cant process packages, come back later"
1007     except AlreadyLockedError, e:
1008         print "Seems to be locked by %s already, skipping..." % (e)
1009
1010 ################################################################################
1011
1012 def end():
1013     accept_count = SummaryStats().accept_count
1014     accept_bytes = SummaryStats().accept_bytes
1015
1016     if accept_count:
1017         sets = "set"
1018         if accept_count > 1:
1019             sets = "sets"
1020         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1021         Logger.log(["total",accept_count,accept_bytes])
1022
1023     if not Options["No-Action"] and not Options["Trainee"]:
1024         Logger.close()
1025
1026 ################################################################################
1027
1028 # def do_comments(dir, opref, npref, line, fn):
1029 #     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1030 #         lines = open("%s/%s" % (dir, comm)).readlines()
1031 #         if len(lines) == 0 or lines[0] != line + "\n": continue
1032 #         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1033 #                                 and x.endswith(".changes") ]
1034 #         changes_files = sort_changes(changes_files)
1035 #         for f in changes_files:
1036 #             f = utils.validate_changes_file_arg(f, 0)
1037 #             if not f: continue
1038 #             print "\n" + f
1039 #             fn(f, "".join(lines[1:]))
1040
1041 #         if opref != npref and not Options["No-Action"]:
1042 #             newcomm = npref + comm[len(opref):]
1043 #             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1044
1045 # ################################################################################
1046
1047 # def comment_accept(changes_file, comments):
1048 #     Upload.pkg.changes_file = changes_file
1049 #     Upload.init_vars()
1050 #     Upload.update_vars()
1051 #     Upload.update_subst()
1052 #     files = Upload.pkg.files
1053
1054 #     if not recheck():
1055 #         return # dak wants to REJECT, crap
1056
1057 #     (new, byhand) = check_status(files)
1058 #     if not new and not byhand:
1059 #         do_accept()
1060
1061 # ################################################################################
1062
1063 # def comment_reject(changes_file, comments):
1064 #     Upload.pkg.changes_file = changes_file
1065 #     Upload.init_vars()
1066 #     Upload.update_vars()
1067 #     Upload.update_subst()
1068
1069 #     if not recheck():
1070 #         pass # dak has its own reasons to reject as well, which is fine
1071
1072 #     reject(comments)
1073 #     print "REJECT\n" + reject_message,
1074 #     if not Options["No-Action"]:
1075 #         Upload.do_reject(0, reject_message)
1076 #         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1077
1078 ################################################################################
1079
1080 def main():
1081     global Options, Logger, Sections, Priorities
1082
1083     cnf = Config()
1084     session = DBConn().session()
1085
1086     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
1087                  ('h',"help","Process-New::Options::Help"),
1088                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
1089                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
1090                  ('t',"trainee","Process-New::Options::Trainee"),
1091                  ('n',"no-action","Process-New::Options::No-Action")]
1092
1093     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
1094         if not cnf.has_key("Process-New::Options::%s" % (i)):
1095             cnf["Process-New::Options::%s" % (i)] = ""
1096
1097     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
1098     if len(changes_files) == 0 and not cnf.get("Process-New::Options::Comments-Dir",""):
1099         changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
1100
1101     Options = cnf.SubTree("Process-New::Options")
1102
1103     if Options["Help"]:
1104         usage()
1105
1106     if not Options["No-Action"]:
1107         try:
1108             Logger = daklog.Logger(cnf, "process-new")
1109         except CantOpenError, e:
1110             Options["Trainee"] = "True"
1111
1112     Sections = Section_Completer(session)
1113     Priorities = Priority_Completer(session)
1114     readline.parse_and_bind("tab: complete")
1115
1116     if len(changes_files) > 1:
1117         sys.stderr.write("Sorting changes...\n")
1118     changes_files = sort_changes(changes_files, session)
1119
1120     # Kill me now? **FIXME**
1121     cnf["Dinstall::Options::No-Mail"] = ""
1122
1123 #     commentsdir = cnf.get("Process-New::Options::Comments-Dir","")
1124 #     if commentsdir:
1125 #        if changes_files != []:
1126 #            sys.stderr.write("Can't specify any changes files if working with comments-dir")
1127 #            sys.exit(1)
1128 #        do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1129 #        do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1130 #     else:
1131     if True:
1132         for changes_file in changes_files:
1133             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1134             if not changes_file:
1135                 continue
1136             print "\n" + changes_file
1137
1138             do_pkg (changes_file, session)
1139
1140     end()
1141
1142 ################################################################################
1143
1144 if __name__ == '__main__':
1145     main()