dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 from __future__ import with_statement
  46
  47 import copy
  48 import errno
  49 import os
  50 import readline
  51 import stat
  52 import sys
  53 import time
  54 import contextlib
  55 import pwd
  56 import apt_pkg, apt_inst
  57 import examine_package
  58
  59 from daklib.dbconn import *
  60 from daklib.queue import *
  61 from daklib import daklog
  62 from daklib import utils
  63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  65 from daklib.summarystats import SummaryStats
  66 from daklib.config import Config
  67
  68 # Globals
  69 Options = None
  70 Logger = None
  71
  72 Priorities = None
  73 Sections = None
  74
  75 ################################################################################
  76 ################################################################################
  77 ################################################################################
  78
  79 def recheck(upload, session):
  80     files = upload.pkg.files
  81
  82     cnf = Config()
  83     for f in files.keys():
  84         # The .orig.tar.gz can disappear out from under us is it's a
  85         # duplicate of one in the archive.
  86         if not files.has_key(f):
  87             continue
  88         # Check that the source still exists
  89         if files[f]["type"] == "deb":
  90             source_version = files[f]["source version"]
  91             source_package = files[f]["source package"]
  92             if not upload.pkg.changes["architecture"].has_key("source") \
  93                and not upload.source_exists(source_package, source_version, upload.pkg.changes["distribution"].keys()):
  94                 source_epochless_version = re_no_epoch.sub('', source_version)
  95                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  96                 found = 0
  97                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
  98                     if cnf.has_key("Dir::Queue::%s" % (q)):
  99                         if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 100                             found = 1
 101                 if not found:
 102                     upload.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
 103
 104         # Version and file overwrite checks
 105         if files[f]["type"] == "deb":
 106             upload.check_binary_against_db(f, session)
 107         elif files[f]["type"] == "dsc":
 108             upload.check_source_against_db(f, session)
 109             upload.check_dsc_against_db(f, session)
 110
 111     if len(upload.rejects) > 0:
 112         answer = "XXX"
 113         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 114             answer = 'S'
 115
 116         print "REJECT\n" + upload.rejects.join("\n"),
 117         prompt = "[R]eject, Skip, Quit ?"
 118
 119         while prompt.find(answer) == -1:
 120             answer = utils.our_raw_input(prompt)
 121             m = re_default_answer.match(prompt)
 122             if answer == "":
 123                 answer = m.group(1)
 124             answer = answer[:1].upper()
 125
 126         if answer == 'R':
 127             upload.do_reject(manual=0, reject_message=upload.rejects.join("\n"))
 128             os.unlink(upload.pkg.changes_file[:-8]+".dak")
 129             return 0
 130         elif answer == 'S':
 131             return 0
 132         elif answer == 'Q':
 133             end()
 134             sys.exit(0)
 135
 136     return 1
 137
 138 ################################################################################
 139
 140 def indiv_sg_compare (a, b):
 141     """Sort by source name, source, version, 'have source', and
 142        finally by filename."""
 143     # Sort by source version
 144     q = apt_pkg.VersionCompare(a["version"], b["version"])
 145     if q:
 146         return -q
 147
 148     # Sort by 'have source'
 149     a_has_source = a["architecture"].get("source")
 150     b_has_source = b["architecture"].get("source")
 151     if a_has_source and not b_has_source:
 152         return -1
 153     elif b_has_source and not a_has_source:
 154         return 1
 155
 156     return cmp(a["filename"], b["filename"])
 157
 158 ############################################################
 159
 160 def sg_compare (a, b):
 161     a = a[1]
 162     b = b[1]
 163     """Sort by have note, source already in database and time of oldest upload."""
 164     # Sort by have note
 165     a_note_state = a["note_state"]
 166     b_note_state = b["note_state"]
 167     if a_note_state < b_note_state:
 168         return -1
 169     elif a_note_state > b_note_state:
 170         return 1
 171     # Sort by source already in database (descending)
 172     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 173     if source_in_database:
 174         return -source_in_database
 175
 176     # Sort by time of oldest upload
 177     return cmp(a["oldest"], b["oldest"])
 178
 179 def sort_changes(changes_files, session):
 180     """Sort into source groups, then sort each source group by version,
 181     have source, filename.  Finally, sort the source groups by have
 182     note, time of oldest upload of each source upload."""
 183     if len(changes_files) == 1:
 184         return changes_files
 185
 186     sorted_list = []
 187     cache = {}
 188     # Read in all the .changes files
 189     for filename in changes_files:
 190         u = Upload()
 191         try:
 192             u.pkg.load_dot_dak(filename)
 193             u.update_subst()
 194             cache[filename] = copy.copy(u.pkg.changes)
 195             cache[filename]["filename"] = filename
 196         except:
 197             sorted_list.append(filename)
 198             break
 199     # Divide the .changes into per-source groups
 200     per_source = {}
 201     for filename in cache.keys():
 202         source = cache[filename]["source"]
 203         if not per_source.has_key(source):
 204             per_source[source] = {}
 205             per_source[source]["list"] = []
 206         per_source[source]["list"].append(cache[filename])
 207     # Determine oldest time and have note status for each source group
 208     for source in per_source.keys():
 209         q = session.query(DBSource).filter_by(source = source).all()
 210         per_source[source]["source_in_database"] = len(q)>0
 211         source_list = per_source[source]["list"]
 212         first = source_list[0]
 213         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 214         have_note = 0
 215         for d in per_source[source]["list"]:
 216             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 217             if mtime < oldest:
 218                 oldest = mtime
 219             have_note += has_new_comment(d["source"], d["version"], session)
 220         per_source[source]["oldest"] = oldest
 221         if not have_note:
 222             per_source[source]["note_state"] = 0; # none
 223         elif have_note < len(source_list):
 224             per_source[source]["note_state"] = 1; # some
 225         else:
 226             per_source[source]["note_state"] = 2; # all
 227         per_source[source]["list"].sort(indiv_sg_compare)
 228     per_source_items = per_source.items()
 229     per_source_items.sort(sg_compare)
 230     for i in per_source_items:
 231         for j in i[1]["list"]:
 232             sorted_list.append(j["filename"])
 233     return sorted_list
 234
 235 ################################################################################
 236
 237 class Section_Completer:
 238     def __init__ (self, session):
 239         self.sections = []
 240         self.matches = []
 241         for s, in session.query(Section.section):
 242             self.sections.append(s)
 243
 244     def complete(self, text, state):
 245         if state == 0:
 246             self.matches = []
 247             n = len(text)
 248             for word in self.sections:
 249                 if word[:n] == text:
 250                     self.matches.append(word)
 251         try:
 252             return self.matches[state]
 253         except IndexError:
 254             return None
 255
 256 ############################################################
 257
 258 class Priority_Completer:
 259     def __init__ (self, session):
 260         self.priorities = []
 261         self.matches = []
 262         for p, in session.query(Priority.priority):
 263             self.priorities.append(p)
 264
 265     def complete(self, text, state):
 266         if state == 0:
 267             self.matches = []
 268             n = len(text)
 269             for word in self.priorities:
 270                 if word[:n] == text:
 271                     self.matches.append(word)
 272         try:
 273             return self.matches[state]
 274         except IndexError:
 275             return None
 276
 277 ################################################################################
 278
 279 def print_new (new, upload, indexed, file=sys.stdout):
 280     check_valid(new)
 281     broken = False
 282     index = 0
 283     for pkg in new.keys():
 284         index += 1
 285         section = new[pkg]["section"]
 286         priority = new[pkg]["priority"]
 287         if new[pkg]["section id"] == -1:
 288             section += "[!]"
 289             broken = True
 290         if new[pkg]["priority id"] == -1:
 291             priority += "[!]"
 292             broken = True
 293         if indexed:
 294             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 295         else:
 296             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 297         line = line.strip()+'\n'
 298         file.write(line)
 299     notes = get_new_comments(upload.pkg.changes.get("source"))
 300     for note in notes:
 301         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 302               % (note.author, note.version, note.notedate, note.comment)
 303         print "-" * 72
 304     return broken, len(notes) > 0
 305
 306 ################################################################################
 307
 308 def index_range (index):
 309     if index == 1:
 310         return "1"
 311     else:
 312         return "1-%s" % (index)
 313
 314 ################################################################################
 315 ################################################################################
 316
 317 def edit_new (new, upload):
 318     # Write the current data to a temporary file
 319     (fd, temp_filename) = utils.temp_filename()
 320     temp_file = os.fdopen(fd, 'w')
 321     print_new (new, upload, indexed=0, file=temp_file)
 322     temp_file.close()
 323     # Spawn an editor on that file
 324     editor = os.environ.get("EDITOR","vi")
 325     result = os.system("%s %s" % (editor, temp_filename))
 326     if result != 0:
 327         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 328     # Read the edited data back in
 329     temp_file = utils.open_file(temp_filename)
 330     lines = temp_file.readlines()
 331     temp_file.close()
 332     os.unlink(temp_filename)
 333     # Parse the new data
 334     for line in lines:
 335         line = line.strip()
 336         if line == "":
 337             continue
 338         s = line.split()
 339         # Pad the list if necessary
 340         s[len(s):3] = [None] * (3-len(s))
 341         (pkg, priority, section) = s[:3]
 342         if not new.has_key(pkg):
 343             utils.warn("Ignoring unknown package '%s'" % (pkg))
 344         else:
 345             # Strip off any invalid markers, print_new will readd them.
 346             if section.endswith("[!]"):
 347                 section = section[:-3]
 348             if priority.endswith("[!]"):
 349                 priority = priority[:-3]
 350             for f in new[pkg]["files"]:
 351                 upload.pkg.files[f]["section"] = section
 352                 upload.pkg.files[f]["priority"] = priority
 353             new[pkg]["section"] = section
 354             new[pkg]["priority"] = priority
 355
 356 ################################################################################
 357
 358 def edit_index (new, upload, index):
 359     priority = new[index]["priority"]
 360     section = new[index]["section"]
 361     ftype = new[index]["type"]
 362     done = 0
 363     while not done:
 364         print "\t".join([index, priority, section])
 365
 366         answer = "XXX"
 367         if ftype != "dsc":
 368             prompt = "[B]oth, Priority, Section, Done ? "
 369         else:
 370             prompt = "[S]ection, Done ? "
 371         edit_priority = edit_section = 0
 372
 373         while prompt.find(answer) == -1:
 374             answer = utils.our_raw_input(prompt)
 375             m = re_default_answer.match(prompt)
 376             if answer == "":
 377                 answer = m.group(1)
 378             answer = answer[:1].upper()
 379
 380         if answer == 'P':
 381             edit_priority = 1
 382         elif answer == 'S':
 383             edit_section = 1
 384         elif answer == 'B':
 385             edit_priority = edit_section = 1
 386         elif answer == 'D':
 387             done = 1
 388
 389         # Edit the priority
 390         if edit_priority:
 391             readline.set_completer(Priorities.complete)
 392             got_priority = 0
 393             while not got_priority:
 394                 new_priority = utils.our_raw_input("New priority: ").strip()
 395                 if new_priority not in Priorities.priorities:
 396                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 397                 else:
 398                     got_priority = 1
 399                     priority = new_priority
 400
 401         # Edit the section
 402         if edit_section:
 403             readline.set_completer(Sections.complete)
 404             got_section = 0
 405             while not got_section:
 406                 new_section = utils.our_raw_input("New section: ").strip()
 407                 if new_section not in Sections.sections:
 408                     print "E: '%s' is not a valid section, try again." % (new_section)
 409                 else:
 410                     got_section = 1
 411                     section = new_section
 412
 413         # Reset the readline completer
 414         readline.set_completer(None)
 415
 416     for f in new[index]["files"]:
 417         upload.pkg.files[f]["section"] = section
 418         upload.pkg.files[f]["priority"] = priority
 419     new[index]["priority"] = priority
 420     new[index]["section"] = section
 421     return new
 422
 423 ################################################################################
 424
 425 def edit_overrides (new, upload, session):
 426     print
 427     done = 0
 428     while not done:
 429         print_new (new, upload, indexed=1)
 430         new_index = {}
 431         index = 0
 432         for i in new.keys():
 433             index += 1
 434             new_index[index] = i
 435
 436         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 437
 438         got_answer = 0
 439         while not got_answer:
 440             answer = utils.our_raw_input(prompt)
 441             if not answer.isdigit():
 442                 answer = answer[:1].upper()
 443             if answer == "E" or answer == "D":
 444                 got_answer = 1
 445             elif re_isanum.match (answer):
 446                 answer = int(answer)
 447                 if (answer < 1) or (answer > index):
 448                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 449                 else:
 450                     got_answer = 1
 451
 452         if answer == 'E':
 453             edit_new(new, upload)
 454         elif answer == 'D':
 455             done = 1
 456         else:
 457             edit_index (new, upload, new_index[answer])
 458
 459     return new
 460
 461 ################################################################################
 462
 463 def edit_note(note, upload, session):
 464     # Write the current data to a temporary file
 465     (fd, temp_filename) = utils.temp_filename()
 466     editor = os.environ.get("EDITOR","vi")
 467     answer = 'E'
 468     while answer == 'E':
 469         os.system("%s %s" % (editor, temp_filename))
 470         temp_file = utils.open_file(temp_filename)
 471         newnote = temp_file.read().rstrip()
 472         temp_file.close()
 473         print "New Note:"
 474         print utils.prefix_multi_line_string(newnote,"  ")
 475         prompt = "[D]one, Edit, Abandon, Quit ?"
 476         answer = "XXX"
 477         while prompt.find(answer) == -1:
 478             answer = utils.our_raw_input(prompt)
 479             m = re_default_answer.search(prompt)
 480             if answer == "":
 481                 answer = m.group(1)
 482             answer = answer[:1].upper()
 483     os.unlink(temp_filename)
 484     if answer == 'A':
 485         return
 486     elif answer == 'Q':
 487         end()
 488         sys.exit(0)
 489
 490     comment = NewComment()
 491     comment.package = upload.pkg.changes["source"]
 492     comment.version = upload.pkg.changes["version"]
 493     comment.comment = newnote
 494     comment.author  = utils.whoami()
 495     comment.trainee = bool(Options["Trainee"])
 496     session.add(comment)
 497     session.commit()
 498
 499 ################################################################################
 500
 501 def check_pkg (upload):
 502     try:
 503         less_fd = os.popen("less -R -", 'w', 0)
 504         stdout_fd = sys.stdout
 505         try:
 506             sys.stdout = less_fd
 507             changes = utils.parse_changes (upload.pkg.changes_file)
 508             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
 509             files = upload.pkg.files
 510             for f in files.keys():
 511                 if files[f].has_key("new"):
 512                     ftype = files[f]["type"]
 513                     if ftype == "deb":
 514                         examine_package.check_deb(changes['distribution'], f)
 515                     elif ftype == "dsc":
 516                         examine_package.check_dsc(changes['distribution'], f)
 517         finally:
 518             examine_package.output_package_relations()
 519             sys.stdout = stdout_fd
 520     except IOError, e:
 521         if e.errno == errno.EPIPE:
 522             utils.warn("[examine_package] Caught EPIPE; skipping.")
 523             pass
 524         else:
 525             raise
 526     except KeyboardInterrupt:
 527         utils.warn("[examine_package] Caught C-c; skipping.")
 528         pass
 529
 530 ################################################################################
 531
 532 ## FIXME: horribly Debian specific
 533
 534 def do_bxa_notification(upload):
 535     files = upload.pkg.files
 536     summary = ""
 537     for f in files.keys():
 538         if files[f]["type"] == "deb":
 539             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 540             summary += "\n"
 541             summary += "Package: %s\n" % (control.Find("Package"))
 542             summary += "Description: %s\n" % (control.Find("Description"))
 543     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 544     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
 545     utils.send_mail(bxa_mail)
 546
 547 ################################################################################
 548
 549 def add_overrides (new, upload, session):
 550     changes = upload.pkg.changes
 551     files = upload.pkg.files
 552     srcpkg = changes.get("source")
 553
 554     for suite in changes["suite"].keys():
 555         suite_id = get_suite(suite).suite_id
 556         for pkg in new.keys():
 557             component_id = get_component(new[pkg]["component"]).component_id
 558             type_id = get_override_type(new[pkg]["type"]).overridetype_id
 559             priority_id = new[pkg]["priority id"]
 560             section_id = new[pkg]["section id"]
 561             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 562             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
 563                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
 564             for f in new[pkg]["files"]:
 565                 if files[f].has_key("new"):
 566                     del files[f]["new"]
 567             del new[pkg]
 568
 569     session.commit()
 570
 571     if Config().FindB("Dinstall::BXANotify"):
 572         do_bxa_notification(upload)
 573
 574 ################################################################################
 575
 576 def prod_maintainer (note, upload):
 577     cnf = Config()
 578     # Here we prepare an editor and get them ready to prod...
 579     (fd, temp_filename) = utils.temp_filename()
 580     temp_file = os.fdopen(fd, 'w')
 581     if len(note) > 0:
 582         for line in note:
 583             temp_file.write(line)
 584     temp_file.close()
 585     editor = os.environ.get("EDITOR","vi")
 586     answer = 'E'
 587     while answer == 'E':
 588         os.system("%s %s" % (editor, temp_filename))
 589         temp_fh = utils.open_file(temp_filename)
 590         prod_message = "".join(temp_fh.readlines())
 591         temp_fh.close()
 592         print "Prod message:"
 593         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 594         prompt = "[P]rod, Edit, Abandon, Quit ?"
 595         answer = "XXX"
 596         while prompt.find(answer) == -1:
 597             answer = utils.our_raw_input(prompt)
 598             m = re_default_answer.search(prompt)
 599             if answer == "":
 600                 answer = m.group(1)
 601             answer = answer[:1].upper()
 602     os.unlink(temp_filename)
 603     if answer == 'A':
 604         return
 605     elif answer == 'Q':
 606         end()
 607         sys.exit(0)
 608     # Otherwise, do the proding...
 609     user_email_address = utils.whoami() + " <%s>" % (
 610         cnf["Dinstall::MyAdminAddress"])
 611
 612     Subst = upload.Subst
 613
 614     Subst["__FROM_ADDRESS__"] = user_email_address
 615     Subst["__PROD_MESSAGE__"] = prod_message
 616     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
 617
 618     prod_mail_message = utils.TemplateSubst(
 619         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 620
 621     # Send the prod mail if appropriate
 622     if not cnf["Dinstall::Options::No-Mail"]:
 623         utils.send_mail(prod_mail_message)
 624
 625     print "Sent proding message"
 626
 627 ################################################################################
 628
 629 def do_new(upload, session):
 630     print "NEW\n"
 631     files = upload.pkg.files
 632     changes = upload.pkg.changes
 633     cnf = Config()
 634
 635     # Make a copy of distribution we can happily trample on
 636     changes["suite"] = copy.copy(changes["distribution"])
 637
 638     # Fix up the list of target suites
 639     for suite in changes["suite"].keys():
 640         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
 641         if override:
 642             (olderr, newerr) = (get_suite(suite, session) == None,
 643                                 get_suite(override, session) == None)
 644             if olderr or newerr:
 645                 (oinv, newinv) = ("", "")
 646                 if olderr: oinv = "invalid "
 647                 if newerr: ninv = "invalid "
 648                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 649                         oinv, suite, ninv, override)
 650             del changes["suite"][suite]
 651             changes["suite"][override] = 1
 652     # Validate suites
 653     for suite in changes["suite"].keys():
 654         if get_suite(suite, session) is None:
 655             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 656
 657     # The main NEW processing loop
 658     done = 0
 659     while not done:
 660         # Find out what's new
 661         new = determine_new(changes, files)
 662
 663         if not new:
 664             break
 665
 666         answer = "XXX"
 667         if Options["No-Action"] or Options["Automatic"]:
 668             answer = 'S'
 669
 670         (broken, note) = print_new(new, upload, indexed=0)
 671         prompt = ""
 672
 673         if not broken and not note:
 674             prompt = "Add overrides, "
 675         if broken:
 676             print "W: [!] marked entries must be fixed before package can be processed."
 677         if note:
 678             print "W: note must be removed before package can be processed."
 679             prompt += "RemOve all notes, Remove note, "
 680
 681         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 682
 683         while prompt.find(answer) == -1:
 684             answer = utils.our_raw_input(prompt)
 685             m = re_default_answer.search(prompt)
 686             if answer == "":
 687                 answer = m.group(1)
 688             answer = answer[:1].upper()
 689
 690         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 691             utils.warn("Trainees can't do that")
 692             continue
 693
 694         if answer == 'A' and not Options["Trainee"]:
 695             try:
 696                 check_daily_lock()
 697                 done = add_overrides (new, upload, session)
 698                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
 699             except CantGetLockError:
 700                 print "Hello? Operator! Give me the number for 911!"
 701                 print "Dinstall in the locked area, cant process packages, come back later"
 702         elif answer == 'C':
 703             check_pkg(upload)
 704         elif answer == 'E' and not Options["Trainee"]:
 705             new = edit_overrides (new, upload, session)
 706         elif answer == 'M' and not Options["Trainee"]:
 707             aborted = upload.do_reject(manual=1,
 708                                        reject_message=Options["Manual-Reject"],
 709                                        note=get_new_comments(changes.get("source", ""), session=session))
 710             if not aborted:
 711                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
 712                 os.unlink(upload.pkg.changes_file[:-8]+".dak")
 713                 done = 1
 714         elif answer == 'N':
 715             edit_note(get_new_comments(changes.get("source", ""), session=session),
 716                       upload, session)
 717         elif answer == 'P' and not Options["Trainee"]:
 718             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
 719                             upload)
 720             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
 721         elif answer == 'R' and not Options["Trainee"]:
 722             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 723             if confirm == "y":
 724                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
 725                     session.delete(c)
 726                 session.commit()
 727         elif answer == 'O' and not Options["Trainee"]:
 728             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 729             if confirm == "y":
 730                 for c in get_new_comments(changes.get("source", ""), session=session):
 731                     session.delete(c)
 732                 session.commit()
 733
 734         elif answer == 'S':
 735             done = 1
 736         elif answer == 'Q':
 737             end()
 738             sys.exit(0)
 739
 740 ################################################################################
 741 ################################################################################
 742 ################################################################################
 743
 744 def usage (exit_code=0):
 745     print """Usage: dak process-new [OPTION]... [CHANGES]...
 746   -a, --automatic           automatic run
 747   -h, --help                show this help and exit.
 748   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 749   -m, --manual-reject=MSG   manual reject with `msg'
 750   -n, --no-action           don't do anything
 751   -t, --trainee             FTP Trainee mode
 752   -V, --version             display the version number and exit"""
 753     sys.exit(exit_code)
 754
 755 ################################################################################
 756
 757 def do_byhand(upload, session):
 758     done = 0
 759     while not done:
 760         files = upload.pkg.files
 761         will_install = 1
 762         byhand = []
 763
 764         for f in files.keys():
 765             if files[f]["type"] == "byhand":
 766                 if os.path.exists(f):
 767                     print "W: %s still present; please process byhand components and try again." % (f)
 768                     will_install = 0
 769                 else:
 770                     byhand.append(f)
 771
 772         answer = "XXXX"
 773         if Options["No-Action"]:
 774             answer = "S"
 775         if will_install:
 776             if Options["Automatic"] and not Options["No-Action"]:
 777                 answer = 'A'
 778             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 779         else:
 780             prompt = "Manual reject, [S]kip, Quit ?"
 781
 782         while prompt.find(answer) == -1:
 783             answer = utils.our_raw_input(prompt)
 784             m = re_default_answer.search(prompt)
 785             if answer == "":
 786                 answer = m.group(1)
 787             answer = answer[:1].upper()
 788
 789         if answer == 'A':
 790             try:
 791                 check_daily_lock()
 792                 done = 1
 793                 for f in byhand:
 794                     del files[f]
 795                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
 796             except CantGetLockError:
 797                 print "Hello? Operator! Give me the number for 911!"
 798                 print "Dinstall in the locked area, cant process packages, come back later"
 799         elif answer == 'M':
 800             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
 801             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
 802             os.unlink(upload.pkg.changes_file[:-8]+".dak")
 803             done = 1
 804         elif answer == 'S':
 805             done = 1
 806         elif answer == 'Q':
 807             end()
 808             sys.exit(0)
 809
 810 ################################################################################
 811
 812 def check_daily_lock():
 813     """
 814     Raises CantGetLockError if the dinstall daily.lock exists.
 815     """
 816
 817     cnf = Config()
 818     try:
 819         os.open(cnf["Process-New::DinstallLockFile"],
 820                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 821     except OSError, e:
 822         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 823             raise CantGetLockError
 824
 825     os.unlink(cnf["Process-New::DinstallLockFile"])
 826
 827
 828 @contextlib.contextmanager
 829 def lock_package(package):
 830     """
 831     Lock C{package} so that noone else jumps in processing it.
 832
 833     @type package: string
 834     @param package: source package name to lock
 835     """
 836
 837     path = os.path.join(Config()["Process-New::LockDir"], package)
 838     try:
 839         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 840     except OSError, e:
 841         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 842             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 843             raise AlreadyLockedError, user
 844
 845     try:
 846         yield fd
 847     finally:
 848         os.unlink(path)
 849
 850 # def move_to_dir (upload, dest, perms=0660, changesperms=0664):
 851 #     utils.move (upload.pkg.changes_file, dest, perms=changesperms)
 852 #     file_keys = upload.pkg.files.keys()
 853 #     for f in file_keys:
 854 #         utils.move (f, dest, perms=perms)
 855
 856 # def is_source_in_queue_dir(qdir):
 857 #     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 858 #                 and x.endswith(".changes") ]
 859 #     for entry in entries:
 860 #         # read the .dak
 861 #         u = queue.Upload(Cnf)
 862 #         u.pkg.changes_file = os.path.join(qdir, entry)
 863 #         u.update_vars()
 864 #         if not u.pkg.changes["architecture"].has_key("source"):
 865 #             # another binary upload, ignore
 866 #             continue
 867 #         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 868 #             # another version, ignore
 869 #             continue
 870 #         # found it!
 871 #         return True
 872 #     return False
 873
 874 # def move_to_holding(suite, queue_dir):
 875 #     print "Moving to %s holding area." % (suite.upper(),)
 876 #     if Options["No-Action"]:
 877 #       return
 878 #     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 879 #     Upload.dump_vars(queue_dir)
 880 #     move_to_dir(queue_dir, perms=0664)
 881 #     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 882
 883 def _accept(upload):
 884     if Options["No-Action"]:
 885         return
 886     (summary, short_summary) = upload.build_summaries()
 887     upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"])
 888     os.unlink(upload.pkg.changes_file[:-8]+".dak")
 889
 890 # def do_accept_stableupdate(upload,suite, q):
 891 #     cnf = Config()
 892 #     queue_dir = cnf["Dir::Queue::%s" % (q,)]
 893 #     if not upload.pkg.changes["architecture"].has_key("source"):
 894 #         # It is not a sourceful upload.  So its source may be either in p-u
 895 #         # holding, in new, in accepted or already installed.
 896 #         if is_source_in_queue_dir(queue_dir):
 897 #             # It's in p-u holding, so move it there.
 898 #             print "Binary-only upload, source in %s." % (q,)
 899 #             move_to_holding(suite, queue_dir)
 900 #         elif Upload.source_exists(Upload.pkg.changes["source"],
 901 #                 Upload.pkg.changes["version"]):
 902 #             # dak tells us that there is source available.  At time of
 903 #             # writing this means that it is installed, so put it into
 904 #             # accepted.
 905 #             print "Binary-only upload, source installed."
 906 #             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 907 #             _accept()
 908 #         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 909 #             # The source is in accepted, the binary cleared NEW: accept it.
 910 #             print "Binary-only upload, source in accepted."
 911 #             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 912 #             _accept()
 913 #         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 914 #             # It's in NEW.  We expect the source to land in p-u holding
 915 #             # pretty soon.
 916 #             print "Binary-only upload, source in new."
 917 #             move_to_holding(suite, queue_dir)
 918 #         elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
 919 #             # It's in newstage.  Accept into the holding area
 920 #             print "Binary-only upload, source in newstage."
 921 #             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 922 #             _accept()
 923 #         else:
 924 #             # No case applicable.  Bail out.  Return will cause the upload
 925 #             # to be skipped.
 926 #             print "ERROR"
 927 #             print "Stable update failed.  Source not found."
 928 #             return
 929 #     else:
 930 #         # We are handling a sourceful upload.  Move to accepted if currently
 931 #         # in p-u holding and to p-u holding otherwise.
 932 #         if is_source_in_queue_dir(queue_dir):
 933 #             print "Sourceful upload in %s, accepting." % (q,)
 934 #             _accept()
 935 #         else:
 936 #             move_to_holding(suite, queue_dir)
 937
 938 def do_accept(upload):
 939     print "ACCEPT"
 940     cnf = Config()
 941     if not Options["No-Action"]:
 942         (summary, short_summary) = upload.build_summaries()
 943 #     if cnf.FindB("Dinstall::SecurityQueueHandling"):
 944 #         upload.dump_vars(cnf["Dir::Queue::Embargoed"])
 945 #         move_to_dir(cnf["Dir::Queue::Embargoed"])
 946 #         upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
 947 #         # Check for override disparities
 948 #         upload.Subst["__SUMMARY__"] = summary
 949 #     else:
 950         # Stable updates need to be copied to proposed-updates holding
 951         # area instead of accepted.  Sourceful uploads need to go
 952         # to it directly, binaries only if the source has not yet been
 953         # accepted into p-u.
 954         for suite, q in [("proposed-updates", "ProposedUpdates"),
 955                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
 956             if not upload.pkg.changes["distribution"].has_key(suite):
 957                 continue
 958             utils.fubar("stable accept not supported yet")
 959 #            return do_accept_stableupdate(suite, q)
 960         # Just a normal upload, accept it...
 961         _accept(upload)
 962
 963 def check_status(files):
 964     new = byhand = 0
 965     for f in files.keys():
 966         if files[f]["type"] == "byhand":
 967             byhand = 1
 968         elif files[f].has_key("new"):
 969             new = 1
 970     return (new, byhand)
 971
 972 def do_pkg(changes_file, session):
 973     u = Upload()
 974     u.pkg.load_dot_dak(changes_file)
 975     u.update_subst()
 976
 977     cnf = Config()
 978     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
 979     if cnf.has_key("Dinstall::Bcc"):
 980         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 981     else:
 982         u.Subst["__BCC__"] = bcc
 983
 984     files = u.pkg.files
 985
 986     try:
 987         with lock_package(u.pkg.changes["source"]):
 988             if not recheck(u, session):
 989                 return
 990
 991             (new, byhand) = check_status(files)
 992             if new or byhand:
 993                 if new:
 994                     do_new(u, session)
 995                 if byhand:
 996                     do_byhand(u, session)
 997                 (new, byhand) = check_status(files)
 998
 999             if not new and not byhand:
1000                 try:
1001                     check_daily_lock()
1002                     do_accept(u)
1003                 except CantGetLockError:
1004                     print "Hello? Operator! Give me the number for 911!"
1005                     print "Dinstall in the locked area, cant process packages, come back later"
1006     except AlreadyLockedError, e:
1007         print "Seems to be locked by %s already, skipping..." % (e)
1008
1009 ################################################################################
1010
1011 def end():
1012     accept_count = SummaryStats().accept_count
1013     accept_bytes = SummaryStats().accept_bytes
1014
1015     if accept_count:
1016         sets = "set"
1017         if accept_count > 1:
1018             sets = "sets"
1019         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1020         Logger.log(["total",accept_count,accept_bytes])
1021
1022     if not Options["No-Action"] and not Options["Trainee"]:
1023         Logger.close()
1024
1025 ################################################################################
1026
1027 # def do_comments(dir, opref, npref, line, fn):
1028 #     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1029 #         lines = open("%s/%s" % (dir, comm)).readlines()
1030 #         if len(lines) == 0 or lines[0] != line + "\n": continue
1031 #         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1032 #                                 and x.endswith(".changes") ]
1033 #         changes_files = sort_changes(changes_files)
1034 #         for f in changes_files:
1035 #             f = utils.validate_changes_file_arg(f, 0)
1036 #             if not f: continue
1037 #             print "\n" + f
1038 #             fn(f, "".join(lines[1:]))
1039
1040 #         if opref != npref and not Options["No-Action"]:
1041 #             newcomm = npref + comm[len(opref):]
1042 #             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1043
1044 # ################################################################################
1045
1046 # def comment_accept(changes_file, comments):
1047 #     Upload.pkg.changes_file = changes_file
1048 #     Upload.init_vars()
1049 #     Upload.update_vars()
1050 #     Upload.update_subst()
1051 #     files = Upload.pkg.files
1052
1053 #     if not recheck():
1054 #         return # dak wants to REJECT, crap
1055
1056 #     (new, byhand) = check_status(files)
1057 #     if not new and not byhand:
1058 #         do_accept()
1059
1060 # ################################################################################
1061
1062 # def comment_reject(changes_file, comments):
1063 #     Upload.pkg.changes_file = changes_file
1064 #     Upload.init_vars()
1065 #     Upload.update_vars()
1066 #     Upload.update_subst()
1067
1068 #     if not recheck():
1069 #         pass # dak has its own reasons to reject as well, which is fine
1070
1071 #     reject(comments)
1072 #     print "REJECT\n" + reject_message,
1073 #     if not Options["No-Action"]:
1074 #         Upload.do_reject(0, reject_message)
1075 #         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1076
1077 ################################################################################
1078
1079 def main():
1080     global Options, Logger, Sections, Priorities
1081
1082     cnf = Config()
1083     session = DBConn().session()
1084
1085     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
1086                  ('h',"help","Process-New::Options::Help"),
1087                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
1088                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
1089                  ('t',"trainee","Process-New::Options::Trainee"),
1090                  ('n',"no-action","Process-New::Options::No-Action")]
1091
1092     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
1093         if not cnf.has_key("Process-New::Options::%s" % (i)):
1094             cnf["Process-New::Options::%s" % (i)] = ""
1095
1096     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
1097     if len(changes_files) == 0 and not cnf.get("Process-New::Options::Comments-Dir",""):
1098         changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
1099
1100     Options = cnf.SubTree("Process-New::Options")
1101
1102     if Options["Help"]:
1103         usage()
1104
1105     if not Options["No-Action"]:
1106         try:
1107             Logger = daklog.Logger(cnf, "process-new")
1108         except CantOpenError, e:
1109             Options["Trainee"] = "True"
1110
1111     Sections = Section_Completer(session)
1112     Priorities = Priority_Completer(session)
1113     readline.parse_and_bind("tab: complete")
1114
1115     if len(changes_files) > 1:
1116         sys.stderr.write("Sorting changes...\n")
1117     changes_files = sort_changes(changes_files, session)
1118
1119     # Kill me now? **FIXME**
1120     cnf["Dinstall::Options::No-Mail"] = ""
1121
1122 #     commentsdir = cnf.get("Process-New::Options::Comments-Dir","")
1123 #     if commentsdir:
1124 #        if changes_files != []:
1125 #            sys.stderr.write("Can't specify any changes files if working with comments-dir")
1126 #            sys.exit(1)
1127 #        do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1128 #        do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1129 #     else:
1130     if True:
1131         for changes_file in changes_files:
1132             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1133             if not changes_file:
1134                 continue
1135             print "\n" + changes_file
1136
1137             do_pkg (changes_file, session)
1138
1139     end()
1140
1141 ################################################################################
1142
1143 if __name__ == '__main__':
1144     main()