dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 from __future__ import with_statement
  45
  46 import copy
  47 import errno
  48 import os
  49 import readline
  50 import stat
  51 import sys
  52 import time
  53 import contextlib
  54 import apt_pkg, apt_inst
  55 import examine_package
  56 from daklib import database
  57 from daklib import logging
  58 from daklib import queue
  59 from daklib import utils
  60 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  61 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError
  62
  63 # Globals
  64 Cnf = None       #: Configuration, apt_pkg.Configuration
  65 Options = None
  66 Upload = None
  67 projectB = None  #: database connection, pgobject
  68 Logger = None
  69
  70 Priorities = None
  71 Sections = None
  72
  73 reject_message = ""
  74
  75 ################################################################################
  76 ################################################################################
  77 ################################################################################
  78
  79 def reject (str, prefix="Rejected: "):
  80     global reject_message
  81     if str:
  82         reject_message += prefix + str + "\n"
  83
  84 def recheck():
  85     global reject_message
  86     files = Upload.pkg.files
  87     reject_message = ""
  88
  89     for f in files.keys():
  90         # The .orig.tar.gz can disappear out from under us is it's a
  91         # duplicate of one in the archive.
  92         if not files.has_key(f):
  93             continue
  94         # Check that the source still exists
  95         if files[f]["type"] == "deb":
  96             source_version = files[f]["source version"]
  97             source_package = files[f]["source package"]
  98             if not Upload.pkg.changes["architecture"].has_key("source") \
  99                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
 100                 source_epochless_version = re_no_epoch.sub('', source_version)
 101                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 102                 found = 0
 103                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
 104                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 105                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 106                             found = 1
 107                 if not found:
 108                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 109
 110         # Version and file overwrite checks
 111         if files[f]["type"] == "deb":
 112             reject(Upload.check_binary_against_db(f), "")
 113         elif files[f]["type"] == "dsc":
 114             reject(Upload.check_source_against_db(f), "")
 115             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 116             reject(reject_msg, "")
 117
 118     if reject_message.find("Rejected") != -1:
 119         answer = "XXX"
 120         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 121             answer = 'S'
 122
 123         print "REJECT\n" + reject_message,
 124         prompt = "[R]eject, Skip, Quit ?"
 125
 126         while prompt.find(answer) == -1:
 127             answer = utils.our_raw_input(prompt)
 128             m = re_default_answer.match(prompt)
 129             if answer == "":
 130                 answer = m.group(1)
 131             answer = answer[:1].upper()
 132
 133         if answer == 'R':
 134             Upload.do_reject(0, reject_message)
 135             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 136             return 0
 137         elif answer == 'S':
 138             return 0
 139         elif answer == 'Q':
 140             end()
 141             sys.exit(0)
 142
 143     return 1
 144
 145 ################################################################################
 146
 147 def indiv_sg_compare (a, b):
 148     """Sort by source name, source, version, 'have source', and
 149        finally by filename."""
 150     # Sort by source version
 151     q = apt_pkg.VersionCompare(a["version"], b["version"])
 152     if q:
 153         return -q
 154
 155     # Sort by 'have source'
 156     a_has_source = a["architecture"].get("source")
 157     b_has_source = b["architecture"].get("source")
 158     if a_has_source and not b_has_source:
 159         return -1
 160     elif b_has_source and not a_has_source:
 161         return 1
 162
 163     return cmp(a["filename"], b["filename"])
 164
 165 ############################################################
 166
 167 def sg_compare (a, b):
 168     a = a[1]
 169     b = b[1]
 170     """Sort by have note, source already in database and time of oldest upload."""
 171     # Sort by have note
 172     a_note_state = a["note_state"]
 173     b_note_state = b["note_state"]
 174     if a_note_state < b_note_state:
 175         return -1
 176     elif a_note_state > b_note_state:
 177         return 1
 178     # Sort by source already in database (descending)
 179     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 180     if source_in_database:
 181         return -source_in_database
 182
 183     # Sort by time of oldest upload
 184     return cmp(a["oldest"], b["oldest"])
 185
 186 def sort_changes(changes_files):
 187     """Sort into source groups, then sort each source group by version,
 188     have source, filename.  Finally, sort the source groups by have
 189     note, time of oldest upload of each source upload."""
 190     if len(changes_files) == 1:
 191         return changes_files
 192
 193     sorted_list = []
 194     cache = {}
 195     # Read in all the .changes files
 196     for filename in changes_files:
 197         try:
 198             Upload.pkg.changes_file = filename
 199             Upload.init_vars()
 200             Upload.update_vars()
 201             cache[filename] = copy.copy(Upload.pkg.changes)
 202             cache[filename]["filename"] = filename
 203         except:
 204             sorted_list.append(filename)
 205             break
 206     # Divide the .changes into per-source groups
 207     per_source = {}
 208     for filename in cache.keys():
 209         source = cache[filename]["source"]
 210         if not per_source.has_key(source):
 211             per_source[source] = {}
 212             per_source[source]["list"] = []
 213         per_source[source]["list"].append(cache[filename])
 214     # Determine oldest time and have note status for each source group
 215     for source in per_source.keys():
 216         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 217         ql = q.getresult()
 218         per_source[source]["source_in_database"] = len(ql)>0
 219         source_list = per_source[source]["list"]
 220         first = source_list[0]
 221         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 222         have_note = 0
 223         for d in per_source[source]["list"]:
 224             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 225             if mtime < oldest:
 226                 oldest = mtime
 227             have_note += (database.has_new_comment(d["source"], d["version"]))
 228         per_source[source]["oldest"] = oldest
 229         if not have_note:
 230             per_source[source]["note_state"] = 0; # none
 231         elif have_note < len(source_list):
 232             per_source[source]["note_state"] = 1; # some
 233         else:
 234             per_source[source]["note_state"] = 2; # all
 235         per_source[source]["list"].sort(indiv_sg_compare)
 236     per_source_items = per_source.items()
 237     per_source_items.sort(sg_compare)
 238     for i in per_source_items:
 239         for j in i[1]["list"]:
 240             sorted_list.append(j["filename"])
 241     return sorted_list
 242
 243 ################################################################################
 244
 245 class Section_Completer:
 246     def __init__ (self):
 247         self.sections = []
 248         self.matches = []
 249         q = projectB.query("SELECT section FROM section")
 250         for i in q.getresult():
 251             self.sections.append(i[0])
 252
 253     def complete(self, text, state):
 254         if state == 0:
 255             self.matches = []
 256             n = len(text)
 257             for word in self.sections:
 258                 if word[:n] == text:
 259                     self.matches.append(word)
 260         try:
 261             return self.matches[state]
 262         except IndexError:
 263             return None
 264
 265 ############################################################
 266
 267 class Priority_Completer:
 268     def __init__ (self):
 269         self.priorities = []
 270         self.matches = []
 271         q = projectB.query("SELECT priority FROM priority")
 272         for i in q.getresult():
 273             self.priorities.append(i[0])
 274
 275     def complete(self, text, state):
 276         if state == 0:
 277             self.matches = []
 278             n = len(text)
 279             for word in self.priorities:
 280                 if word[:n] == text:
 281                     self.matches.append(word)
 282         try:
 283             return self.matches[state]
 284         except IndexError:
 285             return None
 286
 287 ################################################################################
 288
 289 def print_new (new, indexed, file=sys.stdout):
 290     queue.check_valid(new)
 291     broken = 0
 292     index = 0
 293     for pkg in new.keys():
 294         index += 1
 295         section = new[pkg]["section"]
 296         priority = new[pkg]["priority"]
 297         if new[pkg]["section id"] == -1:
 298             section += "[!]"
 299             broken = 1
 300         if new[pkg]["priority id"] == -1:
 301             priority += "[!]"
 302             broken = 1
 303         if indexed:
 304             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 305         else:
 306             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 307         line = line.strip()+'\n'
 308         file.write(line)
 309     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 310     if len(note) > 0:
 311         for line in note:
 312             print line
 313     return broken, note
 314
 315 ################################################################################
 316
 317 def index_range (index):
 318     if index == 1:
 319         return "1"
 320     else:
 321         return "1-%s" % (index)
 322
 323 ################################################################################
 324 ################################################################################
 325
 326 def edit_new (new):
 327     # Write the current data to a temporary file
 328     (fd, temp_filename) = utils.temp_filename()
 329     temp_file = os.fdopen(fd, 'w')
 330     print_new (new, 0, temp_file)
 331     temp_file.close()
 332     # Spawn an editor on that file
 333     editor = os.environ.get("EDITOR","vi")
 334     result = os.system("%s %s" % (editor, temp_filename))
 335     if result != 0:
 336         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 337     # Read the edited data back in
 338     temp_file = utils.open_file(temp_filename)
 339     lines = temp_file.readlines()
 340     temp_file.close()
 341     os.unlink(temp_filename)
 342     # Parse the new data
 343     for line in lines:
 344         line = line.strip()
 345         if line == "":
 346             continue
 347         s = line.split()
 348         # Pad the list if necessary
 349         s[len(s):3] = [None] * (3-len(s))
 350         (pkg, priority, section) = s[:3]
 351         if not new.has_key(pkg):
 352             utils.warn("Ignoring unknown package '%s'" % (pkg))
 353         else:
 354             # Strip off any invalid markers, print_new will readd them.
 355             if section.endswith("[!]"):
 356                 section = section[:-3]
 357             if priority.endswith("[!]"):
 358                 priority = priority[:-3]
 359             for f in new[pkg]["files"]:
 360                 Upload.pkg.files[f]["section"] = section
 361                 Upload.pkg.files[f]["priority"] = priority
 362             new[pkg]["section"] = section
 363             new[pkg]["priority"] = priority
 364
 365 ################################################################################
 366
 367 def edit_index (new, index):
 368     priority = new[index]["priority"]
 369     section = new[index]["section"]
 370     ftype = new[index]["type"]
 371     done = 0
 372     while not done:
 373         print "\t".join([index, priority, section])
 374
 375         answer = "XXX"
 376         if ftype != "dsc":
 377             prompt = "[B]oth, Priority, Section, Done ? "
 378         else:
 379             prompt = "[S]ection, Done ? "
 380         edit_priority = edit_section = 0
 381
 382         while prompt.find(answer) == -1:
 383             answer = utils.our_raw_input(prompt)
 384             m = re_default_answer.match(prompt)
 385             if answer == "":
 386                 answer = m.group(1)
 387             answer = answer[:1].upper()
 388
 389         if answer == 'P':
 390             edit_priority = 1
 391         elif answer == 'S':
 392             edit_section = 1
 393         elif answer == 'B':
 394             edit_priority = edit_section = 1
 395         elif answer == 'D':
 396             done = 1
 397
 398         # Edit the priority
 399         if edit_priority:
 400             readline.set_completer(Priorities.complete)
 401             got_priority = 0
 402             while not got_priority:
 403                 new_priority = utils.our_raw_input("New priority: ").strip()
 404                 if new_priority not in Priorities.priorities:
 405                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 406                 else:
 407                     got_priority = 1
 408                     priority = new_priority
 409
 410         # Edit the section
 411         if edit_section:
 412             readline.set_completer(Sections.complete)
 413             got_section = 0
 414             while not got_section:
 415                 new_section = utils.our_raw_input("New section: ").strip()
 416                 if new_section not in Sections.sections:
 417                     print "E: '%s' is not a valid section, try again." % (new_section)
 418                 else:
 419                     got_section = 1
 420                     section = new_section
 421
 422         # Reset the readline completer
 423         readline.set_completer(None)
 424
 425     for f in new[index]["files"]:
 426         Upload.pkg.files[f]["section"] = section
 427         Upload.pkg.files[f]["priority"] = priority
 428     new[index]["priority"] = priority
 429     new[index]["section"] = section
 430     return new
 431
 432 ################################################################################
 433
 434 def edit_overrides (new):
 435     print
 436     done = 0
 437     while not done:
 438         print_new (new, 1)
 439         new_index = {}
 440         index = 0
 441         for i in new.keys():
 442             index += 1
 443             new_index[index] = i
 444
 445         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 446
 447         got_answer = 0
 448         while not got_answer:
 449             answer = utils.our_raw_input(prompt)
 450             if not answer.isdigit():
 451                 answer = answer[:1].upper()
 452             if answer == "E" or answer == "D":
 453                 got_answer = 1
 454             elif re_isanum.match (answer):
 455                 answer = int(answer)
 456                 if (answer < 1) or (answer > index):
 457                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 458                 else:
 459                     got_answer = 1
 460
 461         if answer == 'E':
 462             edit_new(new)
 463         elif answer == 'D':
 464             done = 1
 465         else:
 466             edit_index (new, new_index[answer])
 467
 468     return new
 469
 470 ################################################################################
 471
 472 def edit_note(note):
 473     # Write the current data to a temporary file
 474     (fd, temp_filename) = utils.temp_filename()
 475     editor = os.environ.get("EDITOR","vi")
 476     answer = 'E'
 477     while answer == 'E':
 478         os.system("%s %s" % (editor, temp_filename))
 479         temp_file = utils.open_file(temp_filename)
 480         newnote = temp_file.read().rstrip()
 481         temp_file.close()
 482         print "New Note:"
 483         print utils.prefix_multi_line_string(newnote,"  ")
 484         prompt = "[D]one, Edit, Abandon, Quit ?"
 485         answer = "XXX"
 486         while prompt.find(answer) == -1:
 487             answer = utils.our_raw_input(prompt)
 488             m = re_default_answer.search(prompt)
 489             if answer == "":
 490                 answer = m.group(1)
 491             answer = answer[:1].upper()
 492     os.unlink(temp_filename)
 493     if answer == 'A':
 494         return
 495     elif answer == 'Q':
 496         end()
 497         sys.exit(0)
 498     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami())
 499
 500 ################################################################################
 501
 502 def check_pkg ():
 503     try:
 504         less_fd = os.popen("less -R -", 'w', 0)
 505         stdout_fd = sys.stdout
 506         try:
 507             sys.stdout = less_fd
 508             changes = utils.parse_changes (Upload.pkg.changes_file)
 509             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 510             files = Upload.pkg.files
 511             for f in files.keys():
 512                 if files[f].has_key("new"):
 513                     ftype = files[f]["type"]
 514                     if ftype == "deb":
 515                         examine_package.check_deb(changes['distribution'], f)
 516                     elif ftype == "dsc":
 517                         examine_package.check_dsc(changes['distribution'], f)
 518         finally:
 519             examine_package.output_package_relations()
 520             sys.stdout = stdout_fd
 521     except IOError, e:
 522         if e.errno == errno.EPIPE:
 523             utils.warn("[examine_package] Caught EPIPE; skipping.")
 524             pass
 525         else:
 526             raise
 527     except KeyboardInterrupt:
 528         utils.warn("[examine_package] Caught C-c; skipping.")
 529         pass
 530
 531 ################################################################################
 532
 533 ## FIXME: horribly Debian specific
 534
 535 def do_bxa_notification():
 536     files = Upload.pkg.files
 537     summary = ""
 538     for f in files.keys():
 539         if files[f]["type"] == "deb":
 540             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 541             summary += "\n"
 542             summary += "Package: %s\n" % (control.Find("Package"))
 543             summary += "Description: %s\n" % (control.Find("Description"))
 544     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 545     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 546     utils.send_mail(bxa_mail)
 547
 548 ################################################################################
 549
 550 def add_overrides (new):
 551     changes = Upload.pkg.changes
 552     files = Upload.pkg.files
 553
 554     projectB.query("BEGIN WORK")
 555     for suite in changes["suite"].keys():
 556         suite_id = database.get_suite_id(suite)
 557         for pkg in new.keys():
 558             component_id = database.get_component_id(new[pkg]["component"])
 559             type_id = database.get_override_type_id(new[pkg]["type"])
 560             priority_id = new[pkg]["priority id"]
 561             section_id = new[pkg]["section id"]
 562             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 563             for f in new[pkg]["files"]:
 564                 if files[f].has_key("new"):
 565                     del files[f]["new"]
 566             del new[pkg]
 567
 568     projectB.query("COMMIT WORK")
 569
 570     if Cnf.FindB("Dinstall::BXANotify"):
 571         do_bxa_notification()
 572
 573 ################################################################################
 574
 575 def prod_maintainer (note):
 576     # Here we prepare an editor and get them ready to prod...
 577     (fd, temp_filename) = utils.temp_filename()
 578     temp_file = os.fdopen(fd, 'w')
 579     if len(note) > 0:
 580         for line in note:
 581             temp_file.write(line)
 582     temp_file.close()
 583     editor = os.environ.get("EDITOR","vi")
 584     answer = 'E'
 585     while answer == 'E':
 586         os.system("%s %s" % (editor, temp_filename))
 587         temp_fh = utils.open_file(temp_filename)
 588         prod_message = "".join(temp_fh.readlines())
 589         temp_fh.close()
 590         print "Prod message:"
 591         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 592         prompt = "[P]rod, Edit, Abandon, Quit ?"
 593         answer = "XXX"
 594         while prompt.find(answer) == -1:
 595             answer = utils.our_raw_input(prompt)
 596             m = re_default_answer.search(prompt)
 597             if answer == "":
 598                 answer = m.group(1)
 599             answer = answer[:1].upper()
 600     os.unlink(temp_filename)
 601     if answer == 'A':
 602         return
 603     elif answer == 'Q':
 604         end()
 605         sys.exit(0)
 606     # Otherwise, do the proding...
 607     user_email_address = utils.whoami() + " <%s>" % (
 608         Cnf["Dinstall::MyAdminAddress"])
 609
 610     Subst = Upload.Subst
 611
 612     Subst["__FROM_ADDRESS__"] = user_email_address
 613     Subst["__PROD_MESSAGE__"] = prod_message
 614     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 615
 616     prod_mail_message = utils.TemplateSubst(
 617         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 618
 619     # Send the prod mail if appropriate
 620     if not Cnf["Dinstall::Options::No-Mail"]:
 621         utils.send_mail(prod_mail_message)
 622
 623     print "Sent proding message"
 624
 625 ################################################################################
 626
 627 def do_new():
 628     print "NEW\n"
 629     files = Upload.pkg.files
 630     changes = Upload.pkg.changes
 631
 632     # Make a copy of distribution we can happily trample on
 633     changes["suite"] = copy.copy(changes["distribution"])
 634
 635     # Fix up the list of target suites
 636     for suite in changes["suite"].keys():
 637         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 638         if override:
 639             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 640               database.get_suite_id(override) == -1)
 641             if olderr or newerr:
 642                 (oinv, newinv) = ("", "")
 643                 if olderr: oinv = "invalid "
 644                 if newerr: ninv = "invalid "
 645                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 646                         oinv, suite, ninv, override)
 647             del changes["suite"][suite]
 648             changes["suite"][override] = 1
 649     # Validate suites
 650     for suite in changes["suite"].keys():
 651         suite_id = database.get_suite_id(suite)
 652         if suite_id == -1:
 653             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 654
 655     # The main NEW processing loop
 656     done = 0
 657     while not done:
 658         # Find out what's new
 659         new = queue.determine_new(changes, files, projectB)
 660
 661         if not new:
 662             break
 663
 664         answer = "XXX"
 665         if Options["No-Action"] or Options["Automatic"]:
 666             answer = 'S'
 667
 668         (broken, note) = print_new(new, 0)
 669         prompt = ""
 670
 671         if not broken and not note:
 672             prompt = "Add overrides, "
 673         if broken:
 674             print "W: [!] marked entries must be fixed before package can be processed."
 675         if note:
 676             print "W: note must be removed before package can be processed."
 677             prompt += "Remove note, "
 678
 679         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 680
 681         while prompt.find(answer) == -1:
 682             answer = utils.our_raw_input(prompt)
 683             m = re_default_answer.search(prompt)
 684             if answer == "":
 685                 answer = m.group(1)
 686             answer = answer[:1].upper()
 687
 688         if answer == 'A' and not Options["Trainee"]:
 689             done = add_overrides (new)
 690         elif answer == 'C':
 691             check_pkg()
 692         elif answer == 'E' and not Options["Trainee"]:
 693             new = edit_overrides (new)
 694         elif answer == 'M' and not Options["Trainee"]:
 695             aborted = Upload.do_reject(manual=1,
 696                                        reject_message=Options["Manual-Reject"],
 697                                        note=database.get_new_comments(changes.get("source", "")))
 698             if not aborted:
 699                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 700                 done = 1
 701         elif answer == 'N':
 702             edit_note(database.get_new_comments(changes.get("source", "")))
 703         elif answer == 'P' and not Options["Trainee"]:
 704             prod_maintainer(database.get_new_comments(changes.get("source", "")))
 705         elif answer == 'R':
 706             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 707             if confirm == "y":
 708                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 709         elif answer == 'S':
 710             done = 1
 711         elif answer == 'Q':
 712             end()
 713             sys.exit(0)
 714
 715 ################################################################################
 716 ################################################################################
 717 ################################################################################
 718
 719 def usage (exit_code=0):
 720     print """Usage: dak process-new [OPTION]... [CHANGES]...
 721   -a, --automatic           automatic run
 722   -h, --help                show this help and exit.
 723   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 724   -m, --manual-reject=MSG   manual reject with `msg'
 725   -n, --no-action           don't do anything
 726   -t, --trainee             FTP Trainee mode
 727   -V, --version             display the version number and exit"""
 728     sys.exit(exit_code)
 729
 730 ################################################################################
 731
 732 def init():
 733     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 734
 735     Cnf = utils.get_conf()
 736
 737     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 738                  ('h',"help","Process-New::Options::Help"),
 739                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 740                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 741                  ('t',"trainee","Process-New::Options::Trainee"),
 742                  ('n',"no-action","Process-New::Options::No-Action")]
 743
 744     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 745         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 746             Cnf["Process-New::Options::%s" % (i)] = ""
 747
 748     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 749     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 750         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 751
 752     Options = Cnf.SubTree("Process-New::Options")
 753
 754     if Options["Help"]:
 755         usage()
 756
 757     Upload = queue.Upload(Cnf)
 758
 759     if not Options["No-Action"]:
 760         try:
 761             Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 762         except CantOpenError, e:
 763             Options["Trainee"] = "Oh yes"
 764
 765     projectB = Upload.projectB
 766
 767     Sections = Section_Completer()
 768     Priorities = Priority_Completer()
 769     readline.parse_and_bind("tab: complete")
 770
 771     return changes_files
 772
 773 ################################################################################
 774
 775 def do_byhand():
 776     done = 0
 777     while not done:
 778         files = Upload.pkg.files
 779         will_install = 1
 780         byhand = []
 781
 782         for f in files.keys():
 783             if files[f]["type"] == "byhand":
 784                 if os.path.exists(f):
 785                     print "W: %s still present; please process byhand components and try again." % (f)
 786                     will_install = 0
 787                 else:
 788                     byhand.append(f)
 789
 790         answer = "XXXX"
 791         if Options["No-Action"]:
 792             answer = "S"
 793         if will_install:
 794             if Options["Automatic"] and not Options["No-Action"]:
 795                 answer = 'A'
 796             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 797         else:
 798             prompt = "Manual reject, [S]kip, Quit ?"
 799
 800         while prompt.find(answer) == -1:
 801             answer = utils.our_raw_input(prompt)
 802             m = re_default_answer.search(prompt)
 803             if answer == "":
 804                 answer = m.group(1)
 805             answer = answer[:1].upper()
 806
 807         if answer == 'A':
 808             done = 1
 809             for f in byhand:
 810                 del files[f]
 811         elif answer == 'M':
 812             Upload.do_reject(1, Options["Manual-Reject"])
 813             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 814             done = 1
 815         elif answer == 'S':
 816             done = 1
 817         elif answer == 'Q':
 818             end()
 819             sys.exit(0)
 820
 821 ################################################################################
 822
 823 def get_accept_lock():
 824     retry = 0
 825     while retry < 10:
 826         try:
 827             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 828             retry = 10
 829         except OSError, e:
 830             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 831                 retry += 1
 832                 if (retry >= 10):
 833                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 834                 else:
 835                     print("Unable to get accepted lock (try %d of 10)" % retry)
 836                 time.sleep(60)
 837             else:
 838                 raise
 839
 840
 841 @contextlib.contextmanager
 842 def lock_package(package):
 843     """
 844     Lock C{package} so that noone else jumps in processing it.
 845
 846     @type package: string
 847     @param package: source package name to lock
 848     """
 849
 850     path = os.path.join(Cnf["Process-New::LockDir"], package)
 851     try:
 852         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 853     except OSError, e:
 854         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 855             raise AlreadyLockedError, e.errno
 856
 857     try:
 858         yield fd
 859     finally:
 860         os.unlink(path)
 861
 862 def move_to_dir (dest, perms=0660, changesperms=0664):
 863     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 864     file_keys = Upload.pkg.files.keys()
 865     for f in file_keys:
 866         utils.move (f, dest, perms=perms)
 867
 868 def is_source_in_queue_dir(qdir):
 869     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 870                 and x.endswith(".changes") ]
 871     for entry in entries:
 872         # read the .dak
 873         u = queue.Upload(Cnf)
 874         u.pkg.changes_file = os.path.join(qdir, entry)
 875         u.update_vars()
 876         if not u.pkg.changes["architecture"].has_key("source"):
 877             # another binary upload, ignore
 878             continue
 879         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 880             # another version, ignore
 881             continue
 882         # found it!
 883         return True
 884     return False
 885
 886 def move_to_holding(suite, queue_dir):
 887     print "Moving to %s holding area." % (suite.upper(),)
 888     if Options["No-Action"]:
 889         return
 890     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 891     Upload.dump_vars(queue_dir)
 892     move_to_dir(queue_dir, perms=0664)
 893     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 894
 895 def _accept():
 896     if Options["No-Action"]:
 897         return
 898     (summary, short_summary) = Upload.build_summaries()
 899     Upload.accept(summary, short_summary)
 900     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 901
 902 def do_accept_stableupdate(suite, q):
 903     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 904     if not Upload.pkg.changes["architecture"].has_key("source"):
 905         # It is not a sourceful upload.  So its source may be either in p-u
 906         # holding, in new, in accepted or already installed.
 907         if is_source_in_queue_dir(queue_dir):
 908             # It's in p-u holding, so move it there.
 909             print "Binary-only upload, source in %s." % (q,)
 910             move_to_holding(suite, queue_dir)
 911         elif Upload.source_exists(Upload.pkg.changes["source"],
 912                 Upload.pkg.changes["version"]):
 913             # dak tells us that there is source available.  At time of
 914             # writing this means that it is installed, so put it into
 915             # accepted.
 916             print "Binary-only upload, source installed."
 917             _accept()
 918         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 919             # The source is in accepted, the binary cleared NEW: accept it.
 920             print "Binary-only upload, source in accepted."
 921             _accept()
 922         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 923             # It's in NEW.  We expect the source to land in p-u holding
 924             # pretty soon.
 925             print "Binary-only upload, source in new."
 926             move_to_holding(suite, queue_dir)
 927         else:
 928             # No case applicable.  Bail out.  Return will cause the upload
 929             # to be skipped.
 930             print "ERROR"
 931             print "Stable update failed.  Source not found."
 932             return
 933     else:
 934         # We are handling a sourceful upload.  Move to accepted if currently
 935         # in p-u holding and to p-u holding otherwise.
 936         if is_source_in_queue_dir(queue_dir):
 937             print "Sourceful upload in %s, accepting." % (q,)
 938             _accept()
 939         else:
 940             move_to_holding(suite, queue_dir)
 941
 942 def do_accept():
 943     print "ACCEPT"
 944     if not Options["No-Action"]:
 945         get_accept_lock()
 946         (summary, short_summary) = Upload.build_summaries()
 947     try:
 948         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 949             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 950             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 951             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 952             # Check for override disparities
 953             Upload.Subst["__SUMMARY__"] = summary
 954         else:
 955             # Stable updates need to be copied to proposed-updates holding
 956             # area instead of accepted.  Sourceful uploads need to go
 957             # to it directly, binaries only if the source has not yet been
 958             # accepted into p-u.
 959             for suite, q in [("proposed-updates", "ProposedUpdates"),
 960                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 961                 if not Upload.pkg.changes["distribution"].has_key(suite):
 962                     continue
 963                 return do_accept_stableupdate(suite, q)
 964             # Just a normal upload, accept it...
 965             _accept()
 966     finally:
 967         if not Options["No-Action"]:
 968             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 969
 970 def check_status(files):
 971     new = byhand = 0
 972     for f in files.keys():
 973         if files[f]["type"] == "byhand":
 974             byhand = 1
 975         elif files[f].has_key("new"):
 976             new = 1
 977     return (new, byhand)
 978
 979 def do_pkg(changes_file):
 980     Upload.pkg.changes_file = changes_file
 981     Upload.init_vars()
 982     Upload.update_vars()
 983     Upload.update_subst()
 984     files = Upload.pkg.files
 985
 986     try:
 987         with lock_package(Upload.pkg.changes["source"]):
 988             if not recheck():
 989                 return
 990
 991             (new, byhand) = check_status(files)
 992             if new or byhand:
 993                 if new:
 994                     do_new()
 995                 if byhand:
 996                     do_byhand()
 997                 (new, byhand) = check_status(files)
 998
 999             if not new and not byhand:
1000                 do_accept()
1001     except AlreadyLockedError, e:
1002         print "Seems to be locked already, skipping..."
1003
1004 ################################################################################
1005
1006 def end():
1007     accept_count = Upload.accept_count
1008     accept_bytes = Upload.accept_bytes
1009
1010     if accept_count:
1011         sets = "set"
1012         if accept_count > 1:
1013             sets = "sets"
1014         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1015         Logger.log(["total",accept_count,accept_bytes])
1016
1017     if not Options["No-Action"] and not Options["Trainee"]:
1018         Logger.close()
1019
1020 ################################################################################
1021
1022 def do_comments(dir, opref, npref, line, fn):
1023     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1024         lines = open("%s/%s" % (dir, comm)).readlines()
1025         if len(lines) == 0 or lines[0] != line + "\n": continue
1026         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1027                                 and x.endswith(".changes") ]
1028         changes_files = sort_changes(changes_files)
1029         for f in changes_files:
1030             f = utils.validate_changes_file_arg(f, 0)
1031             if not f: continue
1032             print "\n" + f
1033             fn(f, "".join(lines[1:]))
1034
1035         if opref != npref and not Options["No-Action"]:
1036             newcomm = npref + comm[len(opref):]
1037             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1038
1039 ################################################################################
1040
1041 def comment_accept(changes_file, comments):
1042     Upload.pkg.changes_file = changes_file
1043     Upload.init_vars()
1044     Upload.update_vars()
1045     Upload.update_subst()
1046     files = Upload.pkg.files
1047
1048     if not recheck():
1049         return # dak wants to REJECT, crap
1050
1051     (new, byhand) = check_status(files)
1052     if not new and not byhand:
1053         do_accept()
1054
1055 ################################################################################
1056
1057 def comment_reject(changes_file, comments):
1058     Upload.pkg.changes_file = changes_file
1059     Upload.init_vars()
1060     Upload.update_vars()
1061     Upload.update_subst()
1062
1063     if not recheck():
1064         pass # dak has its own reasons to reject as well, which is fine
1065
1066     reject(comments)
1067     print "REJECT\n" + reject_message,
1068     if not Options["No-Action"]:
1069         Upload.do_reject(0, reject_message)
1070         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1071
1072 ################################################################################
1073
1074 def main():
1075     changes_files = init()
1076     if len(changes_files) > 50:
1077         sys.stderr.write("Sorting changes...\n")
1078     changes_files = sort_changes(changes_files)
1079
1080     # Kill me now? **FIXME**
1081     Cnf["Dinstall::Options::No-Mail"] = ""
1082     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1083     if Cnf.has_key("Dinstall::Bcc"):
1084         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1085     else:
1086         Upload.Subst["__BCC__"] = bcc
1087
1088     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1089     if commentsdir:
1090         if changes_files != []:
1091             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1092             sys.exit(1)
1093         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1094         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1095     else:
1096         for changes_file in changes_files:
1097             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1098             if not changes_file:
1099                 continue
1100             print "\n" + changes_file
1101             do_pkg (changes_file)
1102
1103     end()
1104
1105 ################################################################################
1106
1107 if __name__ == '__main__':
1108     main()