dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 import copy
  45 import errno
  46 import os
  47 import readline
  48 import stat
  49 import sys
  50 import time
  51 import apt_pkg, apt_inst
  52 import examine_package
  53 from daklib import database
  54 from daklib import logging
  55 from daklib import queue
  56 from daklib import utils
  57 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  58 from daklib.dak_exceptions import CantOpenError
  59
  60 # Globals
  61 Cnf = None       #: Configuration, apt_pkg.Configuration
  62 Options = None
  63 Upload = None
  64 projectB = None  #: database connection, pgobject
  65 Logger = None
  66
  67 Priorities = None
  68 Sections = None
  69
  70 reject_message = ""
  71
  72 ################################################################################
  73 ################################################################################
  74 ################################################################################
  75
  76 def reject (str, prefix="Rejected: "):
  77     global reject_message
  78     if str:
  79         reject_message += prefix + str + "\n"
  80
  81 def recheck():
  82     global reject_message
  83     files = Upload.pkg.files
  84     reject_message = ""
  85
  86     for f in files.keys():
  87         # The .orig.tar.gz can disappear out from under us is it's a
  88         # duplicate of one in the archive.
  89         if not files.has_key(f):
  90             continue
  91         # Check that the source still exists
  92         if files[f]["type"] == "deb":
  93             source_version = files[f]["source version"]
  94             source_package = files[f]["source package"]
  95             if not Upload.pkg.changes["architecture"].has_key("source") \
  96                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  97                 source_epochless_version = re_no_epoch.sub('', source_version)
  98                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  99                 found = 0
 100                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
 101                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 102                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 103                             found = 1
 104                 if not found:
 105                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 106
 107         # Version and file overwrite checks
 108         if files[f]["type"] == "deb":
 109             reject(Upload.check_binary_against_db(f), "")
 110         elif files[f]["type"] == "dsc":
 111             reject(Upload.check_source_against_db(f), "")
 112             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 113             reject(reject_msg, "")
 114
 115     if reject_message.find("Rejected") != -1:
 116         answer = "XXX"
 117         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 118             answer = 'S'
 119
 120         print "REJECT\n" + reject_message,
 121         prompt = "[R]eject, Skip, Quit ?"
 122
 123         while prompt.find(answer) == -1:
 124             answer = utils.our_raw_input(prompt)
 125             m = re_default_answer.match(prompt)
 126             if answer == "":
 127                 answer = m.group(1)
 128             answer = answer[:1].upper()
 129
 130         if answer == 'R':
 131             Upload.do_reject(0, reject_message)
 132             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 133             return 0
 134         elif answer == 'S':
 135             return 0
 136         elif answer == 'Q':
 137             end()
 138             sys.exit(0)
 139
 140     return 1
 141
 142 ################################################################################
 143
 144 def indiv_sg_compare (a, b):
 145     """Sort by source name, source, version, 'have source', and
 146        finally by filename."""
 147     # Sort by source version
 148     q = apt_pkg.VersionCompare(a["version"], b["version"])
 149     if q:
 150         return -q
 151
 152     # Sort by 'have source'
 153     a_has_source = a["architecture"].get("source")
 154     b_has_source = b["architecture"].get("source")
 155     if a_has_source and not b_has_source:
 156         return -1
 157     elif b_has_source and not a_has_source:
 158         return 1
 159
 160     return cmp(a["filename"], b["filename"])
 161
 162 ############################################################
 163
 164 def sg_compare (a, b):
 165     a = a[1]
 166     b = b[1]
 167     """Sort by have note, source already in database and time of oldest upload."""
 168     # Sort by have note
 169     a_note_state = a["note_state"]
 170     b_note_state = b["note_state"]
 171     if a_note_state < b_note_state:
 172         return -1
 173     elif a_note_state > b_note_state:
 174         return 1
 175     # Sort by source already in database (descending)
 176     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 177     if source_in_database:
 178         return -source_in_database
 179
 180     # Sort by time of oldest upload
 181     return cmp(a["oldest"], b["oldest"])
 182
 183 def sort_changes(changes_files):
 184     """Sort into source groups, then sort each source group by version,
 185     have source, filename.  Finally, sort the source groups by have
 186     note, time of oldest upload of each source upload."""
 187     if len(changes_files) == 1:
 188         return changes_files
 189
 190     sorted_list = []
 191     cache = {}
 192     # Read in all the .changes files
 193     for filename in changes_files:
 194         try:
 195             Upload.pkg.changes_file = filename
 196             Upload.init_vars()
 197             Upload.update_vars()
 198             cache[filename] = copy.copy(Upload.pkg.changes)
 199             cache[filename]["filename"] = filename
 200         except:
 201             sorted_list.append(filename)
 202             break
 203     # Divide the .changes into per-source groups
 204     per_source = {}
 205     for filename in cache.keys():
 206         source = cache[filename]["source"]
 207         if not per_source.has_key(source):
 208             per_source[source] = {}
 209             per_source[source]["list"] = []
 210         per_source[source]["list"].append(cache[filename])
 211     # Determine oldest time and have note status for each source group
 212     for source in per_source.keys():
 213         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 214         ql = q.getresult()
 215         per_source[source]["source_in_database"] = len(ql)>0
 216         source_list = per_source[source]["list"]
 217         first = source_list[0]
 218         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 219         have_note = 0
 220         for d in per_source[source]["list"]:
 221             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 222             if mtime < oldest:
 223                 oldest = mtime
 224             have_note += (database.has_new_comment(d["source"], d["version"]))
 225         per_source[source]["oldest"] = oldest
 226         if not have_note:
 227             per_source[source]["note_state"] = 0; # none
 228         elif have_note < len(source_list):
 229             per_source[source]["note_state"] = 1; # some
 230         else:
 231             per_source[source]["note_state"] = 2; # all
 232         per_source[source]["list"].sort(indiv_sg_compare)
 233     per_source_items = per_source.items()
 234     per_source_items.sort(sg_compare)
 235     for i in per_source_items:
 236         for j in i[1]["list"]:
 237             sorted_list.append(j["filename"])
 238     return sorted_list
 239
 240 ################################################################################
 241
 242 class Section_Completer:
 243     def __init__ (self):
 244         self.sections = []
 245         self.matches = []
 246         q = projectB.query("SELECT section FROM section")
 247         for i in q.getresult():
 248             self.sections.append(i[0])
 249
 250     def complete(self, text, state):
 251         if state == 0:
 252             self.matches = []
 253             n = len(text)
 254             for word in self.sections:
 255                 if word[:n] == text:
 256                     self.matches.append(word)
 257         try:
 258             return self.matches[state]
 259         except IndexError:
 260             return None
 261
 262 ############################################################
 263
 264 class Priority_Completer:
 265     def __init__ (self):
 266         self.priorities = []
 267         self.matches = []
 268         q = projectB.query("SELECT priority FROM priority")
 269         for i in q.getresult():
 270             self.priorities.append(i[0])
 271
 272     def complete(self, text, state):
 273         if state == 0:
 274             self.matches = []
 275             n = len(text)
 276             for word in self.priorities:
 277                 if word[:n] == text:
 278                     self.matches.append(word)
 279         try:
 280             return self.matches[state]
 281         except IndexError:
 282             return None
 283
 284 ################################################################################
 285
 286 def print_new (new, indexed, file=sys.stdout):
 287     queue.check_valid(new)
 288     broken = 0
 289     index = 0
 290     for pkg in new.keys():
 291         index += 1
 292         section = new[pkg]["section"]
 293         priority = new[pkg]["priority"]
 294         if new[pkg]["section id"] == -1:
 295             section += "[!]"
 296             broken = 1
 297         if new[pkg]["priority id"] == -1:
 298             priority += "[!]"
 299             broken = 1
 300         if indexed:
 301             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 302         else:
 303             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 304         line = line.strip()+'\n'
 305         file.write(line)
 306     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 307     if len(note) > 0:
 308         for line in note:
 309             print line
 310     return broken, note
 311
 312 ################################################################################
 313
 314 def index_range (index):
 315     if index == 1:
 316         return "1"
 317     else:
 318         return "1-%s" % (index)
 319
 320 ################################################################################
 321 ################################################################################
 322
 323 def edit_new (new):
 324     # Write the current data to a temporary file
 325     (fd, temp_filename) = utils.temp_filename()
 326     temp_file = os.fdopen(fd, 'w')
 327     print_new (new, 0, temp_file)
 328     temp_file.close()
 329     # Spawn an editor on that file
 330     editor = os.environ.get("EDITOR","vi")
 331     result = os.system("%s %s" % (editor, temp_filename))
 332     if result != 0:
 333         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 334     # Read the edited data back in
 335     temp_file = utils.open_file(temp_filename)
 336     lines = temp_file.readlines()
 337     temp_file.close()
 338     os.unlink(temp_filename)
 339     # Parse the new data
 340     for line in lines:
 341         line = line.strip()
 342         if line == "":
 343             continue
 344         s = line.split()
 345         # Pad the list if necessary
 346         s[len(s):3] = [None] * (3-len(s))
 347         (pkg, priority, section) = s[:3]
 348         if not new.has_key(pkg):
 349             utils.warn("Ignoring unknown package '%s'" % (pkg))
 350         else:
 351             # Strip off any invalid markers, print_new will readd them.
 352             if section.endswith("[!]"):
 353                 section = section[:-3]
 354             if priority.endswith("[!]"):
 355                 priority = priority[:-3]
 356             for f in new[pkg]["files"]:
 357                 Upload.pkg.files[f]["section"] = section
 358                 Upload.pkg.files[f]["priority"] = priority
 359             new[pkg]["section"] = section
 360             new[pkg]["priority"] = priority
 361
 362 ################################################################################
 363
 364 def edit_index (new, index):
 365     priority = new[index]["priority"]
 366     section = new[index]["section"]
 367     ftype = new[index]["type"]
 368     done = 0
 369     while not done:
 370         print "\t".join([index, priority, section])
 371
 372         answer = "XXX"
 373         if ftype != "dsc":
 374             prompt = "[B]oth, Priority, Section, Done ? "
 375         else:
 376             prompt = "[S]ection, Done ? "
 377         edit_priority = edit_section = 0
 378
 379         while prompt.find(answer) == -1:
 380             answer = utils.our_raw_input(prompt)
 381             m = re_default_answer.match(prompt)
 382             if answer == "":
 383                 answer = m.group(1)
 384             answer = answer[:1].upper()
 385
 386         if answer == 'P':
 387             edit_priority = 1
 388         elif answer == 'S':
 389             edit_section = 1
 390         elif answer == 'B':
 391             edit_priority = edit_section = 1
 392         elif answer == 'D':
 393             done = 1
 394
 395         # Edit the priority
 396         if edit_priority:
 397             readline.set_completer(Priorities.complete)
 398             got_priority = 0
 399             while not got_priority:
 400                 new_priority = utils.our_raw_input("New priority: ").strip()
 401                 if new_priority not in Priorities.priorities:
 402                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 403                 else:
 404                     got_priority = 1
 405                     priority = new_priority
 406
 407         # Edit the section
 408         if edit_section:
 409             readline.set_completer(Sections.complete)
 410             got_section = 0
 411             while not got_section:
 412                 new_section = utils.our_raw_input("New section: ").strip()
 413                 if new_section not in Sections.sections:
 414                     print "E: '%s' is not a valid section, try again." % (new_section)
 415                 else:
 416                     got_section = 1
 417                     section = new_section
 418
 419         # Reset the readline completer
 420         readline.set_completer(None)
 421
 422     for f in new[index]["files"]:
 423         Upload.pkg.files[f]["section"] = section
 424         Upload.pkg.files[f]["priority"] = priority
 425     new[index]["priority"] = priority
 426     new[index]["section"] = section
 427     return new
 428
 429 ################################################################################
 430
 431 def edit_overrides (new):
 432     print
 433     done = 0
 434     while not done:
 435         print_new (new, 1)
 436         new_index = {}
 437         index = 0
 438         for i in new.keys():
 439             index += 1
 440             new_index[index] = i
 441
 442         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 443
 444         got_answer = 0
 445         while not got_answer:
 446             answer = utils.our_raw_input(prompt)
 447             if not answer.isdigit():
 448                 answer = answer[:1].upper()
 449             if answer == "E" or answer == "D":
 450                 got_answer = 1
 451             elif re_isanum.match (answer):
 452                 answer = int(answer)
 453                 if (answer < 1) or (answer > index):
 454                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 455                 else:
 456                     got_answer = 1
 457
 458         if answer == 'E':
 459             edit_new(new)
 460         elif answer == 'D':
 461             done = 1
 462         else:
 463             edit_index (new, new_index[answer])
 464
 465     return new
 466
 467 ################################################################################
 468
 469 def edit_note(note):
 470     # Write the current data to a temporary file
 471     (fd, temp_filename) = utils.temp_filename()
 472     editor = os.environ.get("EDITOR","vi")
 473     answer = 'E'
 474     while answer == 'E':
 475         os.system("%s %s" % (editor, temp_filename))
 476         temp_file = utils.open_file(temp_filename)
 477         newnote = temp_file.read().rstrip()
 478         temp_file.close()
 479         print "New Note:"
 480         print utils.prefix_multi_line_string(newnote,"  ")
 481         prompt = "[D]one, Edit, Abandon, Quit ?"
 482         answer = "XXX"
 483         while prompt.find(answer) == -1:
 484             answer = utils.our_raw_input(prompt)
 485             m = re_default_answer.search(prompt)
 486             if answer == "":
 487                 answer = m.group(1)
 488             answer = answer[:1].upper()
 489     os.unlink(temp_filename)
 490     if answer == 'A':
 491         return
 492     elif answer == 'Q':
 493         end()
 494         sys.exit(0)
 495     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami())
 496
 497 ################################################################################
 498
 499 def check_pkg ():
 500     try:
 501         less_fd = os.popen("less -R -", 'w', 0)
 502         stdout_fd = sys.stdout
 503         try:
 504             sys.stdout = less_fd
 505             changes = utils.parse_changes (Upload.pkg.changes_file)
 506             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 507             files = Upload.pkg.files
 508             for f in files.keys():
 509                 if files[f].has_key("new"):
 510                     ftype = files[f]["type"]
 511                     if ftype == "deb":
 512                         examine_package.check_deb(changes['distribution'], f)
 513                     elif ftype == "dsc":
 514                         examine_package.check_dsc(changes['distribution'], f)
 515         finally:
 516             examine_package.output_package_relations()
 517             sys.stdout = stdout_fd
 518     except IOError, e:
 519         if e.errno == errno.EPIPE:
 520             utils.warn("[examine_package] Caught EPIPE; skipping.")
 521             pass
 522         else:
 523             raise
 524     except KeyboardInterrupt:
 525         utils.warn("[examine_package] Caught C-c; skipping.")
 526         pass
 527
 528 ################################################################################
 529
 530 ## FIXME: horribly Debian specific
 531
 532 def do_bxa_notification():
 533     files = Upload.pkg.files
 534     summary = ""
 535     for f in files.keys():
 536         if files[f]["type"] == "deb":
 537             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 538             summary += "\n"
 539             summary += "Package: %s\n" % (control.Find("Package"))
 540             summary += "Description: %s\n" % (control.Find("Description"))
 541     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 542     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 543     utils.send_mail(bxa_mail)
 544
 545 ################################################################################
 546
 547 def add_overrides (new):
 548     changes = Upload.pkg.changes
 549     files = Upload.pkg.files
 550
 551     projectB.query("BEGIN WORK")
 552     for suite in changes["suite"].keys():
 553         suite_id = database.get_suite_id(suite)
 554         for pkg in new.keys():
 555             component_id = database.get_component_id(new[pkg]["component"])
 556             type_id = database.get_override_type_id(new[pkg]["type"])
 557             priority_id = new[pkg]["priority id"]
 558             section_id = new[pkg]["section id"]
 559             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 560             for f in new[pkg]["files"]:
 561                 if files[f].has_key("new"):
 562                     del files[f]["new"]
 563             del new[pkg]
 564
 565     projectB.query("COMMIT WORK")
 566
 567     if Cnf.FindB("Dinstall::BXANotify"):
 568         do_bxa_notification()
 569
 570 ################################################################################
 571
 572 def prod_maintainer ():
 573     # Here we prepare an editor and get them ready to prod...
 574     (fd, temp_filename) = utils.temp_filename()
 575     editor = os.environ.get("EDITOR","vi")
 576     answer = 'E'
 577     while answer == 'E':
 578         os.system("%s %s" % (editor, temp_filename))
 579         f = os.fdopen(fd)
 580         prod_message = "".join(f.readlines())
 581         f.close()
 582         print "Prod message:"
 583         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 584         prompt = "[P]rod, Edit, Abandon, Quit ?"
 585         answer = "XXX"
 586         while prompt.find(answer) == -1:
 587             answer = utils.our_raw_input(prompt)
 588             m = re_default_answer.search(prompt)
 589             if answer == "":
 590                 answer = m.group(1)
 591             answer = answer[:1].upper()
 592         os.unlink(temp_filename)
 593         if answer == 'A':
 594             return
 595         elif answer == 'Q':
 596             end()
 597             sys.exit(0)
 598     # Otherwise, do the proding...
 599     user_email_address = utils.whoami() + " <%s>" % (
 600         Cnf["Dinstall::MyAdminAddress"])
 601
 602     Subst = Upload.Subst
 603
 604     Subst["__FROM_ADDRESS__"] = user_email_address
 605     Subst["__PROD_MESSAGE__"] = prod_message
 606     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 607
 608     prod_mail_message = utils.TemplateSubst(
 609         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 610
 611     # Send the prod mail if appropriate
 612     if not Cnf["Dinstall::Options::No-Mail"]:
 613         utils.send_mail(prod_mail_message)
 614
 615     print "Sent proding message"
 616
 617 ################################################################################
 618
 619 def do_new():
 620     print "NEW\n"
 621     files = Upload.pkg.files
 622     changes = Upload.pkg.changes
 623
 624     # Make a copy of distribution we can happily trample on
 625     changes["suite"] = copy.copy(changes["distribution"])
 626
 627     # Fix up the list of target suites
 628     for suite in changes["suite"].keys():
 629         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 630         if override:
 631             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 632               database.get_suite_id(override) == -1)
 633             if olderr or newerr:
 634                 (oinv, newinv) = ("", "")
 635                 if olderr: oinv = "invalid "
 636                 if newerr: ninv = "invalid "
 637                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 638                         oinv, suite, ninv, override)
 639             del changes["suite"][suite]
 640             changes["suite"][override] = 1
 641     # Validate suites
 642     for suite in changes["suite"].keys():
 643         suite_id = database.get_suite_id(suite)
 644         if suite_id == -1:
 645             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 646
 647     # The main NEW processing loop
 648     done = 0
 649     while not done:
 650         # Find out what's new
 651         new = queue.determine_new(changes, files, projectB)
 652
 653         if not new:
 654             break
 655
 656         answer = "XXX"
 657         if Options["No-Action"] or Options["Automatic"]:
 658             answer = 'S'
 659
 660         (broken, note) = print_new(new, 0)
 661         prompt = ""
 662
 663         if not broken and not note:
 664             prompt = "Add overrides, "
 665         if broken:
 666             print "W: [!] marked entries must be fixed before package can be processed."
 667         if note:
 668             print "W: note must be removed before package can be processed."
 669             prompt += "Remove note, "
 670
 671         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 672
 673         while prompt.find(answer) == -1:
 674             answer = utils.our_raw_input(prompt)
 675             m = re_default_answer.search(prompt)
 676             if answer == "":
 677                 answer = m.group(1)
 678             answer = answer[:1].upper()
 679
 680         if answer == 'A' and not Options["Trainee"]:
 681             done = add_overrides (new)
 682         elif answer == 'C':
 683             check_pkg()
 684         elif answer == 'E' and not Options["Trainee"]:
 685             new = edit_overrides (new)
 686         elif answer == 'M' and not Options["Trainee"]:
 687             aborted = Upload.do_reject(manual=1,
 688                                        reject_message=Options["Manual-Reject"],
 689                                        note=database.get_new_comments(changes.get("source", "")))
 690             if not aborted:
 691                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 692                 done = 1
 693         elif answer == 'N':
 694             edit_note(database.get_new_comments(changes.get("source", "")))
 695         elif answer == 'P' and not Options["Trainee"]:
 696             prod_maintainer(database.get_new_comments(changes.get("source", "")))
 697         elif answer == 'R':
 698             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 699             if confirm == "y":
 700                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 701         elif answer == 'S':
 702             done = 1
 703         elif answer == 'Q':
 704             end()
 705             sys.exit(0)
 706
 707 ################################################################################
 708 ################################################################################
 709 ################################################################################
 710
 711 def usage (exit_code=0):
 712     print """Usage: dak process-new [OPTION]... [CHANGES]...
 713   -a, --automatic           automatic run
 714   -h, --help                show this help and exit.
 715   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 716   -m, --manual-reject=MSG   manual reject with `msg'
 717   -n, --no-action           don't do anything
 718   -t, --trainee             FTP Trainee mode
 719   -V, --version             display the version number and exit"""
 720     sys.exit(exit_code)
 721
 722 ################################################################################
 723
 724 def init():
 725     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 726
 727     Cnf = utils.get_conf()
 728
 729     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 730                  ('h',"help","Process-New::Options::Help"),
 731                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 732                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 733                  ('t',"trainee","Process-New::Options::Trainee"),
 734                  ('n',"no-action","Process-New::Options::No-Action")]
 735
 736     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 737         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 738             Cnf["Process-New::Options::%s" % (i)] = ""
 739
 740     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 741     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 742         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 743
 744     Options = Cnf.SubTree("Process-New::Options")
 745
 746     if Options["Help"]:
 747         usage()
 748
 749     Upload = queue.Upload(Cnf)
 750
 751     if not Options["No-Action"]:
 752         try:
 753             Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 754         except CantOpenError, e:
 755             Options["Trainee"] = "Oh yes"
 756
 757     projectB = Upload.projectB
 758
 759     Sections = Section_Completer()
 760     Priorities = Priority_Completer()
 761     readline.parse_and_bind("tab: complete")
 762
 763     return changes_files
 764
 765 ################################################################################
 766
 767 def do_byhand():
 768     done = 0
 769     while not done:
 770         files = Upload.pkg.files
 771         will_install = 1
 772         byhand = []
 773
 774         for f in files.keys():
 775             if files[f]["type"] == "byhand":
 776                 if os.path.exists(f):
 777                     print "W: %s still present; please process byhand components and try again." % (f)
 778                     will_install = 0
 779                 else:
 780                     byhand.append(f)
 781
 782         answer = "XXXX"
 783         if Options["No-Action"]:
 784             answer = "S"
 785         if will_install:
 786             if Options["Automatic"] and not Options["No-Action"]:
 787                 answer = 'A'
 788             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 789         else:
 790             prompt = "Manual reject, [S]kip, Quit ?"
 791
 792         while prompt.find(answer) == -1:
 793             answer = utils.our_raw_input(prompt)
 794             m = re_default_answer.search(prompt)
 795             if answer == "":
 796                 answer = m.group(1)
 797             answer = answer[:1].upper()
 798
 799         if answer == 'A':
 800             done = 1
 801             for f in byhand:
 802                 del files[f]
 803         elif answer == 'M':
 804             Upload.do_reject(1, Options["Manual-Reject"])
 805             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 806             done = 1
 807         elif answer == 'S':
 808             done = 1
 809         elif answer == 'Q':
 810             end()
 811             sys.exit(0)
 812
 813 ################################################################################
 814
 815 def get_accept_lock():
 816     retry = 0
 817     while retry < 10:
 818         try:
 819             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 820             retry = 10
 821         except OSError, e:
 822             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 823                 retry += 1
 824                 if (retry >= 10):
 825                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 826                 else:
 827                     print("Unable to get accepted lock (try %d of 10)" % retry)
 828                 time.sleep(60)
 829             else:
 830                 raise
 831
 832 def move_to_dir (dest, perms=0660, changesperms=0664):
 833     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 834     file_keys = Upload.pkg.files.keys()
 835     for f in file_keys:
 836         utils.move (f, dest, perms=perms)
 837
 838 def is_source_in_queue_dir(qdir):
 839     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 840                 and x.endswith(".changes") ]
 841     for entry in entries:
 842         # read the .dak
 843         u = queue.Upload(Cnf)
 844         u.pkg.changes_file = os.path.join(qdir, entry)
 845         u.update_vars()
 846         if not u.pkg.changes["architecture"].has_key("source"):
 847             # another binary upload, ignore
 848             continue
 849         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 850             # another version, ignore
 851             continue
 852         # found it!
 853         return True
 854     return False
 855
 856 def move_to_holding(suite, queue_dir):
 857     print "Moving to %s holding area." % (suite.upper(),)
 858     if Options["No-Action"]:
 859         return
 860     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 861     Upload.dump_vars(queue_dir)
 862     move_to_dir(queue_dir, perms=0664)
 863     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 864
 865 def _accept():
 866     if Options["No-Action"]:
 867         return
 868     (summary, short_summary) = Upload.build_summaries()
 869     Upload.accept(summary, short_summary)
 870     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 871
 872 def do_accept_stableupdate(suite, q):
 873     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 874     if not Upload.pkg.changes["architecture"].has_key("source"):
 875         # It is not a sourceful upload.  So its source may be either in p-u
 876         # holding, in new, in accepted or already installed.
 877         if is_source_in_queue_dir(queue_dir):
 878             # It's in p-u holding, so move it there.
 879             print "Binary-only upload, source in %s." % (q,)
 880             move_to_holding(suite, queue_dir)
 881         elif Upload.source_exists(Upload.pkg.changes["source"],
 882                 Upload.pkg.changes["version"]):
 883             # dak tells us that there is source available.  At time of
 884             # writing this means that it is installed, so put it into
 885             # accepted.
 886             print "Binary-only upload, source installed."
 887             _accept()
 888         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 889             # The source is in accepted, the binary cleared NEW: accept it.
 890             print "Binary-only upload, source in accepted."
 891             _accept()
 892         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 893             # It's in NEW.  We expect the source to land in p-u holding
 894             # pretty soon.
 895             print "Binary-only upload, source in new."
 896             move_to_holding(suite, queue_dir)
 897         else:
 898             # No case applicable.  Bail out.  Return will cause the upload
 899             # to be skipped.
 900             print "ERROR"
 901             print "Stable update failed.  Source not found."
 902             return
 903     else:
 904         # We are handling a sourceful upload.  Move to accepted if currently
 905         # in p-u holding and to p-u holding otherwise.
 906         if is_source_in_queue_dir(queue_dir):
 907             print "Sourceful upload in %s, accepting." % (q,)
 908             _accept()
 909         else:
 910             move_to_holding(suite, queue_dir)
 911
 912 def do_accept():
 913     print "ACCEPT"
 914     if not Options["No-Action"]:
 915         get_accept_lock()
 916         (summary, short_summary) = Upload.build_summaries()
 917     try:
 918         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 919             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 920             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 921             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 922             # Check for override disparities
 923             Upload.Subst["__SUMMARY__"] = summary
 924         else:
 925             # Stable updates need to be copied to proposed-updates holding
 926             # area instead of accepted.  Sourceful uploads need to go
 927             # to it directly, binaries only if the source has not yet been
 928             # accepted into p-u.
 929             for suite, q in [("proposed-updates", "ProposedUpdates"),
 930                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 931                 if not Upload.pkg.changes["distribution"].has_key(suite):
 932                     continue
 933                 return do_accept_stableupdate(suite, q)
 934             # Just a normal upload, accept it...
 935             _accept()
 936     finally:
 937         if not Options["No-Action"]:
 938             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 939
 940 def check_status(files):
 941     new = byhand = 0
 942     for f in files.keys():
 943         if files[f]["type"] == "byhand":
 944             byhand = 1
 945         elif files[f].has_key("new"):
 946             new = 1
 947     return (new, byhand)
 948
 949 def do_pkg(changes_file):
 950     Upload.pkg.changes_file = changes_file
 951     Upload.init_vars()
 952     Upload.update_vars()
 953     Upload.update_subst()
 954     files = Upload.pkg.files
 955
 956     if not recheck():
 957         return
 958
 959     (new, byhand) = check_status(files)
 960     if new or byhand:
 961         if new:
 962             do_new()
 963         if byhand:
 964             do_byhand()
 965         (new, byhand) = check_status(files)
 966
 967     if not new and not byhand:
 968         do_accept()
 969
 970 ################################################################################
 971
 972 def end():
 973     accept_count = Upload.accept_count
 974     accept_bytes = Upload.accept_bytes
 975
 976     if accept_count:
 977         sets = "set"
 978         if accept_count > 1:
 979             sets = "sets"
 980         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 981         Logger.log(["total",accept_count,accept_bytes])
 982
 983     if not Options["No-Action"] and not Options["Trainee"]:
 984         Logger.close()
 985
 986 ################################################################################
 987
 988 def do_comments(dir, opref, npref, line, fn):
 989     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 990         lines = open("%s/%s" % (dir, comm)).readlines()
 991         if len(lines) == 0 or lines[0] != line + "\n": continue
 992         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 993                                 and x.endswith(".changes") ]
 994         changes_files = sort_changes(changes_files)
 995         for f in changes_files:
 996             f = utils.validate_changes_file_arg(f, 0)
 997             if not f: continue
 998             print "\n" + f
 999             fn(f, "".join(lines[1:]))
1000
1001         if opref != npref and not Options["No-Action"]:
1002             newcomm = npref + comm[len(opref):]
1003             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1004
1005 ################################################################################
1006
1007 def comment_accept(changes_file, comments):
1008     Upload.pkg.changes_file = changes_file
1009     Upload.init_vars()
1010     Upload.update_vars()
1011     Upload.update_subst()
1012     files = Upload.pkg.files
1013
1014     if not recheck():
1015         return # dak wants to REJECT, crap
1016
1017     (new, byhand) = check_status(files)
1018     if not new and not byhand:
1019         do_accept()
1020
1021 ################################################################################
1022
1023 def comment_reject(changes_file, comments):
1024     Upload.pkg.changes_file = changes_file
1025     Upload.init_vars()
1026     Upload.update_vars()
1027     Upload.update_subst()
1028
1029     if not recheck():
1030         pass # dak has its own reasons to reject as well, which is fine
1031
1032     reject(comments)
1033     print "REJECT\n" + reject_message,
1034     if not Options["No-Action"]:
1035         Upload.do_reject(0, reject_message)
1036         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1037
1038 ################################################################################
1039
1040 def main():
1041     changes_files = init()
1042     if len(changes_files) > 50:
1043         sys.stderr.write("Sorting changes...\n")
1044     changes_files = sort_changes(changes_files)
1045
1046     # Kill me now? **FIXME**
1047     Cnf["Dinstall::Options::No-Mail"] = ""
1048     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1049     if Cnf.has_key("Dinstall::Bcc"):
1050         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1051     else:
1052         Upload.Subst["__BCC__"] = bcc
1053
1054     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1055     if commentsdir:
1056         if changes_files != []:
1057             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1058             sys.exit(1)
1059         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1060         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1061     else:
1062         for changes_file in changes_files:
1063             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1064             if not changes_file:
1065                 continue
1066             print "\n" + changes_file
1067             do_pkg (changes_file)
1068
1069     end()
1070
1071 ################################################################################
1072
1073 if __name__ == '__main__':
1074     main()