dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 import copy
  45 import errno
  46 import os
  47 import readline
  48 import stat
  49 import sys
  50 import time
  51 import apt_pkg, apt_inst
  52 import examine_package
  53 from daklib import database
  54 from daklib import logging
  55 from daklib import queue
  56 from daklib import utils
  57 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  58 from daklib.dak_exceptions import CantOpenError
  59
  60 # Globals
  61 Cnf = None       #: Configuration, apt_pkg.Configuration
  62 Options = None
  63 Upload = None
  64 projectB = None  #: database connection, pgobject
  65 Logger = None
  66
  67 Priorities = None
  68 Sections = None
  69
  70 reject_message = ""
  71
  72 ################################################################################
  73 ################################################################################
  74 ################################################################################
  75
  76 def reject (str, prefix="Rejected: "):
  77     global reject_message
  78     if str:
  79         reject_message += prefix + str + "\n"
  80
  81 def recheck():
  82     global reject_message
  83     files = Upload.pkg.files
  84     reject_message = ""
  85
  86     for f in files.keys():
  87         # The .orig.tar.gz can disappear out from under us is it's a
  88         # duplicate of one in the archive.
  89         if not files.has_key(f):
  90             continue
  91         # Check that the source still exists
  92         if files[f]["type"] == "deb":
  93             source_version = files[f]["source version"]
  94             source_package = files[f]["source package"]
  95             if not Upload.pkg.changes["architecture"].has_key("source") \
  96                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  97                 source_epochless_version = re_no_epoch.sub('', source_version)
  98                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  99                 found = 0
 100                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
 101                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 102                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 103                             found = 1
 104                 if not found:
 105                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 106
 107         # Version and file overwrite checks
 108         if files[f]["type"] == "deb":
 109             reject(Upload.check_binary_against_db(f), "")
 110         elif files[f]["type"] == "dsc":
 111             reject(Upload.check_source_against_db(f), "")
 112             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 113             reject(reject_msg, "")
 114
 115     if reject_message.find("Rejected") != -1:
 116         answer = "XXX"
 117         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 118             answer = 'S'
 119
 120         print "REJECT\n" + reject_message,
 121         prompt = "[R]eject, Skip, Quit ?"
 122
 123         while prompt.find(answer) == -1:
 124             answer = utils.our_raw_input(prompt)
 125             m = re_default_answer.match(prompt)
 126             if answer == "":
 127                 answer = m.group(1)
 128             answer = answer[:1].upper()
 129
 130         if answer == 'R':
 131             Upload.do_reject(0, reject_message)
 132             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 133             return 0
 134         elif answer == 'S':
 135             return 0
 136         elif answer == 'Q':
 137             end()
 138             sys.exit(0)
 139
 140     return 1
 141
 142 ################################################################################
 143
 144 def indiv_sg_compare (a, b):
 145     """Sort by source name, source, version, 'have source', and
 146        finally by filename."""
 147     # Sort by source version
 148     q = apt_pkg.VersionCompare(a["version"], b["version"])
 149     if q:
 150         return -q
 151
 152     # Sort by 'have source'
 153     a_has_source = a["architecture"].get("source")
 154     b_has_source = b["architecture"].get("source")
 155     if a_has_source and not b_has_source:
 156         return -1
 157     elif b_has_source and not a_has_source:
 158         return 1
 159
 160     return cmp(a["filename"], b["filename"])
 161
 162 ############################################################
 163
 164 def sg_compare (a, b):
 165     a = a[1]
 166     b = b[1]
 167     """Sort by have note, source already in database and time of oldest upload."""
 168     # Sort by have note
 169     a_note_state = a["note_state"]
 170     b_note_state = b["note_state"]
 171     if a_note_state < b_note_state:
 172         return -1
 173     elif a_note_state > b_note_state:
 174         return 1
 175     # Sort by source already in database (descending)
 176     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 177     if source_in_database:
 178         return -source_in_database
 179
 180     # Sort by time of oldest upload
 181     return cmp(a["oldest"], b["oldest"])
 182
 183 def sort_changes(changes_files):
 184     """Sort into source groups, then sort each source group by version,
 185     have source, filename.  Finally, sort the source groups by have
 186     note, time of oldest upload of each source upload."""
 187     if len(changes_files) == 1:
 188         return changes_files
 189
 190     sorted_list = []
 191     cache = {}
 192     # Read in all the .changes files
 193     for filename in changes_files:
 194         try:
 195             Upload.pkg.changes_file = filename
 196             Upload.init_vars()
 197             Upload.update_vars()
 198             cache[filename] = copy.copy(Upload.pkg.changes)
 199             cache[filename]["filename"] = filename
 200         except:
 201             sorted_list.append(filename)
 202             break
 203     # Divide the .changes into per-source groups
 204     per_source = {}
 205     for filename in cache.keys():
 206         source = cache[filename]["source"]
 207         if not per_source.has_key(source):
 208             per_source[source] = {}
 209             per_source[source]["list"] = []
 210         per_source[source]["list"].append(cache[filename])
 211     # Determine oldest time and have note status for each source group
 212     for source in per_source.keys():
 213         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 214         ql = q.getresult()
 215         per_source[source]["source_in_database"] = len(ql)>0
 216         source_list = per_source[source]["list"]
 217         first = source_list[0]
 218         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 219         have_note = 0
 220         for d in per_source[source]["list"]:
 221             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 222             if mtime < oldest:
 223                 oldest = mtime
 224             have_note += (database.has_new_comment(d["source"], d["version"]))
 225         per_source[source]["oldest"] = oldest
 226         if not have_note:
 227             per_source[source]["note_state"] = 0; # none
 228         elif have_note < len(source_list):
 229             per_source[source]["note_state"] = 1; # some
 230         else:
 231             per_source[source]["note_state"] = 2; # all
 232         per_source[source]["list"].sort(indiv_sg_compare)
 233     per_source_items = per_source.items()
 234     per_source_items.sort(sg_compare)
 235     for i in per_source_items:
 236         for j in i[1]["list"]:
 237             sorted_list.append(j["filename"])
 238     return sorted_list
 239
 240 ################################################################################
 241
 242 class Section_Completer:
 243     def __init__ (self):
 244         self.sections = []
 245         self.matches = []
 246         q = projectB.query("SELECT section FROM section")
 247         for i in q.getresult():
 248             self.sections.append(i[0])
 249
 250     def complete(self, text, state):
 251         if state == 0:
 252             self.matches = []
 253             n = len(text)
 254             for word in self.sections:
 255                 if word[:n] == text:
 256                     self.matches.append(word)
 257         try:
 258             return self.matches[state]
 259         except IndexError:
 260             return None
 261
 262 ############################################################
 263
 264 class Priority_Completer:
 265     def __init__ (self):
 266         self.priorities = []
 267         self.matches = []
 268         q = projectB.query("SELECT priority FROM priority")
 269         for i in q.getresult():
 270             self.priorities.append(i[0])
 271
 272     def complete(self, text, state):
 273         if state == 0:
 274             self.matches = []
 275             n = len(text)
 276             for word in self.priorities:
 277                 if word[:n] == text:
 278                     self.matches.append(word)
 279         try:
 280             return self.matches[state]
 281         except IndexError:
 282             return None
 283
 284 ################################################################################
 285
 286 def print_new (new, indexed, file=sys.stdout):
 287     queue.check_valid(new)
 288     broken = 0
 289     index = 0
 290     for pkg in new.keys():
 291         index += 1
 292         section = new[pkg]["section"]
 293         priority = new[pkg]["priority"]
 294         if new[pkg]["section id"] == -1:
 295             section += "[!]"
 296             broken = 1
 297         if new[pkg]["priority id"] == -1:
 298             priority += "[!]"
 299             broken = 1
 300         if indexed:
 301             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 302         else:
 303             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 304         line = line.strip()+'\n'
 305         file.write(line)
 306     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 307     if len(note) > 0:
 308         for line in note:
 309             print line
 310     return broken, note
 311
 312 ################################################################################
 313
 314 def index_range (index):
 315     if index == 1:
 316         return "1"
 317     else:
 318         return "1-%s" % (index)
 319
 320 ################################################################################
 321 ################################################################################
 322
 323 def edit_new (new):
 324     # Write the current data to a temporary file
 325     (fd, temp_filename) = utils.temp_filename()
 326     temp_file = os.fdopen(fd, 'w')
 327     print_new (new, 0, temp_file)
 328     temp_file.close()
 329     # Spawn an editor on that file
 330     editor = os.environ.get("EDITOR","vi")
 331     result = os.system("%s %s" % (editor, temp_filename))
 332     if result != 0:
 333         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 334     # Read the edited data back in
 335     temp_file = utils.open_file(temp_filename)
 336     lines = temp_file.readlines()
 337     temp_file.close()
 338     os.unlink(temp_filename)
 339     # Parse the new data
 340     for line in lines:
 341         line = line.strip()
 342         if line == "":
 343             continue
 344         s = line.split()
 345         # Pad the list if necessary
 346         s[len(s):3] = [None] * (3-len(s))
 347         (pkg, priority, section) = s[:3]
 348         if not new.has_key(pkg):
 349             utils.warn("Ignoring unknown package '%s'" % (pkg))
 350         else:
 351             # Strip off any invalid markers, print_new will readd them.
 352             if section.endswith("[!]"):
 353                 section = section[:-3]
 354             if priority.endswith("[!]"):
 355                 priority = priority[:-3]
 356             for f in new[pkg]["files"]:
 357                 Upload.pkg.files[f]["section"] = section
 358                 Upload.pkg.files[f]["priority"] = priority
 359             new[pkg]["section"] = section
 360             new[pkg]["priority"] = priority
 361
 362 ################################################################################
 363
 364 def edit_index (new, index):
 365     priority = new[index]["priority"]
 366     section = new[index]["section"]
 367     ftype = new[index]["type"]
 368     done = 0
 369     while not done:
 370         print "\t".join([index, priority, section])
 371
 372         answer = "XXX"
 373         if ftype != "dsc":
 374             prompt = "[B]oth, Priority, Section, Done ? "
 375         else:
 376             prompt = "[S]ection, Done ? "
 377         edit_priority = edit_section = 0
 378
 379         while prompt.find(answer) == -1:
 380             answer = utils.our_raw_input(prompt)
 381             m = re_default_answer.match(prompt)
 382             if answer == "":
 383                 answer = m.group(1)
 384             answer = answer[:1].upper()
 385
 386         if answer == 'P':
 387             edit_priority = 1
 388         elif answer == 'S':
 389             edit_section = 1
 390         elif answer == 'B':
 391             edit_priority = edit_section = 1
 392         elif answer == 'D':
 393             done = 1
 394
 395         # Edit the priority
 396         if edit_priority:
 397             readline.set_completer(Priorities.complete)
 398             got_priority = 0
 399             while not got_priority:
 400                 new_priority = utils.our_raw_input("New priority: ").strip()
 401                 if new_priority not in Priorities.priorities:
 402                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 403                 else:
 404                     got_priority = 1
 405                     priority = new_priority
 406
 407         # Edit the section
 408         if edit_section:
 409             readline.set_completer(Sections.complete)
 410             got_section = 0
 411             while not got_section:
 412                 new_section = utils.our_raw_input("New section: ").strip()
 413                 if new_section not in Sections.sections:
 414                     print "E: '%s' is not a valid section, try again." % (new_section)
 415                 else:
 416                     got_section = 1
 417                     section = new_section
 418
 419         # Reset the readline completer
 420         readline.set_completer(None)
 421
 422     for f in new[index]["files"]:
 423         Upload.pkg.files[f]["section"] = section
 424         Upload.pkg.files[f]["priority"] = priority
 425     new[index]["priority"] = priority
 426     new[index]["section"] = section
 427     return new
 428
 429 ################################################################################
 430
 431 def edit_overrides (new):
 432     print
 433     done = 0
 434     while not done:
 435         print_new (new, 1)
 436         new_index = {}
 437         index = 0
 438         for i in new.keys():
 439             index += 1
 440             new_index[index] = i
 441
 442         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 443
 444         got_answer = 0
 445         while not got_answer:
 446             answer = utils.our_raw_input(prompt)
 447             if not answer.isdigit():
 448                 answer = answer[:1].upper()
 449             if answer == "E" or answer == "D":
 450                 got_answer = 1
 451             elif re_isanum.match (answer):
 452                 answer = int(answer)
 453                 if (answer < 1) or (answer > index):
 454                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 455                 else:
 456                     got_answer = 1
 457
 458         if answer == 'E':
 459             edit_new(new)
 460         elif answer == 'D':
 461             done = 1
 462         else:
 463             edit_index (new, new_index[answer])
 464
 465     return new
 466
 467 ################################################################################
 468
 469 def edit_note(note):
 470     # Write the current data to a temporary file
 471     (fd, temp_filename) = utils.temp_filename()
 472     temp_file = os.fdopen(fd, 'w')
 473     if len(note) > 0:
 474         for line in note:
 475             temp_file.write(line)
 476     temp_file.close()
 477     editor = os.environ.get("EDITOR","vi")
 478     answer = 'E'
 479     while answer == 'E':
 480         os.system("%s %s" % (editor, temp_filename))
 481         temp_file = utils.open_file(temp_filename)
 482         note = temp_file.read().rstrip()
 483         temp_file.close()
 484         print "Note:"
 485         print utils.prefix_multi_line_string(note,"  ")
 486         prompt = "[D]one, Edit, Abandon, Quit ?"
 487         answer = "XXX"
 488         while prompt.find(answer) == -1:
 489             answer = utils.our_raw_input(prompt)
 490             m = re_default_answer.search(prompt)
 491             if answer == "":
 492                 answer = m.group(1)
 493             answer = answer[:1].upper()
 494     os.unlink(temp_filename)
 495     if answer == 'A':
 496         return
 497     elif answer == 'Q':
 498         end()
 499         sys.exit(0)
 500     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], note, utils.whoami())
 501
 502 ################################################################################
 503
 504 def check_pkg ():
 505     try:
 506         less_fd = os.popen("less -R -", 'w', 0)
 507         stdout_fd = sys.stdout
 508         try:
 509             sys.stdout = less_fd
 510             changes = utils.parse_changes (Upload.pkg.changes_file)
 511             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 512             files = Upload.pkg.files
 513             for f in files.keys():
 514                 if files[f].has_key("new"):
 515                     ftype = files[f]["type"]
 516                     if ftype == "deb":
 517                         examine_package.check_deb(changes['distribution'], f)
 518                     elif ftype == "dsc":
 519                         examine_package.check_dsc(changes['distribution'], f)
 520         finally:
 521             examine_package.output_package_relations()
 522             sys.stdout = stdout_fd
 523     except IOError, e:
 524         if e.errno == errno.EPIPE:
 525             utils.warn("[examine_package] Caught EPIPE; skipping.")
 526             pass
 527         else:
 528             raise
 529     except KeyboardInterrupt:
 530         utils.warn("[examine_package] Caught C-c; skipping.")
 531         pass
 532
 533 ################################################################################
 534
 535 ## FIXME: horribly Debian specific
 536
 537 def do_bxa_notification():
 538     files = Upload.pkg.files
 539     summary = ""
 540     for f in files.keys():
 541         if files[f]["type"] == "deb":
 542             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 543             summary += "\n"
 544             summary += "Package: %s\n" % (control.Find("Package"))
 545             summary += "Description: %s\n" % (control.Find("Description"))
 546     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 547     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 548     utils.send_mail(bxa_mail)
 549
 550 ################################################################################
 551
 552 def add_overrides (new):
 553     changes = Upload.pkg.changes
 554     files = Upload.pkg.files
 555
 556     projectB.query("BEGIN WORK")
 557     for suite in changes["suite"].keys():
 558         suite_id = database.get_suite_id(suite)
 559         for pkg in new.keys():
 560             component_id = database.get_component_id(new[pkg]["component"])
 561             type_id = database.get_override_type_id(new[pkg]["type"])
 562             priority_id = new[pkg]["priority id"]
 563             section_id = new[pkg]["section id"]
 564             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 565             for f in new[pkg]["files"]:
 566                 if files[f].has_key("new"):
 567                     del files[f]["new"]
 568             del new[pkg]
 569
 570     projectB.query("COMMIT WORK")
 571
 572     if Cnf.FindB("Dinstall::BXANotify"):
 573         do_bxa_notification()
 574
 575 ################################################################################
 576
 577 def prod_maintainer ():
 578     # Here we prepare an editor and get them ready to prod...
 579     (fd, temp_filename) = utils.temp_filename()
 580     editor = os.environ.get("EDITOR","vi")
 581     answer = 'E'
 582     while answer == 'E':
 583         os.system("%s %s" % (editor, temp_filename))
 584         f = os.fdopen(fd)
 585         prod_message = "".join(f.readlines())
 586         f.close()
 587         print "Prod message:"
 588         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 589         prompt = "[P]rod, Edit, Abandon, Quit ?"
 590         answer = "XXX"
 591         while prompt.find(answer) == -1:
 592             answer = utils.our_raw_input(prompt)
 593             m = re_default_answer.search(prompt)
 594             if answer == "":
 595                 answer = m.group(1)
 596             answer = answer[:1].upper()
 597         os.unlink(temp_filename)
 598         if answer == 'A':
 599             return
 600         elif answer == 'Q':
 601             end()
 602             sys.exit(0)
 603     # Otherwise, do the proding...
 604     user_email_address = utils.whoami() + " <%s>" % (
 605         Cnf["Dinstall::MyAdminAddress"])
 606
 607     Subst = Upload.Subst
 608
 609     Subst["__FROM_ADDRESS__"] = user_email_address
 610     Subst["__PROD_MESSAGE__"] = prod_message
 611     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 612
 613     prod_mail_message = utils.TemplateSubst(
 614         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 615
 616     # Send the prod mail if appropriate
 617     if not Cnf["Dinstall::Options::No-Mail"]:
 618         utils.send_mail(prod_mail_message)
 619
 620     print "Sent proding message"
 621
 622 ################################################################################
 623
 624 def do_new():
 625     print "NEW\n"
 626     files = Upload.pkg.files
 627     changes = Upload.pkg.changes
 628
 629     # Make a copy of distribution we can happily trample on
 630     changes["suite"] = copy.copy(changes["distribution"])
 631
 632     # Fix up the list of target suites
 633     for suite in changes["suite"].keys():
 634         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 635         if override:
 636             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 637               database.get_suite_id(override) == -1)
 638             if olderr or newerr:
 639                 (oinv, newinv) = ("", "")
 640                 if olderr: oinv = "invalid "
 641                 if newerr: ninv = "invalid "
 642                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 643                         oinv, suite, ninv, override)
 644             del changes["suite"][suite]
 645             changes["suite"][override] = 1
 646     # Validate suites
 647     for suite in changes["suite"].keys():
 648         suite_id = database.get_suite_id(suite)
 649         if suite_id == -1:
 650             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 651
 652     # The main NEW processing loop
 653     done = 0
 654     while not done:
 655         # Find out what's new
 656         new = queue.determine_new(changes, files, projectB)
 657
 658         if not new:
 659             break
 660
 661         answer = "XXX"
 662         if Options["No-Action"] or Options["Automatic"]:
 663             answer = 'S'
 664
 665         (broken, note) = print_new(new, 0)
 666         prompt = ""
 667
 668         if not broken and not note:
 669             prompt = "Add overrides, "
 670         if broken:
 671             print "W: [!] marked entries must be fixed before package can be processed."
 672         if note:
 673             print "W: note must be removed before package can be processed."
 674             prompt += "Remove note, "
 675
 676         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 677
 678         while prompt.find(answer) == -1:
 679             answer = utils.our_raw_input(prompt)
 680             m = re_default_answer.search(prompt)
 681             if answer == "":
 682                 answer = m.group(1)
 683             answer = answer[:1].upper()
 684
 685         if answer == 'A' and not Options["Trainee"]:
 686             done = add_overrides (new)
 687         elif answer == 'C':
 688             check_pkg()
 689         elif answer == 'E' and not Options["Trainee"]:
 690             new = edit_overrides (new)
 691         elif answer == 'M' and not Options["Trainee"]:
 692             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 693             if not aborted:
 694                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 695                 done = 1
 696         elif answer == 'N':
 697             edit_note(database.get_new_comments(changes.get("source", "")))
 698         elif answer == 'P' and not Options["Trainee"]:
 699             prod_maintainer()
 700         elif answer == 'R':
 701             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 702             if confirm == "y":
 703                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 704         elif answer == 'S':
 705             done = 1
 706         elif answer == 'Q':
 707             end()
 708             sys.exit(0)
 709
 710 ################################################################################
 711 ################################################################################
 712 ################################################################################
 713
 714 def usage (exit_code=0):
 715     print """Usage: dak process-new [OPTION]... [CHANGES]...
 716   -a, --automatic           automatic run
 717   -h, --help                show this help and exit.
 718   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 719   -m, --manual-reject=MSG   manual reject with `msg'
 720   -n, --no-action           don't do anything
 721   -t, --trainee             FTP Trainee mode
 722   -V, --version             display the version number and exit"""
 723     sys.exit(exit_code)
 724
 725 ################################################################################
 726
 727 def init():
 728     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 729
 730     Cnf = utils.get_conf()
 731
 732     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 733                  ('h',"help","Process-New::Options::Help"),
 734                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 735                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 736                  ('t',"trainee","Process-New::Options::Trainee"),
 737                  ('n',"no-action","Process-New::Options::No-Action")]
 738
 739     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 740         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 741             Cnf["Process-New::Options::%s" % (i)] = ""
 742
 743     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 744     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 745         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 746
 747     Options = Cnf.SubTree("Process-New::Options")
 748
 749     if Options["Help"]:
 750         usage()
 751
 752     Upload = queue.Upload(Cnf)
 753
 754     if not Options["No-Action"]:
 755         try:
 756             Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 757         except CantOpenError, e:
 758             Options["Trainee"] = True
 759
 760     projectB = Upload.projectB
 761
 762     Sections = Section_Completer()
 763     Priorities = Priority_Completer()
 764     readline.parse_and_bind("tab: complete")
 765
 766     return changes_files
 767
 768 ################################################################################
 769
 770 def do_byhand():
 771     done = 0
 772     while not done:
 773         files = Upload.pkg.files
 774         will_install = 1
 775         byhand = []
 776
 777         for f in files.keys():
 778             if files[f]["type"] == "byhand":
 779                 if os.path.exists(f):
 780                     print "W: %s still present; please process byhand components and try again." % (f)
 781                     will_install = 0
 782                 else:
 783                     byhand.append(f)
 784
 785         answer = "XXXX"
 786         if Options["No-Action"]:
 787             answer = "S"
 788         if will_install:
 789             if Options["Automatic"] and not Options["No-Action"]:
 790                 answer = 'A'
 791             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 792         else:
 793             prompt = "Manual reject, [S]kip, Quit ?"
 794
 795         while prompt.find(answer) == -1:
 796             answer = utils.our_raw_input(prompt)
 797             m = re_default_answer.search(prompt)
 798             if answer == "":
 799                 answer = m.group(1)
 800             answer = answer[:1].upper()
 801
 802         if answer == 'A':
 803             done = 1
 804             for f in byhand:
 805                 del files[f]
 806         elif answer == 'M':
 807             Upload.do_reject(1, Options["Manual-Reject"])
 808             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 809             done = 1
 810         elif answer == 'S':
 811             done = 1
 812         elif answer == 'Q':
 813             end()
 814             sys.exit(0)
 815
 816 ################################################################################
 817
 818 def get_accept_lock():
 819     retry = 0
 820     while retry < 10:
 821         try:
 822             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 823             retry = 10
 824         except OSError, e:
 825             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 826                 retry += 1
 827                 if (retry >= 10):
 828                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 829                 else:
 830                     print("Unable to get accepted lock (try %d of 10)" % retry)
 831                 time.sleep(60)
 832             else:
 833                 raise
 834
 835 def move_to_dir (dest, perms=0660, changesperms=0664):
 836     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 837     file_keys = Upload.pkg.files.keys()
 838     for f in file_keys:
 839         utils.move (f, dest, perms=perms)
 840
 841 def is_source_in_queue_dir(qdir):
 842     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 843                 and x.endswith(".changes") ]
 844     for entry in entries:
 845         # read the .dak
 846         u = queue.Upload(Cnf)
 847         u.pkg.changes_file = os.path.join(qdir, entry)
 848         u.update_vars()
 849         if not u.pkg.changes["architecture"].has_key("source"):
 850             # another binary upload, ignore
 851             continue
 852         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 853             # another version, ignore
 854             continue
 855         # found it!
 856         return True
 857     return False
 858
 859 def move_to_holding(suite, queue_dir):
 860     print "Moving to %s holding area." % (suite.upper(),)
 861     if Options["No-Action"]:
 862         return
 863     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 864     Upload.dump_vars(queue_dir)
 865     move_to_dir(queue_dir, perms=0664)
 866     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 867
 868 def _accept():
 869     if Options["No-Action"]:
 870         return
 871     (summary, short_summary) = Upload.build_summaries()
 872     Upload.accept(summary, short_summary)
 873     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 874
 875 def do_accept_stableupdate(suite, q):
 876     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 877     if not Upload.pkg.changes["architecture"].has_key("source"):
 878         # It is not a sourceful upload.  So its source may be either in p-u
 879         # holding, in new, in accepted or already installed.
 880         if is_source_in_queue_dir(queue_dir):
 881             # It's in p-u holding, so move it there.
 882             print "Binary-only upload, source in %s." % (q,)
 883             move_to_holding(suite, queue_dir)
 884         elif Upload.source_exists(Upload.pkg.changes["source"],
 885                 Upload.pkg.changes["version"]):
 886             # dak tells us that there is source available.  At time of
 887             # writing this means that it is installed, so put it into
 888             # accepted.
 889             print "Binary-only upload, source installed."
 890             _accept()
 891         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 892             # The source is in accepted, the binary cleared NEW: accept it.
 893             print "Binary-only upload, source in accepted."
 894             _accept()
 895         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 896             # It's in NEW.  We expect the source to land in p-u holding
 897             # pretty soon.
 898             print "Binary-only upload, source in new."
 899             move_to_holding(suite, queue_dir)
 900         else:
 901             # No case applicable.  Bail out.  Return will cause the upload
 902             # to be skipped.
 903             print "ERROR"
 904             print "Stable update failed.  Source not found."
 905             return
 906     else:
 907         # We are handling a sourceful upload.  Move to accepted if currently
 908         # in p-u holding and to p-u holding otherwise.
 909         if is_source_in_queue_dir(queue_dir):
 910             print "Sourceful upload in %s, accepting." % (q,)
 911             _accept()
 912         else:
 913             move_to_holding(suite, queue_dir)
 914
 915 def do_accept():
 916     print "ACCEPT"
 917     if not Options["No-Action"]:
 918         get_accept_lock()
 919         (summary, short_summary) = Upload.build_summaries()
 920     try:
 921         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 922             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 923             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 924             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 925             # Check for override disparities
 926             Upload.Subst["__SUMMARY__"] = summary
 927         else:
 928             # Stable updates need to be copied to proposed-updates holding
 929             # area instead of accepted.  Sourceful uploads need to go
 930             # to it directly, binaries only if the source has not yet been
 931             # accepted into p-u.
 932             for suite, q in [("proposed-updates", "ProposedUpdates"),
 933                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 934                 if not Upload.pkg.changes["distribution"].has_key(suite):
 935                     continue
 936                 return do_accept_stableupdate(suite, q)
 937             # Just a normal upload, accept it...
 938             _accept()
 939     finally:
 940         if not Options["No-Action"]:
 941             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 942
 943 def check_status(files):
 944     new = byhand = 0
 945     for f in files.keys():
 946         if files[f]["type"] == "byhand":
 947             byhand = 1
 948         elif files[f].has_key("new"):
 949             new = 1
 950     return (new, byhand)
 951
 952 def do_pkg(changes_file):
 953     Upload.pkg.changes_file = changes_file
 954     Upload.init_vars()
 955     Upload.update_vars()
 956     Upload.update_subst()
 957     files = Upload.pkg.files
 958
 959     if not recheck():
 960         return
 961
 962     (new, byhand) = check_status(files)
 963     if new or byhand:
 964         if new:
 965             do_new()
 966         if byhand:
 967             do_byhand()
 968         (new, byhand) = check_status(files)
 969
 970     if not new and not byhand:
 971         do_accept()
 972
 973 ################################################################################
 974
 975 def end():
 976     accept_count = Upload.accept_count
 977     accept_bytes = Upload.accept_bytes
 978
 979     if accept_count:
 980         sets = "set"
 981         if accept_count > 1:
 982             sets = "sets"
 983         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 984         Logger.log(["total",accept_count,accept_bytes])
 985
 986     if not Options["No-Action"]:
 987         Logger.close()
 988
 989 ################################################################################
 990
 991 def do_comments(dir, opref, npref, line, fn):
 992     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 993         lines = open("%s/%s" % (dir, comm)).readlines()
 994         if len(lines) == 0 or lines[0] != line + "\n": continue
 995         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 996                                 and x.endswith(".changes") ]
 997         changes_files = sort_changes(changes_files)
 998         for f in changes_files:
 999             f = utils.validate_changes_file_arg(f, 0)
1000             if not f: continue
1001             print "\n" + f
1002             fn(f, "".join(lines[1:]))
1003
1004         if opref != npref and not Options["No-Action"]:
1005             newcomm = npref + comm[len(opref):]
1006             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1007
1008 ################################################################################
1009
1010 def comment_accept(changes_file, comments):
1011     Upload.pkg.changes_file = changes_file
1012     Upload.init_vars()
1013     Upload.update_vars()
1014     Upload.update_subst()
1015     files = Upload.pkg.files
1016
1017     if not recheck():
1018         return # dak wants to REJECT, crap
1019
1020     (new, byhand) = check_status(files)
1021     if not new and not byhand:
1022         do_accept()
1023
1024 ################################################################################
1025
1026 def comment_reject(changes_file, comments):
1027     Upload.pkg.changes_file = changes_file
1028     Upload.init_vars()
1029     Upload.update_vars()
1030     Upload.update_subst()
1031
1032     if not recheck():
1033         pass # dak has its own reasons to reject as well, which is fine
1034
1035     reject(comments)
1036     print "REJECT\n" + reject_message,
1037     if not Options["No-Action"]:
1038         Upload.do_reject(0, reject_message)
1039         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1040
1041 ################################################################################
1042
1043 def main():
1044     changes_files = init()
1045     if len(changes_files) > 50:
1046         sys.stderr.write("Sorting changes...\n")
1047     changes_files = sort_changes(changes_files)
1048
1049     # Kill me now? **FIXME**
1050     Cnf["Dinstall::Options::No-Mail"] = ""
1051     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1052     if Cnf.has_key("Dinstall::Bcc"):
1053         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1054     else:
1055         Upload.Subst["__BCC__"] = bcc
1056
1057     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1058     if commentsdir:
1059         if changes_files != []:
1060             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1061             sys.exit(1)
1062         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1063         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1064     else:
1065         for changes_file in changes_files:
1066             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1067             if not changes_file:
1068                 continue
1069             print "\n" + changes_file
1070             do_pkg (changes_file)
1071
1072     end()
1073
1074 ################################################################################
1075
1076 if __name__ == '__main__':
1077     main()