dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @license: GNU General Public License version 2 or later
   9 """
  10 # This program is free software; you can redistribute it and/or modify
  11 # it under the terms of the GNU General Public License as published by
  12 # the Free Software Foundation; either version 2 of the License, or
  13 # (at your option) any later version.
  14
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU General Public License for more details.
  19
  20 # You should have received a copy of the GNU General Public License
  21 # along with this program; if not, write to the Free Software
  22 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  23
  24 ################################################################################
  25
  26 # 23:12|<aj> I will not hush!
  27 # 23:12|<elmo> :>
  28 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  29 # 23:13|<aj> I shall not be silenced!
  30 # 23:13|<aj> The world shall know!
  31 # 23:13|<aj> The world *must* know!
  32 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  33 # 23:13|<aj> yay powerpuff girls!!
  34 # 23:13|<aj> buttercup's my favourite, who's yours?
  35 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  36 # 23:14|<aj> *AREN'T YOU*?!
  37 # 23:15|<aj> I will not be treated like this.
  38 # 23:15|<aj> I shall have my revenge.
  39 # 23:15|<aj> I SHALL!!!
  40
  41 ################################################################################
  42
  43 import copy
  44 import errno
  45 import os
  46 import readline
  47 import stat
  48 import sys
  49 import time
  50 import apt_pkg, apt_inst
  51 import examine_package
  52 from daklib import database
  53 from daklib import logging
  54 from daklib import queue
  55 from daklib import utils
  56 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  57
  58 # Globals
  59 Cnf = None       #: Configuration, apt_pkg.Configuration
  60 Options = None
  61 Upload = None
  62 projectB = None  #: database connection, pgobject
  63 Logger = None
  64
  65 Priorities = None
  66 Sections = None
  67
  68 reject_message = ""
  69
  70 ################################################################################
  71 ################################################################################
  72 ################################################################################
  73
  74 def reject (str, prefix="Rejected: "):
  75     global reject_message
  76     if str:
  77         reject_message += prefix + str + "\n"
  78
  79 def recheck():
  80     global reject_message
  81     files = Upload.pkg.files
  82     reject_message = ""
  83
  84     for f in files.keys():
  85         # The .orig.tar.gz can disappear out from under us is it's a
  86         # duplicate of one in the archive.
  87         if not files.has_key(f):
  88             continue
  89         # Check that the source still exists
  90         if files[f]["type"] == "deb":
  91             source_version = files[f]["source version"]
  92             source_package = files[f]["source package"]
  93             if not Upload.pkg.changes["architecture"].has_key("source") \
  94                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  95                 source_epochless_version = re_no_epoch.sub('', source_version)
  96                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  97                 found = 0
  98                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
  99                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 100                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 101                             found = 1
 102                 if not found:
 103                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 104
 105         # Version and file overwrite checks
 106         if files[f]["type"] == "deb":
 107             reject(Upload.check_binary_against_db(f), "")
 108         elif files[f]["type"] == "dsc":
 109             reject(Upload.check_source_against_db(f), "")
 110             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 111             reject(reject_msg, "")
 112
 113     if reject_message.find("Rejected") != -1:
 114         answer = "XXX"
 115         if Options["No-Action"] or Options["Automatic"]:
 116             answer = 'S'
 117
 118         print "REJECT\n" + reject_message,
 119         prompt = "[R]eject, Skip, Quit ?"
 120
 121         while prompt.find(answer) == -1:
 122             answer = utils.our_raw_input(prompt)
 123             m = re_default_answer.match(prompt)
 124             if answer == "":
 125                 answer = m.group(1)
 126             answer = answer[:1].upper()
 127
 128         if answer == 'R':
 129             Upload.do_reject(0, reject_message)
 130             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 131             return 0
 132         elif answer == 'S':
 133             return 0
 134         elif answer == 'Q':
 135             end()
 136             sys.exit(0)
 137
 138     return 1
 139
 140 ################################################################################
 141
 142 def indiv_sg_compare (a, b):
 143     """Sort by source name, source, version, 'have source', and
 144        finally by filename."""
 145     # Sort by source version
 146     q = apt_pkg.VersionCompare(a["version"], b["version"])
 147     if q:
 148         return -q
 149
 150     # Sort by 'have source'
 151     a_has_source = a["architecture"].get("source")
 152     b_has_source = b["architecture"].get("source")
 153     if a_has_source and not b_has_source:
 154         return -1
 155     elif b_has_source and not a_has_source:
 156         return 1
 157
 158     return cmp(a["filename"], b["filename"])
 159
 160 ############################################################
 161
 162 def sg_compare (a, b):
 163     a = a[1]
 164     b = b[1]
 165     """Sort by have note, source already in database and time of oldest upload."""
 166     # Sort by have note
 167     a_note_state = a["note_state"]
 168     b_note_state = b["note_state"]
 169     if a_note_state < b_note_state:
 170         return -1
 171     elif a_note_state > b_note_state:
 172         return 1
 173     # Sort by source already in database (descending)
 174     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 175     if source_in_database:
 176         return -source_in_database
 177
 178     # Sort by time of oldest upload
 179     return cmp(a["oldest"], b["oldest"])
 180
 181 def sort_changes(changes_files):
 182     """Sort into source groups, then sort each source group by version,
 183     have source, filename.  Finally, sort the source groups by have
 184     note, time of oldest upload of each source upload."""
 185     if len(changes_files) == 1:
 186         return changes_files
 187
 188     sorted_list = []
 189     cache = {}
 190     # Read in all the .changes files
 191     for filename in changes_files:
 192         try:
 193             Upload.pkg.changes_file = filename
 194             Upload.init_vars()
 195             Upload.update_vars()
 196             cache[filename] = copy.copy(Upload.pkg.changes)
 197             cache[filename]["filename"] = filename
 198         except:
 199             sorted_list.append(filename)
 200             break
 201     # Divide the .changes into per-source groups
 202     per_source = {}
 203     for filename in cache.keys():
 204         source = cache[filename]["source"]
 205         if not per_source.has_key(source):
 206             per_source[source] = {}
 207             per_source[source]["list"] = []
 208         per_source[source]["list"].append(cache[filename])
 209     # Determine oldest time and have note status for each source group
 210     for source in per_source.keys():
 211         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 212         ql = q.getresult()
 213         per_source[source]["source_in_database"] = len(ql)>0
 214         source_list = per_source[source]["list"]
 215         first = source_list[0]
 216         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 217         have_note = 0
 218         for d in per_source[source]["list"]:
 219             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 220             if mtime < oldest:
 221                 oldest = mtime
 222             have_note += (d.has_key("process-new note"))
 223         per_source[source]["oldest"] = oldest
 224         if not have_note:
 225             per_source[source]["note_state"] = 0; # none
 226         elif have_note < len(source_list):
 227             per_source[source]["note_state"] = 1; # some
 228         else:
 229             per_source[source]["note_state"] = 2; # all
 230         per_source[source]["list"].sort(indiv_sg_compare)
 231     per_source_items = per_source.items()
 232     per_source_items.sort(sg_compare)
 233     for i in per_source_items:
 234         for j in i[1]["list"]:
 235             sorted_list.append(j["filename"])
 236     return sorted_list
 237
 238 ################################################################################
 239
 240 class Section_Completer:
 241     def __init__ (self):
 242         self.sections = []
 243         self.matches = []
 244         q = projectB.query("SELECT section FROM section")
 245         for i in q.getresult():
 246             self.sections.append(i[0])
 247
 248     def complete(self, text, state):
 249         if state == 0:
 250             self.matches = []
 251             n = len(text)
 252             for word in self.sections:
 253                 if word[:n] == text:
 254                     self.matches.append(word)
 255         try:
 256             return self.matches[state]
 257         except IndexError:
 258             return None
 259
 260 ############################################################
 261
 262 class Priority_Completer:
 263     def __init__ (self):
 264         self.priorities = []
 265         self.matches = []
 266         q = projectB.query("SELECT priority FROM priority")
 267         for i in q.getresult():
 268             self.priorities.append(i[0])
 269
 270     def complete(self, text, state):
 271         if state == 0:
 272             self.matches = []
 273             n = len(text)
 274             for word in self.priorities:
 275                 if word[:n] == text:
 276                     self.matches.append(word)
 277         try:
 278             return self.matches[state]
 279         except IndexError:
 280             return None
 281
 282 ################################################################################
 283
 284 def print_new (new, indexed, file=sys.stdout):
 285     queue.check_valid(new)
 286     broken = 0
 287     index = 0
 288     for pkg in new.keys():
 289         index += 1
 290         section = new[pkg]["section"]
 291         priority = new[pkg]["priority"]
 292         if new[pkg]["section id"] == -1:
 293             section += "[!]"
 294             broken = 1
 295         if new[pkg]["priority id"] == -1:
 296             priority += "[!]"
 297             broken = 1
 298         if indexed:
 299             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 300         else:
 301             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 302         line = line.strip()+'\n'
 303         file.write(line)
 304     note = Upload.pkg.changes.get("process-new note")
 305     if note:
 306         print "*"*75
 307         print note
 308         print "*"*75
 309     return broken, note
 310
 311 ################################################################################
 312
 313 def index_range (index):
 314     if index == 1:
 315         return "1"
 316     else:
 317         return "1-%s" % (index)
 318
 319 ################################################################################
 320 ################################################################################
 321
 322 def edit_new (new):
 323     # Write the current data to a temporary file
 324     (fd, temp_filename) = utils.temp_filename()
 325     temp_file = os.fdopen(fd, 'w')
 326     print_new (new, 0, temp_file)
 327     temp_file.close()
 328     # Spawn an editor on that file
 329     editor = os.environ.get("EDITOR","vi")
 330     result = os.system("%s %s" % (editor, temp_filename))
 331     if result != 0:
 332         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 333     # Read the edited data back in
 334     temp_file = utils.open_file(temp_filename)
 335     lines = temp_file.readlines()
 336     temp_file.close()
 337     os.unlink(temp_filename)
 338     # Parse the new data
 339     for line in lines:
 340         line = line.strip()
 341         if line == "":
 342             continue
 343         s = line.split()
 344         # Pad the list if necessary
 345         s[len(s):3] = [None] * (3-len(s))
 346         (pkg, priority, section) = s[:3]
 347         if not new.has_key(pkg):
 348             utils.warn("Ignoring unknown package '%s'" % (pkg))
 349         else:
 350             # Strip off any invalid markers, print_new will readd them.
 351             if section.endswith("[!]"):
 352                 section = section[:-3]
 353             if priority.endswith("[!]"):
 354                 priority = priority[:-3]
 355             for f in new[pkg]["files"]:
 356                 Upload.pkg.files[f]["section"] = section
 357                 Upload.pkg.files[f]["priority"] = priority
 358             new[pkg]["section"] = section
 359             new[pkg]["priority"] = priority
 360
 361 ################################################################################
 362
 363 def edit_index (new, index):
 364     priority = new[index]["priority"]
 365     section = new[index]["section"]
 366     ftype = new[index]["type"]
 367     done = 0
 368     while not done:
 369         print "\t".join([index, priority, section])
 370
 371         answer = "XXX"
 372         if ftype != "dsc":
 373             prompt = "[B]oth, Priority, Section, Done ? "
 374         else:
 375             prompt = "[S]ection, Done ? "
 376         edit_priority = edit_section = 0
 377
 378         while prompt.find(answer) == -1:
 379             answer = utils.our_raw_input(prompt)
 380             m = re_default_answer.match(prompt)
 381             if answer == "":
 382                 answer = m.group(1)
 383             answer = answer[:1].upper()
 384
 385         if answer == 'P':
 386             edit_priority = 1
 387         elif answer == 'S':
 388             edit_section = 1
 389         elif answer == 'B':
 390             edit_priority = edit_section = 1
 391         elif answer == 'D':
 392             done = 1
 393
 394         # Edit the priority
 395         if edit_priority:
 396             readline.set_completer(Priorities.complete)
 397             got_priority = 0
 398             while not got_priority:
 399                 new_priority = utils.our_raw_input("New priority: ").strip()
 400                 if new_priority not in Priorities.priorities:
 401                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 402                 else:
 403                     got_priority = 1
 404                     priority = new_priority
 405
 406         # Edit the section
 407         if edit_section:
 408             readline.set_completer(Sections.complete)
 409             got_section = 0
 410             while not got_section:
 411                 new_section = utils.our_raw_input("New section: ").strip()
 412                 if new_section not in Sections.sections:
 413                     print "E: '%s' is not a valid section, try again." % (new_section)
 414                 else:
 415                     got_section = 1
 416                     section = new_section
 417
 418         # Reset the readline completer
 419         readline.set_completer(None)
 420
 421     for f in new[index]["files"]:
 422         Upload.pkg.files[f]["section"] = section
 423         Upload.pkg.files[f]["priority"] = priority
 424     new[index]["priority"] = priority
 425     new[index]["section"] = section
 426     return new
 427
 428 ################################################################################
 429
 430 def edit_overrides (new):
 431     print
 432     done = 0
 433     while not done:
 434         print_new (new, 1)
 435         new_index = {}
 436         index = 0
 437         for i in new.keys():
 438             index += 1
 439             new_index[index] = i
 440
 441         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 442
 443         got_answer = 0
 444         while not got_answer:
 445             answer = utils.our_raw_input(prompt)
 446             if not answer.isdigit():
 447                 answer = answer[:1].upper()
 448             if answer == "E" or answer == "D":
 449                 got_answer = 1
 450             elif re_isanum.match (answer):
 451                 answer = int(answer)
 452                 if (answer < 1) or (answer > index):
 453                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 454                 else:
 455                     got_answer = 1
 456
 457         if answer == 'E':
 458             edit_new(new)
 459         elif answer == 'D':
 460             done = 1
 461         else:
 462             edit_index (new, new_index[answer])
 463
 464     return new
 465
 466 ################################################################################
 467
 468 def edit_note(note):
 469     # Write the current data to a temporary file
 470     (fd, temp_filename) = utils.temp_filename()
 471     temp_file = os.fdopen(fd, 'w')
 472     temp_file.write(note)
 473     temp_file.close()
 474     editor = os.environ.get("EDITOR","vi")
 475     answer = 'E'
 476     while answer == 'E':
 477         os.system("%s %s" % (editor, temp_filename))
 478         temp_file = utils.open_file(temp_filename)
 479         note = temp_file.read().rstrip()
 480         temp_file.close()
 481         print "Note:"
 482         print utils.prefix_multi_line_string(note,"  ")
 483         prompt = "[D]one, Edit, Abandon, Quit ?"
 484         answer = "XXX"
 485         while prompt.find(answer) == -1:
 486             answer = utils.our_raw_input(prompt)
 487             m = re_default_answer.search(prompt)
 488             if answer == "":
 489                 answer = m.group(1)
 490             answer = answer[:1].upper()
 491     os.unlink(temp_filename)
 492     if answer == 'A':
 493         return
 494     elif answer == 'Q':
 495         end()
 496         sys.exit(0)
 497     Upload.pkg.changes["process-new note"] = note
 498     Upload.dump_vars(Cnf["Dir::Queue::New"])
 499
 500 ################################################################################
 501
 502 def check_pkg ():
 503     try:
 504         less_fd = os.popen("less -R -", 'w', 0)
 505         stdout_fd = sys.stdout
 506         try:
 507             sys.stdout = less_fd
 508             changes = utils.parse_changes (Upload.pkg.changes_file)
 509             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 510             files = Upload.pkg.files
 511             for f in files.keys():
 512                 if files[f].has_key("new"):
 513                     ftype = files[f]["type"]
 514                     if ftype == "deb":
 515                         examine_package.check_deb(changes['distribution'], f)
 516                     elif ftype == "dsc":
 517                         examine_package.check_dsc(changes['distribution'], f)
 518         finally:
 519             examine_package.output_package_relations()
 520             sys.stdout = stdout_fd
 521     except IOError, e:
 522         if e.errno == errno.EPIPE:
 523             utils.warn("[examine_package] Caught EPIPE; skipping.")
 524             pass
 525         else:
 526             raise
 527     except KeyboardInterrupt:
 528         utils.warn("[examine_package] Caught C-c; skipping.")
 529         pass
 530
 531 ################################################################################
 532
 533 ## FIXME: horribly Debian specific
 534
 535 def do_bxa_notification():
 536     files = Upload.pkg.files
 537     summary = ""
 538     for f in files.keys():
 539         if files[f]["type"] == "deb":
 540             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 541             summary += "\n"
 542             summary += "Package: %s\n" % (control.Find("Package"))
 543             summary += "Description: %s\n" % (control.Find("Description"))
 544     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 545     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 546     utils.send_mail(bxa_mail)
 547
 548 ################################################################################
 549
 550 def add_overrides (new):
 551     changes = Upload.pkg.changes
 552     files = Upload.pkg.files
 553
 554     projectB.query("BEGIN WORK")
 555     for suite in changes["suite"].keys():
 556         suite_id = database.get_suite_id(suite)
 557         for pkg in new.keys():
 558             component_id = database.get_component_id(new[pkg]["component"])
 559             type_id = database.get_override_type_id(new[pkg]["type"])
 560             priority_id = new[pkg]["priority id"]
 561             section_id = new[pkg]["section id"]
 562             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 563             for f in new[pkg]["files"]:
 564                 if files[f].has_key("new"):
 565                     del files[f]["new"]
 566             del new[pkg]
 567
 568     projectB.query("COMMIT WORK")
 569
 570     if Cnf.FindB("Dinstall::BXANotify"):
 571         do_bxa_notification()
 572
 573 ################################################################################
 574
 575 def prod_maintainer ():
 576     # Here we prepare an editor and get them ready to prod...
 577     (fd, temp_filename) = utils.temp_filename()
 578     editor = os.environ.get("EDITOR","vi")
 579     answer = 'E'
 580     while answer == 'E':
 581         os.system("%s %s" % (editor, temp_filename))
 582         f = os.fdopen(fd)
 583         prod_message = "".join(f.readlines())
 584         f.close()
 585         print "Prod message:"
 586         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 587         prompt = "[P]rod, Edit, Abandon, Quit ?"
 588         answer = "XXX"
 589         while prompt.find(answer) == -1:
 590             answer = utils.our_raw_input(prompt)
 591             m = re_default_answer.search(prompt)
 592             if answer == "":
 593                 answer = m.group(1)
 594             answer = answer[:1].upper()
 595         os.unlink(temp_filename)
 596         if answer == 'A':
 597             return
 598         elif answer == 'Q':
 599             end()
 600             sys.exit(0)
 601     # Otherwise, do the proding...
 602     user_email_address = utils.whoami() + " <%s>" % (
 603         Cnf["Dinstall::MyAdminAddress"])
 604
 605     Subst = Upload.Subst
 606
 607     Subst["__FROM_ADDRESS__"] = user_email_address
 608     Subst["__PROD_MESSAGE__"] = prod_message
 609     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 610
 611     prod_mail_message = utils.TemplateSubst(
 612         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 613
 614     # Send the prod mail if appropriate
 615     if not Cnf["Dinstall::Options::No-Mail"]:
 616         utils.send_mail(prod_mail_message)
 617
 618     print "Sent proding message"
 619
 620 ################################################################################
 621
 622 def do_new():
 623     print "NEW\n"
 624     files = Upload.pkg.files
 625     changes = Upload.pkg.changes
 626
 627     # Make a copy of distribution we can happily trample on
 628     changes["suite"] = copy.copy(changes["distribution"])
 629
 630     # Fix up the list of target suites
 631     for suite in changes["suite"].keys():
 632         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 633         if override:
 634             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 635               database.get_suite_id(override) == -1)
 636             if olderr or newerr:
 637                 (oinv, newinv) = ("", "")
 638                 if olderr: oinv = "invalid "
 639                 if newerr: ninv = "invalid "
 640                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 641                         oinv, suite, ninv, override)
 642             del changes["suite"][suite]
 643             changes["suite"][override] = 1
 644     # Validate suites
 645     for suite in changes["suite"].keys():
 646         suite_id = database.get_suite_id(suite)
 647         if suite_id == -1:
 648             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 649
 650     # The main NEW processing loop
 651     done = 0
 652     while not done:
 653         # Find out what's new
 654         new = queue.determine_new(changes, files, projectB)
 655
 656         if not new:
 657             break
 658
 659         answer = "XXX"
 660         if Options["No-Action"] or Options["Automatic"]:
 661             answer = 'S'
 662
 663         (broken, note) = print_new(new, 0)
 664         prompt = ""
 665
 666         if not broken and not note:
 667             prompt = "Add overrides, "
 668         if broken:
 669             print "W: [!] marked entries must be fixed before package can be processed."
 670         if note:
 671             print "W: note must be removed before package can be processed."
 672             prompt += "Remove note, "
 673
 674         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 675
 676         while prompt.find(answer) == -1:
 677             answer = utils.our_raw_input(prompt)
 678             m = re_default_answer.search(prompt)
 679             if answer == "":
 680                 answer = m.group(1)
 681             answer = answer[:1].upper()
 682
 683         if answer == 'A':
 684             done = add_overrides (new)
 685         elif answer == 'C':
 686             check_pkg()
 687         elif answer == 'E':
 688             new = edit_overrides (new)
 689         elif answer == 'M':
 690             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 691             if not aborted:
 692                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 693                 done = 1
 694         elif answer == 'N':
 695             edit_note(changes.get("process-new note", ""))
 696         elif answer == 'P':
 697             prod_maintainer()
 698         elif answer == 'R':
 699             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 700             if confirm == "y":
 701                 del changes["process-new note"]
 702         elif answer == 'S':
 703             done = 1
 704         elif answer == 'Q':
 705             end()
 706             sys.exit(0)
 707
 708 ################################################################################
 709 ################################################################################
 710 ################################################################################
 711
 712 def usage (exit_code=0):
 713     print """Usage: dak process-new [OPTION]... [CHANGES]...
 714   -a, --automatic           automatic run
 715   -h, --help                show this help and exit.
 716   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 717   -m, --manual-reject=MSG   manual reject with `msg'
 718   -n, --no-action           don't do anything
 719   -V, --version             display the version number and exit"""
 720     sys.exit(exit_code)
 721
 722 ################################################################################
 723
 724 def init():
 725     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 726
 727     Cnf = utils.get_conf()
 728
 729     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 730                  ('h',"help","Process-New::Options::Help"),
 731                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 732                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 733                  ('n',"no-action","Process-New::Options::No-Action")]
 734
 735     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 736         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 737             Cnf["Process-New::Options::%s" % (i)] = ""
 738
 739     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 740     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 741         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 742
 743     Options = Cnf.SubTree("Process-New::Options")
 744
 745     if Options["Help"]:
 746         usage()
 747
 748     Upload = queue.Upload(Cnf)
 749
 750     if not Options["No-Action"]:
 751         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 752
 753     projectB = Upload.projectB
 754
 755     Sections = Section_Completer()
 756     Priorities = Priority_Completer()
 757     readline.parse_and_bind("tab: complete")
 758
 759     return changes_files
 760
 761 ################################################################################
 762
 763 def do_byhand():
 764     done = 0
 765     while not done:
 766         files = Upload.pkg.files
 767         will_install = 1
 768         byhand = []
 769
 770         for f in files.keys():
 771             if files[f]["type"] == "byhand":
 772                 if os.path.exists(f):
 773                     print "W: %s still present; please process byhand components and try again." % (f)
 774                     will_install = 0
 775                 else:
 776                     byhand.append(f)
 777
 778         answer = "XXXX"
 779         if Options["No-Action"]:
 780             answer = "S"
 781         if will_install:
 782             if Options["Automatic"] and not Options["No-Action"]:
 783                 answer = 'A'
 784             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 785         else:
 786             prompt = "Manual reject, [S]kip, Quit ?"
 787
 788         while prompt.find(answer) == -1:
 789             answer = utils.our_raw_input(prompt)
 790             m = re_default_answer.search(prompt)
 791             if answer == "":
 792                 answer = m.group(1)
 793             answer = answer[:1].upper()
 794
 795         if answer == 'A':
 796             done = 1
 797             for f in byhand:
 798                 del files[f]
 799         elif answer == 'M':
 800             Upload.do_reject(1, Options["Manual-Reject"])
 801             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 802             done = 1
 803         elif answer == 'S':
 804             done = 1
 805         elif answer == 'Q':
 806             end()
 807             sys.exit(0)
 808
 809 ################################################################################
 810
 811 def get_accept_lock():
 812     retry = 0
 813     while retry < 10:
 814         try:
 815             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 816             retry = 10
 817         except OSError, e:
 818             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 819                 retry += 1
 820                 if (retry >= 10):
 821                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 822                 else:
 823                     print("Unable to get accepted lock (try %d of 10)" % retry)
 824                 time.sleep(60)
 825             else:
 826                 raise
 827
 828 def move_to_dir (dest, perms=0660, changesperms=0664):
 829     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 830     file_keys = Upload.pkg.files.keys()
 831     for f in file_keys:
 832         utils.move (f, dest, perms=perms)
 833
 834 def is_source_in_queue_dir(qdir):
 835     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 836                 and x.endswith(".changes") ]
 837     for entry in entries:
 838         # read the .dak
 839         u = queue.Upload(Cnf)
 840         u.pkg.changes_file = os.path.join(qdir, entry)
 841         u.update_vars()
 842         if not u.pkg.changes["architecture"].has_key("source"):
 843             # another binary upload, ignore
 844             continue
 845         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 846             # another version, ignore
 847             continue
 848         # found it!
 849         return True
 850     return False
 851
 852 def move_to_holding(suite, queue_dir):
 853     print "Moving to %s holding area." % (suite.upper(),)
 854     if Options["No-Action"]:
 855         return
 856     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 857     Upload.dump_vars(queue_dir)
 858     move_to_dir(queue_dir, perms=0664)
 859     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 860
 861 def _accept():
 862     if Options["No-Action"]:
 863         return
 864     (summary, short_summary) = Upload.build_summaries()
 865     Upload.accept(summary, short_summary)
 866     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 867
 868 def do_accept_stableupdate(suite, q):
 869     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 870     if not Upload.pkg.changes["architecture"].has_key("source"):
 871         # It is not a sourceful upload.  So its source may be either in p-u
 872         # holding, in new, in accepted or already installed.
 873         if is_source_in_queue_dir(queue_dir):
 874             # It's in p-u holding, so move it there.
 875             print "Binary-only upload, source in %s." % (q,)
 876             move_to_holding(suite, queue_dir)
 877         elif Upload.source_exists(Upload.pkg.changes["source"],
 878                 Upload.pkg.changes["version"]):
 879             # dak tells us that there is source available.  At time of
 880             # writing this means that it is installed, so put it into
 881             # accepted.
 882             print "Binary-only upload, source installed."
 883             _accept()
 884         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 885             # The source is in accepted, the binary cleared NEW: accept it.
 886             print "Binary-only upload, source in accepted."
 887             _accept()
 888         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 889             # It's in NEW.  We expect the source to land in p-u holding
 890             # pretty soon.
 891             print "Binary-only upload, source in new."
 892             move_to_holding(suite, queue_dir)
 893         else:
 894             # No case applicable.  Bail out.  Return will cause the upload
 895             # to be skipped.
 896             print "ERROR"
 897             print "Stable update failed.  Source not found."
 898             return
 899     else:
 900         # We are handling a sourceful upload.  Move to accepted if currently
 901         # in p-u holding and to p-u holding otherwise.
 902         if is_source_in_queue_dir(queue_dir):
 903             print "Sourceful upload in %s, accepting." % (q,)
 904             _accept()
 905         else:
 906             move_to_holding(suite, queue_dir)
 907
 908 def do_accept():
 909     print "ACCEPT"
 910     if not Options["No-Action"]:
 911         get_accept_lock()
 912         (summary, short_summary) = Upload.build_summaries()
 913     try:
 914         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 915             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 916             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 917             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 918             # Check for override disparities
 919             Upload.Subst["__SUMMARY__"] = summary
 920         else:
 921             # Stable updates need to be copied to proposed-updates holding
 922             # area instead of accepted.  Sourceful uploads need to go
 923             # to it directly, binaries only if the source has not yet been
 924             # accepted into p-u.
 925             for suite, q in [("proposed-updates", "ProposedUpdates"),
 926                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 927                 if not Upload.pkg.changes["distribution"].has_key(suite):
 928                     continue
 929                 return do_accept_stableupdate(suite, q)
 930             # Just a normal upload, accept it...
 931             _accept()
 932     finally:
 933         if not Options["No-Action"]:
 934             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 935
 936 def check_status(files):
 937     new = byhand = 0
 938     for f in files.keys():
 939         if files[f]["type"] == "byhand":
 940             byhand = 1
 941         elif files[f].has_key("new"):
 942             new = 1
 943     return (new, byhand)
 944
 945 def do_pkg(changes_file):
 946     Upload.pkg.changes_file = changes_file
 947     Upload.init_vars()
 948     Upload.update_vars()
 949     Upload.update_subst()
 950     files = Upload.pkg.files
 951
 952     if not recheck():
 953         return
 954
 955     (new, byhand) = check_status(files)
 956     if new or byhand:
 957         if new:
 958             do_new()
 959         if byhand:
 960             do_byhand()
 961         (new, byhand) = check_status(files)
 962
 963     if not new and not byhand:
 964         do_accept()
 965
 966 ################################################################################
 967
 968 def end():
 969     accept_count = Upload.accept_count
 970     accept_bytes = Upload.accept_bytes
 971
 972     if accept_count:
 973         sets = "set"
 974         if accept_count > 1:
 975             sets = "sets"
 976         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 977         Logger.log(["total",accept_count,accept_bytes])
 978
 979     if not Options["No-Action"]:
 980         Logger.close()
 981
 982 ################################################################################
 983
 984 def do_comments(dir, opref, npref, line, fn):
 985     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 986         lines = open("%s/%s" % (dir, comm)).readlines()
 987         if len(lines) == 0 or lines[0] != line + "\n": continue
 988         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 989                                 and x.endswith(".changes") ]
 990         changes_files = sort_changes(changes_files)
 991         for f in changes_files:
 992             f = utils.validate_changes_file_arg(f, 0)
 993             if not f: continue
 994             print "\n" + f
 995             fn(f, "".join(lines[1:]))
 996
 997         if opref != npref and not Options["No-Action"]:
 998             newcomm = npref + comm[len(opref):]
 999             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1000
1001 ################################################################################
1002
1003 def comment_accept(changes_file, comments):
1004     Upload.pkg.changes_file = changes_file
1005     Upload.init_vars()
1006     Upload.update_vars()
1007     Upload.update_subst()
1008     files = Upload.pkg.files
1009
1010     if not recheck():
1011         return # dak wants to REJECT, crap
1012
1013     (new, byhand) = check_status(files)
1014     if not new and not byhand:
1015         do_accept()
1016
1017 ################################################################################
1018
1019 def comment_reject(changes_file, comments):
1020     Upload.pkg.changes_file = changes_file
1021     Upload.init_vars()
1022     Upload.update_vars()
1023     Upload.update_subst()
1024
1025     if not recheck():
1026         pass # dak has its own reasons to reject as well, which is fine
1027
1028     reject(comments)
1029     print "REJECT\n" + reject_message,
1030     if not Options["No-Action"]:
1031         Upload.do_reject(0, reject_message)
1032         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1033
1034 ################################################################################
1035
1036 def main():
1037     changes_files = init()
1038     if len(changes_files) > 50:
1039         sys.stderr.write("Sorting changes...\n")
1040     changes_files = sort_changes(changes_files)
1041
1042     # Kill me now? **FIXME**
1043     Cnf["Dinstall::Options::No-Mail"] = ""
1044     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1045     if Cnf.has_key("Dinstall::Bcc"):
1046         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1047     else:
1048         Upload.Subst["__BCC__"] = bcc
1049
1050     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1051     if commentsdir:
1052         if changes_files != []:
1053             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1054             sys.exit(1)
1055         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1056         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1057     else:
1058         for changes_file in changes_files:
1059             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1060             if not changes_file:
1061                 continue
1062             print "\n" + changes_file
1063             do_pkg (changes_file)
1064
1065     end()
1066
1067 ################################################################################
1068
1069 if __name__ == '__main__':
1070     main()