dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @license: GNU General Public License version 2 or later
   9 """
  10 # This program is free software; you can redistribute it and/or modify
  11 # it under the terms of the GNU General Public License as published by
  12 # the Free Software Foundation; either version 2 of the License, or
  13 # (at your option) any later version.
  14
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU General Public License for more details.
  19
  20 # You should have received a copy of the GNU General Public License
  21 # along with this program; if not, write to the Free Software
  22 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  23
  24 ################################################################################
  25
  26 # 23:12|<aj> I will not hush!
  27 # 23:12|<elmo> :>
  28 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  29 # 23:13|<aj> I shall not be silenced!
  30 # 23:13|<aj> The world shall know!
  31 # 23:13|<aj> The world *must* know!
  32 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  33 # 23:13|<aj> yay powerpuff girls!!
  34 # 23:13|<aj> buttercup's my favourite, who's yours?
  35 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  36 # 23:14|<aj> *AREN'T YOU*?!
  37 # 23:15|<aj> I will not be treated like this.
  38 # 23:15|<aj> I shall have my revenge.
  39 # 23:15|<aj> I SHALL!!!
  40
  41 ################################################################################
  42
  43 import copy
  44 import errno
  45 import os
  46 import readline
  47 import stat
  48 import sys
  49 import time
  50 import apt_pkg, apt_inst
  51 import examine_package
  52 from daklib import database
  53 from daklib import logging
  54 from daklib import queue
  55 from daklib import utils
  56 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  57
  58 # Globals
  59 Cnf = None       #: Configuration, apt_pkg.Configuration
  60 Options = None
  61 Upload = None
  62 projectB = None  #: database connection, pgobject
  63 Logger = None
  64
  65 Priorities = None
  66 Sections = None
  67
  68 reject_message = ""
  69
  70 ################################################################################
  71 ################################################################################
  72 ################################################################################
  73
  74 def reject (str, prefix="Rejected: "):
  75     global reject_message
  76     if str:
  77         reject_message += prefix + str + "\n"
  78
  79 def recheck():
  80     global reject_message
  81     files = Upload.pkg.files
  82     reject_message = ""
  83
  84     for f in files.keys():
  85         # The .orig.tar.gz can disappear out from under us is it's a
  86         # duplicate of one in the archive.
  87         if not files.has_key(f):
  88             continue
  89         # Check that the source still exists
  90         if files[f]["type"] == "deb":
  91             source_version = files[f]["source version"]
  92             source_package = files[f]["source package"]
  93             if not Upload.pkg.changes["architecture"].has_key("source") \
  94                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  95                 source_epochless_version = re_no_epoch.sub('', source_version)
  96                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  97                 found = 0
  98                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
  99                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 100                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 101                             found = 1
 102                 if not found:
 103                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 104
 105         # Version and file overwrite checks
 106         if files[f]["type"] == "deb":
 107             reject(Upload.check_binary_against_db(f), "")
 108         elif files[f]["type"] == "dsc":
 109             reject(Upload.check_source_against_db(f), "")
 110             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 111             reject(reject_msg, "")
 112
 113     if reject_message.find("Rejected") != -1:
 114         answer = "XXX"
 115         if Options["No-Action"] or Options["Automatic"]:
 116             answer = 'S'
 117
 118         print "REJECT\n" + reject_message,
 119         prompt = "[R]eject, Skip, Quit ?"
 120
 121         while prompt.find(answer) == -1:
 122             answer = utils.our_raw_input(prompt)
 123             m = re_default_answer.match(prompt)
 124             if answer == "":
 125                 answer = m.group(1)
 126             answer = answer[:1].upper()
 127
 128         if answer == 'R':
 129             Upload.do_reject(0, reject_message)
 130             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 131             return 0
 132         elif answer == 'S':
 133             return 0
 134         elif answer == 'Q':
 135             end()
 136             sys.exit(0)
 137
 138     return 1
 139
 140 ################################################################################
 141
 142 def indiv_sg_compare (a, b):
 143     """Sort by source name, source, version, 'have source', and
 144        finally by filename."""
 145     # Sort by source version
 146     q = apt_pkg.VersionCompare(a["version"], b["version"])
 147     if q:
 148         return -q
 149
 150     # Sort by 'have source'
 151     a_has_source = a["architecture"].get("source")
 152     b_has_source = b["architecture"].get("source")
 153     if a_has_source and not b_has_source:
 154         return -1
 155     elif b_has_source and not a_has_source:
 156         return 1
 157
 158     return cmp(a["filename"], b["filename"])
 159
 160 ############################################################
 161
 162 def sg_compare (a, b):
 163     a = a[1]
 164     b = b[1]
 165     """Sort by have note, source already in database and time of oldest upload."""
 166     # Sort by have note
 167     a_note_state = a["note_state"]
 168     b_note_state = b["note_state"]
 169     if a_note_state < b_note_state:
 170         return -1
 171     elif a_note_state > b_note_state:
 172         return 1
 173     # Sort by source already in database (descending)
 174     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 175     if source_in_database:
 176         return -source_in_database
 177
 178     # Sort by time of oldest upload
 179     return cmp(a["oldest"], b["oldest"])
 180
 181 def sort_changes(changes_files):
 182     """Sort into source groups, then sort each source group by version,
 183     have source, filename.  Finally, sort the source groups by have
 184     note, time of oldest upload of each source upload."""
 185     if len(changes_files) == 1:
 186         return changes_files
 187
 188     sorted_list = []
 189     cache = {}
 190     # Read in all the .changes files
 191     for filename in changes_files:
 192         try:
 193             Upload.pkg.changes_file = filename
 194             Upload.init_vars()
 195             Upload.update_vars()
 196             cache[filename] = copy.copy(Upload.pkg.changes)
 197             cache[filename]["filename"] = filename
 198         except:
 199             sorted_list.append(filename)
 200             break
 201     # Divide the .changes into per-source groups
 202     per_source = {}
 203     for filename in cache.keys():
 204         source = cache[filename]["source"]
 205         if not per_source.has_key(source):
 206             per_source[source] = {}
 207             per_source[source]["list"] = []
 208         per_source[source]["list"].append(cache[filename])
 209     # Determine oldest time and have note status for each source group
 210     for source in per_source.keys():
 211         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 212         ql = q.getresult()
 213         per_source[source]["source_in_database"] = len(ql)>0
 214         source_list = per_source[source]["list"]
 215         first = source_list[0]
 216         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 217         have_note = 0
 218         for d in per_source[source]["list"]:
 219             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 220             if mtime < oldest:
 221                 oldest = mtime
 222             have_note += (d.has_key("process-new note"))
 223         per_source[source]["oldest"] = oldest
 224         if not have_note:
 225             per_source[source]["note_state"] = 0; # none
 226         elif have_note < len(source_list):
 227             per_source[source]["note_state"] = 1; # some
 228         else:
 229             per_source[source]["note_state"] = 2; # all
 230         per_source[source]["list"].sort(indiv_sg_compare)
 231     per_source_items = per_source.items()
 232     per_source_items.sort(sg_compare)
 233     for i in per_source_items:
 234         for j in i[1]["list"]:
 235             sorted_list.append(j["filename"])
 236     return sorted_list
 237
 238 ################################################################################
 239
 240 class Section_Completer:
 241     def __init__ (self):
 242         self.sections = []
 243         self.matches = []
 244         q = projectB.query("SELECT section FROM section")
 245         for i in q.getresult():
 246             self.sections.append(i[0])
 247
 248     def complete(self, text, state):
 249         if state == 0:
 250             self.matches = []
 251             n = len(text)
 252             for word in self.sections:
 253                 if word[:n] == text:
 254                     self.matches.append(word)
 255         try:
 256             return self.matches[state]
 257         except IndexError:
 258             return None
 259
 260 ############################################################
 261
 262 class Priority_Completer:
 263     def __init__ (self):
 264         self.priorities = []
 265         self.matches = []
 266         q = projectB.query("SELECT priority FROM priority")
 267         for i in q.getresult():
 268             self.priorities.append(i[0])
 269
 270     def complete(self, text, state):
 271         if state == 0:
 272             self.matches = []
 273             n = len(text)
 274             for word in self.priorities:
 275                 if word[:n] == text:
 276                     self.matches.append(word)
 277         try:
 278             return self.matches[state]
 279         except IndexError:
 280             return None
 281
 282 ################################################################################
 283
 284 def print_new (new, indexed, file=sys.stdout):
 285     queue.check_valid(new)
 286     broken = 0
 287     index = 0
 288     for pkg in new.keys():
 289         index += 1
 290         section = new[pkg]["section"]
 291         priority = new[pkg]["priority"]
 292         if new[pkg]["section id"] == -1:
 293             section += "[!]"
 294             broken = 1
 295         if new[pkg]["priority id"] == -1:
 296             priority += "[!]"
 297             broken = 1
 298         if indexed:
 299             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 300         else:
 301             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 302         line = line.strip()+'\n'
 303         file.write(line)
 304     note = Upload.pkg.changes.get("process-new note")
 305     if note:
 306         print "*"*75
 307         print note
 308         print "*"*75
 309     return broken, note
 310
 311 ################################################################################
 312
 313 def index_range (index):
 314     if index == 1:
 315         return "1"
 316     else:
 317         return "1-%s" % (index)
 318
 319 ################################################################################
 320 ################################################################################
 321
 322 def edit_new (new):
 323     # Write the current data to a temporary file
 324     (fd, temp_filename) = utils.temp_filename()
 325     temp_file = os.fdopen(fd, 'w')
 326     print_new (new, 0, temp_file)
 327     temp_file.close()
 328     # Spawn an editor on that file
 329     editor = os.environ.get("EDITOR","vi")
 330     result = os.system("%s %s" % (editor, temp_filename))
 331     if result != 0:
 332         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 333     # Read the edited data back in
 334     temp_file = utils.open_file(temp_filename)
 335     lines = temp_file.readlines()
 336     temp_file.close()
 337     os.unlink(temp_filename)
 338     # Parse the new data
 339     for line in lines:
 340         line = line.strip()
 341         if line == "":
 342             continue
 343         s = line.split()
 344         # Pad the list if necessary
 345         s[len(s):3] = [None] * (3-len(s))
 346         (pkg, priority, section) = s[:3]
 347         if not new.has_key(pkg):
 348             utils.warn("Ignoring unknown package '%s'" % (pkg))
 349         else:
 350             # Strip off any invalid markers, print_new will readd them.
 351             if section.endswith("[!]"):
 352                 section = section[:-3]
 353             if priority.endswith("[!]"):
 354                 priority = priority[:-3]
 355             for f in new[pkg]["files"]:
 356                 Upload.pkg.files[f]["section"] = section
 357                 Upload.pkg.files[f]["priority"] = priority
 358             new[pkg]["section"] = section
 359             new[pkg]["priority"] = priority
 360
 361 ################################################################################
 362
 363 def edit_index (new, index):
 364     priority = new[index]["priority"]
 365     section = new[index]["section"]
 366     ftype = new[index]["type"]
 367     done = 0
 368     while not done:
 369         print "\t".join([index, priority, section])
 370
 371         answer = "XXX"
 372         if ftype != "dsc":
 373             prompt = "[B]oth, Priority, Section, Done ? "
 374         else:
 375             prompt = "[S]ection, Done ? "
 376         edit_priority = edit_section = 0
 377
 378         while prompt.find(answer) == -1:
 379             answer = utils.our_raw_input(prompt)
 380             m = re_default_answer.match(prompt)
 381             if answer == "":
 382                 answer = m.group(1)
 383             answer = answer[:1].upper()
 384
 385         if answer == 'P':
 386             edit_priority = 1
 387         elif answer == 'S':
 388             edit_section = 1
 389         elif answer == 'B':
 390             edit_priority = edit_section = 1
 391         elif answer == 'D':
 392             done = 1
 393
 394         # Edit the priority
 395         if edit_priority:
 396             readline.set_completer(Priorities.complete)
 397             got_priority = 0
 398             while not got_priority:
 399                 new_priority = utils.our_raw_input("New priority: ").strip()
 400                 if new_priority not in Priorities.priorities:
 401                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 402                 else:
 403                     got_priority = 1
 404                     priority = new_priority
 405
 406         # Edit the section
 407         if edit_section:
 408             readline.set_completer(Sections.complete)
 409             got_section = 0
 410             while not got_section:
 411                 new_section = utils.our_raw_input("New section: ").strip()
 412                 if new_section not in Sections.sections:
 413                     print "E: '%s' is not a valid section, try again." % (new_section)
 414                 else:
 415                     got_section = 1
 416                     section = new_section
 417
 418         # Reset the readline completer
 419         readline.set_completer(None)
 420
 421     for f in new[index]["files"]:
 422         Upload.pkg.files[f]["section"] = section
 423         Upload.pkg.files[f]["priority"] = priority
 424     new[index]["priority"] = priority
 425     new[index]["section"] = section
 426     return new
 427
 428 ################################################################################
 429
 430 def edit_overrides (new):
 431     print
 432     done = 0
 433     while not done:
 434         print_new (new, 1)
 435         new_index = {}
 436         index = 0
 437         for i in new.keys():
 438             index += 1
 439             new_index[index] = i
 440
 441         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 442
 443         got_answer = 0
 444         while not got_answer:
 445             answer = utils.our_raw_input(prompt)
 446             if not answer.isdigit():
 447                 answer = answer[:1].upper()
 448             if answer == "E" or answer == "D":
 449                 got_answer = 1
 450             elif re_isanum.match (answer):
 451                 answer = int(answer)
 452                 if (answer < 1) or (answer > index):
 453                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 454                 else:
 455                     got_answer = 1
 456
 457         if answer == 'E':
 458             edit_new(new)
 459         elif answer == 'D':
 460             done = 1
 461         else:
 462             edit_index (new, new_index[answer])
 463
 464     return new
 465
 466 ################################################################################
 467
 468 def edit_note(note):
 469     # Write the current data to a temporary file
 470     (fd, temp_filename) = utils.temp_filename()
 471     temp_file = os.fdopen(fd, 'w')
 472     temp_file.write(note)
 473     temp_file.close()
 474     editor = os.environ.get("EDITOR","vi")
 475     answer = 'E'
 476     while answer == 'E':
 477         os.system("%s %s" % (editor, temp_filename))
 478         temp_file = utils.open_file(temp_filename)
 479         note = temp_file.read().rstrip()
 480         temp_file.close()
 481         print "Note:"
 482         print utils.prefix_multi_line_string(note,"  ")
 483         prompt = "[D]one, Edit, Abandon, Quit ?"
 484         answer = "XXX"
 485         while prompt.find(answer) == -1:
 486             answer = utils.our_raw_input(prompt)
 487             m = re_default_answer.search(prompt)
 488             if answer == "":
 489                 answer = m.group(1)
 490             answer = answer[:1].upper()
 491     os.unlink(temp_filename)
 492     if answer == 'A':
 493         return
 494     elif answer == 'Q':
 495         end()
 496         sys.exit(0)
 497     Upload.pkg.changes["process-new note"] = note
 498     Upload.dump_vars(Cnf["Dir::Queue::New"])
 499
 500 ################################################################################
 501
 502 def check_pkg ():
 503     try:
 504         less_fd = os.popen("less -R -", 'w', 0)
 505         stdout_fd = sys.stdout
 506         try:
 507             sys.stdout = less_fd
 508             changes = utils.parse_changes (Upload.pkg.changes_file)
 509             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 510             files = Upload.pkg.files
 511             for f in files.keys():
 512                 if files[f].has_key("new"):
 513                     ftype = files[f]["type"]
 514                     if ftype == "deb":
 515                         examine_package.check_deb(changes['distribution'], f)
 516                     elif ftype == "dsc":
 517                         examine_package.check_dsc(changes['distribution'], f)
 518         finally:
 519             sys.stdout = stdout_fd
 520     except IOError, e:
 521         if e.errno == errno.EPIPE:
 522             utils.warn("[examine_package] Caught EPIPE; skipping.")
 523             pass
 524         else:
 525             raise
 526     except KeyboardInterrupt:
 527         utils.warn("[examine_package] Caught C-c; skipping.")
 528         pass
 529
 530 ################################################################################
 531
 532 ## FIXME: horribly Debian specific
 533
 534 def do_bxa_notification():
 535     files = Upload.pkg.files
 536     summary = ""
 537     for f in files.keys():
 538         if files[f]["type"] == "deb":
 539             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 540             summary += "\n"
 541             summary += "Package: %s\n" % (control.Find("Package"))
 542             summary += "Description: %s\n" % (control.Find("Description"))
 543     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 544     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 545     utils.send_mail(bxa_mail)
 546
 547 ################################################################################
 548
 549 def add_overrides (new):
 550     changes = Upload.pkg.changes
 551     files = Upload.pkg.files
 552
 553     projectB.query("BEGIN WORK")
 554     for suite in changes["suite"].keys():
 555         suite_id = database.get_suite_id(suite)
 556         for pkg in new.keys():
 557             component_id = database.get_component_id(new[pkg]["component"])
 558             type_id = database.get_override_type_id(new[pkg]["type"])
 559             priority_id = new[pkg]["priority id"]
 560             section_id = new[pkg]["section id"]
 561             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 562             for f in new[pkg]["files"]:
 563                 if files[f].has_key("new"):
 564                     del files[f]["new"]
 565             del new[pkg]
 566
 567     projectB.query("COMMIT WORK")
 568
 569     if Cnf.FindB("Dinstall::BXANotify"):
 570         do_bxa_notification()
 571
 572 ################################################################################
 573
 574 def prod_maintainer ():
 575     # Here we prepare an editor and get them ready to prod...
 576     (fd, temp_filename) = utils.temp_filename()
 577     editor = os.environ.get("EDITOR","vi")
 578     answer = 'E'
 579     while answer == 'E':
 580         os.system("%s %s" % (editor, temp_filename))
 581         f = os.fdopen(fd)
 582         prod_message = "".join(f.readlines())
 583         f.close()
 584         print "Prod message:"
 585         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 586         prompt = "[P]rod, Edit, Abandon, Quit ?"
 587         answer = "XXX"
 588         while prompt.find(answer) == -1:
 589             answer = utils.our_raw_input(prompt)
 590             m = re_default_answer.search(prompt)
 591             if answer == "":
 592                 answer = m.group(1)
 593             answer = answer[:1].upper()
 594         os.unlink(temp_filename)
 595         if answer == 'A':
 596             return
 597         elif answer == 'Q':
 598             end()
 599             sys.exit(0)
 600     # Otherwise, do the proding...
 601     user_email_address = utils.whoami() + " <%s>" % (
 602         Cnf["Dinstall::MyAdminAddress"])
 603
 604     Subst = Upload.Subst
 605
 606     Subst["__FROM_ADDRESS__"] = user_email_address
 607     Subst["__PROD_MESSAGE__"] = prod_message
 608     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 609
 610     prod_mail_message = utils.TemplateSubst(
 611         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 612
 613     # Send the prod mail if appropriate
 614     if not Cnf["Dinstall::Options::No-Mail"]:
 615         utils.send_mail(prod_mail_message)
 616
 617     print "Sent proding message"
 618
 619 ################################################################################
 620
 621 def do_new():
 622     print "NEW\n"
 623     files = Upload.pkg.files
 624     changes = Upload.pkg.changes
 625
 626     # Make a copy of distribution we can happily trample on
 627     changes["suite"] = copy.copy(changes["distribution"])
 628
 629     # Fix up the list of target suites
 630     for suite in changes["suite"].keys():
 631         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 632         if override:
 633             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 634               database.get_suite_id(override) == -1)
 635             if olderr or newerr:
 636                 (oinv, newinv) = ("", "")
 637                 if olderr: oinv = "invalid "
 638                 if newerr: ninv = "invalid "
 639                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 640                         oinv, suite, ninv, override)
 641             del changes["suite"][suite]
 642             changes["suite"][override] = 1
 643     # Validate suites
 644     for suite in changes["suite"].keys():
 645         suite_id = database.get_suite_id(suite)
 646         if suite_id == -1:
 647             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 648
 649     # The main NEW processing loop
 650     done = 0
 651     while not done:
 652         # Find out what's new
 653         new = queue.determine_new(changes, files, projectB)
 654
 655         if not new:
 656             break
 657
 658         answer = "XXX"
 659         if Options["No-Action"] or Options["Automatic"]:
 660             answer = 'S'
 661
 662         (broken, note) = print_new(new, 0)
 663         prompt = ""
 664
 665         if not broken and not note:
 666             prompt = "Add overrides, "
 667         if broken:
 668             print "W: [!] marked entries must be fixed before package can be processed."
 669         if note:
 670             print "W: note must be removed before package can be processed."
 671             prompt += "Remove note, "
 672
 673         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 674
 675         while prompt.find(answer) == -1:
 676             answer = utils.our_raw_input(prompt)
 677             m = re_default_answer.search(prompt)
 678             if answer == "":
 679                 answer = m.group(1)
 680             answer = answer[:1].upper()
 681
 682         if answer == 'A':
 683             done = add_overrides (new)
 684         elif answer == 'C':
 685             check_pkg()
 686         elif answer == 'E':
 687             new = edit_overrides (new)
 688         elif answer == 'M':
 689             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 690             if not aborted:
 691                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 692                 done = 1
 693         elif answer == 'N':
 694             edit_note(changes.get("process-new note", ""))
 695         elif answer == 'P':
 696             prod_maintainer()
 697         elif answer == 'R':
 698             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 699             if confirm == "y":
 700                 del changes["process-new note"]
 701         elif answer == 'S':
 702             done = 1
 703         elif answer == 'Q':
 704             end()
 705             sys.exit(0)
 706
 707 ################################################################################
 708 ################################################################################
 709 ################################################################################
 710
 711 def usage (exit_code=0):
 712     print """Usage: dak process-new [OPTION]... [CHANGES]...
 713   -a, --automatic           automatic run
 714   -h, --help                show this help and exit.
 715   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 716   -m, --manual-reject=MSG   manual reject with `msg'
 717   -n, --no-action           don't do anything
 718   -V, --version             display the version number and exit"""
 719     sys.exit(exit_code)
 720
 721 ################################################################################
 722
 723 def init():
 724     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 725
 726     Cnf = utils.get_conf()
 727
 728     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 729                  ('h',"help","Process-New::Options::Help"),
 730                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 731                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 732                  ('n',"no-action","Process-New::Options::No-Action")]
 733
 734     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 735         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 736             Cnf["Process-New::Options::%s" % (i)] = ""
 737
 738     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 739     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 740         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 741
 742     Options = Cnf.SubTree("Process-New::Options")
 743
 744     if Options["Help"]:
 745         usage()
 746
 747     Upload = queue.Upload(Cnf)
 748
 749     if not Options["No-Action"]:
 750         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 751
 752     projectB = Upload.projectB
 753
 754     Sections = Section_Completer()
 755     Priorities = Priority_Completer()
 756     readline.parse_and_bind("tab: complete")
 757
 758     return changes_files
 759
 760 ################################################################################
 761
 762 def do_byhand():
 763     done = 0
 764     while not done:
 765         files = Upload.pkg.files
 766         will_install = 1
 767         byhand = []
 768
 769         for f in files.keys():
 770             if files[f]["type"] == "byhand":
 771                 if os.path.exists(f):
 772                     print "W: %s still present; please process byhand components and try again." % (f)
 773                     will_install = 0
 774                 else:
 775                     byhand.append(f)
 776
 777         answer = "XXXX"
 778         if Options["No-Action"]:
 779             answer = "S"
 780         if will_install:
 781             if Options["Automatic"] and not Options["No-Action"]:
 782                 answer = 'A'
 783             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 784         else:
 785             prompt = "Manual reject, [S]kip, Quit ?"
 786
 787         while prompt.find(answer) == -1:
 788             answer = utils.our_raw_input(prompt)
 789             m = re_default_answer.search(prompt)
 790             if answer == "":
 791                 answer = m.group(1)
 792             answer = answer[:1].upper()
 793
 794         if answer == 'A':
 795             done = 1
 796             for f in byhand:
 797                 del files[f]
 798         elif answer == 'M':
 799             Upload.do_reject(1, Options["Manual-Reject"])
 800             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 801             done = 1
 802         elif answer == 'S':
 803             done = 1
 804         elif answer == 'Q':
 805             end()
 806             sys.exit(0)
 807
 808 ################################################################################
 809
 810 def get_accept_lock():
 811     retry = 0
 812     while retry < 10:
 813         try:
 814             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 815             retry = 10
 816         except OSError, e:
 817             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 818                 retry += 1
 819                 if (retry >= 10):
 820                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 821                 else:
 822                     print("Unable to get accepted lock (try %d of 10)" % retry)
 823                 time.sleep(60)
 824             else:
 825                 raise
 826
 827 def move_to_dir (dest, perms=0660, changesperms=0664):
 828     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 829     file_keys = Upload.pkg.files.keys()
 830     for f in file_keys:
 831         utils.move (f, dest, perms=perms)
 832
 833 def is_source_in_queue_dir(qdir):
 834     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 835                 and x.endswith(".changes") ]
 836     for entry in entries:
 837         # read the .dak
 838         u = queue.Upload(Cnf)
 839         u.pkg.changes_file = os.path.join(qdir, entry)
 840         u.update_vars()
 841         if not u.pkg.changes["architecture"].has_key("source"):
 842             # another binary upload, ignore
 843             continue
 844         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 845             # another version, ignore
 846             continue
 847         # found it!
 848         return True
 849     return False
 850
 851 def move_to_holding(suite, queue_dir):
 852     print "Moving to %s holding area." % (suite.upper(),)
 853     if Options["No-Action"]:
 854         return
 855     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 856     Upload.dump_vars(queue_dir)
 857     move_to_dir(queue_dir)
 858     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 859
 860 def _accept():
 861     if Options["No-Action"]:
 862         return
 863     (summary, short_summary) = Upload.build_summaries()
 864     Upload.accept(summary, short_summary)
 865     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 866
 867 def do_accept_stableupdate(suite, q):
 868     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 869     if not Upload.pkg.changes["architecture"].has_key("source"):
 870         # It is not a sourceful upload.  So its source may be either in p-u
 871         # holding, in new, in accepted or already installed.
 872         if is_source_in_queue_dir(queue_dir):
 873             # It's in p-u holding, so move it there.
 874             print "Binary-only upload, source in %s." % (q,)
 875             move_to_holding(suite, queue_dir)
 876         elif Upload.source_exists(Upload.pkg.changes["source"],
 877                 Upload.pkg.changes["version"]):
 878             # dak tells us that there is source available.  At time of
 879             # writing this means that it is installed, so put it into
 880             # accepted.
 881             print "Binary-only upload, source installed."
 882             _accept()
 883         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 884             # The source is in accepted, the binary cleared NEW: accept it.
 885             print "Binary-only upload, source in accepted."
 886             _accept()
 887         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 888             # It's in NEW.  We expect the source to land in p-u holding
 889             # pretty soon.
 890             print "Binary-only upload, source in new."
 891             move_to_holding(suite, queue_dir)
 892         else:
 893             # No case applicable.  Bail out.  Return will cause the upload
 894             # to be skipped.
 895             print "ERROR"
 896             print "Stable update failed.  Source not found."
 897             return
 898     else:
 899         # We are handling a sourceful upload.  Move to accepted if currently
 900         # in p-u holding and to p-u holding otherwise.
 901         if is_source_in_queue_dir(queue_dir):
 902             print "Sourceful upload in %s, accepting." % (q,)
 903             _accept()
 904         else:
 905             move_to_holding(suite, queue_dir)
 906
 907 def do_accept():
 908     print "ACCEPT"
 909     if not Options["No-Action"]:
 910         get_accept_lock()
 911         (summary, short_summary) = Upload.build_summaries()
 912     try:
 913         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 914             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 915             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 916             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 917             # Check for override disparities
 918             Upload.Subst["__SUMMARY__"] = summary
 919         else:
 920             # Stable updates need to be copied to proposed-updates holding
 921             # area instead of accepted.  Sourceful uploads need to go
 922             # to it directly, binaries only if the source has not yet been
 923             # accepted into p-u.
 924             for suite, q in [("proposed-updates", "ProposedUpdates"),
 925                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 926                 if not Upload.pkg.changes["distribution"].has_key(suite):
 927                     continue
 928                 return do_accept_stableupdate(suite, q)
 929             # Just a normal upload, accept it...
 930             _accept()
 931     finally:
 932         if not Options["No-Action"]:
 933             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 934
 935 def check_status(files):
 936     new = byhand = 0
 937     for f in files.keys():
 938         if files[f]["type"] == "byhand":
 939             byhand = 1
 940         elif files[f].has_key("new"):
 941             new = 1
 942     return (new, byhand)
 943
 944 def do_pkg(changes_file):
 945     Upload.pkg.changes_file = changes_file
 946     Upload.init_vars()
 947     Upload.update_vars()
 948     Upload.update_subst()
 949     files = Upload.pkg.files
 950
 951     if not recheck():
 952         return
 953
 954     (new, byhand) = check_status(files)
 955     if new or byhand:
 956         if new:
 957             do_new()
 958         if byhand:
 959             do_byhand()
 960         (new, byhand) = check_status(files)
 961
 962     if not new and not byhand:
 963         do_accept()
 964
 965 ################################################################################
 966
 967 def end():
 968     accept_count = Upload.accept_count
 969     accept_bytes = Upload.accept_bytes
 970
 971     if accept_count:
 972         sets = "set"
 973         if accept_count > 1:
 974             sets = "sets"
 975         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 976         Logger.log(["total",accept_count,accept_bytes])
 977
 978     if not Options["No-Action"]:
 979         Logger.close()
 980
 981 ################################################################################
 982
 983 def do_comments(dir, opref, npref, line, fn):
 984     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 985         lines = open("%s/%s" % (dir, comm)).readlines()
 986         if len(lines) == 0 or lines[0] != line + "\n": continue
 987         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 988                                 and x.endswith(".changes") ]
 989         changes_files = sort_changes(changes_files)
 990         for f in changes_files:
 991             f = utils.validate_changes_file_arg(f, 0)
 992             if not f: continue
 993             print "\n" + f
 994             fn(f, "".join(lines[1:]))
 995
 996         if opref != npref and not Options["No-Action"]:
 997             newcomm = npref + comm[len(opref):]
 998             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 999
1000 ################################################################################
1001
1002 def comment_accept(changes_file, comments):
1003     Upload.pkg.changes_file = changes_file
1004     Upload.init_vars()
1005     Upload.update_vars()
1006     Upload.update_subst()
1007     files = Upload.pkg.files
1008
1009     if not recheck():
1010         return # dak wants to REJECT, crap
1011
1012     (new, byhand) = check_status(files)
1013     if not new and not byhand:
1014         do_accept()
1015
1016 ################################################################################
1017
1018 def comment_reject(changes_file, comments):
1019     Upload.pkg.changes_file = changes_file
1020     Upload.init_vars()
1021     Upload.update_vars()
1022     Upload.update_subst()
1023
1024     if not recheck():
1025         pass # dak has its own reasons to reject as well, which is fine
1026
1027     reject(comments)
1028     print "REJECT\n" + reject_message,
1029     if not Options["No-Action"]:
1030         Upload.do_reject(0, reject_message)
1031         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1032
1033 ################################################################################
1034
1035 def main():
1036     changes_files = init()
1037     if len(changes_files) > 50:
1038         sys.stderr.write("Sorting changes...\n")
1039     changes_files = sort_changes(changes_files)
1040
1041     # Kill me now? **FIXME**
1042     Cnf["Dinstall::Options::No-Mail"] = ""
1043     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1044     if Cnf.has_key("Dinstall::Bcc"):
1045         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1046     else:
1047         Upload.Subst["__BCC__"] = bcc
1048
1049     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1050     if commentsdir:
1051         if changes_files != []:
1052             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1053             sys.exit(1)
1054         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1055         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1056     else:
1057         for changes_file in changes_files:
1058             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1059             if not changes_file:
1060                 continue
1061             print "\n" + changes_file
1062             do_pkg (changes_file)
1063
1064     end()
1065
1066 ################################################################################
1067
1068 if __name__ == '__main__':
1069     main()