dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 import copy
  45 import errno
  46 import os
  47 import readline
  48 import stat
  49 import sys
  50 import time
  51 import apt_pkg, apt_inst
  52 import examine_package
  53 from daklib import database
  54 from daklib import logging
  55 from daklib import queue
  56 from daklib import utils
  57 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  58
  59 # Globals
  60 Cnf = None       #: Configuration, apt_pkg.Configuration
  61 Options = None
  62 Upload = None
  63 projectB = None  #: database connection, pgobject
  64 Logger = None
  65
  66 Priorities = None
  67 Sections = None
  68
  69 reject_message = ""
  70
  71 ################################################################################
  72 ################################################################################
  73 ################################################################################
  74
  75 def reject (str, prefix="Rejected: "):
  76     global reject_message
  77     if str:
  78         reject_message += prefix + str + "\n"
  79
  80 def recheck():
  81     global reject_message
  82     files = Upload.pkg.files
  83     reject_message = ""
  84
  85     for f in files.keys():
  86         # The .orig.tar.gz can disappear out from under us is it's a
  87         # duplicate of one in the archive.
  88         if not files.has_key(f):
  89             continue
  90         # Check that the source still exists
  91         if files[f]["type"] == "deb":
  92             source_version = files[f]["source version"]
  93             source_package = files[f]["source package"]
  94             if not Upload.pkg.changes["architecture"].has_key("source") \
  95                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  96                 source_epochless_version = re_no_epoch.sub('', source_version)
  97                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  98                 found = 0
  99                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
 100                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 101                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 102                             found = 1
 103                 if not found:
 104                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 105
 106         # Version and file overwrite checks
 107         if files[f]["type"] == "deb":
 108             reject(Upload.check_binary_against_db(f), "")
 109         elif files[f]["type"] == "dsc":
 110             reject(Upload.check_source_against_db(f), "")
 111             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 112             reject(reject_msg, "")
 113
 114     if reject_message.find("Rejected") != -1:
 115         answer = "XXX"
 116         if Options["No-Action"] or Options["Automatic"]:
 117             answer = 'S'
 118
 119         print "REJECT\n" + reject_message,
 120         prompt = "[R]eject, Skip, Quit ?"
 121
 122         while prompt.find(answer) == -1:
 123             answer = utils.our_raw_input(prompt)
 124             m = re_default_answer.match(prompt)
 125             if answer == "":
 126                 answer = m.group(1)
 127             answer = answer[:1].upper()
 128
 129         if answer == 'R':
 130             Upload.do_reject(0, reject_message)
 131             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 132             return 0
 133         elif answer == 'S':
 134             return 0
 135         elif answer == 'Q':
 136             end()
 137             sys.exit(0)
 138
 139     return 1
 140
 141 ################################################################################
 142
 143 def indiv_sg_compare (a, b):
 144     """Sort by source name, source, version, 'have source', and
 145        finally by filename."""
 146     # Sort by source version
 147     q = apt_pkg.VersionCompare(a["version"], b["version"])
 148     if q:
 149         return -q
 150
 151     # Sort by 'have source'
 152     a_has_source = a["architecture"].get("source")
 153     b_has_source = b["architecture"].get("source")
 154     if a_has_source and not b_has_source:
 155         return -1
 156     elif b_has_source and not a_has_source:
 157         return 1
 158
 159     return cmp(a["filename"], b["filename"])
 160
 161 ############################################################
 162
 163 def sg_compare (a, b):
 164     a = a[1]
 165     b = b[1]
 166     """Sort by have note, source already in database and time of oldest upload."""
 167     # Sort by have note
 168     a_note_state = a["note_state"]
 169     b_note_state = b["note_state"]
 170     if a_note_state < b_note_state:
 171         return -1
 172     elif a_note_state > b_note_state:
 173         return 1
 174     # Sort by source already in database (descending)
 175     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 176     if source_in_database:
 177         return -source_in_database
 178
 179     # Sort by time of oldest upload
 180     return cmp(a["oldest"], b["oldest"])
 181
 182 def sort_changes(changes_files):
 183     """Sort into source groups, then sort each source group by version,
 184     have source, filename.  Finally, sort the source groups by have
 185     note, time of oldest upload of each source upload."""
 186     if len(changes_files) == 1:
 187         return changes_files
 188
 189     sorted_list = []
 190     cache = {}
 191     # Read in all the .changes files
 192     for filename in changes_files:
 193         try:
 194             Upload.pkg.changes_file = filename
 195             Upload.init_vars()
 196             Upload.update_vars()
 197             cache[filename] = copy.copy(Upload.pkg.changes)
 198             cache[filename]["filename"] = filename
 199         except:
 200             sorted_list.append(filename)
 201             break
 202     # Divide the .changes into per-source groups
 203     per_source = {}
 204     for filename in cache.keys():
 205         source = cache[filename]["source"]
 206         if not per_source.has_key(source):
 207             per_source[source] = {}
 208             per_source[source]["list"] = []
 209         per_source[source]["list"].append(cache[filename])
 210     # Determine oldest time and have note status for each source group
 211     for source in per_source.keys():
 212         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 213         ql = q.getresult()
 214         per_source[source]["source_in_database"] = len(ql)>0
 215         source_list = per_source[source]["list"]
 216         first = source_list[0]
 217         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 218         have_note = 0
 219         for d in per_source[source]["list"]:
 220             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 221             if mtime < oldest:
 222                 oldest = mtime
 223             have_note += (database.has_new_comment(d["source"], d["version"]))
 224         per_source[source]["oldest"] = oldest
 225         if not have_note:
 226             per_source[source]["note_state"] = 0; # none
 227         elif have_note < len(source_list):
 228             per_source[source]["note_state"] = 1; # some
 229         else:
 230             per_source[source]["note_state"] = 2; # all
 231         per_source[source]["list"].sort(indiv_sg_compare)
 232     per_source_items = per_source.items()
 233     per_source_items.sort(sg_compare)
 234     for i in per_source_items:
 235         for j in i[1]["list"]:
 236             sorted_list.append(j["filename"])
 237     return sorted_list
 238
 239 ################################################################################
 240
 241 class Section_Completer:
 242     def __init__ (self):
 243         self.sections = []
 244         self.matches = []
 245         q = projectB.query("SELECT section FROM section")
 246         for i in q.getresult():
 247             self.sections.append(i[0])
 248
 249     def complete(self, text, state):
 250         if state == 0:
 251             self.matches = []
 252             n = len(text)
 253             for word in self.sections:
 254                 if word[:n] == text:
 255                     self.matches.append(word)
 256         try:
 257             return self.matches[state]
 258         except IndexError:
 259             return None
 260
 261 ############################################################
 262
 263 class Priority_Completer:
 264     def __init__ (self):
 265         self.priorities = []
 266         self.matches = []
 267         q = projectB.query("SELECT priority FROM priority")
 268         for i in q.getresult():
 269             self.priorities.append(i[0])
 270
 271     def complete(self, text, state):
 272         if state == 0:
 273             self.matches = []
 274             n = len(text)
 275             for word in self.priorities:
 276                 if word[:n] == text:
 277                     self.matches.append(word)
 278         try:
 279             return self.matches[state]
 280         except IndexError:
 281             return None
 282
 283 ################################################################################
 284
 285 def print_new (new, indexed, file=sys.stdout):
 286     queue.check_valid(new)
 287     broken = 0
 288     index = 0
 289     for pkg in new.keys():
 290         index += 1
 291         section = new[pkg]["section"]
 292         priority = new[pkg]["priority"]
 293         if new[pkg]["section id"] == -1:
 294             section += "[!]"
 295             broken = 1
 296         if new[pkg]["priority id"] == -1:
 297             priority += "[!]"
 298             broken = 1
 299         if indexed:
 300             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 301         else:
 302             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 303         line = line.strip()+'\n'
 304         file.write(line)
 305     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 306     if len(note) > 0:
 307         for line in note:
 308             print line
 309     return broken, note
 310
 311 ################################################################################
 312
 313 def index_range (index):
 314     if index == 1:
 315         return "1"
 316     else:
 317         return "1-%s" % (index)
 318
 319 ################################################################################
 320 ################################################################################
 321
 322 def edit_new (new):
 323     # Write the current data to a temporary file
 324     (fd, temp_filename) = utils.temp_filename()
 325     temp_file = os.fdopen(fd, 'w')
 326     print_new (new, 0, temp_file)
 327     temp_file.close()
 328     # Spawn an editor on that file
 329     editor = os.environ.get("EDITOR","vi")
 330     result = os.system("%s %s" % (editor, temp_filename))
 331     if result != 0:
 332         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 333     # Read the edited data back in
 334     temp_file = utils.open_file(temp_filename)
 335     lines = temp_file.readlines()
 336     temp_file.close()
 337     os.unlink(temp_filename)
 338     # Parse the new data
 339     for line in lines:
 340         line = line.strip()
 341         if line == "":
 342             continue
 343         s = line.split()
 344         # Pad the list if necessary
 345         s[len(s):3] = [None] * (3-len(s))
 346         (pkg, priority, section) = s[:3]
 347         if not new.has_key(pkg):
 348             utils.warn("Ignoring unknown package '%s'" % (pkg))
 349         else:
 350             # Strip off any invalid markers, print_new will readd them.
 351             if section.endswith("[!]"):
 352                 section = section[:-3]
 353             if priority.endswith("[!]"):
 354                 priority = priority[:-3]
 355             for f in new[pkg]["files"]:
 356                 Upload.pkg.files[f]["section"] = section
 357                 Upload.pkg.files[f]["priority"] = priority
 358             new[pkg]["section"] = section
 359             new[pkg]["priority"] = priority
 360
 361 ################################################################################
 362
 363 def edit_index (new, index):
 364     priority = new[index]["priority"]
 365     section = new[index]["section"]
 366     ftype = new[index]["type"]
 367     done = 0
 368     while not done:
 369         print "\t".join([index, priority, section])
 370
 371         answer = "XXX"
 372         if ftype != "dsc":
 373             prompt = "[B]oth, Priority, Section, Done ? "
 374         else:
 375             prompt = "[S]ection, Done ? "
 376         edit_priority = edit_section = 0
 377
 378         while prompt.find(answer) == -1:
 379             answer = utils.our_raw_input(prompt)
 380             m = re_default_answer.match(prompt)
 381             if answer == "":
 382                 answer = m.group(1)
 383             answer = answer[:1].upper()
 384
 385         if answer == 'P':
 386             edit_priority = 1
 387         elif answer == 'S':
 388             edit_section = 1
 389         elif answer == 'B':
 390             edit_priority = edit_section = 1
 391         elif answer == 'D':
 392             done = 1
 393
 394         # Edit the priority
 395         if edit_priority:
 396             readline.set_completer(Priorities.complete)
 397             got_priority = 0
 398             while not got_priority:
 399                 new_priority = utils.our_raw_input("New priority: ").strip()
 400                 if new_priority not in Priorities.priorities:
 401                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 402                 else:
 403                     got_priority = 1
 404                     priority = new_priority
 405
 406         # Edit the section
 407         if edit_section:
 408             readline.set_completer(Sections.complete)
 409             got_section = 0
 410             while not got_section:
 411                 new_section = utils.our_raw_input("New section: ").strip()
 412                 if new_section not in Sections.sections:
 413                     print "E: '%s' is not a valid section, try again." % (new_section)
 414                 else:
 415                     got_section = 1
 416                     section = new_section
 417
 418         # Reset the readline completer
 419         readline.set_completer(None)
 420
 421     for f in new[index]["files"]:
 422         Upload.pkg.files[f]["section"] = section
 423         Upload.pkg.files[f]["priority"] = priority
 424     new[index]["priority"] = priority
 425     new[index]["section"] = section
 426     return new
 427
 428 ################################################################################
 429
 430 def edit_overrides (new):
 431     print
 432     done = 0
 433     while not done:
 434         print_new (new, 1)
 435         new_index = {}
 436         index = 0
 437         for i in new.keys():
 438             index += 1
 439             new_index[index] = i
 440
 441         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 442
 443         got_answer = 0
 444         while not got_answer:
 445             answer = utils.our_raw_input(prompt)
 446             if not answer.isdigit():
 447                 answer = answer[:1].upper()
 448             if answer == "E" or answer == "D":
 449                 got_answer = 1
 450             elif re_isanum.match (answer):
 451                 answer = int(answer)
 452                 if (answer < 1) or (answer > index):
 453                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 454                 else:
 455                     got_answer = 1
 456
 457         if answer == 'E':
 458             edit_new(new)
 459         elif answer == 'D':
 460             done = 1
 461         else:
 462             edit_index (new, new_index[answer])
 463
 464     return new
 465
 466 ################################################################################
 467
 468 def edit_note(note):
 469     # Write the current data to a temporary file
 470     (fd, temp_filename) = utils.temp_filename()
 471     temp_file = os.fdopen(fd, 'w')
 472     temp_file.write(note)
 473     temp_file.close()
 474     editor = os.environ.get("EDITOR","vi")
 475     answer = 'E'
 476     while answer == 'E':
 477         os.system("%s %s" % (editor, temp_filename))
 478         temp_file = utils.open_file(temp_filename)
 479         note = temp_file.read().rstrip()
 480         temp_file.close()
 481         print "Note:"
 482         print utils.prefix_multi_line_string(note,"  ")
 483         prompt = "[D]one, Edit, Abandon, Quit ?"
 484         answer = "XXX"
 485         while prompt.find(answer) == -1:
 486             answer = utils.our_raw_input(prompt)
 487             m = re_default_answer.search(prompt)
 488             if answer == "":
 489                 answer = m.group(1)
 490             answer = answer[:1].upper()
 491     os.unlink(temp_filename)
 492     if answer == 'A':
 493         return
 494     elif answer == 'Q':
 495         end()
 496         sys.exit(0)
 497     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], note, utils.whoami())
 498
 499 ################################################################################
 500
 501 def check_pkg ():
 502     try:
 503         less_fd = os.popen("less -R -", 'w', 0)
 504         stdout_fd = sys.stdout
 505         try:
 506             sys.stdout = less_fd
 507             changes = utils.parse_changes (Upload.pkg.changes_file)
 508             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 509             files = Upload.pkg.files
 510             for f in files.keys():
 511                 if files[f].has_key("new"):
 512                     ftype = files[f]["type"]
 513                     if ftype == "deb":
 514                         examine_package.check_deb(changes['distribution'], f)
 515                     elif ftype == "dsc":
 516                         examine_package.check_dsc(changes['distribution'], f)
 517         finally:
 518             examine_package.output_package_relations()
 519             sys.stdout = stdout_fd
 520     except IOError, e:
 521         if e.errno == errno.EPIPE:
 522             utils.warn("[examine_package] Caught EPIPE; skipping.")
 523             pass
 524         else:
 525             raise
 526     except KeyboardInterrupt:
 527         utils.warn("[examine_package] Caught C-c; skipping.")
 528         pass
 529
 530 ################################################################################
 531
 532 ## FIXME: horribly Debian specific
 533
 534 def do_bxa_notification():
 535     files = Upload.pkg.files
 536     summary = ""
 537     for f in files.keys():
 538         if files[f]["type"] == "deb":
 539             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 540             summary += "\n"
 541             summary += "Package: %s\n" % (control.Find("Package"))
 542             summary += "Description: %s\n" % (control.Find("Description"))
 543     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 544     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 545     utils.send_mail(bxa_mail)
 546
 547 ################################################################################
 548
 549 def add_overrides (new):
 550     changes = Upload.pkg.changes
 551     files = Upload.pkg.files
 552
 553     projectB.query("BEGIN WORK")
 554     for suite in changes["suite"].keys():
 555         suite_id = database.get_suite_id(suite)
 556         for pkg in new.keys():
 557             component_id = database.get_component_id(new[pkg]["component"])
 558             type_id = database.get_override_type_id(new[pkg]["type"])
 559             priority_id = new[pkg]["priority id"]
 560             section_id = new[pkg]["section id"]
 561             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 562             for f in new[pkg]["files"]:
 563                 if files[f].has_key("new"):
 564                     del files[f]["new"]
 565             del new[pkg]
 566
 567     projectB.query("COMMIT WORK")
 568
 569     if Cnf.FindB("Dinstall::BXANotify"):
 570         do_bxa_notification()
 571
 572 ################################################################################
 573
 574 def prod_maintainer ():
 575     # Here we prepare an editor and get them ready to prod...
 576     (fd, temp_filename) = utils.temp_filename()
 577     editor = os.environ.get("EDITOR","vi")
 578     answer = 'E'
 579     while answer == 'E':
 580         os.system("%s %s" % (editor, temp_filename))
 581         f = os.fdopen(fd)
 582         prod_message = "".join(f.readlines())
 583         f.close()
 584         print "Prod message:"
 585         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 586         prompt = "[P]rod, Edit, Abandon, Quit ?"
 587         answer = "XXX"
 588         while prompt.find(answer) == -1:
 589             answer = utils.our_raw_input(prompt)
 590             m = re_default_answer.search(prompt)
 591             if answer == "":
 592                 answer = m.group(1)
 593             answer = answer[:1].upper()
 594         os.unlink(temp_filename)
 595         if answer == 'A':
 596             return
 597         elif answer == 'Q':
 598             end()
 599             sys.exit(0)
 600     # Otherwise, do the proding...
 601     user_email_address = utils.whoami() + " <%s>" % (
 602         Cnf["Dinstall::MyAdminAddress"])
 603
 604     Subst = Upload.Subst
 605
 606     Subst["__FROM_ADDRESS__"] = user_email_address
 607     Subst["__PROD_MESSAGE__"] = prod_message
 608     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 609
 610     prod_mail_message = utils.TemplateSubst(
 611         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 612
 613     # Send the prod mail if appropriate
 614     if not Cnf["Dinstall::Options::No-Mail"]:
 615         utils.send_mail(prod_mail_message)
 616
 617     print "Sent proding message"
 618
 619 ################################################################################
 620
 621 def do_new():
 622     print "NEW\n"
 623     files = Upload.pkg.files
 624     changes = Upload.pkg.changes
 625
 626     # Make a copy of distribution we can happily trample on
 627     changes["suite"] = copy.copy(changes["distribution"])
 628
 629     # Fix up the list of target suites
 630     for suite in changes["suite"].keys():
 631         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 632         if override:
 633             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 634               database.get_suite_id(override) == -1)
 635             if olderr or newerr:
 636                 (oinv, newinv) = ("", "")
 637                 if olderr: oinv = "invalid "
 638                 if newerr: ninv = "invalid "
 639                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 640                         oinv, suite, ninv, override)
 641             del changes["suite"][suite]
 642             changes["suite"][override] = 1
 643     # Validate suites
 644     for suite in changes["suite"].keys():
 645         suite_id = database.get_suite_id(suite)
 646         if suite_id == -1:
 647             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 648
 649     # The main NEW processing loop
 650     done = 0
 651     while not done:
 652         # Find out what's new
 653         new = queue.determine_new(changes, files, projectB)
 654
 655         if not new:
 656             break
 657
 658         answer = "XXX"
 659         if Options["No-Action"] or Options["Automatic"]:
 660             answer = 'S'
 661
 662         (broken, note) = print_new(new, 0)
 663         prompt = ""
 664
 665         if not broken and not note:
 666             prompt = "Add overrides, "
 667         if broken:
 668             print "W: [!] marked entries must be fixed before package can be processed."
 669         if note:
 670             print "W: note must be removed before package can be processed."
 671             prompt += "Remove note, "
 672
 673         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 674
 675         while prompt.find(answer) == -1:
 676             answer = utils.our_raw_input(prompt)
 677             m = re_default_answer.search(prompt)
 678             if answer == "":
 679                 answer = m.group(1)
 680             answer = answer[:1].upper()
 681
 682         if answer == 'A':
 683             done = add_overrides (new)
 684         elif answer == 'C':
 685             check_pkg()
 686         elif answer == 'E':
 687             new = edit_overrides (new)
 688         elif answer == 'M':
 689             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 690             if not aborted:
 691                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 692                 done = 1
 693         elif answer == 'N':
 694             edit_note(database.get_new_comments(changes.get("source", "")))
 695         elif answer == 'P':
 696             prod_maintainer()
 697         elif answer == 'R':
 698             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 699             if confirm == "y":
 700                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 701         elif answer == 'S':
 702             done = 1
 703         elif answer == 'Q':
 704             end()
 705             sys.exit(0)
 706
 707 ################################################################################
 708 ################################################################################
 709 ################################################################################
 710
 711 def usage (exit_code=0):
 712     print """Usage: dak process-new [OPTION]... [CHANGES]...
 713   -a, --automatic           automatic run
 714   -h, --help                show this help and exit.
 715   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 716   -m, --manual-reject=MSG   manual reject with `msg'
 717   -n, --no-action           don't do anything
 718   -V, --version             display the version number and exit"""
 719     sys.exit(exit_code)
 720
 721 ################################################################################
 722
 723 def init():
 724     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 725
 726     Cnf = utils.get_conf()
 727
 728     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 729                  ('h',"help","Process-New::Options::Help"),
 730                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 731                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 732                  ('n',"no-action","Process-New::Options::No-Action")]
 733
 734     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 735         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 736             Cnf["Process-New::Options::%s" % (i)] = ""
 737
 738     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 739     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 740         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 741
 742     Options = Cnf.SubTree("Process-New::Options")
 743
 744     if Options["Help"]:
 745         usage()
 746
 747     Upload = queue.Upload(Cnf)
 748
 749     if not Options["No-Action"]:
 750         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 751
 752     projectB = Upload.projectB
 753
 754     Sections = Section_Completer()
 755     Priorities = Priority_Completer()
 756     readline.parse_and_bind("tab: complete")
 757
 758     return changes_files
 759
 760 ################################################################################
 761
 762 def do_byhand():
 763     done = 0
 764     while not done:
 765         files = Upload.pkg.files
 766         will_install = 1
 767         byhand = []
 768
 769         for f in files.keys():
 770             if files[f]["type"] == "byhand":
 771                 if os.path.exists(f):
 772                     print "W: %s still present; please process byhand components and try again." % (f)
 773                     will_install = 0
 774                 else:
 775                     byhand.append(f)
 776
 777         answer = "XXXX"
 778         if Options["No-Action"]:
 779             answer = "S"
 780         if will_install:
 781             if Options["Automatic"] and not Options["No-Action"]:
 782                 answer = 'A'
 783             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 784         else:
 785             prompt = "Manual reject, [S]kip, Quit ?"
 786
 787         while prompt.find(answer) == -1:
 788             answer = utils.our_raw_input(prompt)
 789             m = re_default_answer.search(prompt)
 790             if answer == "":
 791                 answer = m.group(1)
 792             answer = answer[:1].upper()
 793
 794         if answer == 'A':
 795             done = 1
 796             for f in byhand:
 797                 del files[f]
 798         elif answer == 'M':
 799             Upload.do_reject(1, Options["Manual-Reject"])
 800             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 801             done = 1
 802         elif answer == 'S':
 803             done = 1
 804         elif answer == 'Q':
 805             end()
 806             sys.exit(0)
 807
 808 ################################################################################
 809
 810 def get_accept_lock():
 811     retry = 0
 812     while retry < 10:
 813         try:
 814             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 815             retry = 10
 816         except OSError, e:
 817             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 818                 retry += 1
 819                 if (retry >= 10):
 820                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 821                 else:
 822                     print("Unable to get accepted lock (try %d of 10)" % retry)
 823                 time.sleep(60)
 824             else:
 825                 raise
 826
 827 def move_to_dir (dest, perms=0660, changesperms=0664):
 828     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 829     file_keys = Upload.pkg.files.keys()
 830     for f in file_keys:
 831         utils.move (f, dest, perms=perms)
 832
 833 def is_source_in_queue_dir(qdir):
 834     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 835                 and x.endswith(".changes") ]
 836     for entry in entries:
 837         # read the .dak
 838         u = queue.Upload(Cnf)
 839         u.pkg.changes_file = os.path.join(qdir, entry)
 840         u.update_vars()
 841         if not u.pkg.changes["architecture"].has_key("source"):
 842             # another binary upload, ignore
 843             continue
 844         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 845             # another version, ignore
 846             continue
 847         # found it!
 848         return True
 849     return False
 850
 851 def move_to_holding(suite, queue_dir):
 852     print "Moving to %s holding area." % (suite.upper(),)
 853     if Options["No-Action"]:
 854         return
 855     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 856     Upload.dump_vars(queue_dir)
 857     move_to_dir(queue_dir, perms=0664)
 858     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 859
 860 def _accept():
 861     if Options["No-Action"]:
 862         return
 863     (summary, short_summary) = Upload.build_summaries()
 864     Upload.accept(summary, short_summary)
 865     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 866
 867 def do_accept_stableupdate(suite, q):
 868     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 869     if not Upload.pkg.changes["architecture"].has_key("source"):
 870         # It is not a sourceful upload.  So its source may be either in p-u
 871         # holding, in new, in accepted or already installed.
 872         if is_source_in_queue_dir(queue_dir):
 873             # It's in p-u holding, so move it there.
 874             print "Binary-only upload, source in %s." % (q,)
 875             move_to_holding(suite, queue_dir)
 876         elif Upload.source_exists(Upload.pkg.changes["source"],
 877                 Upload.pkg.changes["version"]):
 878             # dak tells us that there is source available.  At time of
 879             # writing this means that it is installed, so put it into
 880             # accepted.
 881             print "Binary-only upload, source installed."
 882             _accept()
 883         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 884             # The source is in accepted, the binary cleared NEW: accept it.
 885             print "Binary-only upload, source in accepted."
 886             _accept()
 887         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 888             # It's in NEW.  We expect the source to land in p-u holding
 889             # pretty soon.
 890             print "Binary-only upload, source in new."
 891             move_to_holding(suite, queue_dir)
 892         else:
 893             # No case applicable.  Bail out.  Return will cause the upload
 894             # to be skipped.
 895             print "ERROR"
 896             print "Stable update failed.  Source not found."
 897             return
 898     else:
 899         # We are handling a sourceful upload.  Move to accepted if currently
 900         # in p-u holding and to p-u holding otherwise.
 901         if is_source_in_queue_dir(queue_dir):
 902             print "Sourceful upload in %s, accepting." % (q,)
 903             _accept()
 904         else:
 905             move_to_holding(suite, queue_dir)
 906
 907 def do_accept():
 908     print "ACCEPT"
 909     if not Options["No-Action"]:
 910         get_accept_lock()
 911         (summary, short_summary) = Upload.build_summaries()
 912     try:
 913         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 914             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 915             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 916             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 917             # Check for override disparities
 918             Upload.Subst["__SUMMARY__"] = summary
 919         else:
 920             # Stable updates need to be copied to proposed-updates holding
 921             # area instead of accepted.  Sourceful uploads need to go
 922             # to it directly, binaries only if the source has not yet been
 923             # accepted into p-u.
 924             for suite, q in [("proposed-updates", "ProposedUpdates"),
 925                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 926                 if not Upload.pkg.changes["distribution"].has_key(suite):
 927                     continue
 928                 return do_accept_stableupdate(suite, q)
 929             # Just a normal upload, accept it...
 930             _accept()
 931     finally:
 932         if not Options["No-Action"]:
 933             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 934
 935 def check_status(files):
 936     new = byhand = 0
 937     for f in files.keys():
 938         if files[f]["type"] == "byhand":
 939             byhand = 1
 940         elif files[f].has_key("new"):
 941             new = 1
 942     return (new, byhand)
 943
 944 def do_pkg(changes_file):
 945     Upload.pkg.changes_file = changes_file
 946     Upload.init_vars()
 947     Upload.update_vars()
 948     Upload.update_subst()
 949     files = Upload.pkg.files
 950
 951     if not recheck():
 952         return
 953
 954     (new, byhand) = check_status(files)
 955     if new or byhand:
 956         if new:
 957             do_new()
 958         if byhand:
 959             do_byhand()
 960         (new, byhand) = check_status(files)
 961
 962     if not new and not byhand:
 963         do_accept()
 964
 965 ################################################################################
 966
 967 def end():
 968     accept_count = Upload.accept_count
 969     accept_bytes = Upload.accept_bytes
 970
 971     if accept_count:
 972         sets = "set"
 973         if accept_count > 1:
 974             sets = "sets"
 975         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 976         Logger.log(["total",accept_count,accept_bytes])
 977
 978     if not Options["No-Action"]:
 979         Logger.close()
 980
 981 ################################################################################
 982
 983 def do_comments(dir, opref, npref, line, fn):
 984     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 985         lines = open("%s/%s" % (dir, comm)).readlines()
 986         if len(lines) == 0 or lines[0] != line + "\n": continue
 987         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 988                                 and x.endswith(".changes") ]
 989         changes_files = sort_changes(changes_files)
 990         for f in changes_files:
 991             f = utils.validate_changes_file_arg(f, 0)
 992             if not f: continue
 993             print "\n" + f
 994             fn(f, "".join(lines[1:]))
 995
 996         if opref != npref and not Options["No-Action"]:
 997             newcomm = npref + comm[len(opref):]
 998             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 999
1000 ################################################################################
1001
1002 def comment_accept(changes_file, comments):
1003     Upload.pkg.changes_file = changes_file
1004     Upload.init_vars()
1005     Upload.update_vars()
1006     Upload.update_subst()
1007     files = Upload.pkg.files
1008
1009     if not recheck():
1010         return # dak wants to REJECT, crap
1011
1012     (new, byhand) = check_status(files)
1013     if not new and not byhand:
1014         do_accept()
1015
1016 ################################################################################
1017
1018 def comment_reject(changes_file, comments):
1019     Upload.pkg.changes_file = changes_file
1020     Upload.init_vars()
1021     Upload.update_vars()
1022     Upload.update_subst()
1023
1024     if not recheck():
1025         pass # dak has its own reasons to reject as well, which is fine
1026
1027     reject(comments)
1028     print "REJECT\n" + reject_message,
1029     if not Options["No-Action"]:
1030         Upload.do_reject(0, reject_message)
1031         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1032
1033 ################################################################################
1034
1035 def main():
1036     changes_files = init()
1037     if len(changes_files) > 50:
1038         sys.stderr.write("Sorting changes...\n")
1039     changes_files = sort_changes(changes_files)
1040
1041     # Kill me now? **FIXME**
1042     Cnf["Dinstall::Options::No-Mail"] = ""
1043     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1044     if Cnf.has_key("Dinstall::Bcc"):
1045         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1046     else:
1047         Upload.Subst["__BCC__"] = bcc
1048
1049     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1050     if commentsdir:
1051         if changes_files != []:
1052             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1053             sys.exit(1)
1054         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1055         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1056     else:
1057         for changes_file in changes_files:
1058             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1059             if not changes_file:
1060                 continue
1061             print "\n" + changes_file
1062             do_pkg (changes_file)
1063
1064     end()
1065
1066 ################################################################################
1067
1068 if __name__ == '__main__':
1069     main()