dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 import copy
  45 import errno
  46 import os
  47 import readline
  48 import stat
  49 import sys
  50 import time
  51 import apt_pkg, apt_inst
  52 import examine_package
  53 from daklib import database
  54 from daklib import logging
  55 from daklib import queue
  56 from daklib import utils
  57 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  58
  59 # Globals
  60 Cnf = None       #: Configuration, apt_pkg.Configuration
  61 Options = None
  62 Upload = None
  63 projectB = None  #: database connection, pgobject
  64 Logger = None
  65
  66 Priorities = None
  67 Sections = None
  68
  69 reject_message = ""
  70
  71 ################################################################################
  72 ################################################################################
  73 ################################################################################
  74
  75 def reject (str, prefix="Rejected: "):
  76     global reject_message
  77     if str:
  78         reject_message += prefix + str + "\n"
  79
  80 def recheck():
  81     global reject_message
  82     files = Upload.pkg.files
  83     reject_message = ""
  84
  85     for f in files.keys():
  86         # The .orig.tar.gz can disappear out from under us is it's a
  87         # duplicate of one in the archive.
  88         if not files.has_key(f):
  89             continue
  90         # Check that the source still exists
  91         if files[f]["type"] == "deb":
  92             source_version = files[f]["source version"]
  93             source_package = files[f]["source package"]
  94             if not Upload.pkg.changes["architecture"].has_key("source") \
  95                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  96                 source_epochless_version = re_no_epoch.sub('', source_version)
  97                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  98                 found = 0
  99                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
 100                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 101                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 102                             found = 1
 103                 if not found:
 104                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 105
 106         # Version and file overwrite checks
 107         if files[f]["type"] == "deb":
 108             reject(Upload.check_binary_against_db(f), "")
 109         elif files[f]["type"] == "dsc":
 110             reject(Upload.check_source_against_db(f), "")
 111             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 112             reject(reject_msg, "")
 113
 114     if reject_message.find("Rejected") != -1:
 115         answer = "XXX"
 116         if Options["No-Action"] or Options["Automatic"]:
 117             answer = 'S'
 118
 119         print "REJECT\n" + reject_message,
 120         prompt = "[R]eject, Skip, Quit ?"
 121
 122         while prompt.find(answer) == -1:
 123             answer = utils.our_raw_input(prompt)
 124             m = re_default_answer.match(prompt)
 125             if answer == "":
 126                 answer = m.group(1)
 127             answer = answer[:1].upper()
 128
 129         if answer == 'R':
 130             Upload.do_reject(0, reject_message)
 131             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 132             return 0
 133         elif answer == 'S':
 134             return 0
 135         elif answer == 'Q':
 136             end()
 137             sys.exit(0)
 138
 139     return 1
 140
 141 ################################################################################
 142
 143 def indiv_sg_compare (a, b):
 144     """Sort by source name, source, version, 'have source', and
 145        finally by filename."""
 146     # Sort by source version
 147     q = apt_pkg.VersionCompare(a["version"], b["version"])
 148     if q:
 149         return -q
 150
 151     # Sort by 'have source'
 152     a_has_source = a["architecture"].get("source")
 153     b_has_source = b["architecture"].get("source")
 154     if a_has_source and not b_has_source:
 155         return -1
 156     elif b_has_source and not a_has_source:
 157         return 1
 158
 159     return cmp(a["filename"], b["filename"])
 160
 161 ############################################################
 162
 163 def sg_compare (a, b):
 164     a = a[1]
 165     b = b[1]
 166     """Sort by have note, source already in database and time of oldest upload."""
 167     # Sort by have note
 168     a_note_state = a["note_state"]
 169     b_note_state = b["note_state"]
 170     if a_note_state < b_note_state:
 171         return -1
 172     elif a_note_state > b_note_state:
 173         return 1
 174     # Sort by source already in database (descending)
 175     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 176     if source_in_database:
 177         return -source_in_database
 178
 179     # Sort by time of oldest upload
 180     return cmp(a["oldest"], b["oldest"])
 181
 182 def sort_changes(changes_files):
 183     """Sort into source groups, then sort each source group by version,
 184     have source, filename.  Finally, sort the source groups by have
 185     note, time of oldest upload of each source upload."""
 186     if len(changes_files) == 1:
 187         return changes_files
 188
 189     sorted_list = []
 190     cache = {}
 191     # Read in all the .changes files
 192     for filename in changes_files:
 193         try:
 194             Upload.pkg.changes_file = filename
 195             Upload.init_vars()
 196             Upload.update_vars()
 197             cache[filename] = copy.copy(Upload.pkg.changes)
 198             cache[filename]["filename"] = filename
 199         except:
 200             sorted_list.append(filename)
 201             break
 202     # Divide the .changes into per-source groups
 203     per_source = {}
 204     for filename in cache.keys():
 205         source = cache[filename]["source"]
 206         if not per_source.has_key(source):
 207             per_source[source] = {}
 208             per_source[source]["list"] = []
 209         per_source[source]["list"].append(cache[filename])
 210     # Determine oldest time and have note status for each source group
 211     for source in per_source.keys():
 212         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 213         ql = q.getresult()
 214         per_source[source]["source_in_database"] = len(ql)>0
 215         source_list = per_source[source]["list"]
 216         first = source_list[0]
 217         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 218         have_note = 0
 219         for d in per_source[source]["list"]:
 220             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 221             if mtime < oldest:
 222                 oldest = mtime
 223             have_note += (database.has_new_comment(d["source"], d["version"]))
 224         per_source[source]["oldest"] = oldest
 225         if not have_note:
 226             per_source[source]["note_state"] = 0; # none
 227         elif have_note < len(source_list):
 228             per_source[source]["note_state"] = 1; # some
 229         else:
 230             per_source[source]["note_state"] = 2; # all
 231         per_source[source]["list"].sort(indiv_sg_compare)
 232     per_source_items = per_source.items()
 233     per_source_items.sort(sg_compare)
 234     for i in per_source_items:
 235         for j in i[1]["list"]:
 236             sorted_list.append(j["filename"])
 237     return sorted_list
 238
 239 ################################################################################
 240
 241 class Section_Completer:
 242     def __init__ (self):
 243         self.sections = []
 244         self.matches = []
 245         q = projectB.query("SELECT section FROM section")
 246         for i in q.getresult():
 247             self.sections.append(i[0])
 248
 249     def complete(self, text, state):
 250         if state == 0:
 251             self.matches = []
 252             n = len(text)
 253             for word in self.sections:
 254                 if word[:n] == text:
 255                     self.matches.append(word)
 256         try:
 257             return self.matches[state]
 258         except IndexError:
 259             return None
 260
 261 ############################################################
 262
 263 class Priority_Completer:
 264     def __init__ (self):
 265         self.priorities = []
 266         self.matches = []
 267         q = projectB.query("SELECT priority FROM priority")
 268         for i in q.getresult():
 269             self.priorities.append(i[0])
 270
 271     def complete(self, text, state):
 272         if state == 0:
 273             self.matches = []
 274             n = len(text)
 275             for word in self.priorities:
 276                 if word[:n] == text:
 277                     self.matches.append(word)
 278         try:
 279             return self.matches[state]
 280         except IndexError:
 281             return None
 282
 283 ################################################################################
 284
 285 def print_new (new, indexed, file=sys.stdout):
 286     queue.check_valid(new)
 287     broken = 0
 288     index = 0
 289     for pkg in new.keys():
 290         index += 1
 291         section = new[pkg]["section"]
 292         priority = new[pkg]["priority"]
 293         if new[pkg]["section id"] == -1:
 294             section += "[!]"
 295             broken = 1
 296         if new[pkg]["priority id"] == -1:
 297             priority += "[!]"
 298             broken = 1
 299         if indexed:
 300             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 301         else:
 302             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 303         line = line.strip()+'\n'
 304         file.write(line)
 305     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 306     if len(note) > 0:
 307         for line in note:
 308             print line
 309     return broken, note
 310
 311 ################################################################################
 312
 313 def index_range (index):
 314     if index == 1:
 315         return "1"
 316     else:
 317         return "1-%s" % (index)
 318
 319 ################################################################################
 320 ################################################################################
 321
 322 def edit_new (new):
 323     # Write the current data to a temporary file
 324     (fd, temp_filename) = utils.temp_filename()
 325     temp_file = os.fdopen(fd, 'w')
 326     print_new (new, 0, temp_file)
 327     temp_file.close()
 328     # Spawn an editor on that file
 329     editor = os.environ.get("EDITOR","vi")
 330     result = os.system("%s %s" % (editor, temp_filename))
 331     if result != 0:
 332         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 333     # Read the edited data back in
 334     temp_file = utils.open_file(temp_filename)
 335     lines = temp_file.readlines()
 336     temp_file.close()
 337     os.unlink(temp_filename)
 338     # Parse the new data
 339     for line in lines:
 340         line = line.strip()
 341         if line == "":
 342             continue
 343         s = line.split()
 344         # Pad the list if necessary
 345         s[len(s):3] = [None] * (3-len(s))
 346         (pkg, priority, section) = s[:3]
 347         if not new.has_key(pkg):
 348             utils.warn("Ignoring unknown package '%s'" % (pkg))
 349         else:
 350             # Strip off any invalid markers, print_new will readd them.
 351             if section.endswith("[!]"):
 352                 section = section[:-3]
 353             if priority.endswith("[!]"):
 354                 priority = priority[:-3]
 355             for f in new[pkg]["files"]:
 356                 Upload.pkg.files[f]["section"] = section
 357                 Upload.pkg.files[f]["priority"] = priority
 358             new[pkg]["section"] = section
 359             new[pkg]["priority"] = priority
 360
 361 ################################################################################
 362
 363 def edit_index (new, index):
 364     priority = new[index]["priority"]
 365     section = new[index]["section"]
 366     ftype = new[index]["type"]
 367     done = 0
 368     while not done:
 369         print "\t".join([index, priority, section])
 370
 371         answer = "XXX"
 372         if ftype != "dsc":
 373             prompt = "[B]oth, Priority, Section, Done ? "
 374         else:
 375             prompt = "[S]ection, Done ? "
 376         edit_priority = edit_section = 0
 377
 378         while prompt.find(answer) == -1:
 379             answer = utils.our_raw_input(prompt)
 380             m = re_default_answer.match(prompt)
 381             if answer == "":
 382                 answer = m.group(1)
 383             answer = answer[:1].upper()
 384
 385         if answer == 'P':
 386             edit_priority = 1
 387         elif answer == 'S':
 388             edit_section = 1
 389         elif answer == 'B':
 390             edit_priority = edit_section = 1
 391         elif answer == 'D':
 392             done = 1
 393
 394         # Edit the priority
 395         if edit_priority:
 396             readline.set_completer(Priorities.complete)
 397             got_priority = 0
 398             while not got_priority:
 399                 new_priority = utils.our_raw_input("New priority: ").strip()
 400                 if new_priority not in Priorities.priorities:
 401                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 402                 else:
 403                     got_priority = 1
 404                     priority = new_priority
 405
 406         # Edit the section
 407         if edit_section:
 408             readline.set_completer(Sections.complete)
 409             got_section = 0
 410             while not got_section:
 411                 new_section = utils.our_raw_input("New section: ").strip()
 412                 if new_section not in Sections.sections:
 413                     print "E: '%s' is not a valid section, try again." % (new_section)
 414                 else:
 415                     got_section = 1
 416                     section = new_section
 417
 418         # Reset the readline completer
 419         readline.set_completer(None)
 420
 421     for f in new[index]["files"]:
 422         Upload.pkg.files[f]["section"] = section
 423         Upload.pkg.files[f]["priority"] = priority
 424     new[index]["priority"] = priority
 425     new[index]["section"] = section
 426     return new
 427
 428 ################################################################################
 429
 430 def edit_overrides (new):
 431     print
 432     done = 0
 433     while not done:
 434         print_new (new, 1)
 435         new_index = {}
 436         index = 0
 437         for i in new.keys():
 438             index += 1
 439             new_index[index] = i
 440
 441         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 442
 443         got_answer = 0
 444         while not got_answer:
 445             answer = utils.our_raw_input(prompt)
 446             if not answer.isdigit():
 447                 answer = answer[:1].upper()
 448             if answer == "E" or answer == "D":
 449                 got_answer = 1
 450             elif re_isanum.match (answer):
 451                 answer = int(answer)
 452                 if (answer < 1) or (answer > index):
 453                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 454                 else:
 455                     got_answer = 1
 456
 457         if answer == 'E':
 458             edit_new(new)
 459         elif answer == 'D':
 460             done = 1
 461         else:
 462             edit_index (new, new_index[answer])
 463
 464     return new
 465
 466 ################################################################################
 467
 468 def edit_note(note):
 469     # Write the current data to a temporary file
 470     (fd, temp_filename) = utils.temp_filename()
 471     temp_file = os.fdopen(fd, 'w')
 472     if len(note) > 0:
 473         for line in note:
 474             temp_file.write(line)
 475     temp_file.close()
 476     editor = os.environ.get("EDITOR","vi")
 477     answer = 'E'
 478     while answer == 'E':
 479         os.system("%s %s" % (editor, temp_filename))
 480         temp_file = utils.open_file(temp_filename)
 481         note = temp_file.read().rstrip()
 482         temp_file.close()
 483         print "Note:"
 484         print utils.prefix_multi_line_string(note,"  ")
 485         prompt = "[D]one, Edit, Abandon, Quit ?"
 486         answer = "XXX"
 487         while prompt.find(answer) == -1:
 488             answer = utils.our_raw_input(prompt)
 489             m = re_default_answer.search(prompt)
 490             if answer == "":
 491                 answer = m.group(1)
 492             answer = answer[:1].upper()
 493     os.unlink(temp_filename)
 494     if answer == 'A':
 495         return
 496     elif answer == 'Q':
 497         end()
 498         sys.exit(0)
 499     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], note, utils.whoami())
 500
 501 ################################################################################
 502
 503 def check_pkg ():
 504     try:
 505         less_fd = os.popen("less -R -", 'w', 0)
 506         stdout_fd = sys.stdout
 507         try:
 508             sys.stdout = less_fd
 509             changes = utils.parse_changes (Upload.pkg.changes_file)
 510             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 511             files = Upload.pkg.files
 512             for f in files.keys():
 513                 if files[f].has_key("new"):
 514                     ftype = files[f]["type"]
 515                     if ftype == "deb":
 516                         examine_package.check_deb(changes['distribution'], f)
 517                     elif ftype == "dsc":
 518                         examine_package.check_dsc(changes['distribution'], f)
 519         finally:
 520             examine_package.output_package_relations()
 521             sys.stdout = stdout_fd
 522     except IOError, e:
 523         if e.errno == errno.EPIPE:
 524             utils.warn("[examine_package] Caught EPIPE; skipping.")
 525             pass
 526         else:
 527             raise
 528     except KeyboardInterrupt:
 529         utils.warn("[examine_package] Caught C-c; skipping.")
 530         pass
 531
 532 ################################################################################
 533
 534 ## FIXME: horribly Debian specific
 535
 536 def do_bxa_notification():
 537     files = Upload.pkg.files
 538     summary = ""
 539     for f in files.keys():
 540         if files[f]["type"] == "deb":
 541             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 542             summary += "\n"
 543             summary += "Package: %s\n" % (control.Find("Package"))
 544             summary += "Description: %s\n" % (control.Find("Description"))
 545     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 546     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 547     utils.send_mail(bxa_mail)
 548
 549 ################################################################################
 550
 551 def add_overrides (new):
 552     changes = Upload.pkg.changes
 553     files = Upload.pkg.files
 554
 555     projectB.query("BEGIN WORK")
 556     for suite in changes["suite"].keys():
 557         suite_id = database.get_suite_id(suite)
 558         for pkg in new.keys():
 559             component_id = database.get_component_id(new[pkg]["component"])
 560             type_id = database.get_override_type_id(new[pkg]["type"])
 561             priority_id = new[pkg]["priority id"]
 562             section_id = new[pkg]["section id"]
 563             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 564             for f in new[pkg]["files"]:
 565                 if files[f].has_key("new"):
 566                     del files[f]["new"]
 567             del new[pkg]
 568
 569     projectB.query("COMMIT WORK")
 570
 571     if Cnf.FindB("Dinstall::BXANotify"):
 572         do_bxa_notification()
 573
 574 ################################################################################
 575
 576 def prod_maintainer ():
 577     # Here we prepare an editor and get them ready to prod...
 578     (fd, temp_filename) = utils.temp_filename()
 579     editor = os.environ.get("EDITOR","vi")
 580     answer = 'E'
 581     while answer == 'E':
 582         os.system("%s %s" % (editor, temp_filename))
 583         f = os.fdopen(fd)
 584         prod_message = "".join(f.readlines())
 585         f.close()
 586         print "Prod message:"
 587         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 588         prompt = "[P]rod, Edit, Abandon, Quit ?"
 589         answer = "XXX"
 590         while prompt.find(answer) == -1:
 591             answer = utils.our_raw_input(prompt)
 592             m = re_default_answer.search(prompt)
 593             if answer == "":
 594                 answer = m.group(1)
 595             answer = answer[:1].upper()
 596         os.unlink(temp_filename)
 597         if answer == 'A':
 598             return
 599         elif answer == 'Q':
 600             end()
 601             sys.exit(0)
 602     # Otherwise, do the proding...
 603     user_email_address = utils.whoami() + " <%s>" % (
 604         Cnf["Dinstall::MyAdminAddress"])
 605
 606     Subst = Upload.Subst
 607
 608     Subst["__FROM_ADDRESS__"] = user_email_address
 609     Subst["__PROD_MESSAGE__"] = prod_message
 610     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 611
 612     prod_mail_message = utils.TemplateSubst(
 613         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 614
 615     # Send the prod mail if appropriate
 616     if not Cnf["Dinstall::Options::No-Mail"]:
 617         utils.send_mail(prod_mail_message)
 618
 619     print "Sent proding message"
 620
 621 ################################################################################
 622
 623 def do_new():
 624     print "NEW\n"
 625     files = Upload.pkg.files
 626     changes = Upload.pkg.changes
 627
 628     # Make a copy of distribution we can happily trample on
 629     changes["suite"] = copy.copy(changes["distribution"])
 630
 631     # Fix up the list of target suites
 632     for suite in changes["suite"].keys():
 633         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 634         if override:
 635             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 636               database.get_suite_id(override) == -1)
 637             if olderr or newerr:
 638                 (oinv, newinv) = ("", "")
 639                 if olderr: oinv = "invalid "
 640                 if newerr: ninv = "invalid "
 641                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 642                         oinv, suite, ninv, override)
 643             del changes["suite"][suite]
 644             changes["suite"][override] = 1
 645     # Validate suites
 646     for suite in changes["suite"].keys():
 647         suite_id = database.get_suite_id(suite)
 648         if suite_id == -1:
 649             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 650
 651     # The main NEW processing loop
 652     done = 0
 653     while not done:
 654         # Find out what's new
 655         new = queue.determine_new(changes, files, projectB)
 656
 657         if not new:
 658             break
 659
 660         answer = "XXX"
 661         if Options["No-Action"] or Options["Automatic"]:
 662             answer = 'S'
 663
 664         (broken, note) = print_new(new, 0)
 665         prompt = ""
 666
 667         if not broken and not note:
 668             prompt = "Add overrides, "
 669         if broken:
 670             print "W: [!] marked entries must be fixed before package can be processed."
 671         if note:
 672             print "W: note must be removed before package can be processed."
 673             prompt += "Remove note, "
 674
 675         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 676
 677         while prompt.find(answer) == -1:
 678             answer = utils.our_raw_input(prompt)
 679             m = re_default_answer.search(prompt)
 680             if answer == "":
 681                 answer = m.group(1)
 682             answer = answer[:1].upper()
 683
 684         if answer == 'A':
 685             done = add_overrides (new)
 686         elif answer == 'C':
 687             check_pkg()
 688         elif answer == 'E':
 689             new = edit_overrides (new)
 690         elif answer == 'M':
 691             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 692             if not aborted:
 693                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 694                 done = 1
 695         elif answer == 'N':
 696             edit_note(database.get_new_comments(changes.get("source", "")))
 697         elif answer == 'P':
 698             prod_maintainer()
 699         elif answer == 'R':
 700             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 701             if confirm == "y":
 702                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 703         elif answer == 'S':
 704             done = 1
 705         elif answer == 'Q':
 706             end()
 707             sys.exit(0)
 708
 709 ################################################################################
 710 ################################################################################
 711 ################################################################################
 712
 713 def usage (exit_code=0):
 714     print """Usage: dak process-new [OPTION]... [CHANGES]...
 715   -a, --automatic           automatic run
 716   -h, --help                show this help and exit.
 717   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 718   -m, --manual-reject=MSG   manual reject with `msg'
 719   -n, --no-action           don't do anything
 720   -V, --version             display the version number and exit"""
 721     sys.exit(exit_code)
 722
 723 ################################################################################
 724
 725 def init():
 726     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 727
 728     Cnf = utils.get_conf()
 729
 730     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 731                  ('h',"help","Process-New::Options::Help"),
 732                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 733                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 734                  ('n',"no-action","Process-New::Options::No-Action")]
 735
 736     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 737         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 738             Cnf["Process-New::Options::%s" % (i)] = ""
 739
 740     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 741     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 742         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 743
 744     Options = Cnf.SubTree("Process-New::Options")
 745
 746     if Options["Help"]:
 747         usage()
 748
 749     Upload = queue.Upload(Cnf)
 750
 751     if not Options["No-Action"]:
 752         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 753
 754     projectB = Upload.projectB
 755
 756     Sections = Section_Completer()
 757     Priorities = Priority_Completer()
 758     readline.parse_and_bind("tab: complete")
 759
 760     return changes_files
 761
 762 ################################################################################
 763
 764 def do_byhand():
 765     done = 0
 766     while not done:
 767         files = Upload.pkg.files
 768         will_install = 1
 769         byhand = []
 770
 771         for f in files.keys():
 772             if files[f]["type"] == "byhand":
 773                 if os.path.exists(f):
 774                     print "W: %s still present; please process byhand components and try again." % (f)
 775                     will_install = 0
 776                 else:
 777                     byhand.append(f)
 778
 779         answer = "XXXX"
 780         if Options["No-Action"]:
 781             answer = "S"
 782         if will_install:
 783             if Options["Automatic"] and not Options["No-Action"]:
 784                 answer = 'A'
 785             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 786         else:
 787             prompt = "Manual reject, [S]kip, Quit ?"
 788
 789         while prompt.find(answer) == -1:
 790             answer = utils.our_raw_input(prompt)
 791             m = re_default_answer.search(prompt)
 792             if answer == "":
 793                 answer = m.group(1)
 794             answer = answer[:1].upper()
 795
 796         if answer == 'A':
 797             done = 1
 798             for f in byhand:
 799                 del files[f]
 800         elif answer == 'M':
 801             Upload.do_reject(1, Options["Manual-Reject"])
 802             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 803             done = 1
 804         elif answer == 'S':
 805             done = 1
 806         elif answer == 'Q':
 807             end()
 808             sys.exit(0)
 809
 810 ################################################################################
 811
 812 def get_accept_lock():
 813     retry = 0
 814     while retry < 10:
 815         try:
 816             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 817             retry = 10
 818         except OSError, e:
 819             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 820                 retry += 1
 821                 if (retry >= 10):
 822                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 823                 else:
 824                     print("Unable to get accepted lock (try %d of 10)" % retry)
 825                 time.sleep(60)
 826             else:
 827                 raise
 828
 829 def move_to_dir (dest, perms=0660, changesperms=0664):
 830     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 831     file_keys = Upload.pkg.files.keys()
 832     for f in file_keys:
 833         utils.move (f, dest, perms=perms)
 834
 835 def is_source_in_queue_dir(qdir):
 836     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 837                 and x.endswith(".changes") ]
 838     for entry in entries:
 839         # read the .dak
 840         u = queue.Upload(Cnf)
 841         u.pkg.changes_file = os.path.join(qdir, entry)
 842         u.update_vars()
 843         if not u.pkg.changes["architecture"].has_key("source"):
 844             # another binary upload, ignore
 845             continue
 846         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 847             # another version, ignore
 848             continue
 849         # found it!
 850         return True
 851     return False
 852
 853 def move_to_holding(suite, queue_dir):
 854     print "Moving to %s holding area." % (suite.upper(),)
 855     if Options["No-Action"]:
 856         return
 857     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 858     Upload.dump_vars(queue_dir)
 859     move_to_dir(queue_dir, perms=0664)
 860     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 861
 862 def _accept():
 863     if Options["No-Action"]:
 864         return
 865     (summary, short_summary) = Upload.build_summaries()
 866     Upload.accept(summary, short_summary)
 867     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 868
 869 def do_accept_stableupdate(suite, q):
 870     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 871     if not Upload.pkg.changes["architecture"].has_key("source"):
 872         # It is not a sourceful upload.  So its source may be either in p-u
 873         # holding, in new, in accepted or already installed.
 874         if is_source_in_queue_dir(queue_dir):
 875             # It's in p-u holding, so move it there.
 876             print "Binary-only upload, source in %s." % (q,)
 877             move_to_holding(suite, queue_dir)
 878         elif Upload.source_exists(Upload.pkg.changes["source"],
 879                 Upload.pkg.changes["version"]):
 880             # dak tells us that there is source available.  At time of
 881             # writing this means that it is installed, so put it into
 882             # accepted.
 883             print "Binary-only upload, source installed."
 884             _accept()
 885         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 886             # The source is in accepted, the binary cleared NEW: accept it.
 887             print "Binary-only upload, source in accepted."
 888             _accept()
 889         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 890             # It's in NEW.  We expect the source to land in p-u holding
 891             # pretty soon.
 892             print "Binary-only upload, source in new."
 893             move_to_holding(suite, queue_dir)
 894         else:
 895             # No case applicable.  Bail out.  Return will cause the upload
 896             # to be skipped.
 897             print "ERROR"
 898             print "Stable update failed.  Source not found."
 899             return
 900     else:
 901         # We are handling a sourceful upload.  Move to accepted if currently
 902         # in p-u holding and to p-u holding otherwise.
 903         if is_source_in_queue_dir(queue_dir):
 904             print "Sourceful upload in %s, accepting." % (q,)
 905             _accept()
 906         else:
 907             move_to_holding(suite, queue_dir)
 908
 909 def do_accept():
 910     print "ACCEPT"
 911     if not Options["No-Action"]:
 912         get_accept_lock()
 913         (summary, short_summary) = Upload.build_summaries()
 914     try:
 915         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 916             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 917             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 918             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 919             # Check for override disparities
 920             Upload.Subst["__SUMMARY__"] = summary
 921         else:
 922             # Stable updates need to be copied to proposed-updates holding
 923             # area instead of accepted.  Sourceful uploads need to go
 924             # to it directly, binaries only if the source has not yet been
 925             # accepted into p-u.
 926             for suite, q in [("proposed-updates", "ProposedUpdates"),
 927                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 928                 if not Upload.pkg.changes["distribution"].has_key(suite):
 929                     continue
 930                 return do_accept_stableupdate(suite, q)
 931             # Just a normal upload, accept it...
 932             _accept()
 933     finally:
 934         if not Options["No-Action"]:
 935             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 936
 937 def check_status(files):
 938     new = byhand = 0
 939     for f in files.keys():
 940         if files[f]["type"] == "byhand":
 941             byhand = 1
 942         elif files[f].has_key("new"):
 943             new = 1
 944     return (new, byhand)
 945
 946 def do_pkg(changes_file):
 947     Upload.pkg.changes_file = changes_file
 948     Upload.init_vars()
 949     Upload.update_vars()
 950     Upload.update_subst()
 951     files = Upload.pkg.files
 952
 953     if not recheck():
 954         return
 955
 956     (new, byhand) = check_status(files)
 957     if new or byhand:
 958         if new:
 959             do_new()
 960         if byhand:
 961             do_byhand()
 962         (new, byhand) = check_status(files)
 963
 964     if not new and not byhand:
 965         do_accept()
 966
 967 ################################################################################
 968
 969 def end():
 970     accept_count = Upload.accept_count
 971     accept_bytes = Upload.accept_bytes
 972
 973     if accept_count:
 974         sets = "set"
 975         if accept_count > 1:
 976             sets = "sets"
 977         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 978         Logger.log(["total",accept_count,accept_bytes])
 979
 980     if not Options["No-Action"]:
 981         Logger.close()
 982
 983 ################################################################################
 984
 985 def do_comments(dir, opref, npref, line, fn):
 986     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 987         lines = open("%s/%s" % (dir, comm)).readlines()
 988         if len(lines) == 0 or lines[0] != line + "\n": continue
 989         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 990                                 and x.endswith(".changes") ]
 991         changes_files = sort_changes(changes_files)
 992         for f in changes_files:
 993             f = utils.validate_changes_file_arg(f, 0)
 994             if not f: continue
 995             print "\n" + f
 996             fn(f, "".join(lines[1:]))
 997
 998         if opref != npref and not Options["No-Action"]:
 999             newcomm = npref + comm[len(opref):]
1000             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1001
1002 ################################################################################
1003
1004 def comment_accept(changes_file, comments):
1005     Upload.pkg.changes_file = changes_file
1006     Upload.init_vars()
1007     Upload.update_vars()
1008     Upload.update_subst()
1009     files = Upload.pkg.files
1010
1011     if not recheck():
1012         return # dak wants to REJECT, crap
1013
1014     (new, byhand) = check_status(files)
1015     if not new and not byhand:
1016         do_accept()
1017
1018 ################################################################################
1019
1020 def comment_reject(changes_file, comments):
1021     Upload.pkg.changes_file = changes_file
1022     Upload.init_vars()
1023     Upload.update_vars()
1024     Upload.update_subst()
1025
1026     if not recheck():
1027         pass # dak has its own reasons to reject as well, which is fine
1028
1029     reject(comments)
1030     print "REJECT\n" + reject_message,
1031     if not Options["No-Action"]:
1032         Upload.do_reject(0, reject_message)
1033         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1034
1035 ################################################################################
1036
1037 def main():
1038     changes_files = init()
1039     if len(changes_files) > 50:
1040         sys.stderr.write("Sorting changes...\n")
1041     changes_files = sort_changes(changes_files)
1042
1043     # Kill me now? **FIXME**
1044     Cnf["Dinstall::Options::No-Mail"] = ""
1045     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1046     if Cnf.has_key("Dinstall::Bcc"):
1047         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1048     else:
1049         Upload.Subst["__BCC__"] = bcc
1050
1051     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1052     if commentsdir:
1053         if changes_files != []:
1054             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1055             sys.exit(1)
1056         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1057         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1058     else:
1059         for changes_file in changes_files:
1060             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1061             if not changes_file:
1062                 continue
1063             print "\n" + changes_file
1064             do_pkg (changes_file)
1065
1066     end()
1067
1068 ################################################################################
1069
1070 if __name__ == '__main__':
1071     main()