dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 import copy
  45 import errno
  46 import os
  47 import readline
  48 import stat
  49 import sys
  50 import time
  51 import apt_pkg, apt_inst
  52 import examine_package
  53 from daklib import database
  54 from daklib import logging
  55 from daklib import queue
  56 from daklib import utils
  57 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  58 from daklib.dak_exceptions import CantOpenError
  59
  60 # Globals
  61 Cnf = None       #: Configuration, apt_pkg.Configuration
  62 Options = None
  63 Upload = None
  64 projectB = None  #: database connection, pgobject
  65 Logger = None
  66
  67 Priorities = None
  68 Sections = None
  69
  70 reject_message = ""
  71
  72 ################################################################################
  73 ################################################################################
  74 ################################################################################
  75
  76 def reject (str, prefix="Rejected: "):
  77     global reject_message
  78     if str:
  79         reject_message += prefix + str + "\n"
  80
  81 def recheck():
  82     global reject_message
  83     files = Upload.pkg.files
  84     reject_message = ""
  85
  86     for f in files.keys():
  87         # The .orig.tar.gz can disappear out from under us is it's a
  88         # duplicate of one in the archive.
  89         if not files.has_key(f):
  90             continue
  91         # Check that the source still exists
  92         if files[f]["type"] == "deb":
  93             source_version = files[f]["source version"]
  94             source_package = files[f]["source package"]
  95             if not Upload.pkg.changes["architecture"].has_key("source") \
  96                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  97                 source_epochless_version = re_no_epoch.sub('', source_version)
  98                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  99                 found = 0
 100                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
 101                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 102                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 103                             found = 1
 104                 if not found:
 105                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 106
 107         # Version and file overwrite checks
 108         if files[f]["type"] == "deb":
 109             reject(Upload.check_binary_against_db(f), "")
 110         elif files[f]["type"] == "dsc":
 111             reject(Upload.check_source_against_db(f), "")
 112             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 113             reject(reject_msg, "")
 114
 115     if reject_message.find("Rejected") != -1:
 116         answer = "XXX"
 117         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 118             answer = 'S'
 119
 120         print "REJECT\n" + reject_message,
 121         prompt = "[R]eject, Skip, Quit ?"
 122
 123         while prompt.find(answer) == -1:
 124             answer = utils.our_raw_input(prompt)
 125             m = re_default_answer.match(prompt)
 126             if answer == "":
 127                 answer = m.group(1)
 128             answer = answer[:1].upper()
 129
 130         if answer == 'R':
 131             Upload.do_reject(0, reject_message)
 132             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 133             return 0
 134         elif answer == 'S':
 135             return 0
 136         elif answer == 'Q':
 137             end()
 138             sys.exit(0)
 139
 140     return 1
 141
 142 ################################################################################
 143
 144 def indiv_sg_compare (a, b):
 145     """Sort by source name, source, version, 'have source', and
 146        finally by filename."""
 147     # Sort by source version
 148     q = apt_pkg.VersionCompare(a["version"], b["version"])
 149     if q:
 150         return -q
 151
 152     # Sort by 'have source'
 153     a_has_source = a["architecture"].get("source")
 154     b_has_source = b["architecture"].get("source")
 155     if a_has_source and not b_has_source:
 156         return -1
 157     elif b_has_source and not a_has_source:
 158         return 1
 159
 160     return cmp(a["filename"], b["filename"])
 161
 162 ############################################################
 163
 164 def sg_compare (a, b):
 165     a = a[1]
 166     b = b[1]
 167     """Sort by have note, source already in database and time of oldest upload."""
 168     # Sort by have note
 169     a_note_state = a["note_state"]
 170     b_note_state = b["note_state"]
 171     if a_note_state < b_note_state:
 172         return -1
 173     elif a_note_state > b_note_state:
 174         return 1
 175     # Sort by source already in database (descending)
 176     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 177     if source_in_database:
 178         return -source_in_database
 179
 180     # Sort by time of oldest upload
 181     return cmp(a["oldest"], b["oldest"])
 182
 183 def sort_changes(changes_files):
 184     """Sort into source groups, then sort each source group by version,
 185     have source, filename.  Finally, sort the source groups by have
 186     note, time of oldest upload of each source upload."""
 187     if len(changes_files) == 1:
 188         return changes_files
 189
 190     sorted_list = []
 191     cache = {}
 192     # Read in all the .changes files
 193     for filename in changes_files:
 194         try:
 195             Upload.pkg.changes_file = filename
 196             Upload.init_vars()
 197             Upload.update_vars()
 198             cache[filename] = copy.copy(Upload.pkg.changes)
 199             cache[filename]["filename"] = filename
 200         except:
 201             sorted_list.append(filename)
 202             break
 203     # Divide the .changes into per-source groups
 204     per_source = {}
 205     for filename in cache.keys():
 206         source = cache[filename]["source"]
 207         if not per_source.has_key(source):
 208             per_source[source] = {}
 209             per_source[source]["list"] = []
 210         per_source[source]["list"].append(cache[filename])
 211     # Determine oldest time and have note status for each source group
 212     for source in per_source.keys():
 213         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 214         ql = q.getresult()
 215         per_source[source]["source_in_database"] = len(ql)>0
 216         source_list = per_source[source]["list"]
 217         first = source_list[0]
 218         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 219         have_note = 0
 220         for d in per_source[source]["list"]:
 221             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 222             if mtime < oldest:
 223                 oldest = mtime
 224             have_note += (database.has_new_comment(d["source"], d["version"]))
 225         per_source[source]["oldest"] = oldest
 226         if not have_note:
 227             per_source[source]["note_state"] = 0; # none
 228         elif have_note < len(source_list):
 229             per_source[source]["note_state"] = 1; # some
 230         else:
 231             per_source[source]["note_state"] = 2; # all
 232         per_source[source]["list"].sort(indiv_sg_compare)
 233     per_source_items = per_source.items()
 234     per_source_items.sort(sg_compare)
 235     for i in per_source_items:
 236         for j in i[1]["list"]:
 237             sorted_list.append(j["filename"])
 238     return sorted_list
 239
 240 ################################################################################
 241
 242 class Section_Completer:
 243     def __init__ (self):
 244         self.sections = []
 245         self.matches = []
 246         q = projectB.query("SELECT section FROM section")
 247         for i in q.getresult():
 248             self.sections.append(i[0])
 249
 250     def complete(self, text, state):
 251         if state == 0:
 252             self.matches = []
 253             n = len(text)
 254             for word in self.sections:
 255                 if word[:n] == text:
 256                     self.matches.append(word)
 257         try:
 258             return self.matches[state]
 259         except IndexError:
 260             return None
 261
 262 ############################################################
 263
 264 class Priority_Completer:
 265     def __init__ (self):
 266         self.priorities = []
 267         self.matches = []
 268         q = projectB.query("SELECT priority FROM priority")
 269         for i in q.getresult():
 270             self.priorities.append(i[0])
 271
 272     def complete(self, text, state):
 273         if state == 0:
 274             self.matches = []
 275             n = len(text)
 276             for word in self.priorities:
 277                 if word[:n] == text:
 278                     self.matches.append(word)
 279         try:
 280             return self.matches[state]
 281         except IndexError:
 282             return None
 283
 284 ################################################################################
 285
 286 def print_new (new, indexed, file=sys.stdout):
 287     queue.check_valid(new)
 288     broken = 0
 289     index = 0
 290     for pkg in new.keys():
 291         index += 1
 292         section = new[pkg]["section"]
 293         priority = new[pkg]["priority"]
 294         if new[pkg]["section id"] == -1:
 295             section += "[!]"
 296             broken = 1
 297         if new[pkg]["priority id"] == -1:
 298             priority += "[!]"
 299             broken = 1
 300         if indexed:
 301             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 302         else:
 303             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 304         line = line.strip()+'\n'
 305         file.write(line)
 306     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 307     if len(note) > 0:
 308         for line in note:
 309             print line
 310     return broken, note
 311
 312 ################################################################################
 313
 314 def index_range (index):
 315     if index == 1:
 316         return "1"
 317     else:
 318         return "1-%s" % (index)
 319
 320 ################################################################################
 321 ################################################################################
 322
 323 def edit_new (new):
 324     # Write the current data to a temporary file
 325     (fd, temp_filename) = utils.temp_filename()
 326     temp_file = os.fdopen(fd, 'w')
 327     print_new (new, 0, temp_file)
 328     temp_file.close()
 329     # Spawn an editor on that file
 330     editor = os.environ.get("EDITOR","vi")
 331     result = os.system("%s %s" % (editor, temp_filename))
 332     if result != 0:
 333         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 334     # Read the edited data back in
 335     temp_file = utils.open_file(temp_filename)
 336     lines = temp_file.readlines()
 337     temp_file.close()
 338     os.unlink(temp_filename)
 339     # Parse the new data
 340     for line in lines:
 341         line = line.strip()
 342         if line == "":
 343             continue
 344         s = line.split()
 345         # Pad the list if necessary
 346         s[len(s):3] = [None] * (3-len(s))
 347         (pkg, priority, section) = s[:3]
 348         if not new.has_key(pkg):
 349             utils.warn("Ignoring unknown package '%s'" % (pkg))
 350         else:
 351             # Strip off any invalid markers, print_new will readd them.
 352             if section.endswith("[!]"):
 353                 section = section[:-3]
 354             if priority.endswith("[!]"):
 355                 priority = priority[:-3]
 356             for f in new[pkg]["files"]:
 357                 Upload.pkg.files[f]["section"] = section
 358                 Upload.pkg.files[f]["priority"] = priority
 359             new[pkg]["section"] = section
 360             new[pkg]["priority"] = priority
 361
 362 ################################################################################
 363
 364 def edit_index (new, index):
 365     priority = new[index]["priority"]
 366     section = new[index]["section"]
 367     ftype = new[index]["type"]
 368     done = 0
 369     while not done:
 370         print "\t".join([index, priority, section])
 371
 372         answer = "XXX"
 373         if ftype != "dsc":
 374             prompt = "[B]oth, Priority, Section, Done ? "
 375         else:
 376             prompt = "[S]ection, Done ? "
 377         edit_priority = edit_section = 0
 378
 379         while prompt.find(answer) == -1:
 380             answer = utils.our_raw_input(prompt)
 381             m = re_default_answer.match(prompt)
 382             if answer == "":
 383                 answer = m.group(1)
 384             answer = answer[:1].upper()
 385
 386         if answer == 'P':
 387             edit_priority = 1
 388         elif answer == 'S':
 389             edit_section = 1
 390         elif answer == 'B':
 391             edit_priority = edit_section = 1
 392         elif answer == 'D':
 393             done = 1
 394
 395         # Edit the priority
 396         if edit_priority:
 397             readline.set_completer(Priorities.complete)
 398             got_priority = 0
 399             while not got_priority:
 400                 new_priority = utils.our_raw_input("New priority: ").strip()
 401                 if new_priority not in Priorities.priorities:
 402                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 403                 else:
 404                     got_priority = 1
 405                     priority = new_priority
 406
 407         # Edit the section
 408         if edit_section:
 409             readline.set_completer(Sections.complete)
 410             got_section = 0
 411             while not got_section:
 412                 new_section = utils.our_raw_input("New section: ").strip()
 413                 if new_section not in Sections.sections:
 414                     print "E: '%s' is not a valid section, try again." % (new_section)
 415                 else:
 416                     got_section = 1
 417                     section = new_section
 418
 419         # Reset the readline completer
 420         readline.set_completer(None)
 421
 422     for f in new[index]["files"]:
 423         Upload.pkg.files[f]["section"] = section
 424         Upload.pkg.files[f]["priority"] = priority
 425     new[index]["priority"] = priority
 426     new[index]["section"] = section
 427     return new
 428
 429 ################################################################################
 430
 431 def edit_overrides (new):
 432     print
 433     done = 0
 434     while not done:
 435         print_new (new, 1)
 436         new_index = {}
 437         index = 0
 438         for i in new.keys():
 439             index += 1
 440             new_index[index] = i
 441
 442         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 443
 444         got_answer = 0
 445         while not got_answer:
 446             answer = utils.our_raw_input(prompt)
 447             if not answer.isdigit():
 448                 answer = answer[:1].upper()
 449             if answer == "E" or answer == "D":
 450                 got_answer = 1
 451             elif re_isanum.match (answer):
 452                 answer = int(answer)
 453                 if (answer < 1) or (answer > index):
 454                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 455                 else:
 456                     got_answer = 1
 457
 458         if answer == 'E':
 459             edit_new(new)
 460         elif answer == 'D':
 461             done = 1
 462         else:
 463             edit_index (new, new_index[answer])
 464
 465     return new
 466
 467 ################################################################################
 468
 469 def edit_note(note):
 470     # Write the current data to a temporary file
 471     (fd, temp_filename) = utils.temp_filename()
 472     editor = os.environ.get("EDITOR","vi")
 473     answer = 'E'
 474     while answer == 'E':
 475         os.system("%s %s" % (editor, temp_filename))
 476         temp_file = utils.open_file(temp_filename)
 477         newnote = temp_file.read().rstrip()
 478         temp_file.close()
 479         print "New Note:"
 480         print utils.prefix_multi_line_string(newnote,"  ")
 481         prompt = "[D]one, Edit, Abandon, Quit ?"
 482         answer = "XXX"
 483         while prompt.find(answer) == -1:
 484             answer = utils.our_raw_input(prompt)
 485             m = re_default_answer.search(prompt)
 486             if answer == "":
 487                 answer = m.group(1)
 488             answer = answer[:1].upper()
 489     os.unlink(temp_filename)
 490     if answer == 'A':
 491         return
 492     elif answer == 'Q':
 493         end()
 494         sys.exit(0)
 495     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami())
 496
 497 ################################################################################
 498
 499 def check_pkg ():
 500     try:
 501         less_fd = os.popen("less -R -", 'w', 0)
 502         stdout_fd = sys.stdout
 503         try:
 504             sys.stdout = less_fd
 505             changes = utils.parse_changes (Upload.pkg.changes_file)
 506             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 507             files = Upload.pkg.files
 508             for f in files.keys():
 509                 if files[f].has_key("new"):
 510                     ftype = files[f]["type"]
 511                     if ftype == "deb":
 512                         examine_package.check_deb(changes['distribution'], f)
 513                     elif ftype == "dsc":
 514                         examine_package.check_dsc(changes['distribution'], f)
 515         finally:
 516             examine_package.output_package_relations()
 517             sys.stdout = stdout_fd
 518     except IOError, e:
 519         if e.errno == errno.EPIPE:
 520             utils.warn("[examine_package] Caught EPIPE; skipping.")
 521             pass
 522         else:
 523             raise
 524     except KeyboardInterrupt:
 525         utils.warn("[examine_package] Caught C-c; skipping.")
 526         pass
 527
 528 ################################################################################
 529
 530 ## FIXME: horribly Debian specific
 531
 532 def do_bxa_notification():
 533     files = Upload.pkg.files
 534     summary = ""
 535     for f in files.keys():
 536         if files[f]["type"] == "deb":
 537             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 538             summary += "\n"
 539             summary += "Package: %s\n" % (control.Find("Package"))
 540             summary += "Description: %s\n" % (control.Find("Description"))
 541     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 542     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 543     utils.send_mail(bxa_mail)
 544
 545 ################################################################################
 546
 547 def add_overrides (new):
 548     changes = Upload.pkg.changes
 549     files = Upload.pkg.files
 550
 551     projectB.query("BEGIN WORK")
 552     for suite in changes["suite"].keys():
 553         suite_id = database.get_suite_id(suite)
 554         for pkg in new.keys():
 555             component_id = database.get_component_id(new[pkg]["component"])
 556             type_id = database.get_override_type_id(new[pkg]["type"])
 557             priority_id = new[pkg]["priority id"]
 558             section_id = new[pkg]["section id"]
 559             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 560             for f in new[pkg]["files"]:
 561                 if files[f].has_key("new"):
 562                     del files[f]["new"]
 563             del new[pkg]
 564
 565     projectB.query("COMMIT WORK")
 566
 567     if Cnf.FindB("Dinstall::BXANotify"):
 568         do_bxa_notification()
 569
 570 ################################################################################
 571
 572 def prod_maintainer (note):
 573     # Here we prepare an editor and get them ready to prod...
 574     (fd, temp_filename) = utils.temp_filename()
 575     temp_file = os.fdopen(fd, 'w')
 576     if len(note) > 0:
 577         for line in note:
 578             temp_file.write(line)
 579     temp_file.close()
 580     editor = os.environ.get("EDITOR","vi")
 581     answer = 'E'
 582     while answer == 'E':
 583         os.system("%s %s" % (editor, temp_filename))
 584         temp_fh = utils.open_file(temp_filename)
 585         prod_message = "".join(temp_fh.readlines())
 586         temp_fh.close()
 587         print "Prod message:"
 588         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 589         prompt = "[P]rod, Edit, Abandon, Quit ?"
 590         answer = "XXX"
 591         while prompt.find(answer) == -1:
 592             answer = utils.our_raw_input(prompt)
 593             m = re_default_answer.search(prompt)
 594             if answer == "":
 595                 answer = m.group(1)
 596             answer = answer[:1].upper()
 597         os.unlink(temp_filename)
 598         if answer == 'A':
 599             return
 600         elif answer == 'Q':
 601             end()
 602             sys.exit(0)
 603     # Otherwise, do the proding...
 604     user_email_address = utils.whoami() + " <%s>" % (
 605         Cnf["Dinstall::MyAdminAddress"])
 606
 607     Subst = Upload.Subst
 608
 609     Subst["__FROM_ADDRESS__"] = user_email_address
 610     Subst["__PROD_MESSAGE__"] = prod_message
 611     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 612
 613     prod_mail_message = utils.TemplateSubst(
 614         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 615
 616     # Send the prod mail if appropriate
 617     if not Cnf["Dinstall::Options::No-Mail"]:
 618         utils.send_mail(prod_mail_message)
 619
 620     print "Sent proding message"
 621
 622 ################################################################################
 623
 624 def do_new():
 625     print "NEW\n"
 626     files = Upload.pkg.files
 627     changes = Upload.pkg.changes
 628
 629     # Make a copy of distribution we can happily trample on
 630     changes["suite"] = copy.copy(changes["distribution"])
 631
 632     # Fix up the list of target suites
 633     for suite in changes["suite"].keys():
 634         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 635         if override:
 636             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 637               database.get_suite_id(override) == -1)
 638             if olderr or newerr:
 639                 (oinv, newinv) = ("", "")
 640                 if olderr: oinv = "invalid "
 641                 if newerr: ninv = "invalid "
 642                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 643                         oinv, suite, ninv, override)
 644             del changes["suite"][suite]
 645             changes["suite"][override] = 1
 646     # Validate suites
 647     for suite in changes["suite"].keys():
 648         suite_id = database.get_suite_id(suite)
 649         if suite_id == -1:
 650             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 651
 652     # The main NEW processing loop
 653     done = 0
 654     while not done:
 655         # Find out what's new
 656         new = queue.determine_new(changes, files, projectB)
 657
 658         if not new:
 659             break
 660
 661         answer = "XXX"
 662         if Options["No-Action"] or Options["Automatic"]:
 663             answer = 'S'
 664
 665         (broken, note) = print_new(new, 0)
 666         prompt = ""
 667
 668         if not broken and not note:
 669             prompt = "Add overrides, "
 670         if broken:
 671             print "W: [!] marked entries must be fixed before package can be processed."
 672         if note:
 673             print "W: note must be removed before package can be processed."
 674             prompt += "Remove note, "
 675
 676         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 677
 678         while prompt.find(answer) == -1:
 679             answer = utils.our_raw_input(prompt)
 680             m = re_default_answer.search(prompt)
 681             if answer == "":
 682                 answer = m.group(1)
 683             answer = answer[:1].upper()
 684
 685         if answer == 'A' and not Options["Trainee"]:
 686             done = add_overrides (new)
 687         elif answer == 'C':
 688             check_pkg()
 689         elif answer == 'E' and not Options["Trainee"]:
 690             new = edit_overrides (new)
 691         elif answer == 'M' and not Options["Trainee"]:
 692             aborted = Upload.do_reject(manual=1,
 693                                        reject_message=Options["Manual-Reject"],
 694                                        note=database.get_new_comments(changes.get("source", "")))
 695             if not aborted:
 696                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 697                 done = 1
 698         elif answer == 'N':
 699             edit_note(database.get_new_comments(changes.get("source", "")))
 700         elif answer == 'P' and not Options["Trainee"]:
 701             prod_maintainer(database.get_new_comments(changes.get("source", "")))
 702         elif answer == 'R':
 703             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 704             if confirm == "y":
 705                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 706         elif answer == 'S':
 707             done = 1
 708         elif answer == 'Q':
 709             end()
 710             sys.exit(0)
 711
 712 ################################################################################
 713 ################################################################################
 714 ################################################################################
 715
 716 def usage (exit_code=0):
 717     print """Usage: dak process-new [OPTION]... [CHANGES]...
 718   -a, --automatic           automatic run
 719   -h, --help                show this help and exit.
 720   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 721   -m, --manual-reject=MSG   manual reject with `msg'
 722   -n, --no-action           don't do anything
 723   -t, --trainee             FTP Trainee mode
 724   -V, --version             display the version number and exit"""
 725     sys.exit(exit_code)
 726
 727 ################################################################################
 728
 729 def init():
 730     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 731
 732     Cnf = utils.get_conf()
 733
 734     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 735                  ('h',"help","Process-New::Options::Help"),
 736                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 737                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 738                  ('t',"trainee","Process-New::Options::Trainee"),
 739                  ('n',"no-action","Process-New::Options::No-Action")]
 740
 741     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 742         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 743             Cnf["Process-New::Options::%s" % (i)] = ""
 744
 745     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 746     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 747         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 748
 749     Options = Cnf.SubTree("Process-New::Options")
 750
 751     if Options["Help"]:
 752         usage()
 753
 754     Upload = queue.Upload(Cnf)
 755
 756     if not Options["No-Action"]:
 757         try:
 758             Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 759         except CantOpenError, e:
 760             Options["Trainee"] = "Oh yes"
 761
 762     projectB = Upload.projectB
 763
 764     Sections = Section_Completer()
 765     Priorities = Priority_Completer()
 766     readline.parse_and_bind("tab: complete")
 767
 768     return changes_files
 769
 770 ################################################################################
 771
 772 def do_byhand():
 773     done = 0
 774     while not done:
 775         files = Upload.pkg.files
 776         will_install = 1
 777         byhand = []
 778
 779         for f in files.keys():
 780             if files[f]["type"] == "byhand":
 781                 if os.path.exists(f):
 782                     print "W: %s still present; please process byhand components and try again." % (f)
 783                     will_install = 0
 784                 else:
 785                     byhand.append(f)
 786
 787         answer = "XXXX"
 788         if Options["No-Action"]:
 789             answer = "S"
 790         if will_install:
 791             if Options["Automatic"] and not Options["No-Action"]:
 792                 answer = 'A'
 793             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 794         else:
 795             prompt = "Manual reject, [S]kip, Quit ?"
 796
 797         while prompt.find(answer) == -1:
 798             answer = utils.our_raw_input(prompt)
 799             m = re_default_answer.search(prompt)
 800             if answer == "":
 801                 answer = m.group(1)
 802             answer = answer[:1].upper()
 803
 804         if answer == 'A':
 805             done = 1
 806             for f in byhand:
 807                 del files[f]
 808         elif answer == 'M':
 809             Upload.do_reject(1, Options["Manual-Reject"])
 810             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 811             done = 1
 812         elif answer == 'S':
 813             done = 1
 814         elif answer == 'Q':
 815             end()
 816             sys.exit(0)
 817
 818 ################################################################################
 819
 820 def get_accept_lock():
 821     retry = 0
 822     while retry < 10:
 823         try:
 824             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 825             retry = 10
 826         except OSError, e:
 827             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 828                 retry += 1
 829                 if (retry >= 10):
 830                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 831                 else:
 832                     print("Unable to get accepted lock (try %d of 10)" % retry)
 833                 time.sleep(60)
 834             else:
 835                 raise
 836
 837 def move_to_dir (dest, perms=0660, changesperms=0664):
 838     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 839     file_keys = Upload.pkg.files.keys()
 840     for f in file_keys:
 841         utils.move (f, dest, perms=perms)
 842
 843 def is_source_in_queue_dir(qdir):
 844     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 845                 and x.endswith(".changes") ]
 846     for entry in entries:
 847         # read the .dak
 848         u = queue.Upload(Cnf)
 849         u.pkg.changes_file = os.path.join(qdir, entry)
 850         u.update_vars()
 851         if not u.pkg.changes["architecture"].has_key("source"):
 852             # another binary upload, ignore
 853             continue
 854         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 855             # another version, ignore
 856             continue
 857         # found it!
 858         return True
 859     return False
 860
 861 def move_to_holding(suite, queue_dir):
 862     print "Moving to %s holding area." % (suite.upper(),)
 863     if Options["No-Action"]:
 864         return
 865     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 866     Upload.dump_vars(queue_dir)
 867     move_to_dir(queue_dir, perms=0664)
 868     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 869
 870 def _accept():
 871     if Options["No-Action"]:
 872         return
 873     (summary, short_summary) = Upload.build_summaries()
 874     Upload.accept(summary, short_summary)
 875     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 876
 877 def do_accept_stableupdate(suite, q):
 878     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 879     if not Upload.pkg.changes["architecture"].has_key("source"):
 880         # It is not a sourceful upload.  So its source may be either in p-u
 881         # holding, in new, in accepted or already installed.
 882         if is_source_in_queue_dir(queue_dir):
 883             # It's in p-u holding, so move it there.
 884             print "Binary-only upload, source in %s." % (q,)
 885             move_to_holding(suite, queue_dir)
 886         elif Upload.source_exists(Upload.pkg.changes["source"],
 887                 Upload.pkg.changes["version"]):
 888             # dak tells us that there is source available.  At time of
 889             # writing this means that it is installed, so put it into
 890             # accepted.
 891             print "Binary-only upload, source installed."
 892             _accept()
 893         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 894             # The source is in accepted, the binary cleared NEW: accept it.
 895             print "Binary-only upload, source in accepted."
 896             _accept()
 897         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 898             # It's in NEW.  We expect the source to land in p-u holding
 899             # pretty soon.
 900             print "Binary-only upload, source in new."
 901             move_to_holding(suite, queue_dir)
 902         else:
 903             # No case applicable.  Bail out.  Return will cause the upload
 904             # to be skipped.
 905             print "ERROR"
 906             print "Stable update failed.  Source not found."
 907             return
 908     else:
 909         # We are handling a sourceful upload.  Move to accepted if currently
 910         # in p-u holding and to p-u holding otherwise.
 911         if is_source_in_queue_dir(queue_dir):
 912             print "Sourceful upload in %s, accepting." % (q,)
 913             _accept()
 914         else:
 915             move_to_holding(suite, queue_dir)
 916
 917 def do_accept():
 918     print "ACCEPT"
 919     if not Options["No-Action"]:
 920         get_accept_lock()
 921         (summary, short_summary) = Upload.build_summaries()
 922     try:
 923         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 924             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 925             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 926             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 927             # Check for override disparities
 928             Upload.Subst["__SUMMARY__"] = summary
 929         else:
 930             # Stable updates need to be copied to proposed-updates holding
 931             # area instead of accepted.  Sourceful uploads need to go
 932             # to it directly, binaries only if the source has not yet been
 933             # accepted into p-u.
 934             for suite, q in [("proposed-updates", "ProposedUpdates"),
 935                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 936                 if not Upload.pkg.changes["distribution"].has_key(suite):
 937                     continue
 938                 return do_accept_stableupdate(suite, q)
 939             # Just a normal upload, accept it...
 940             _accept()
 941     finally:
 942         if not Options["No-Action"]:
 943             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 944
 945 def check_status(files):
 946     new = byhand = 0
 947     for f in files.keys():
 948         if files[f]["type"] == "byhand":
 949             byhand = 1
 950         elif files[f].has_key("new"):
 951             new = 1
 952     return (new, byhand)
 953
 954 def do_pkg(changes_file):
 955     Upload.pkg.changes_file = changes_file
 956     Upload.init_vars()
 957     Upload.update_vars()
 958     Upload.update_subst()
 959     files = Upload.pkg.files
 960
 961     if not recheck():
 962         return
 963
 964     (new, byhand) = check_status(files)
 965     if new or byhand:
 966         if new:
 967             do_new()
 968         if byhand:
 969             do_byhand()
 970         (new, byhand) = check_status(files)
 971
 972     if not new and not byhand:
 973         do_accept()
 974
 975 ################################################################################
 976
 977 def end():
 978     accept_count = Upload.accept_count
 979     accept_bytes = Upload.accept_bytes
 980
 981     if accept_count:
 982         sets = "set"
 983         if accept_count > 1:
 984             sets = "sets"
 985         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 986         Logger.log(["total",accept_count,accept_bytes])
 987
 988     if not Options["No-Action"] and not Options["Trainee"]:
 989         Logger.close()
 990
 991 ################################################################################
 992
 993 def do_comments(dir, opref, npref, line, fn):
 994     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 995         lines = open("%s/%s" % (dir, comm)).readlines()
 996         if len(lines) == 0 or lines[0] != line + "\n": continue
 997         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 998                                 and x.endswith(".changes") ]
 999         changes_files = sort_changes(changes_files)
1000         for f in changes_files:
1001             f = utils.validate_changes_file_arg(f, 0)
1002             if not f: continue
1003             print "\n" + f
1004             fn(f, "".join(lines[1:]))
1005
1006         if opref != npref and not Options["No-Action"]:
1007             newcomm = npref + comm[len(opref):]
1008             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1009
1010 ################################################################################
1011
1012 def comment_accept(changes_file, comments):
1013     Upload.pkg.changes_file = changes_file
1014     Upload.init_vars()
1015     Upload.update_vars()
1016     Upload.update_subst()
1017     files = Upload.pkg.files
1018
1019     if not recheck():
1020         return # dak wants to REJECT, crap
1021
1022     (new, byhand) = check_status(files)
1023     if not new and not byhand:
1024         do_accept()
1025
1026 ################################################################################
1027
1028 def comment_reject(changes_file, comments):
1029     Upload.pkg.changes_file = changes_file
1030     Upload.init_vars()
1031     Upload.update_vars()
1032     Upload.update_subst()
1033
1034     if not recheck():
1035         pass # dak has its own reasons to reject as well, which is fine
1036
1037     reject(comments)
1038     print "REJECT\n" + reject_message,
1039     if not Options["No-Action"]:
1040         Upload.do_reject(0, reject_message)
1041         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1042
1043 ################################################################################
1044
1045 def main():
1046     changes_files = init()
1047     if len(changes_files) > 50:
1048         sys.stderr.write("Sorting changes...\n")
1049     changes_files = sort_changes(changes_files)
1050
1051     # Kill me now? **FIXME**
1052     Cnf["Dinstall::Options::No-Mail"] = ""
1053     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1054     if Cnf.has_key("Dinstall::Bcc"):
1055         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1056     else:
1057         Upload.Subst["__BCC__"] = bcc
1058
1059     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1060     if commentsdir:
1061         if changes_files != []:
1062             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1063             sys.exit(1)
1064         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1065         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1066     else:
1067         for changes_file in changes_files:
1068             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1069             if not changes_file:
1070                 continue
1071             print "\n" + changes_file
1072             do_pkg (changes_file)
1073
1074     end()
1075
1076 ################################################################################
1077
1078 if __name__ == '__main__':
1079     main()