dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 from __future__ import with_statement
  45
  46 import copy
  47 import errno
  48 import os
  49 import readline
  50 import stat
  51 import sys
  52 import time
  53 import contextlib
  54 import pwd
  55 import apt_pkg, apt_inst
  56 import examine_package
  57 from daklib import database
  58 from daklib import logging
  59 from daklib import queue
  60 from daklib import utils
  61 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  62 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  63
  64 # Globals
  65 Cnf = None       #: Configuration, apt_pkg.Configuration
  66 Options = None
  67 Upload = None
  68 projectB = None  #: database connection, pgobject
  69 Logger = None
  70
  71 Priorities = None
  72 Sections = None
  73
  74 reject_message = ""
  75
  76 ################################################################################
  77 ################################################################################
  78 ################################################################################
  79
  80 def reject (str, prefix="Rejected: "):
  81     global reject_message
  82     if str:
  83         reject_message += prefix + str + "\n"
  84
  85 def recheck():
  86     global reject_message
  87     files = Upload.pkg.files
  88     reject_message = ""
  89
  90     for f in files.keys():
  91         # The .orig.tar.gz can disappear out from under us is it's a
  92         # duplicate of one in the archive.
  93         if not files.has_key(f):
  94             continue
  95         # Check that the source still exists
  96         if files[f]["type"] == "deb":
  97             source_version = files[f]["source version"]
  98             source_package = files[f]["source package"]
  99             if not Upload.pkg.changes["architecture"].has_key("source") \
 100                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
 101                 source_epochless_version = re_no_epoch.sub('', source_version)
 102                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 103                 found = 0
 104                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
 105                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 106                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 107                             found = 1
 108                 if not found:
 109                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 110
 111         # Version and file overwrite checks
 112         if files[f]["type"] == "deb":
 113             reject(Upload.check_binary_against_db(f), "")
 114         elif files[f]["type"] == "dsc":
 115             reject(Upload.check_source_against_db(f), "")
 116             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 117             reject(reject_msg, "")
 118
 119     if reject_message.find("Rejected") != -1:
 120         answer = "XXX"
 121         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 122             answer = 'S'
 123
 124         print "REJECT\n" + reject_message,
 125         prompt = "[R]eject, Skip, Quit ?"
 126
 127         while prompt.find(answer) == -1:
 128             answer = utils.our_raw_input(prompt)
 129             m = re_default_answer.match(prompt)
 130             if answer == "":
 131                 answer = m.group(1)
 132             answer = answer[:1].upper()
 133
 134         if answer == 'R':
 135             Upload.do_reject(0, reject_message)
 136             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 137             return 0
 138         elif answer == 'S':
 139             return 0
 140         elif answer == 'Q':
 141             end()
 142             sys.exit(0)
 143
 144     return 1
 145
 146 ################################################################################
 147
 148 def indiv_sg_compare (a, b):
 149     """Sort by source name, source, version, 'have source', and
 150        finally by filename."""
 151     # Sort by source version
 152     q = apt_pkg.VersionCompare(a["version"], b["version"])
 153     if q:
 154         return -q
 155
 156     # Sort by 'have source'
 157     a_has_source = a["architecture"].get("source")
 158     b_has_source = b["architecture"].get("source")
 159     if a_has_source and not b_has_source:
 160         return -1
 161     elif b_has_source and not a_has_source:
 162         return 1
 163
 164     return cmp(a["filename"], b["filename"])
 165
 166 ############################################################
 167
 168 def sg_compare (a, b):
 169     a = a[1]
 170     b = b[1]
 171     """Sort by have note, source already in database and time of oldest upload."""
 172     # Sort by have note
 173     a_note_state = a["note_state"]
 174     b_note_state = b["note_state"]
 175     if a_note_state < b_note_state:
 176         return -1
 177     elif a_note_state > b_note_state:
 178         return 1
 179     # Sort by source already in database (descending)
 180     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 181     if source_in_database:
 182         return -source_in_database
 183
 184     # Sort by time of oldest upload
 185     return cmp(a["oldest"], b["oldest"])
 186
 187 def sort_changes(changes_files):
 188     """Sort into source groups, then sort each source group by version,
 189     have source, filename.  Finally, sort the source groups by have
 190     note, time of oldest upload of each source upload."""
 191     if len(changes_files) == 1:
 192         return changes_files
 193
 194     sorted_list = []
 195     cache = {}
 196     # Read in all the .changes files
 197     for filename in changes_files:
 198         try:
 199             Upload.pkg.changes_file = filename
 200             Upload.init_vars()
 201             Upload.update_vars()
 202             cache[filename] = copy.copy(Upload.pkg.changes)
 203             cache[filename]["filename"] = filename
 204         except:
 205             sorted_list.append(filename)
 206             break
 207     # Divide the .changes into per-source groups
 208     per_source = {}
 209     for filename in cache.keys():
 210         source = cache[filename]["source"]
 211         if not per_source.has_key(source):
 212             per_source[source] = {}
 213             per_source[source]["list"] = []
 214         per_source[source]["list"].append(cache[filename])
 215     # Determine oldest time and have note status for each source group
 216     for source in per_source.keys():
 217         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 218         ql = q.getresult()
 219         per_source[source]["source_in_database"] = len(ql)>0
 220         source_list = per_source[source]["list"]
 221         first = source_list[0]
 222         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 223         have_note = 0
 224         for d in per_source[source]["list"]:
 225             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 226             if mtime < oldest:
 227                 oldest = mtime
 228             have_note += (database.has_new_comment(d["source"], d["version"]))
 229         per_source[source]["oldest"] = oldest
 230         if not have_note:
 231             per_source[source]["note_state"] = 0; # none
 232         elif have_note < len(source_list):
 233             per_source[source]["note_state"] = 1; # some
 234         else:
 235             per_source[source]["note_state"] = 2; # all
 236         per_source[source]["list"].sort(indiv_sg_compare)
 237     per_source_items = per_source.items()
 238     per_source_items.sort(sg_compare)
 239     for i in per_source_items:
 240         for j in i[1]["list"]:
 241             sorted_list.append(j["filename"])
 242     return sorted_list
 243
 244 ################################################################################
 245
 246 class Section_Completer:
 247     def __init__ (self):
 248         self.sections = []
 249         self.matches = []
 250         q = projectB.query("SELECT section FROM section")
 251         for i in q.getresult():
 252             self.sections.append(i[0])
 253
 254     def complete(self, text, state):
 255         if state == 0:
 256             self.matches = []
 257             n = len(text)
 258             for word in self.sections:
 259                 if word[:n] == text:
 260                     self.matches.append(word)
 261         try:
 262             return self.matches[state]
 263         except IndexError:
 264             return None
 265
 266 ############################################################
 267
 268 class Priority_Completer:
 269     def __init__ (self):
 270         self.priorities = []
 271         self.matches = []
 272         q = projectB.query("SELECT priority FROM priority")
 273         for i in q.getresult():
 274             self.priorities.append(i[0])
 275
 276     def complete(self, text, state):
 277         if state == 0:
 278             self.matches = []
 279             n = len(text)
 280             for word in self.priorities:
 281                 if word[:n] == text:
 282                     self.matches.append(word)
 283         try:
 284             return self.matches[state]
 285         except IndexError:
 286             return None
 287
 288 ################################################################################
 289
 290 def print_new (new, indexed, file=sys.stdout):
 291     queue.check_valid(new)
 292     broken = 0
 293     index = 0
 294     for pkg in new.keys():
 295         index += 1
 296         section = new[pkg]["section"]
 297         priority = new[pkg]["priority"]
 298         if new[pkg]["section id"] == -1:
 299             section += "[!]"
 300             broken = 1
 301         if new[pkg]["priority id"] == -1:
 302             priority += "[!]"
 303             broken = 1
 304         if indexed:
 305             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 306         else:
 307             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 308         line = line.strip()+'\n'
 309         file.write(line)
 310     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 311     if len(note) > 0:
 312         for line in note:
 313             print line
 314     return broken, note
 315
 316 ################################################################################
 317
 318 def index_range (index):
 319     if index == 1:
 320         return "1"
 321     else:
 322         return "1-%s" % (index)
 323
 324 ################################################################################
 325 ################################################################################
 326
 327 def edit_new (new):
 328     # Write the current data to a temporary file
 329     (fd, temp_filename) = utils.temp_filename()
 330     temp_file = os.fdopen(fd, 'w')
 331     print_new (new, 0, temp_file)
 332     temp_file.close()
 333     # Spawn an editor on that file
 334     editor = os.environ.get("EDITOR","vi")
 335     result = os.system("%s %s" % (editor, temp_filename))
 336     if result != 0:
 337         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 338     # Read the edited data back in
 339     temp_file = utils.open_file(temp_filename)
 340     lines = temp_file.readlines()
 341     temp_file.close()
 342     os.unlink(temp_filename)
 343     # Parse the new data
 344     for line in lines:
 345         line = line.strip()
 346         if line == "":
 347             continue
 348         s = line.split()
 349         # Pad the list if necessary
 350         s[len(s):3] = [None] * (3-len(s))
 351         (pkg, priority, section) = s[:3]
 352         if not new.has_key(pkg):
 353             utils.warn("Ignoring unknown package '%s'" % (pkg))
 354         else:
 355             # Strip off any invalid markers, print_new will readd them.
 356             if section.endswith("[!]"):
 357                 section = section[:-3]
 358             if priority.endswith("[!]"):
 359                 priority = priority[:-3]
 360             for f in new[pkg]["files"]:
 361                 Upload.pkg.files[f]["section"] = section
 362                 Upload.pkg.files[f]["priority"] = priority
 363             new[pkg]["section"] = section
 364             new[pkg]["priority"] = priority
 365
 366 ################################################################################
 367
 368 def edit_index (new, index):
 369     priority = new[index]["priority"]
 370     section = new[index]["section"]
 371     ftype = new[index]["type"]
 372     done = 0
 373     while not done:
 374         print "\t".join([index, priority, section])
 375
 376         answer = "XXX"
 377         if ftype != "dsc":
 378             prompt = "[B]oth, Priority, Section, Done ? "
 379         else:
 380             prompt = "[S]ection, Done ? "
 381         edit_priority = edit_section = 0
 382
 383         while prompt.find(answer) == -1:
 384             answer = utils.our_raw_input(prompt)
 385             m = re_default_answer.match(prompt)
 386             if answer == "":
 387                 answer = m.group(1)
 388             answer = answer[:1].upper()
 389
 390         if answer == 'P':
 391             edit_priority = 1
 392         elif answer == 'S':
 393             edit_section = 1
 394         elif answer == 'B':
 395             edit_priority = edit_section = 1
 396         elif answer == 'D':
 397             done = 1
 398
 399         # Edit the priority
 400         if edit_priority:
 401             readline.set_completer(Priorities.complete)
 402             got_priority = 0
 403             while not got_priority:
 404                 new_priority = utils.our_raw_input("New priority: ").strip()
 405                 if new_priority not in Priorities.priorities:
 406                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 407                 else:
 408                     got_priority = 1
 409                     priority = new_priority
 410
 411         # Edit the section
 412         if edit_section:
 413             readline.set_completer(Sections.complete)
 414             got_section = 0
 415             while not got_section:
 416                 new_section = utils.our_raw_input("New section: ").strip()
 417                 if new_section not in Sections.sections:
 418                     print "E: '%s' is not a valid section, try again." % (new_section)
 419                 else:
 420                     got_section = 1
 421                     section = new_section
 422
 423         # Reset the readline completer
 424         readline.set_completer(None)
 425
 426     for f in new[index]["files"]:
 427         Upload.pkg.files[f]["section"] = section
 428         Upload.pkg.files[f]["priority"] = priority
 429     new[index]["priority"] = priority
 430     new[index]["section"] = section
 431     return new
 432
 433 ################################################################################
 434
 435 def edit_overrides (new):
 436     print
 437     done = 0
 438     while not done:
 439         print_new (new, 1)
 440         new_index = {}
 441         index = 0
 442         for i in new.keys():
 443             index += 1
 444             new_index[index] = i
 445
 446         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 447
 448         got_answer = 0
 449         while not got_answer:
 450             answer = utils.our_raw_input(prompt)
 451             if not answer.isdigit():
 452                 answer = answer[:1].upper()
 453             if answer == "E" or answer == "D":
 454                 got_answer = 1
 455             elif re_isanum.match (answer):
 456                 answer = int(answer)
 457                 if (answer < 1) or (answer > index):
 458                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 459                 else:
 460                     got_answer = 1
 461
 462         if answer == 'E':
 463             edit_new(new)
 464         elif answer == 'D':
 465             done = 1
 466         else:
 467             edit_index (new, new_index[answer])
 468
 469     return new
 470
 471 ################################################################################
 472
 473 def edit_note(note):
 474     # Write the current data to a temporary file
 475     (fd, temp_filename) = utils.temp_filename()
 476     editor = os.environ.get("EDITOR","vi")
 477     answer = 'E'
 478     while answer == 'E':
 479         os.system("%s %s" % (editor, temp_filename))
 480         temp_file = utils.open_file(temp_filename)
 481         newnote = temp_file.read().rstrip()
 482         temp_file.close()
 483         print "New Note:"
 484         print utils.prefix_multi_line_string(newnote,"  ")
 485         prompt = "[D]one, Edit, Abandon, Quit ?"
 486         answer = "XXX"
 487         while prompt.find(answer) == -1:
 488             answer = utils.our_raw_input(prompt)
 489             m = re_default_answer.search(prompt)
 490             if answer == "":
 491                 answer = m.group(1)
 492             answer = answer[:1].upper()
 493     os.unlink(temp_filename)
 494     if answer == 'A':
 495         return
 496     elif answer == 'Q':
 497         end()
 498         sys.exit(0)
 499     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami())
 500
 501 ################################################################################
 502
 503 def check_pkg ():
 504     try:
 505         less_fd = os.popen("less -R -", 'w', 0)
 506         stdout_fd = sys.stdout
 507         try:
 508             sys.stdout = less_fd
 509             changes = utils.parse_changes (Upload.pkg.changes_file)
 510             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 511             files = Upload.pkg.files
 512             for f in files.keys():
 513                 if files[f].has_key("new"):
 514                     ftype = files[f]["type"]
 515                     if ftype == "deb":
 516                         examine_package.check_deb(changes['distribution'], f)
 517                     elif ftype == "dsc":
 518                         examine_package.check_dsc(changes['distribution'], f)
 519         finally:
 520             examine_package.output_package_relations()
 521             sys.stdout = stdout_fd
 522     except IOError, e:
 523         if e.errno == errno.EPIPE:
 524             utils.warn("[examine_package] Caught EPIPE; skipping.")
 525             pass
 526         else:
 527             raise
 528     except KeyboardInterrupt:
 529         utils.warn("[examine_package] Caught C-c; skipping.")
 530         pass
 531
 532 ################################################################################
 533
 534 ## FIXME: horribly Debian specific
 535
 536 def do_bxa_notification():
 537     files = Upload.pkg.files
 538     summary = ""
 539     for f in files.keys():
 540         if files[f]["type"] == "deb":
 541             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 542             summary += "\n"
 543             summary += "Package: %s\n" % (control.Find("Package"))
 544             summary += "Description: %s\n" % (control.Find("Description"))
 545     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 546     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 547     utils.send_mail(bxa_mail)
 548
 549 ################################################################################
 550
 551 def add_overrides (new):
 552     changes = Upload.pkg.changes
 553     files = Upload.pkg.files
 554
 555     projectB.query("BEGIN WORK")
 556     for suite in changes["suite"].keys():
 557         suite_id = database.get_suite_id(suite)
 558         for pkg in new.keys():
 559             component_id = database.get_component_id(new[pkg]["component"])
 560             type_id = database.get_override_type_id(new[pkg]["type"])
 561             priority_id = new[pkg]["priority id"]
 562             section_id = new[pkg]["section id"]
 563             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 564             for f in new[pkg]["files"]:
 565                 if files[f].has_key("new"):
 566                     del files[f]["new"]
 567             del new[pkg]
 568
 569     projectB.query("COMMIT WORK")
 570
 571     if Cnf.FindB("Dinstall::BXANotify"):
 572         do_bxa_notification()
 573
 574 ################################################################################
 575
 576 def prod_maintainer (note):
 577     # Here we prepare an editor and get them ready to prod...
 578     (fd, temp_filename) = utils.temp_filename()
 579     temp_file = os.fdopen(fd, 'w')
 580     if len(note) > 0:
 581         for line in note:
 582             temp_file.write(line)
 583     temp_file.close()
 584     editor = os.environ.get("EDITOR","vi")
 585     answer = 'E'
 586     while answer == 'E':
 587         os.system("%s %s" % (editor, temp_filename))
 588         temp_fh = utils.open_file(temp_filename)
 589         prod_message = "".join(temp_fh.readlines())
 590         temp_fh.close()
 591         print "Prod message:"
 592         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 593         prompt = "[P]rod, Edit, Abandon, Quit ?"
 594         answer = "XXX"
 595         while prompt.find(answer) == -1:
 596             answer = utils.our_raw_input(prompt)
 597             m = re_default_answer.search(prompt)
 598             if answer == "":
 599                 answer = m.group(1)
 600             answer = answer[:1].upper()
 601     os.unlink(temp_filename)
 602     if answer == 'A':
 603         return
 604     elif answer == 'Q':
 605         end()
 606         sys.exit(0)
 607     # Otherwise, do the proding...
 608     user_email_address = utils.whoami() + " <%s>" % (
 609         Cnf["Dinstall::MyAdminAddress"])
 610
 611     Subst = Upload.Subst
 612
 613     Subst["__FROM_ADDRESS__"] = user_email_address
 614     Subst["__PROD_MESSAGE__"] = prod_message
 615     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 616
 617     prod_mail_message = utils.TemplateSubst(
 618         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 619
 620     # Send the prod mail if appropriate
 621     if not Cnf["Dinstall::Options::No-Mail"]:
 622         utils.send_mail(prod_mail_message)
 623
 624     print "Sent proding message"
 625
 626 ################################################################################
 627
 628 def do_new():
 629     print "NEW\n"
 630     files = Upload.pkg.files
 631     changes = Upload.pkg.changes
 632
 633     # Make a copy of distribution we can happily trample on
 634     changes["suite"] = copy.copy(changes["distribution"])
 635
 636     # Fix up the list of target suites
 637     for suite in changes["suite"].keys():
 638         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 639         if override:
 640             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 641               database.get_suite_id(override) == -1)
 642             if olderr or newerr:
 643                 (oinv, newinv) = ("", "")
 644                 if olderr: oinv = "invalid "
 645                 if newerr: ninv = "invalid "
 646                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 647                         oinv, suite, ninv, override)
 648             del changes["suite"][suite]
 649             changes["suite"][override] = 1
 650     # Validate suites
 651     for suite in changes["suite"].keys():
 652         suite_id = database.get_suite_id(suite)
 653         if suite_id == -1:
 654             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 655
 656     # The main NEW processing loop
 657     done = 0
 658     while not done:
 659         # Find out what's new
 660         new = queue.determine_new(changes, files, projectB)
 661
 662         if not new:
 663             break
 664
 665         answer = "XXX"
 666         if Options["No-Action"] or Options["Automatic"]:
 667             answer = 'S'
 668
 669         (broken, note) = print_new(new, 0)
 670         prompt = ""
 671
 672         if not broken and not note:
 673             prompt = "Add overrides, "
 674         if broken:
 675             print "W: [!] marked entries must be fixed before package can be processed."
 676         if note:
 677             print "W: note must be removed before package can be processed."
 678             prompt += "Remove note, "
 679
 680         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 681
 682         while prompt.find(answer) == -1:
 683             try:
 684                 answer = utils.our_raw_input(prompt)
 685             except IOError, e:
 686                 if e.errno == errno.EPIPE:
 687                     utils.warn("[process_new] Caught EPIPE; skipping.")
 688                     pass
 689                 else:
 690                     raise
 691             m = re_default_answer.search(prompt)
 692             if answer == "":
 693                 answer = m.group(1)
 694             answer = answer[:1].upper()
 695
 696         if answer == 'A' and not Options["Trainee"]:
 697             try:
 698                 check_daily_lock()
 699                 done = add_overrides (new)
 700             except CantGetLockError:
 701                 print "Hello? Operator! Give me the number for 911!"
 702                 print "Dinstall in the locked area, cant process packages, come back later"
 703         elif answer == 'C':
 704             check_pkg()
 705         elif answer == 'E' and not Options["Trainee"]:
 706             new = edit_overrides (new)
 707         elif answer == 'M' and not Options["Trainee"]:
 708             aborted = Upload.do_reject(manual=1,
 709                                        reject_message=Options["Manual-Reject"],
 710                                        note=database.get_new_comments(changes.get("source", "")))
 711             if not aborted:
 712                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 713                 done = 1
 714         elif answer == 'N':
 715             edit_note(database.get_new_comments(changes.get("source", "")))
 716         elif answer == 'P' and not Options["Trainee"]:
 717             prod_maintainer(database.get_new_comments(changes.get("source", "")))
 718         elif answer == 'R' and not Options["Trainee"]:
 719             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 720             if confirm == "y":
 721                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 722         elif answer == 'S':
 723             done = 1
 724         elif answer == 'Q':
 725             end()
 726             sys.exit(0)
 727
 728 ################################################################################
 729 ################################################################################
 730 ################################################################################
 731
 732 def usage (exit_code=0):
 733     print """Usage: dak process-new [OPTION]... [CHANGES]...
 734   -a, --automatic           automatic run
 735   -h, --help                show this help and exit.
 736   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 737   -m, --manual-reject=MSG   manual reject with `msg'
 738   -n, --no-action           don't do anything
 739   -t, --trainee             FTP Trainee mode
 740   -V, --version             display the version number and exit"""
 741     sys.exit(exit_code)
 742
 743 ################################################################################
 744
 745 def init():
 746     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 747
 748     Cnf = utils.get_conf()
 749
 750     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 751                  ('h',"help","Process-New::Options::Help"),
 752                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 753                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 754                  ('t',"trainee","Process-New::Options::Trainee"),
 755                  ('n',"no-action","Process-New::Options::No-Action")]
 756
 757     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 758         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 759             Cnf["Process-New::Options::%s" % (i)] = ""
 760
 761     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 762     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 763         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 764
 765     Options = Cnf.SubTree("Process-New::Options")
 766
 767     if Options["Help"]:
 768         usage()
 769
 770     Upload = queue.Upload(Cnf)
 771
 772     if not Options["No-Action"]:
 773         try:
 774             Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 775         except CantOpenError, e:
 776             Options["Trainee"] = "Oh yes"
 777
 778     projectB = Upload.projectB
 779
 780     Sections = Section_Completer()
 781     Priorities = Priority_Completer()
 782     readline.parse_and_bind("tab: complete")
 783
 784     return changes_files
 785
 786 ################################################################################
 787
 788 def do_byhand():
 789     done = 0
 790     while not done:
 791         files = Upload.pkg.files
 792         will_install = 1
 793         byhand = []
 794
 795         for f in files.keys():
 796             if files[f]["type"] == "byhand":
 797                 if os.path.exists(f):
 798                     print "W: %s still present; please process byhand components and try again." % (f)
 799                     will_install = 0
 800                 else:
 801                     byhand.append(f)
 802
 803         answer = "XXXX"
 804         if Options["No-Action"]:
 805             answer = "S"
 806         if will_install:
 807             if Options["Automatic"] and not Options["No-Action"]:
 808                 answer = 'A'
 809             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 810         else:
 811             prompt = "Manual reject, [S]kip, Quit ?"
 812
 813         while prompt.find(answer) == -1:
 814             answer = utils.our_raw_input(prompt)
 815             m = re_default_answer.search(prompt)
 816             if answer == "":
 817                 answer = m.group(1)
 818             answer = answer[:1].upper()
 819
 820         if answer == 'A':
 821             try:
 822                 check_daily_lock()
 823                 done = 1
 824                 for f in byhand:
 825                     del files[f]
 826             except CantGetLockError:
 827                 print "Hello? Operator! Give me the number for 911!"
 828                 print "Dinstall in the locked area, cant process packages, come back later"
 829         elif answer == 'M':
 830             Upload.do_reject(1, Options["Manual-Reject"])
 831             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 832             done = 1
 833         elif answer == 'S':
 834             done = 1
 835         elif answer == 'Q':
 836             end()
 837             sys.exit(0)
 838
 839 ################################################################################
 840
 841 def check_daily_lock():
 842     """
 843     Raises CantGetLockError if the dinstall daily.lock exists.
 844     """
 845
 846     try:
 847         os.open(Cnf["Process-New::DinstallLockFile"],  os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 848     except OSError, e:
 849         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 850             raise CantGetLockError
 851
 852     os.unlink(Cnf["Process-New::DinstallLockFile"])
 853
 854
 855 @contextlib.contextmanager
 856 def lock_package(package):
 857     """
 858     Lock C{package} so that noone else jumps in processing it.
 859
 860     @type package: string
 861     @param package: source package name to lock
 862     """
 863
 864     path = os.path.join(Cnf["Process-New::LockDir"], package)
 865     try:
 866         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 867     except OSError, e:
 868         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 869             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 870             raise AlreadyLockedError, user
 871
 872     try:
 873         yield fd
 874     finally:
 875         os.unlink(path)
 876
 877 def move_to_dir (dest, perms=0660, changesperms=0664):
 878     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 879     file_keys = Upload.pkg.files.keys()
 880     for f in file_keys:
 881         utils.move (f, dest, perms=perms)
 882
 883 def is_source_in_queue_dir(qdir):
 884     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 885                 and x.endswith(".changes") ]
 886     for entry in entries:
 887         # read the .dak
 888         u = queue.Upload(Cnf)
 889         u.pkg.changes_file = os.path.join(qdir, entry)
 890         u.update_vars()
 891         if not u.pkg.changes["architecture"].has_key("source"):
 892             # another binary upload, ignore
 893             continue
 894         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 895             # another version, ignore
 896             continue
 897         # found it!
 898         return True
 899     return False
 900
 901 def move_to_holding(suite, queue_dir):
 902     print "Moving to %s holding area." % (suite.upper(),)
 903     if Options["No-Action"]:
 904         return
 905     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 906     Upload.dump_vars(queue_dir)
 907     move_to_dir(queue_dir, perms=0664)
 908     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 909
 910 def _accept():
 911     if Options["No-Action"]:
 912         return
 913     (summary, short_summary) = Upload.build_summaries()
 914     Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"])
 915     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 916
 917 def do_accept_stableupdate(suite, q):
 918     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 919     if not Upload.pkg.changes["architecture"].has_key("source"):
 920         # It is not a sourceful upload.  So its source may be either in p-u
 921         # holding, in new, in accepted or already installed.
 922         if is_source_in_queue_dir(queue_dir):
 923             # It's in p-u holding, so move it there.
 924             print "Binary-only upload, source in %s." % (q,)
 925             move_to_holding(suite, queue_dir)
 926         elif Upload.source_exists(Upload.pkg.changes["source"],
 927                 Upload.pkg.changes["version"]):
 928             # dak tells us that there is source available.  At time of
 929             # writing this means that it is installed, so put it into
 930             # accepted.
 931             print "Binary-only upload, source installed."
 932             _accept()
 933         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 934             # The source is in accepted, the binary cleared NEW: accept it.
 935             print "Binary-only upload, source in accepted."
 936             _accept()
 937         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 938             # It's in NEW.  We expect the source to land in p-u holding
 939             # pretty soon.
 940             print "Binary-only upload, source in new."
 941             move_to_holding(suite, queue_dir)
 942         else:
 943             # No case applicable.  Bail out.  Return will cause the upload
 944             # to be skipped.
 945             print "ERROR"
 946             print "Stable update failed.  Source not found."
 947             return
 948     else:
 949         # We are handling a sourceful upload.  Move to accepted if currently
 950         # in p-u holding and to p-u holding otherwise.
 951         if is_source_in_queue_dir(queue_dir):
 952             print "Sourceful upload in %s, accepting." % (q,)
 953             _accept()
 954         else:
 955             move_to_holding(suite, queue_dir)
 956
 957 def do_accept():
 958     print "ACCEPT"
 959     if not Options["No-Action"]:
 960         (summary, short_summary) = Upload.build_summaries()
 961     if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 962         Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 963         move_to_dir(Cnf["Dir::Queue::Embargoed"])
 964         Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 965         # Check for override disparities
 966         Upload.Subst["__SUMMARY__"] = summary
 967     else:
 968         # Stable updates need to be copied to proposed-updates holding
 969         # area instead of accepted.  Sourceful uploads need to go
 970         # to it directly, binaries only if the source has not yet been
 971         # accepted into p-u.
 972         for suite, q in [("proposed-updates", "ProposedUpdates"),
 973                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
 974             if not Upload.pkg.changes["distribution"].has_key(suite):
 975                 continue
 976             return do_accept_stableupdate(suite, q)
 977         # Just a normal upload, accept it...
 978         _accept()
 979
 980 def check_status(files):
 981     new = byhand = 0
 982     for f in files.keys():
 983         if files[f]["type"] == "byhand":
 984             byhand = 1
 985         elif files[f].has_key("new"):
 986             new = 1
 987     return (new, byhand)
 988
 989 def do_pkg(changes_file):
 990     Upload.pkg.changes_file = changes_file
 991     Upload.init_vars()
 992     Upload.update_vars()
 993     Upload.update_subst()
 994     files = Upload.pkg.files
 995
 996     try:
 997         with lock_package(Upload.pkg.changes["source"]):
 998             if not recheck():
 999                 return
1000
1001             (new, byhand) = check_status(files)
1002             if new or byhand:
1003                 if new:
1004                     do_new()
1005                 if byhand:
1006                     do_byhand()
1007                 (new, byhand) = check_status(files)
1008
1009             if not new and not byhand:
1010                 try:
1011                     check_daily_lock()
1012                     do_accept()
1013                 except CantGetLockError:
1014                     print "Hello? Operator! Give me the number for 911!"
1015                     print "Dinstall in the locked area, cant process packages, come back later"
1016     except AlreadyLockedError, e:
1017         print "Seems to be locked by %s already, skipping..." % (e)
1018
1019 ################################################################################
1020
1021 def end():
1022     accept_count = Upload.accept_count
1023     accept_bytes = Upload.accept_bytes
1024
1025     if accept_count:
1026         sets = "set"
1027         if accept_count > 1:
1028             sets = "sets"
1029         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1030         Logger.log(["total",accept_count,accept_bytes])
1031
1032     if not Options["No-Action"] and not Options["Trainee"]:
1033         Logger.close()
1034
1035 ################################################################################
1036
1037 def do_comments(dir, opref, npref, line, fn):
1038     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1039         lines = open("%s/%s" % (dir, comm)).readlines()
1040         if len(lines) == 0 or lines[0] != line + "\n": continue
1041         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1042                                 and x.endswith(".changes") ]
1043         changes_files = sort_changes(changes_files)
1044         for f in changes_files:
1045             f = utils.validate_changes_file_arg(f, 0)
1046             if not f: continue
1047             print "\n" + f
1048             fn(f, "".join(lines[1:]))
1049
1050         if opref != npref and not Options["No-Action"]:
1051             newcomm = npref + comm[len(opref):]
1052             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1053
1054 ################################################################################
1055
1056 def comment_accept(changes_file, comments):
1057     Upload.pkg.changes_file = changes_file
1058     Upload.init_vars()
1059     Upload.update_vars()
1060     Upload.update_subst()
1061     files = Upload.pkg.files
1062
1063     if not recheck():
1064         return # dak wants to REJECT, crap
1065
1066     (new, byhand) = check_status(files)
1067     if not new and not byhand:
1068         do_accept()
1069
1070 ################################################################################
1071
1072 def comment_reject(changes_file, comments):
1073     Upload.pkg.changes_file = changes_file
1074     Upload.init_vars()
1075     Upload.update_vars()
1076     Upload.update_subst()
1077
1078     if not recheck():
1079         pass # dak has its own reasons to reject as well, which is fine
1080
1081     reject(comments)
1082     print "REJECT\n" + reject_message,
1083     if not Options["No-Action"]:
1084         Upload.do_reject(0, reject_message)
1085         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1086
1087 ################################################################################
1088
1089 def main():
1090     changes_files = init()
1091     if len(changes_files) > 50:
1092         sys.stderr.write("Sorting changes...\n")
1093     changes_files = sort_changes(changes_files)
1094
1095     # Kill me now? **FIXME**
1096     Cnf["Dinstall::Options::No-Mail"] = ""
1097     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1098     if Cnf.has_key("Dinstall::Bcc"):
1099         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1100     else:
1101         Upload.Subst["__BCC__"] = bcc
1102
1103     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1104     if commentsdir:
1105         if changes_files != []:
1106             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1107             sys.exit(1)
1108         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1109         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1110     else:
1111         for changes_file in changes_files:
1112             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1113             if not changes_file:
1114                 continue
1115             print "\n" + changes_file
1116
1117             do_pkg (changes_file)
1118
1119     end()
1120
1121 ################################################################################
1122
1123 if __name__ == '__main__':
1124     main()