dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 from __future__ import with_statement
  45
  46 import copy
  47 import errno
  48 import os
  49 import readline
  50 import stat
  51 import sys
  52 import time
  53 import contextlib
  54 import pwd
  55 import apt_pkg, apt_inst
  56 import examine_package
  57 from daklib import database
  58 from daklib import logging
  59 from daklib import queue
  60 from daklib import utils
  61 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  62 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  63
  64 # Globals
  65 Cnf = None       #: Configuration, apt_pkg.Configuration
  66 Options = None
  67 Upload = None
  68 projectB = None  #: database connection, pgobject
  69 Logger = None
  70
  71 Priorities = None
  72 Sections = None
  73
  74 reject_message = ""
  75
  76 ################################################################################
  77 ################################################################################
  78 ################################################################################
  79
  80 def reject (str, prefix="Rejected: "):
  81     global reject_message
  82     if str:
  83         reject_message += prefix + str + "\n"
  84
  85 def recheck():
  86     global reject_message
  87     files = Upload.pkg.files
  88     reject_message = ""
  89
  90     for f in files.keys():
  91         # The .orig.tar.gz can disappear out from under us is it's a
  92         # duplicate of one in the archive.
  93         if not files.has_key(f):
  94             continue
  95         # Check that the source still exists
  96         if files[f]["type"] == "deb":
  97             source_version = files[f]["source version"]
  98             source_package = files[f]["source package"]
  99             if not Upload.pkg.changes["architecture"].has_key("source") \
 100                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
 101                 source_epochless_version = re_no_epoch.sub('', source_version)
 102                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 103                 found = 0
 104                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
 105                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 106                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 107                             found = 1
 108                 if not found:
 109                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 110
 111         # Version and file overwrite checks
 112         if files[f]["type"] == "deb":
 113             reject(Upload.check_binary_against_db(f), "")
 114         elif files[f]["type"] == "dsc":
 115             reject(Upload.check_source_against_db(f), "")
 116             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 117             reject(reject_msg, "")
 118
 119     if reject_message.find("Rejected") != -1:
 120         answer = "XXX"
 121         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 122             answer = 'S'
 123
 124         print "REJECT\n" + reject_message,
 125         prompt = "[R]eject, Skip, Quit ?"
 126
 127         while prompt.find(answer) == -1:
 128             answer = utils.our_raw_input(prompt)
 129             m = re_default_answer.match(prompt)
 130             if answer == "":
 131                 answer = m.group(1)
 132             answer = answer[:1].upper()
 133
 134         if answer == 'R':
 135             Upload.do_reject(0, reject_message)
 136             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 137             return 0
 138         elif answer == 'S':
 139             return 0
 140         elif answer == 'Q':
 141             end()
 142             sys.exit(0)
 143
 144     return 1
 145
 146 ################################################################################
 147
 148 def indiv_sg_compare (a, b):
 149     """Sort by source name, source, version, 'have source', and
 150        finally by filename."""
 151     # Sort by source version
 152     q = apt_pkg.VersionCompare(a["version"], b["version"])
 153     if q:
 154         return -q
 155
 156     # Sort by 'have source'
 157     a_has_source = a["architecture"].get("source")
 158     b_has_source = b["architecture"].get("source")
 159     if a_has_source and not b_has_source:
 160         return -1
 161     elif b_has_source and not a_has_source:
 162         return 1
 163
 164     return cmp(a["filename"], b["filename"])
 165
 166 ############################################################
 167
 168 def sg_compare (a, b):
 169     a = a[1]
 170     b = b[1]
 171     """Sort by have note, source already in database and time of oldest upload."""
 172     # Sort by have note
 173     a_note_state = a["note_state"]
 174     b_note_state = b["note_state"]
 175     if a_note_state < b_note_state:
 176         return -1
 177     elif a_note_state > b_note_state:
 178         return 1
 179     # Sort by source already in database (descending)
 180     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 181     if source_in_database:
 182         return -source_in_database
 183
 184     # Sort by time of oldest upload
 185     return cmp(a["oldest"], b["oldest"])
 186
 187 def sort_changes(changes_files):
 188     """Sort into source groups, then sort each source group by version,
 189     have source, filename.  Finally, sort the source groups by have
 190     note, time of oldest upload of each source upload."""
 191     if len(changes_files) == 1:
 192         return changes_files
 193
 194     sorted_list = []
 195     cache = {}
 196     # Read in all the .changes files
 197     for filename in changes_files:
 198         try:
 199             Upload.pkg.changes_file = filename
 200             Upload.init_vars()
 201             Upload.update_vars()
 202             cache[filename] = copy.copy(Upload.pkg.changes)
 203             cache[filename]["filename"] = filename
 204         except:
 205             sorted_list.append(filename)
 206             break
 207     # Divide the .changes into per-source groups
 208     per_source = {}
 209     for filename in cache.keys():
 210         source = cache[filename]["source"]
 211         if not per_source.has_key(source):
 212             per_source[source] = {}
 213             per_source[source]["list"] = []
 214         per_source[source]["list"].append(cache[filename])
 215     # Determine oldest time and have note status for each source group
 216     for source in per_source.keys():
 217         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 218         ql = q.getresult()
 219         per_source[source]["source_in_database"] = len(ql)>0
 220         source_list = per_source[source]["list"]
 221         first = source_list[0]
 222         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 223         have_note = 0
 224         for d in per_source[source]["list"]:
 225             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 226             if mtime < oldest:
 227                 oldest = mtime
 228             have_note += (database.has_new_comment(d["source"], d["version"], True))
 229         per_source[source]["oldest"] = oldest
 230         if not have_note:
 231             per_source[source]["note_state"] = 0; # none
 232         elif have_note < len(source_list):
 233             per_source[source]["note_state"] = 1; # some
 234         else:
 235             per_source[source]["note_state"] = 2; # all
 236         per_source[source]["list"].sort(indiv_sg_compare)
 237     per_source_items = per_source.items()
 238     per_source_items.sort(sg_compare)
 239     for i in per_source_items:
 240         for j in i[1]["list"]:
 241             sorted_list.append(j["filename"])
 242     return sorted_list
 243
 244 ################################################################################
 245
 246 class Section_Completer:
 247     def __init__ (self):
 248         self.sections = []
 249         self.matches = []
 250         q = projectB.query("SELECT section FROM section")
 251         for i in q.getresult():
 252             self.sections.append(i[0])
 253
 254     def complete(self, text, state):
 255         if state == 0:
 256             self.matches = []
 257             n = len(text)
 258             for word in self.sections:
 259                 if word[:n] == text:
 260                     self.matches.append(word)
 261         try:
 262             return self.matches[state]
 263         except IndexError:
 264             return None
 265
 266 ############################################################
 267
 268 class Priority_Completer:
 269     def __init__ (self):
 270         self.priorities = []
 271         self.matches = []
 272         q = projectB.query("SELECT priority FROM priority")
 273         for i in q.getresult():
 274             self.priorities.append(i[0])
 275
 276     def complete(self, text, state):
 277         if state == 0:
 278             self.matches = []
 279             n = len(text)
 280             for word in self.priorities:
 281                 if word[:n] == text:
 282                     self.matches.append(word)
 283         try:
 284             return self.matches[state]
 285         except IndexError:
 286             return None
 287
 288 ################################################################################
 289
 290 def print_new (new, indexed, file=sys.stdout):
 291     queue.check_valid(new)
 292     broken = 0
 293     index = 0
 294     for pkg in new.keys():
 295         index += 1
 296         section = new[pkg]["section"]
 297         priority = new[pkg]["priority"]
 298         if new[pkg]["section id"] == -1:
 299             section += "[!]"
 300             broken = 1
 301         if new[pkg]["priority id"] == -1:
 302             priority += "[!]"
 303             broken = 1
 304         if indexed:
 305             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 306         else:
 307             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 308         line = line.strip()+'\n'
 309         file.write(line)
 310     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 311     if len(note) > 0:
 312         for line in note:
 313             print line
 314     return broken, note
 315
 316 ################################################################################
 317
 318 def index_range (index):
 319     if index == 1:
 320         return "1"
 321     else:
 322         return "1-%s" % (index)
 323
 324 ################################################################################
 325 ################################################################################
 326
 327 def edit_new (new):
 328     # Write the current data to a temporary file
 329     (fd, temp_filename) = utils.temp_filename()
 330     temp_file = os.fdopen(fd, 'w')
 331     print_new (new, 0, temp_file)
 332     temp_file.close()
 333     # Spawn an editor on that file
 334     editor = os.environ.get("EDITOR","vi")
 335     result = os.system("%s %s" % (editor, temp_filename))
 336     if result != 0:
 337         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 338     # Read the edited data back in
 339     temp_file = utils.open_file(temp_filename)
 340     lines = temp_file.readlines()
 341     temp_file.close()
 342     os.unlink(temp_filename)
 343     # Parse the new data
 344     for line in lines:
 345         line = line.strip()
 346         if line == "":
 347             continue
 348         s = line.split()
 349         # Pad the list if necessary
 350         s[len(s):3] = [None] * (3-len(s))
 351         (pkg, priority, section) = s[:3]
 352         if not new.has_key(pkg):
 353             utils.warn("Ignoring unknown package '%s'" % (pkg))
 354         else:
 355             # Strip off any invalid markers, print_new will readd them.
 356             if section.endswith("[!]"):
 357                 section = section[:-3]
 358             if priority.endswith("[!]"):
 359                 priority = priority[:-3]
 360             for f in new[pkg]["files"]:
 361                 Upload.pkg.files[f]["section"] = section
 362                 Upload.pkg.files[f]["priority"] = priority
 363             new[pkg]["section"] = section
 364             new[pkg]["priority"] = priority
 365
 366 ################################################################################
 367
 368 def edit_index (new, index):
 369     priority = new[index]["priority"]
 370     section = new[index]["section"]
 371     ftype = new[index]["type"]
 372     done = 0
 373     while not done:
 374         print "\t".join([index, priority, section])
 375
 376         answer = "XXX"
 377         if ftype != "dsc":
 378             prompt = "[B]oth, Priority, Section, Done ? "
 379         else:
 380             prompt = "[S]ection, Done ? "
 381         edit_priority = edit_section = 0
 382
 383         while prompt.find(answer) == -1:
 384             answer = utils.our_raw_input(prompt)
 385             m = re_default_answer.match(prompt)
 386             if answer == "":
 387                 answer = m.group(1)
 388             answer = answer[:1].upper()
 389
 390         if answer == 'P':
 391             edit_priority = 1
 392         elif answer == 'S':
 393             edit_section = 1
 394         elif answer == 'B':
 395             edit_priority = edit_section = 1
 396         elif answer == 'D':
 397             done = 1
 398
 399         # Edit the priority
 400         if edit_priority:
 401             readline.set_completer(Priorities.complete)
 402             got_priority = 0
 403             while not got_priority:
 404                 new_priority = utils.our_raw_input("New priority: ").strip()
 405                 if new_priority not in Priorities.priorities:
 406                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 407                 else:
 408                     got_priority = 1
 409                     priority = new_priority
 410
 411         # Edit the section
 412         if edit_section:
 413             readline.set_completer(Sections.complete)
 414             got_section = 0
 415             while not got_section:
 416                 new_section = utils.our_raw_input("New section: ").strip()
 417                 if new_section not in Sections.sections:
 418                     print "E: '%s' is not a valid section, try again." % (new_section)
 419                 else:
 420                     got_section = 1
 421                     section = new_section
 422
 423         # Reset the readline completer
 424         readline.set_completer(None)
 425
 426     for f in new[index]["files"]:
 427         Upload.pkg.files[f]["section"] = section
 428         Upload.pkg.files[f]["priority"] = priority
 429     new[index]["priority"] = priority
 430     new[index]["section"] = section
 431     return new
 432
 433 ################################################################################
 434
 435 def edit_overrides (new):
 436     print
 437     done = 0
 438     while not done:
 439         print_new (new, 1)
 440         new_index = {}
 441         index = 0
 442         for i in new.keys():
 443             index += 1
 444             new_index[index] = i
 445
 446         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 447
 448         got_answer = 0
 449         while not got_answer:
 450             answer = utils.our_raw_input(prompt)
 451             if not answer.isdigit():
 452                 answer = answer[:1].upper()
 453             if answer == "E" or answer == "D":
 454                 got_answer = 1
 455             elif re_isanum.match (answer):
 456                 answer = int(answer)
 457                 if (answer < 1) or (answer > index):
 458                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 459                 else:
 460                     got_answer = 1
 461
 462         if answer == 'E':
 463             edit_new(new)
 464         elif answer == 'D':
 465             done = 1
 466         else:
 467             edit_index (new, new_index[answer])
 468
 469     return new
 470
 471 ################################################################################
 472
 473 def edit_note(note):
 474     # Write the current data to a temporary file
 475     (fd, temp_filename) = utils.temp_filename()
 476     editor = os.environ.get("EDITOR","vi")
 477     answer = 'E'
 478     while answer == 'E':
 479         os.system("%s %s" % (editor, temp_filename))
 480         temp_file = utils.open_file(temp_filename)
 481         newnote = temp_file.read().rstrip()
 482         temp_file.close()
 483         print "New Note:"
 484         print utils.prefix_multi_line_string(newnote,"  ")
 485         prompt = "[D]one, Edit, Abandon, Quit ?"
 486         answer = "XXX"
 487         while prompt.find(answer) == -1:
 488             answer = utils.our_raw_input(prompt)
 489             m = re_default_answer.search(prompt)
 490             if answer == "":
 491                 answer = m.group(1)
 492             answer = answer[:1].upper()
 493     os.unlink(temp_filename)
 494     if answer == 'A':
 495         return
 496     elif answer == 'Q':
 497         end()
 498         sys.exit(0)
 499
 500     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami(), bool(Options["Trainee"]))
 501
 502 ################################################################################
 503
 504 def check_pkg ():
 505     try:
 506         less_fd = os.popen("less -R -", 'w', 0)
 507         stdout_fd = sys.stdout
 508         try:
 509             sys.stdout = less_fd
 510             changes = utils.parse_changes (Upload.pkg.changes_file)
 511             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 512             files = Upload.pkg.files
 513             for f in files.keys():
 514                 if files[f].has_key("new"):
 515                     ftype = files[f]["type"]
 516                     if ftype == "deb":
 517                         examine_package.check_deb(changes['distribution'], f)
 518                     elif ftype == "dsc":
 519                         examine_package.check_dsc(changes['distribution'], f)
 520         finally:
 521             examine_package.output_package_relations()
 522             sys.stdout = stdout_fd
 523     except IOError, e:
 524         if e.errno == errno.EPIPE:
 525             utils.warn("[examine_package] Caught EPIPE; skipping.")
 526             pass
 527         else:
 528             raise
 529     except KeyboardInterrupt:
 530         utils.warn("[examine_package] Caught C-c; skipping.")
 531         pass
 532
 533 ################################################################################
 534
 535 ## FIXME: horribly Debian specific
 536
 537 def do_bxa_notification():
 538     files = Upload.pkg.files
 539     summary = ""
 540     for f in files.keys():
 541         if files[f]["type"] == "deb":
 542             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 543             summary += "\n"
 544             summary += "Package: %s\n" % (control.Find("Package"))
 545             summary += "Description: %s\n" % (control.Find("Description"))
 546     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 547     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 548     utils.send_mail(bxa_mail)
 549
 550 ################################################################################
 551
 552 def add_overrides (new):
 553     changes = Upload.pkg.changes
 554     files = Upload.pkg.files
 555     srcpkg = changes.get("source")
 556
 557     projectB.query("BEGIN WORK")
 558     for suite in changes["suite"].keys():
 559         suite_id = database.get_suite_id(suite)
 560         for pkg in new.keys():
 561             component_id = database.get_component_id(new[pkg]["component"])
 562             type_id = database.get_override_type_id(new[pkg]["type"])
 563             priority_id = new[pkg]["priority id"]
 564             section_id = new[pkg]["section id"]
 565             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 566             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 567             for f in new[pkg]["files"]:
 568                 if files[f].has_key("new"):
 569                     del files[f]["new"]
 570             del new[pkg]
 571
 572     projectB.query("COMMIT WORK")
 573
 574     if Cnf.FindB("Dinstall::BXANotify"):
 575         do_bxa_notification()
 576
 577 ################################################################################
 578
 579 def prod_maintainer (note):
 580     # Here we prepare an editor and get them ready to prod...
 581     (fd, temp_filename) = utils.temp_filename()
 582     temp_file = os.fdopen(fd, 'w')
 583     if len(note) > 0:
 584         for line in note:
 585             temp_file.write(line)
 586     temp_file.close()
 587     editor = os.environ.get("EDITOR","vi")
 588     answer = 'E'
 589     while answer == 'E':
 590         os.system("%s %s" % (editor, temp_filename))
 591         temp_fh = utils.open_file(temp_filename)
 592         prod_message = "".join(temp_fh.readlines())
 593         temp_fh.close()
 594         print "Prod message:"
 595         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 596         prompt = "[P]rod, Edit, Abandon, Quit ?"
 597         answer = "XXX"
 598         while prompt.find(answer) == -1:
 599             answer = utils.our_raw_input(prompt)
 600             m = re_default_answer.search(prompt)
 601             if answer == "":
 602                 answer = m.group(1)
 603             answer = answer[:1].upper()
 604     os.unlink(temp_filename)
 605     if answer == 'A':
 606         return
 607     elif answer == 'Q':
 608         end()
 609         sys.exit(0)
 610     # Otherwise, do the proding...
 611     user_email_address = utils.whoami() + " <%s>" % (
 612         Cnf["Dinstall::MyAdminAddress"])
 613
 614     Subst = Upload.Subst
 615
 616     Subst["__FROM_ADDRESS__"] = user_email_address
 617     Subst["__PROD_MESSAGE__"] = prod_message
 618     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 619
 620     prod_mail_message = utils.TemplateSubst(
 621         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 622
 623     # Send the prod mail if appropriate
 624     if not Cnf["Dinstall::Options::No-Mail"]:
 625         utils.send_mail(prod_mail_message)
 626
 627     print "Sent proding message"
 628
 629 ################################################################################
 630
 631 def do_new():
 632     print "NEW\n"
 633     files = Upload.pkg.files
 634     changes = Upload.pkg.changes
 635
 636     # Make a copy of distribution we can happily trample on
 637     changes["suite"] = copy.copy(changes["distribution"])
 638
 639     # Fix up the list of target suites
 640     for suite in changes["suite"].keys():
 641         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 642         if override:
 643             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 644               database.get_suite_id(override) == -1)
 645             if olderr or newerr:
 646                 (oinv, newinv) = ("", "")
 647                 if olderr: oinv = "invalid "
 648                 if newerr: ninv = "invalid "
 649                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 650                         oinv, suite, ninv, override)
 651             del changes["suite"][suite]
 652             changes["suite"][override] = 1
 653     # Validate suites
 654     for suite in changes["suite"].keys():
 655         suite_id = database.get_suite_id(suite)
 656         if suite_id == -1:
 657             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 658
 659     # The main NEW processing loop
 660     done = 0
 661     while not done:
 662         # Find out what's new
 663         new = queue.determine_new(changes, files, projectB)
 664
 665         if not new:
 666             break
 667
 668         answer = "XXX"
 669         if Options["No-Action"] or Options["Automatic"]:
 670             answer = 'S'
 671
 672         (broken, note) = print_new(new, 0)
 673         prompt = ""
 674
 675         if not broken and not note:
 676             prompt = "Add overrides, "
 677         if broken:
 678             print "W: [!] marked entries must be fixed before package can be processed."
 679         if note:
 680             print "W: note must be removed before package can be processed."
 681             prompt += "RemOve all notes, Remove note, "
 682
 683         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 684
 685         while prompt.find(answer) == -1:
 686             answer = utils.our_raw_input(prompt)
 687             m = re_default_answer.search(prompt)
 688             if answer == "":
 689                 answer = m.group(1)
 690             answer = answer[:1].upper()
 691
 692         if answer == 'A' and not Options["Trainee"]:
 693             try:
 694                 check_daily_lock()
 695                 done = add_overrides (new)
 696                 Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 697             except CantGetLockError:
 698                 print "Hello? Operator! Give me the number for 911!"
 699                 print "Dinstall in the locked area, cant process packages, come back later"
 700         elif answer == 'C':
 701             check_pkg()
 702         elif answer == 'E' and not Options["Trainee"]:
 703             new = edit_overrides (new)
 704         elif answer == 'M' and not Options["Trainee"]:
 705             aborted = Upload.do_reject(manual=1,
 706                                        reject_message=Options["Manual-Reject"],
 707                                        note=database.get_new_comments(changes.get("source", "")))
 708             if not aborted:
 709                 Logger.log([utils.getusername(), "NEW REJECT: %s" % (Upload.pkg.changes_file)])
 710                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 711                 done = 1
 712         elif answer == 'N':
 713             edit_note(database.get_new_comments(changes.get("source", "")))
 714         elif answer == 'P' and not Options["Trainee"]:
 715             prod_maintainer(database.get_new_comments(changes.get("source", "")))
 716             Logger.log([utils.getusername(), "NEW PROD: %s" % (Upload.pkg.changes_file)])
 717         elif answer == 'R' and not Options["Trainee"]:
 718             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 719             if confirm == "y":
 720                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 721         elif answer == 'O' and not Options["Trainee"]:
 722             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 723             if confirm == "y":
 724                 database.delete_all_new_comments(changes.get("source"))
 725         elif answer == 'S':
 726             done = 1
 727         elif answer == 'Q':
 728             end()
 729             sys.exit(0)
 730
 731 ################################################################################
 732 ################################################################################
 733 ################################################################################
 734
 735 def usage (exit_code=0):
 736     print """Usage: dak process-new [OPTION]... [CHANGES]...
 737   -a, --automatic           automatic run
 738   -h, --help                show this help and exit.
 739   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 740   -m, --manual-reject=MSG   manual reject with `msg'
 741   -n, --no-action           don't do anything
 742   -t, --trainee             FTP Trainee mode
 743   -V, --version             display the version number and exit"""
 744     sys.exit(exit_code)
 745
 746 ################################################################################
 747
 748 def init():
 749     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 750
 751     Cnf = utils.get_conf()
 752
 753     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 754                  ('h',"help","Process-New::Options::Help"),
 755                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 756                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 757                  ('t',"trainee","Process-New::Options::Trainee"),
 758                  ('n',"no-action","Process-New::Options::No-Action")]
 759
 760     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 761         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 762             Cnf["Process-New::Options::%s" % (i)] = ""
 763
 764     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 765     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 766         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 767
 768     Options = Cnf.SubTree("Process-New::Options")
 769
 770     if Options["Help"]:
 771         usage()
 772
 773     Upload = queue.Upload(Cnf)
 774
 775     if not Options["No-Action"]:
 776         try:
 777             Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 778         except CantOpenError, e:
 779             Options["Trainee"] = "True"
 780
 781     projectB = Upload.projectB
 782
 783     Sections = Section_Completer()
 784     Priorities = Priority_Completer()
 785     readline.parse_and_bind("tab: complete")
 786
 787     return changes_files
 788
 789 ################################################################################
 790
 791 def do_byhand():
 792     done = 0
 793     while not done:
 794         files = Upload.pkg.files
 795         will_install = 1
 796         byhand = []
 797
 798         for f in files.keys():
 799             if files[f]["type"] == "byhand":
 800                 if os.path.exists(f):
 801                     print "W: %s still present; please process byhand components and try again." % (f)
 802                     will_install = 0
 803                 else:
 804                     byhand.append(f)
 805
 806         answer = "XXXX"
 807         if Options["No-Action"]:
 808             answer = "S"
 809         if will_install:
 810             if Options["Automatic"] and not Options["No-Action"]:
 811                 answer = 'A'
 812             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 813         else:
 814             prompt = "Manual reject, [S]kip, Quit ?"
 815
 816         while prompt.find(answer) == -1:
 817             answer = utils.our_raw_input(prompt)
 818             m = re_default_answer.search(prompt)
 819             if answer == "":
 820                 answer = m.group(1)
 821             answer = answer[:1].upper()
 822
 823         if answer == 'A':
 824             try:
 825                 check_daily_lock()
 826                 done = 1
 827                 for f in byhand:
 828                     del files[f]
 829                 Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (Upload.pkg.changes_file)])
 830             except CantGetLockError:
 831                 print "Hello? Operator! Give me the number for 911!"
 832                 print "Dinstall in the locked area, cant process packages, come back later"
 833         elif answer == 'M':
 834             Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (Upload.pkg.changes_file)])
 835             Upload.do_reject(1, Options["Manual-Reject"])
 836             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 837             done = 1
 838         elif answer == 'S':
 839             done = 1
 840         elif answer == 'Q':
 841             end()
 842             sys.exit(0)
 843
 844 ################################################################################
 845
 846 def check_daily_lock():
 847     """
 848     Raises CantGetLockError if the dinstall daily.lock exists.
 849     """
 850
 851     try:
 852         os.open(Cnf["Process-New::DinstallLockFile"],  os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 853     except OSError, e:
 854         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 855             raise CantGetLockError
 856
 857     os.unlink(Cnf["Process-New::DinstallLockFile"])
 858
 859
 860 @contextlib.contextmanager
 861 def lock_package(package):
 862     """
 863     Lock C{package} so that noone else jumps in processing it.
 864
 865     @type package: string
 866     @param package: source package name to lock
 867     """
 868
 869     path = os.path.join(Cnf["Process-New::LockDir"], package)
 870     try:
 871         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 872     except OSError, e:
 873         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 874             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 875             raise AlreadyLockedError, user
 876
 877     try:
 878         yield fd
 879     finally:
 880         os.unlink(path)
 881
 882 def move_to_dir (dest, perms=0660, changesperms=0664):
 883     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 884     file_keys = Upload.pkg.files.keys()
 885     for f in file_keys:
 886         utils.move (f, dest, perms=perms)
 887
 888 def is_source_in_queue_dir(qdir):
 889     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 890                 and x.endswith(".changes") ]
 891     for entry in entries:
 892         # read the .dak
 893         u = queue.Upload(Cnf)
 894         u.pkg.changes_file = os.path.join(qdir, entry)
 895         u.update_vars()
 896         if not u.pkg.changes["architecture"].has_key("source"):
 897             # another binary upload, ignore
 898             continue
 899         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 900             # another version, ignore
 901             continue
 902         # found it!
 903         return True
 904     return False
 905
 906 def move_to_holding(suite, queue_dir):
 907     print "Moving to %s holding area." % (suite.upper(),)
 908     if Options["No-Action"]:
 909         return
 910     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 911     Upload.dump_vars(queue_dir)
 912     move_to_dir(queue_dir, perms=0664)
 913     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 914
 915 def _accept():
 916     if Options["No-Action"]:
 917         return
 918     (summary, short_summary) = Upload.build_summaries()
 919     Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"])
 920     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 921
 922 def do_accept_stableupdate(suite, q):
 923     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 924     if not Upload.pkg.changes["architecture"].has_key("source"):
 925         # It is not a sourceful upload.  So its source may be either in p-u
 926         # holding, in new, in accepted or already installed.
 927         if is_source_in_queue_dir(queue_dir):
 928             # It's in p-u holding, so move it there.
 929             print "Binary-only upload, source in %s." % (q,)
 930             move_to_holding(suite, queue_dir)
 931         elif Upload.source_exists(Upload.pkg.changes["source"],
 932                 Upload.pkg.changes["version"]):
 933             # dak tells us that there is source available.  At time of
 934             # writing this means that it is installed, so put it into
 935             # accepted.
 936             print "Binary-only upload, source installed."
 937             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 938             _accept()
 939         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 940             # The source is in accepted, the binary cleared NEW: accept it.
 941             print "Binary-only upload, source in accepted."
 942             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 943             _accept()
 944         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 945             # It's in NEW.  We expect the source to land in p-u holding
 946             # pretty soon.
 947             print "Binary-only upload, source in new."
 948             move_to_holding(suite, queue_dir)
 949         elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
 950             # It's in newstage.  Accept into the holding area
 951             print "Binary-only upload, source in newstage."
 952             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 953             _accept()
 954         else:
 955             # No case applicable.  Bail out.  Return will cause the upload
 956             # to be skipped.
 957             print "ERROR"
 958             print "Stable update failed.  Source not found."
 959             return
 960     else:
 961         # We are handling a sourceful upload.  Move to accepted if currently
 962         # in p-u holding and to p-u holding otherwise.
 963         if is_source_in_queue_dir(queue_dir):
 964             print "Sourceful upload in %s, accepting." % (q,)
 965             _accept()
 966         else:
 967             move_to_holding(suite, queue_dir)
 968
 969 def do_accept():
 970     print "ACCEPT"
 971     if not Options["No-Action"]:
 972         (summary, short_summary) = Upload.build_summaries()
 973     if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 974         Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 975         move_to_dir(Cnf["Dir::Queue::Embargoed"])
 976         Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 977         # Check for override disparities
 978         Upload.Subst["__SUMMARY__"] = summary
 979     else:
 980         # Stable updates need to be copied to proposed-updates holding
 981         # area instead of accepted.  Sourceful uploads need to go
 982         # to it directly, binaries only if the source has not yet been
 983         # accepted into p-u.
 984         for suite, q in [("proposed-updates", "ProposedUpdates"),
 985                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
 986             if not Upload.pkg.changes["distribution"].has_key(suite):
 987                 continue
 988             return do_accept_stableupdate(suite, q)
 989         # Just a normal upload, accept it...
 990         _accept()
 991
 992 def check_status(files):
 993     new = byhand = 0
 994     for f in files.keys():
 995         if files[f]["type"] == "byhand":
 996             byhand = 1
 997         elif files[f].has_key("new"):
 998             new = 1
 999     return (new, byhand)
1000
1001 def do_pkg(changes_file):
1002     Upload.pkg.changes_file = changes_file
1003     Upload.init_vars()
1004     Upload.update_vars()
1005     Upload.update_subst()
1006     files = Upload.pkg.files
1007
1008     try:
1009         with lock_package(Upload.pkg.changes["source"]):
1010             if not recheck():
1011                 return
1012
1013             (new, byhand) = check_status(files)
1014             if new or byhand:
1015                 if new:
1016                     do_new()
1017                 if byhand:
1018                     do_byhand()
1019                 (new, byhand) = check_status(files)
1020
1021             if not new and not byhand:
1022                 try:
1023                     check_daily_lock()
1024                     do_accept()
1025                 except CantGetLockError:
1026                     print "Hello? Operator! Give me the number for 911!"
1027                     print "Dinstall in the locked area, cant process packages, come back later"
1028     except AlreadyLockedError, e:
1029         print "Seems to be locked by %s already, skipping..." % (e)
1030
1031 ################################################################################
1032
1033 def end():
1034     accept_count = Upload.accept_count
1035     accept_bytes = Upload.accept_bytes
1036
1037     if accept_count:
1038         sets = "set"
1039         if accept_count > 1:
1040             sets = "sets"
1041         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1042         Logger.log([utils.getusername(), "total",accept_count,accept_bytes])
1043
1044     if not Options["No-Action"] and not Options["Trainee"]:
1045         Logger.close()
1046
1047 ################################################################################
1048
1049 def do_comments(dir, opref, npref, line, fn):
1050     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1051         lines = open("%s/%s" % (dir, comm)).readlines()
1052         if len(lines) == 0 or lines[0] != line + "\n": continue
1053         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1054                                 and x.endswith(".changes") ]
1055         changes_files = sort_changes(changes_files)
1056         for f in changes_files:
1057             f = utils.validate_changes_file_arg(f, 0)
1058             if not f: continue
1059             print "\n" + f
1060             fn(f, "".join(lines[1:]))
1061
1062         if opref != npref and not Options["No-Action"]:
1063             newcomm = npref + comm[len(opref):]
1064             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1065
1066 ################################################################################
1067
1068 def comment_accept(changes_file, comments):
1069     Upload.pkg.changes_file = changes_file
1070     Upload.init_vars()
1071     Upload.update_vars()
1072     Upload.update_subst()
1073     files = Upload.pkg.files
1074
1075     if not recheck():
1076         return # dak wants to REJECT, crap
1077
1078     (new, byhand) = check_status(files)
1079     if not new and not byhand:
1080         do_accept()
1081
1082 ################################################################################
1083
1084 def comment_reject(changes_file, comments):
1085     Upload.pkg.changes_file = changes_file
1086     Upload.init_vars()
1087     Upload.update_vars()
1088     Upload.update_subst()
1089
1090     if not recheck():
1091         pass # dak has its own reasons to reject as well, which is fine
1092
1093     reject(comments)
1094     print "REJECT\n" + reject_message,
1095     if not Options["No-Action"]:
1096         Upload.do_reject(0, reject_message)
1097         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1098
1099 ################################################################################
1100
1101 def main():
1102     changes_files = init()
1103     if len(changes_files) > 50:
1104         sys.stderr.write("Sorting changes...\n")
1105     changes_files = sort_changes(changes_files)
1106
1107     # Kill me now? **FIXME**
1108     Cnf["Dinstall::Options::No-Mail"] = ""
1109     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1110     if Cnf.has_key("Dinstall::Bcc"):
1111         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1112     else:
1113         Upload.Subst["__BCC__"] = bcc
1114
1115     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1116     if commentsdir:
1117         if changes_files != []:
1118             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1119             sys.exit(1)
1120         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1121         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1122     else:
1123         for changes_file in changes_files:
1124             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1125             if not changes_file:
1126                 continue
1127             print "\n" + changes_file
1128
1129             do_pkg (changes_file)
1130
1131     end()
1132
1133 ################################################################################
1134
1135 if __name__ == '__main__':
1136     main()