dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 from __future__ import with_statement
  45
  46 import copy
  47 import errno
  48 import os
  49 import readline
  50 import stat
  51 import sys
  52 import time
  53 import contextlib
  54 import pwd
  55 import apt_pkg, apt_inst
  56 import examine_package
  57
  58 from daklib import database
  59 from daklib import daklog
  60 from daklib import queue
  61 from daklib import utils
  62 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  63 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  64 from daklib.summarystats import SummaryStats
  65
  66 # Globals
  67 Cnf = None       #: Configuration, apt_pkg.Configuration
  68 Options = None
  69 Upload = None
  70 projectB = None  #: database connection, pgobject
  71 Logger = None
  72
  73 Priorities = None
  74 Sections = None
  75
  76 reject_message = ""
  77
  78 ################################################################################
  79 ################################################################################
  80 ################################################################################
  81
  82 def reject (str, prefix="Rejected: "):
  83     global reject_message
  84     if str:
  85         reject_message += prefix + str + "\n"
  86
  87 def recheck():
  88     global reject_message
  89     files = Upload.pkg.files
  90     reject_message = ""
  91
  92     for f in files.keys():
  93         # The .orig.tar.gz can disappear out from under us is it's a
  94         # duplicate of one in the archive.
  95         if not files.has_key(f):
  96             continue
  97         # Check that the source still exists
  98         if files[f]["type"] == "deb":
  99             source_version = files[f]["source version"]
 100             source_package = files[f]["source package"]
 101             if not Upload.pkg.changes["architecture"].has_key("source") \
 102                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
 103                 source_epochless_version = re_no_epoch.sub('', source_version)
 104                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 105                 found = 0
 106                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
 107                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 108                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 109                             found = 1
 110                 if not found:
 111                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 112
 113         # Version and file overwrite checks
 114         if files[f]["type"] == "deb":
 115             reject(Upload.check_binary_against_db(f), "")
 116         elif files[f]["type"] == "dsc":
 117             reject(Upload.check_source_against_db(f), "")
 118             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 119             reject(reject_msg, "")
 120
 121     if reject_message.find("Rejected") != -1:
 122         answer = "XXX"
 123         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 124             answer = 'S'
 125
 126         print "REJECT\n" + reject_message,
 127         prompt = "[R]eject, Skip, Quit ?"
 128
 129         while prompt.find(answer) == -1:
 130             answer = utils.our_raw_input(prompt)
 131             m = re_default_answer.match(prompt)
 132             if answer == "":
 133                 answer = m.group(1)
 134             answer = answer[:1].upper()
 135
 136         if answer == 'R':
 137             Upload.do_reject(0, reject_message)
 138             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 139             return 0
 140         elif answer == 'S':
 141             return 0
 142         elif answer == 'Q':
 143             end()
 144             sys.exit(0)
 145
 146     return 1
 147
 148 ################################################################################
 149
 150 def indiv_sg_compare (a, b):
 151     """Sort by source name, source, version, 'have source', and
 152        finally by filename."""
 153     # Sort by source version
 154     q = apt_pkg.VersionCompare(a["version"], b["version"])
 155     if q:
 156         return -q
 157
 158     # Sort by 'have source'
 159     a_has_source = a["architecture"].get("source")
 160     b_has_source = b["architecture"].get("source")
 161     if a_has_source and not b_has_source:
 162         return -1
 163     elif b_has_source and not a_has_source:
 164         return 1
 165
 166     return cmp(a["filename"], b["filename"])
 167
 168 ############################################################
 169
 170 def sg_compare (a, b):
 171     a = a[1]
 172     b = b[1]
 173     """Sort by have note, source already in database and time of oldest upload."""
 174     # Sort by have note
 175     a_note_state = a["note_state"]
 176     b_note_state = b["note_state"]
 177     if a_note_state < b_note_state:
 178         return -1
 179     elif a_note_state > b_note_state:
 180         return 1
 181     # Sort by source already in database (descending)
 182     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 183     if source_in_database:
 184         return -source_in_database
 185
 186     # Sort by time of oldest upload
 187     return cmp(a["oldest"], b["oldest"])
 188
 189 def sort_changes(changes_files):
 190     """Sort into source groups, then sort each source group by version,
 191     have source, filename.  Finally, sort the source groups by have
 192     note, time of oldest upload of each source upload."""
 193     if len(changes_files) == 1:
 194         return changes_files
 195
 196     sorted_list = []
 197     cache = {}
 198     # Read in all the .changes files
 199     for filename in changes_files:
 200         try:
 201             Upload.pkg.changes_file = filename
 202             Upload.init_vars()
 203             Upload.update_vars()
 204             cache[filename] = copy.copy(Upload.pkg.changes)
 205             cache[filename]["filename"] = filename
 206         except:
 207             sorted_list.append(filename)
 208             break
 209     # Divide the .changes into per-source groups
 210     per_source = {}
 211     for filename in cache.keys():
 212         source = cache[filename]["source"]
 213         if not per_source.has_key(source):
 214             per_source[source] = {}
 215             per_source[source]["list"] = []
 216         per_source[source]["list"].append(cache[filename])
 217     # Determine oldest time and have note status for each source group
 218     for source in per_source.keys():
 219         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 220         ql = q.getresult()
 221         per_source[source]["source_in_database"] = len(ql)>0
 222         source_list = per_source[source]["list"]
 223         first = source_list[0]
 224         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 225         have_note = 0
 226         for d in per_source[source]["list"]:
 227             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 228             if mtime < oldest:
 229                 oldest = mtime
 230             have_note += (database.has_new_comment(d["source"], d["version"], True))
 231         per_source[source]["oldest"] = oldest
 232         if not have_note:
 233             per_source[source]["note_state"] = 0; # none
 234         elif have_note < len(source_list):
 235             per_source[source]["note_state"] = 1; # some
 236         else:
 237             per_source[source]["note_state"] = 2; # all
 238         per_source[source]["list"].sort(indiv_sg_compare)
 239     per_source_items = per_source.items()
 240     per_source_items.sort(sg_compare)
 241     for i in per_source_items:
 242         for j in i[1]["list"]:
 243             sorted_list.append(j["filename"])
 244     return sorted_list
 245
 246 ################################################################################
 247
 248 class Section_Completer:
 249     def __init__ (self):
 250         self.sections = []
 251         self.matches = []
 252         q = projectB.query("SELECT section FROM section")
 253         for i in q.getresult():
 254             self.sections.append(i[0])
 255
 256     def complete(self, text, state):
 257         if state == 0:
 258             self.matches = []
 259             n = len(text)
 260             for word in self.sections:
 261                 if word[:n] == text:
 262                     self.matches.append(word)
 263         try:
 264             return self.matches[state]
 265         except IndexError:
 266             return None
 267
 268 ############################################################
 269
 270 class Priority_Completer:
 271     def __init__ (self):
 272         self.priorities = []
 273         self.matches = []
 274         q = projectB.query("SELECT priority FROM priority")
 275         for i in q.getresult():
 276             self.priorities.append(i[0])
 277
 278     def complete(self, text, state):
 279         if state == 0:
 280             self.matches = []
 281             n = len(text)
 282             for word in self.priorities:
 283                 if word[:n] == text:
 284                     self.matches.append(word)
 285         try:
 286             return self.matches[state]
 287         except IndexError:
 288             return None
 289
 290 ################################################################################
 291
 292 def print_new (new, indexed, file=sys.stdout):
 293     queue.check_valid(new)
 294     broken = 0
 295     index = 0
 296     for pkg in new.keys():
 297         index += 1
 298         section = new[pkg]["section"]
 299         priority = new[pkg]["priority"]
 300         if new[pkg]["section id"] == -1:
 301             section += "[!]"
 302             broken = 1
 303         if new[pkg]["priority id"] == -1:
 304             priority += "[!]"
 305             broken = 1
 306         if indexed:
 307             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 308         else:
 309             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 310         line = line.strip()+'\n'
 311         file.write(line)
 312     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 313     if len(note) > 0:
 314         for line in note:
 315             print line
 316     return broken, note
 317
 318 ################################################################################
 319
 320 def index_range (index):
 321     if index == 1:
 322         return "1"
 323     else:
 324         return "1-%s" % (index)
 325
 326 ################################################################################
 327 ################################################################################
 328
 329 def edit_new (new):
 330     # Write the current data to a temporary file
 331     (fd, temp_filename) = utils.temp_filename()
 332     temp_file = os.fdopen(fd, 'w')
 333     print_new (new, 0, temp_file)
 334     temp_file.close()
 335     # Spawn an editor on that file
 336     editor = os.environ.get("EDITOR","vi")
 337     result = os.system("%s %s" % (editor, temp_filename))
 338     if result != 0:
 339         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 340     # Read the edited data back in
 341     temp_file = utils.open_file(temp_filename)
 342     lines = temp_file.readlines()
 343     temp_file.close()
 344     os.unlink(temp_filename)
 345     # Parse the new data
 346     for line in lines:
 347         line = line.strip()
 348         if line == "":
 349             continue
 350         s = line.split()
 351         # Pad the list if necessary
 352         s[len(s):3] = [None] * (3-len(s))
 353         (pkg, priority, section) = s[:3]
 354         if not new.has_key(pkg):
 355             utils.warn("Ignoring unknown package '%s'" % (pkg))
 356         else:
 357             # Strip off any invalid markers, print_new will readd them.
 358             if section.endswith("[!]"):
 359                 section = section[:-3]
 360             if priority.endswith("[!]"):
 361                 priority = priority[:-3]
 362             for f in new[pkg]["files"]:
 363                 Upload.pkg.files[f]["section"] = section
 364                 Upload.pkg.files[f]["priority"] = priority
 365             new[pkg]["section"] = section
 366             new[pkg]["priority"] = priority
 367
 368 ################################################################################
 369
 370 def edit_index (new, index):
 371     priority = new[index]["priority"]
 372     section = new[index]["section"]
 373     ftype = new[index]["type"]
 374     done = 0
 375     while not done:
 376         print "\t".join([index, priority, section])
 377
 378         answer = "XXX"
 379         if ftype != "dsc":
 380             prompt = "[B]oth, Priority, Section, Done ? "
 381         else:
 382             prompt = "[S]ection, Done ? "
 383         edit_priority = edit_section = 0
 384
 385         while prompt.find(answer) == -1:
 386             answer = utils.our_raw_input(prompt)
 387             m = re_default_answer.match(prompt)
 388             if answer == "":
 389                 answer = m.group(1)
 390             answer = answer[:1].upper()
 391
 392         if answer == 'P':
 393             edit_priority = 1
 394         elif answer == 'S':
 395             edit_section = 1
 396         elif answer == 'B':
 397             edit_priority = edit_section = 1
 398         elif answer == 'D':
 399             done = 1
 400
 401         # Edit the priority
 402         if edit_priority:
 403             readline.set_completer(Priorities.complete)
 404             got_priority = 0
 405             while not got_priority:
 406                 new_priority = utils.our_raw_input("New priority: ").strip()
 407                 if new_priority not in Priorities.priorities:
 408                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 409                 else:
 410                     got_priority = 1
 411                     priority = new_priority
 412
 413         # Edit the section
 414         if edit_section:
 415             readline.set_completer(Sections.complete)
 416             got_section = 0
 417             while not got_section:
 418                 new_section = utils.our_raw_input("New section: ").strip()
 419                 if new_section not in Sections.sections:
 420                     print "E: '%s' is not a valid section, try again." % (new_section)
 421                 else:
 422                     got_section = 1
 423                     section = new_section
 424
 425         # Reset the readline completer
 426         readline.set_completer(None)
 427
 428     for f in new[index]["files"]:
 429         Upload.pkg.files[f]["section"] = section
 430         Upload.pkg.files[f]["priority"] = priority
 431     new[index]["priority"] = priority
 432     new[index]["section"] = section
 433     return new
 434
 435 ################################################################################
 436
 437 def edit_overrides (new):
 438     print
 439     done = 0
 440     while not done:
 441         print_new (new, 1)
 442         new_index = {}
 443         index = 0
 444         for i in new.keys():
 445             index += 1
 446             new_index[index] = i
 447
 448         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 449
 450         got_answer = 0
 451         while not got_answer:
 452             answer = utils.our_raw_input(prompt)
 453             if not answer.isdigit():
 454                 answer = answer[:1].upper()
 455             if answer == "E" or answer == "D":
 456                 got_answer = 1
 457             elif re_isanum.match (answer):
 458                 answer = int(answer)
 459                 if (answer < 1) or (answer > index):
 460                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 461                 else:
 462                     got_answer = 1
 463
 464         if answer == 'E':
 465             edit_new(new)
 466         elif answer == 'D':
 467             done = 1
 468         else:
 469             edit_index (new, new_index[answer])
 470
 471     return new
 472
 473 ################################################################################
 474
 475 def edit_note(note):
 476     # Write the current data to a temporary file
 477     (fd, temp_filename) = utils.temp_filename()
 478     editor = os.environ.get("EDITOR","vi")
 479     answer = 'E'
 480     while answer == 'E':
 481         os.system("%s %s" % (editor, temp_filename))
 482         temp_file = utils.open_file(temp_filename)
 483         newnote = temp_file.read().rstrip()
 484         temp_file.close()
 485         print "New Note:"
 486         print utils.prefix_multi_line_string(newnote,"  ")
 487         prompt = "[D]one, Edit, Abandon, Quit ?"
 488         answer = "XXX"
 489         while prompt.find(answer) == -1:
 490             answer = utils.our_raw_input(prompt)
 491             m = re_default_answer.search(prompt)
 492             if answer == "":
 493                 answer = m.group(1)
 494             answer = answer[:1].upper()
 495     os.unlink(temp_filename)
 496     if answer == 'A':
 497         return
 498     elif answer == 'Q':
 499         end()
 500         sys.exit(0)
 501
 502     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami(), bool(Options["Trainee"]))
 503
 504 ################################################################################
 505
 506 def check_pkg ():
 507     try:
 508         less_fd = os.popen("less -R -", 'w', 0)
 509         stdout_fd = sys.stdout
 510         try:
 511             sys.stdout = less_fd
 512             changes = utils.parse_changes (Upload.pkg.changes_file)
 513             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 514             files = Upload.pkg.files
 515             for f in files.keys():
 516                 if files[f].has_key("new"):
 517                     ftype = files[f]["type"]
 518                     if ftype == "deb":
 519                         examine_package.check_deb(changes['distribution'], f)
 520                     elif ftype == "dsc":
 521                         examine_package.check_dsc(changes['distribution'], f)
 522         finally:
 523             examine_package.output_package_relations()
 524             sys.stdout = stdout_fd
 525     except IOError, e:
 526         if e.errno == errno.EPIPE:
 527             utils.warn("[examine_package] Caught EPIPE; skipping.")
 528             pass
 529         else:
 530             raise
 531     except KeyboardInterrupt:
 532         utils.warn("[examine_package] Caught C-c; skipping.")
 533         pass
 534
 535 ################################################################################
 536
 537 ## FIXME: horribly Debian specific
 538
 539 def do_bxa_notification():
 540     files = Upload.pkg.files
 541     summary = ""
 542     for f in files.keys():
 543         if files[f]["type"] == "deb":
 544             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 545             summary += "\n"
 546             summary += "Package: %s\n" % (control.Find("Package"))
 547             summary += "Description: %s\n" % (control.Find("Description"))
 548     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 549     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 550     utils.send_mail(bxa_mail)
 551
 552 ################################################################################
 553
 554 def add_overrides (new):
 555     changes = Upload.pkg.changes
 556     files = Upload.pkg.files
 557     srcpkg = changes.get("source")
 558
 559     projectB.query("BEGIN WORK")
 560     for suite in changes["suite"].keys():
 561         suite_id = database.get_suite_id(suite)
 562         for pkg in new.keys():
 563             component_id = database.get_component_id(new[pkg]["component"])
 564             type_id = database.get_override_type_id(new[pkg]["type"])
 565             priority_id = new[pkg]["priority id"]
 566             section_id = new[pkg]["section id"]
 567             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 568             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 569             for f in new[pkg]["files"]:
 570                 if files[f].has_key("new"):
 571                     del files[f]["new"]
 572             del new[pkg]
 573
 574     projectB.query("COMMIT WORK")
 575
 576     if Cnf.FindB("Dinstall::BXANotify"):
 577         do_bxa_notification()
 578
 579 ################################################################################
 580
 581 def prod_maintainer (note):
 582     # Here we prepare an editor and get them ready to prod...
 583     (fd, temp_filename) = utils.temp_filename()
 584     temp_file = os.fdopen(fd, 'w')
 585     if len(note) > 0:
 586         for line in note:
 587             temp_file.write(line)
 588     temp_file.close()
 589     editor = os.environ.get("EDITOR","vi")
 590     answer = 'E'
 591     while answer == 'E':
 592         os.system("%s %s" % (editor, temp_filename))
 593         temp_fh = utils.open_file(temp_filename)
 594         prod_message = "".join(temp_fh.readlines())
 595         temp_fh.close()
 596         print "Prod message:"
 597         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 598         prompt = "[P]rod, Edit, Abandon, Quit ?"
 599         answer = "XXX"
 600         while prompt.find(answer) == -1:
 601             answer = utils.our_raw_input(prompt)
 602             m = re_default_answer.search(prompt)
 603             if answer == "":
 604                 answer = m.group(1)
 605             answer = answer[:1].upper()
 606     os.unlink(temp_filename)
 607     if answer == 'A':
 608         return
 609     elif answer == 'Q':
 610         end()
 611         sys.exit(0)
 612     # Otherwise, do the proding...
 613     user_email_address = utils.whoami() + " <%s>" % (
 614         Cnf["Dinstall::MyAdminAddress"])
 615
 616     Subst = Upload.Subst
 617
 618     Subst["__FROM_ADDRESS__"] = user_email_address
 619     Subst["__PROD_MESSAGE__"] = prod_message
 620     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 621
 622     prod_mail_message = utils.TemplateSubst(
 623         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 624
 625     # Send the prod mail if appropriate
 626     if not Cnf["Dinstall::Options::No-Mail"]:
 627         utils.send_mail(prod_mail_message)
 628
 629     print "Sent proding message"
 630
 631 ################################################################################
 632
 633 def do_new():
 634     print "NEW\n"
 635     files = Upload.pkg.files
 636     changes = Upload.pkg.changes
 637
 638     # Make a copy of distribution we can happily trample on
 639     changes["suite"] = copy.copy(changes["distribution"])
 640
 641     # Fix up the list of target suites
 642     for suite in changes["suite"].keys():
 643         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 644         if override:
 645             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 646               database.get_suite_id(override) == -1)
 647             if olderr or newerr:
 648                 (oinv, newinv) = ("", "")
 649                 if olderr: oinv = "invalid "
 650                 if newerr: ninv = "invalid "
 651                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 652                         oinv, suite, ninv, override)
 653             del changes["suite"][suite]
 654             changes["suite"][override] = 1
 655     # Validate suites
 656     for suite in changes["suite"].keys():
 657         suite_id = database.get_suite_id(suite)
 658         if suite_id == -1:
 659             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 660
 661     # The main NEW processing loop
 662     done = 0
 663     while not done:
 664         # Find out what's new
 665         new = queue.determine_new(changes, files, projectB)
 666
 667         if not new:
 668             break
 669
 670         answer = "XXX"
 671         if Options["No-Action"] or Options["Automatic"]:
 672             answer = 'S'
 673
 674         (broken, note) = print_new(new, 0)
 675         prompt = ""
 676
 677         if not broken and not note:
 678             prompt = "Add overrides, "
 679         if broken:
 680             print "W: [!] marked entries must be fixed before package can be processed."
 681         if note:
 682             print "W: note must be removed before package can be processed."
 683             prompt += "RemOve all notes, Remove note, "
 684
 685         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 686
 687         while prompt.find(answer) == -1:
 688             answer = utils.our_raw_input(prompt)
 689             m = re_default_answer.search(prompt)
 690             if answer == "":
 691                 answer = m.group(1)
 692             answer = answer[:1].upper()
 693
 694         if answer == 'A' and not Options["Trainee"]:
 695             try:
 696                 check_daily_lock()
 697                 done = add_overrides (new)
 698                 Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 699             except CantGetLockError:
 700                 print "Hello? Operator! Give me the number for 911!"
 701                 print "Dinstall in the locked area, cant process packages, come back later"
 702         elif answer == 'C':
 703             check_pkg()
 704         elif answer == 'E' and not Options["Trainee"]:
 705             new = edit_overrides (new)
 706         elif answer == 'M' and not Options["Trainee"]:
 707             aborted = Upload.do_reject(manual=1,
 708                                        reject_message=Options["Manual-Reject"],
 709                                        note=database.get_new_comments(changes.get("source", "")))
 710             if not aborted:
 711                 Logger.log([utils.getusername(), "NEW REJECT: %s" % (Upload.pkg.changes_file)])
 712                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 713                 done = 1
 714         elif answer == 'N':
 715             edit_note(database.get_new_comments(changes.get("source", "")))
 716         elif answer == 'P' and not Options["Trainee"]:
 717             prod_maintainer(database.get_new_comments(changes.get("source", "")))
 718             Logger.log([utils.getusername(), "NEW PROD: %s" % (Upload.pkg.changes_file)])
 719         elif answer == 'R' and not Options["Trainee"]:
 720             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 721             if confirm == "y":
 722                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 723         elif answer == 'O' and not Options["Trainee"]:
 724             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 725             if confirm == "y":
 726                 database.delete_all_new_comments(changes.get("source"))
 727         elif answer == 'S':
 728             done = 1
 729         elif answer == 'Q':
 730             end()
 731             sys.exit(0)
 732
 733 ################################################################################
 734 ################################################################################
 735 ################################################################################
 736
 737 def usage (exit_code=0):
 738     print """Usage: dak process-new [OPTION]... [CHANGES]...
 739   -a, --automatic           automatic run
 740   -h, --help                show this help and exit.
 741   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 742   -m, --manual-reject=MSG   manual reject with `msg'
 743   -n, --no-action           don't do anything
 744   -t, --trainee             FTP Trainee mode
 745   -V, --version             display the version number and exit"""
 746     sys.exit(exit_code)
 747
 748 ################################################################################
 749
 750 def init():
 751     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 752
 753     Cnf = utils.get_conf()
 754
 755     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 756                  ('h',"help","Process-New::Options::Help"),
 757                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 758                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 759                  ('t',"trainee","Process-New::Options::Trainee"),
 760                  ('n',"no-action","Process-New::Options::No-Action")]
 761
 762     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 763         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 764             Cnf["Process-New::Options::%s" % (i)] = ""
 765
 766     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 767     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 768         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 769
 770     Options = Cnf.SubTree("Process-New::Options")
 771
 772     if Options["Help"]:
 773         usage()
 774
 775     Upload = queue.Upload(Cnf)
 776
 777     if not Options["No-Action"]:
 778         try:
 779             Logger = Upload.Logger = daklog.Logger(Cnf, "process-new")
 780         except CantOpenError, e:
 781             Options["Trainee"] = "True"
 782
 783     projectB = Upload.projectB
 784
 785     Sections = Section_Completer()
 786     Priorities = Priority_Completer()
 787     readline.parse_and_bind("tab: complete")
 788
 789     return changes_files
 790
 791 ################################################################################
 792
 793 def do_byhand():
 794     done = 0
 795     while not done:
 796         files = Upload.pkg.files
 797         will_install = 1
 798         byhand = []
 799
 800         for f in files.keys():
 801             if files[f]["type"] == "byhand":
 802                 if os.path.exists(f):
 803                     print "W: %s still present; please process byhand components and try again." % (f)
 804                     will_install = 0
 805                 else:
 806                     byhand.append(f)
 807
 808         answer = "XXXX"
 809         if Options["No-Action"]:
 810             answer = "S"
 811         if will_install:
 812             if Options["Automatic"] and not Options["No-Action"]:
 813                 answer = 'A'
 814             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 815         else:
 816             prompt = "Manual reject, [S]kip, Quit ?"
 817
 818         while prompt.find(answer) == -1:
 819             answer = utils.our_raw_input(prompt)
 820             m = re_default_answer.search(prompt)
 821             if answer == "":
 822                 answer = m.group(1)
 823             answer = answer[:1].upper()
 824
 825         if answer == 'A':
 826             try:
 827                 check_daily_lock()
 828                 done = 1
 829                 for f in byhand:
 830                     del files[f]
 831                 Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (Upload.pkg.changes_file)])
 832             except CantGetLockError:
 833                 print "Hello? Operator! Give me the number for 911!"
 834                 print "Dinstall in the locked area, cant process packages, come back later"
 835         elif answer == 'M':
 836             Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (Upload.pkg.changes_file)])
 837             Upload.do_reject(1, Options["Manual-Reject"])
 838             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 839             done = 1
 840         elif answer == 'S':
 841             done = 1
 842         elif answer == 'Q':
 843             end()
 844             sys.exit(0)
 845
 846 ################################################################################
 847
 848 def check_daily_lock():
 849     """
 850     Raises CantGetLockError if the dinstall daily.lock exists.
 851     """
 852
 853     try:
 854         os.open(Cnf["Process-New::DinstallLockFile"],  os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 855     except OSError, e:
 856         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 857             raise CantGetLockError
 858
 859     os.unlink(Cnf["Process-New::DinstallLockFile"])
 860
 861
 862 @contextlib.contextmanager
 863 def lock_package(package):
 864     """
 865     Lock C{package} so that noone else jumps in processing it.
 866
 867     @type package: string
 868     @param package: source package name to lock
 869     """
 870
 871     path = os.path.join(Cnf["Process-New::LockDir"], package)
 872     try:
 873         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 874     except OSError, e:
 875         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 876             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 877             raise AlreadyLockedError, user
 878
 879     try:
 880         yield fd
 881     finally:
 882         os.unlink(path)
 883
 884 def move_to_dir (dest, perms=0660, changesperms=0664):
 885     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 886     file_keys = Upload.pkg.files.keys()
 887     for f in file_keys:
 888         utils.move (f, dest, perms=perms)
 889
 890 def is_source_in_queue_dir(qdir):
 891     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 892                 and x.endswith(".changes") ]
 893     for entry in entries:
 894         # read the .dak
 895         u = queue.Upload(Cnf)
 896         u.pkg.changes_file = os.path.join(qdir, entry)
 897         u.update_vars()
 898         if not u.pkg.changes["architecture"].has_key("source"):
 899             # another binary upload, ignore
 900             continue
 901         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 902             # another version, ignore
 903             continue
 904         # found it!
 905         return True
 906     return False
 907
 908 def move_to_holding(suite, queue_dir):
 909     print "Moving to %s holding area." % (suite.upper(),)
 910     if Options["No-Action"]:
 911         return
 912     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 913     Upload.dump_vars(queue_dir)
 914     move_to_dir(queue_dir, perms=0664)
 915     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 916
 917 def _accept():
 918     if Options["No-Action"]:
 919         return
 920     (summary, short_summary) = Upload.build_summaries()
 921     Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"])
 922     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 923
 924 def do_accept_stableupdate(suite, q):
 925     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 926     if not Upload.pkg.changes["architecture"].has_key("source"):
 927         # It is not a sourceful upload.  So its source may be either in p-u
 928         # holding, in new, in accepted or already installed.
 929         if is_source_in_queue_dir(queue_dir):
 930             # It's in p-u holding, so move it there.
 931             print "Binary-only upload, source in %s." % (q,)
 932             move_to_holding(suite, queue_dir)
 933         elif Upload.source_exists(Upload.pkg.changes["source"],
 934                 Upload.pkg.changes["version"]):
 935             # dak tells us that there is source available.  At time of
 936             # writing this means that it is installed, so put it into
 937             # accepted.
 938             print "Binary-only upload, source installed."
 939             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 940             _accept()
 941         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 942             # The source is in accepted, the binary cleared NEW: accept it.
 943             print "Binary-only upload, source in accepted."
 944             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 945             _accept()
 946         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 947             # It's in NEW.  We expect the source to land in p-u holding
 948             # pretty soon.
 949             print "Binary-only upload, source in new."
 950             move_to_holding(suite, queue_dir)
 951         elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
 952             # It's in newstage.  Accept into the holding area
 953             print "Binary-only upload, source in newstage."
 954             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 955             _accept()
 956         else:
 957             # No case applicable.  Bail out.  Return will cause the upload
 958             # to be skipped.
 959             print "ERROR"
 960             print "Stable update failed.  Source not found."
 961             return
 962     else:
 963         # We are handling a sourceful upload.  Move to accepted if currently
 964         # in p-u holding and to p-u holding otherwise.
 965         if is_source_in_queue_dir(queue_dir):
 966             print "Sourceful upload in %s, accepting." % (q,)
 967             _accept()
 968         else:
 969             move_to_holding(suite, queue_dir)
 970
 971 def do_accept():
 972     print "ACCEPT"
 973     if not Options["No-Action"]:
 974         (summary, short_summary) = Upload.build_summaries()
 975     if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 976         Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 977         move_to_dir(Cnf["Dir::Queue::Embargoed"])
 978         Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 979         # Check for override disparities
 980         Upload.Subst["__SUMMARY__"] = summary
 981     else:
 982         # Stable updates need to be copied to proposed-updates holding
 983         # area instead of accepted.  Sourceful uploads need to go
 984         # to it directly, binaries only if the source has not yet been
 985         # accepted into p-u.
 986         for suite, q in [("proposed-updates", "ProposedUpdates"),
 987                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
 988             if not Upload.pkg.changes["distribution"].has_key(suite):
 989                 continue
 990             return do_accept_stableupdate(suite, q)
 991         # Just a normal upload, accept it...
 992         _accept()
 993
 994 def check_status(files):
 995     new = byhand = 0
 996     for f in files.keys():
 997         if files[f]["type"] == "byhand":
 998             byhand = 1
 999         elif files[f].has_key("new"):
1000             new = 1
1001     return (new, byhand)
1002
1003 def do_pkg(changes_file):
1004     Upload.pkg.changes_file = changes_file
1005     Upload.init_vars()
1006     Upload.update_vars()
1007     Upload.update_subst()
1008     files = Upload.pkg.files
1009
1010     try:
1011         with lock_package(Upload.pkg.changes["source"]):
1012             if not recheck():
1013                 return
1014
1015             (new, byhand) = check_status(files)
1016             if new or byhand:
1017                 if new:
1018                     do_new()
1019                 if byhand:
1020                     do_byhand()
1021                 (new, byhand) = check_status(files)
1022
1023             if not new and not byhand:
1024                 try:
1025                     check_daily_lock()
1026                     do_accept()
1027                 except CantGetLockError:
1028                     print "Hello? Operator! Give me the number for 911!"
1029                     print "Dinstall in the locked area, cant process packages, come back later"
1030     except AlreadyLockedError, e:
1031         print "Seems to be locked by %s already, skipping..." % (e)
1032
1033 ################################################################################
1034
1035 def end():
1036     accept_count = SummaryStats().accept_count
1037     accept_bytes = SummaryStats().accept_bytes
1038
1039     if accept_count:
1040         sets = "set"
1041         if accept_count > 1:
1042             sets = "sets"
1043         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1044         Logger.log([utils.getusername(), "total",accept_count,accept_bytes])
1045
1046     if not Options["No-Action"] and not Options["Trainee"]:
1047         Logger.close()
1048
1049 ################################################################################
1050
1051 def do_comments(dir, opref, npref, line, fn):
1052     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1053         lines = open("%s/%s" % (dir, comm)).readlines()
1054         if len(lines) == 0 or lines[0] != line + "\n": continue
1055         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1056                                 and x.endswith(".changes") ]
1057         changes_files = sort_changes(changes_files)
1058         for f in changes_files:
1059             f = utils.validate_changes_file_arg(f, 0)
1060             if not f: continue
1061             print "\n" + f
1062             fn(f, "".join(lines[1:]))
1063
1064         if opref != npref and not Options["No-Action"]:
1065             newcomm = npref + comm[len(opref):]
1066             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1067
1068 ################################################################################
1069
1070 def comment_accept(changes_file, comments):
1071     Upload.pkg.changes_file = changes_file
1072     Upload.init_vars()
1073     Upload.update_vars()
1074     Upload.update_subst()
1075     files = Upload.pkg.files
1076
1077     if not recheck():
1078         return # dak wants to REJECT, crap
1079
1080     (new, byhand) = check_status(files)
1081     if not new and not byhand:
1082         do_accept()
1083
1084 ################################################################################
1085
1086 def comment_reject(changes_file, comments):
1087     Upload.pkg.changes_file = changes_file
1088     Upload.init_vars()
1089     Upload.update_vars()
1090     Upload.update_subst()
1091
1092     if not recheck():
1093         pass # dak has its own reasons to reject as well, which is fine
1094
1095     reject(comments)
1096     print "REJECT\n" + reject_message,
1097     if not Options["No-Action"]:
1098         Upload.do_reject(0, reject_message)
1099         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1100
1101 ################################################################################
1102
1103 def main():
1104     changes_files = init()
1105     if len(changes_files) > 50:
1106         sys.stderr.write("Sorting changes...\n")
1107     changes_files = sort_changes(changes_files)
1108
1109     # Kill me now? **FIXME**
1110     Cnf["Dinstall::Options::No-Mail"] = ""
1111     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1112     if Cnf.has_key("Dinstall::Bcc"):
1113         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1114     else:
1115         Upload.Subst["__BCC__"] = bcc
1116
1117     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1118     if commentsdir:
1119         if changes_files != []:
1120             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1121             sys.exit(1)
1122         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1123         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1124     else:
1125         for changes_file in changes_files:
1126             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1127             if not changes_file:
1128                 continue
1129             print "\n" + changes_file
1130
1131             do_pkg (changes_file)
1132
1133     end()
1134
1135 ################################################################################
1136
1137 if __name__ == '__main__':
1138     main()