dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 from __future__ import with_statement
  45
  46 import copy
  47 import errno
  48 import os
  49 import readline
  50 import stat
  51 import sys
  52 import time
  53 import contextlib
  54 import pwd
  55 import apt_pkg, apt_inst
  56 import examine_package
  57
  58 from daklib import database
  59 from daklib import daklog
  60 from daklib import queue
  61 from daklib import utils
  62 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  63 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  64 from daklib.summarystats import SummaryStats
  65
  66 # Globals
  67 Cnf = None       #: Configuration, apt_pkg.Configuration
  68 Options = None
  69 Upload = None
  70 projectB = None  #: database connection, pgobject
  71 Logger = None
  72
  73 Priorities = None
  74 Sections = None
  75
  76 reject_message = ""
  77
  78 ################################################################################
  79 ################################################################################
  80 ################################################################################
  81
  82 def reject (str, prefix="Rejected: "):
  83     global reject_message
  84     if str:
  85         reject_message += prefix + str + "\n"
  86
  87 def recheck():
  88     global reject_message
  89     files = Upload.pkg.files
  90     reject_message = ""
  91
  92     for f in files.keys():
  93         # The .orig.tar.gz can disappear out from under us is it's a
  94         # duplicate of one in the archive.
  95         if not files.has_key(f):
  96             continue
  97         # Check that the source still exists
  98         if files[f]["type"] == "deb":
  99             source_version = files[f]["source version"]
 100             source_package = files[f]["source package"]
 101             if not Upload.pkg.changes["architecture"].has_key("source") \
 102                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
 103                 source_epochless_version = re_no_epoch.sub('', source_version)
 104                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 105                 found = 0
 106                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
 107                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 108                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 109                             found = 1
 110                 if not found:
 111                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 112
 113         # Version and file overwrite checks
 114         if files[f]["type"] == "deb":
 115             reject(Upload.check_binary_against_db(f), "")
 116         elif files[f]["type"] == "dsc":
 117             reject(Upload.check_source_against_db(f), "")
 118             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 119             reject(reject_msg, "")
 120
 121     if reject_message.find("Rejected") != -1:
 122         answer = "XXX"
 123         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 124             answer = 'S'
 125
 126         print "REJECT\n" + reject_message,
 127         prompt = "[R]eject, Skip, Quit ?"
 128
 129         while prompt.find(answer) == -1:
 130             answer = utils.our_raw_input(prompt)
 131             m = re_default_answer.match(prompt)
 132             if answer == "":
 133                 answer = m.group(1)
 134             answer = answer[:1].upper()
 135
 136         if answer == 'R':
 137             Upload.do_reject(0, reject_message)
 138             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 139             return 0
 140         elif answer == 'S':
 141             return 0
 142         elif answer == 'Q':
 143             end()
 144             sys.exit(0)
 145
 146     return 1
 147
 148 ################################################################################
 149
 150 def indiv_sg_compare (a, b):
 151     """Sort by source name, source, version, 'have source', and
 152        finally by filename."""
 153     # Sort by source version
 154     q = apt_pkg.VersionCompare(a["version"], b["version"])
 155     if q:
 156         return -q
 157
 158     # Sort by 'have source'
 159     a_has_source = a["architecture"].get("source")
 160     b_has_source = b["architecture"].get("source")
 161     if a_has_source and not b_has_source:
 162         return -1
 163     elif b_has_source and not a_has_source:
 164         return 1
 165
 166     return cmp(a["filename"], b["filename"])
 167
 168 ############################################################
 169
 170 def sg_compare (a, b):
 171     a = a[1]
 172     b = b[1]
 173     """Sort by have note, source already in database and time of oldest upload."""
 174     # Sort by have note
 175     a_note_state = a["note_state"]
 176     b_note_state = b["note_state"]
 177     if a_note_state < b_note_state:
 178         return -1
 179     elif a_note_state > b_note_state:
 180         return 1
 181     # Sort by source already in database (descending)
 182     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 183     if source_in_database:
 184         return -source_in_database
 185
 186     # Sort by time of oldest upload
 187     return cmp(a["oldest"], b["oldest"])
 188
 189 def sort_changes(changes_files):
 190     """Sort into source groups, then sort each source group by version,
 191     have source, filename.  Finally, sort the source groups by have
 192     note, time of oldest upload of each source upload."""
 193     if len(changes_files) == 1:
 194         return changes_files
 195
 196     sorted_list = []
 197     cache = {}
 198     # Read in all the .changes files
 199     for filename in changes_files:
 200         try:
 201             Upload.pkg.changes_file = filename
 202             Upload.init_vars()
 203             Upload.update_vars()
 204             cache[filename] = copy.copy(Upload.pkg.changes)
 205             cache[filename]["filename"] = filename
 206         except:
 207             sorted_list.append(filename)
 208             break
 209     # Divide the .changes into per-source groups
 210     per_source = {}
 211     for filename in cache.keys():
 212         source = cache[filename]["source"]
 213         if not per_source.has_key(source):
 214             per_source[source] = {}
 215             per_source[source]["list"] = []
 216         per_source[source]["list"].append(cache[filename])
 217     # Determine oldest time and have note status for each source group
 218     for source in per_source.keys():
 219         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 220         ql = q.getresult()
 221         per_source[source]["source_in_database"] = len(ql)>0
 222         source_list = per_source[source]["list"]
 223         first = source_list[0]
 224         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 225         have_note = 0
 226         for d in per_source[source]["list"]:
 227             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 228             if mtime < oldest:
 229                 oldest = mtime
 230             have_note += (database.has_new_comment(d["source"], d["version"], True))
 231         per_source[source]["oldest"] = oldest
 232         if not have_note:
 233             per_source[source]["note_state"] = 0; # none
 234         elif have_note < len(source_list):
 235             per_source[source]["note_state"] = 1; # some
 236         else:
 237             per_source[source]["note_state"] = 2; # all
 238         per_source[source]["list"].sort(indiv_sg_compare)
 239     per_source_items = per_source.items()
 240     per_source_items.sort(sg_compare)
 241     for i in per_source_items:
 242         for j in i[1]["list"]:
 243             sorted_list.append(j["filename"])
 244     return sorted_list
 245
 246 ################################################################################
 247
 248 class Section_Completer:
 249     def __init__ (self):
 250         self.sections = []
 251         self.matches = []
 252         q = projectB.query("SELECT section FROM section")
 253         for i in q.getresult():
 254             self.sections.append(i[0])
 255
 256     def complete(self, text, state):
 257         if state == 0:
 258             self.matches = []
 259             n = len(text)
 260             for word in self.sections:
 261                 if word[:n] == text:
 262                     self.matches.append(word)
 263         try:
 264             return self.matches[state]
 265         except IndexError:
 266             return None
 267
 268 ############################################################
 269
 270 class Priority_Completer:
 271     def __init__ (self):
 272         self.priorities = []
 273         self.matches = []
 274         q = projectB.query("SELECT priority FROM priority")
 275         for i in q.getresult():
 276             self.priorities.append(i[0])
 277
 278     def complete(self, text, state):
 279         if state == 0:
 280             self.matches = []
 281             n = len(text)
 282             for word in self.priorities:
 283                 if word[:n] == text:
 284                     self.matches.append(word)
 285         try:
 286             return self.matches[state]
 287         except IndexError:
 288             return None
 289
 290 ################################################################################
 291
 292 def print_new (new, indexed, file=sys.stdout):
 293     queue.check_valid(new)
 294     broken = 0
 295     index = 0
 296     for pkg in new.keys():
 297         index += 1
 298         section = new[pkg]["section"]
 299         priority = new[pkg]["priority"]
 300         if new[pkg]["section id"] == -1:
 301             section += "[!]"
 302             broken = 1
 303         if new[pkg]["priority id"] == -1:
 304             priority += "[!]"
 305             broken = 1
 306         if indexed:
 307             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 308         else:
 309             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 310         line = line.strip()+'\n'
 311         file.write(line)
 312     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 313     if len(note) > 0:
 314         for line in note:
 315             print line
 316     return broken, note
 317
 318 ################################################################################
 319
 320 def index_range (index):
 321     if index == 1:
 322         return "1"
 323     else:
 324         return "1-%s" % (index)
 325
 326 ################################################################################
 327 ################################################################################
 328
 329 def edit_new (new):
 330     # Write the current data to a temporary file
 331     (fd, temp_filename) = utils.temp_filename()
 332     temp_file = os.fdopen(fd, 'w')
 333     print_new (new, 0, temp_file)
 334     temp_file.close()
 335     # Spawn an editor on that file
 336     editor = os.environ.get("EDITOR","vi")
 337     result = os.system("%s %s" % (editor, temp_filename))
 338     if result != 0:
 339         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 340     # Read the edited data back in
 341     temp_file = utils.open_file(temp_filename)
 342     lines = temp_file.readlines()
 343     temp_file.close()
 344     os.unlink(temp_filename)
 345     # Parse the new data
 346     for line in lines:
 347         line = line.strip()
 348         if line == "":
 349             continue
 350         s = line.split()
 351         # Pad the list if necessary
 352         s[len(s):3] = [None] * (3-len(s))
 353         (pkg, priority, section) = s[:3]
 354         if not new.has_key(pkg):
 355             utils.warn("Ignoring unknown package '%s'" % (pkg))
 356         else:
 357             # Strip off any invalid markers, print_new will readd them.
 358             if section.endswith("[!]"):
 359                 section = section[:-3]
 360             if priority.endswith("[!]"):
 361                 priority = priority[:-3]
 362             for f in new[pkg]["files"]:
 363                 Upload.pkg.files[f]["section"] = section
 364                 Upload.pkg.files[f]["priority"] = priority
 365             new[pkg]["section"] = section
 366             new[pkg]["priority"] = priority
 367
 368 ################################################################################
 369
 370 def edit_index (new, index):
 371     priority = new[index]["priority"]
 372     section = new[index]["section"]
 373     ftype = new[index]["type"]
 374     done = 0
 375     while not done:
 376         print "\t".join([index, priority, section])
 377
 378         answer = "XXX"
 379         if ftype != "dsc":
 380             prompt = "[B]oth, Priority, Section, Done ? "
 381         else:
 382             prompt = "[S]ection, Done ? "
 383         edit_priority = edit_section = 0
 384
 385         while prompt.find(answer) == -1:
 386             answer = utils.our_raw_input(prompt)
 387             m = re_default_answer.match(prompt)
 388             if answer == "":
 389                 answer = m.group(1)
 390             answer = answer[:1].upper()
 391
 392         if answer == 'P':
 393             edit_priority = 1
 394         elif answer == 'S':
 395             edit_section = 1
 396         elif answer == 'B':
 397             edit_priority = edit_section = 1
 398         elif answer == 'D':
 399             done = 1
 400
 401         # Edit the priority
 402         if edit_priority:
 403             readline.set_completer(Priorities.complete)
 404             got_priority = 0
 405             while not got_priority:
 406                 new_priority = utils.our_raw_input("New priority: ").strip()
 407                 if new_priority not in Priorities.priorities:
 408                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 409                 else:
 410                     got_priority = 1
 411                     priority = new_priority
 412
 413         # Edit the section
 414         if edit_section:
 415             readline.set_completer(Sections.complete)
 416             got_section = 0
 417             while not got_section:
 418                 new_section = utils.our_raw_input("New section: ").strip()
 419                 if new_section not in Sections.sections:
 420                     print "E: '%s' is not a valid section, try again." % (new_section)
 421                 else:
 422                     got_section = 1
 423                     section = new_section
 424
 425         # Reset the readline completer
 426         readline.set_completer(None)
 427
 428     for f in new[index]["files"]:
 429         Upload.pkg.files[f]["section"] = section
 430         Upload.pkg.files[f]["priority"] = priority
 431     new[index]["priority"] = priority
 432     new[index]["section"] = section
 433     return new
 434
 435 ################################################################################
 436
 437 def edit_overrides (new):
 438     print
 439     done = 0
 440     while not done:
 441         print_new (new, 1)
 442         new_index = {}
 443         index = 0
 444         for i in new.keys():
 445             index += 1
 446             new_index[index] = i
 447
 448         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 449
 450         got_answer = 0
 451         while not got_answer:
 452             answer = utils.our_raw_input(prompt)
 453             if not answer.isdigit():
 454                 answer = answer[:1].upper()
 455             if answer == "E" or answer == "D":
 456                 got_answer = 1
 457             elif re_isanum.match (answer):
 458                 answer = int(answer)
 459                 if (answer < 1) or (answer > index):
 460                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 461                 else:
 462                     got_answer = 1
 463
 464         if answer == 'E':
 465             edit_new(new)
 466         elif answer == 'D':
 467             done = 1
 468         else:
 469             edit_index (new, new_index[answer])
 470
 471     return new
 472
 473 ################################################################################
 474
 475 def edit_note(note):
 476     # Write the current data to a temporary file
 477     (fd, temp_filename) = utils.temp_filename()
 478     editor = os.environ.get("EDITOR","vi")
 479     answer = 'E'
 480     while answer == 'E':
 481         os.system("%s %s" % (editor, temp_filename))
 482         temp_file = utils.open_file(temp_filename)
 483         newnote = temp_file.read().rstrip()
 484         temp_file.close()
 485         print "New Note:"
 486         print utils.prefix_multi_line_string(newnote,"  ")
 487         prompt = "[D]one, Edit, Abandon, Quit ?"
 488         answer = "XXX"
 489         while prompt.find(answer) == -1:
 490             answer = utils.our_raw_input(prompt)
 491             m = re_default_answer.search(prompt)
 492             if answer == "":
 493                 answer = m.group(1)
 494             answer = answer[:1].upper()
 495     os.unlink(temp_filename)
 496     if answer == 'A':
 497         return
 498     elif answer == 'Q':
 499         end()
 500         sys.exit(0)
 501
 502     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami(), bool(Options["Trainee"]))
 503
 504 ################################################################################
 505
 506 def check_pkg ():
 507     try:
 508         less_fd = os.popen("less -R -", 'w', 0)
 509         stdout_fd = sys.stdout
 510         try:
 511             sys.stdout = less_fd
 512             changes = utils.parse_changes (Upload.pkg.changes_file)
 513             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 514             files = Upload.pkg.files
 515             for f in files.keys():
 516                 if files[f].has_key("new"):
 517                     ftype = files[f]["type"]
 518                     if ftype == "deb":
 519                         examine_package.check_deb(changes['distribution'], f)
 520                     elif ftype == "dsc":
 521                         examine_package.check_dsc(changes['distribution'], f)
 522         finally:
 523             examine_package.output_package_relations()
 524             sys.stdout = stdout_fd
 525     except IOError, e:
 526         if e.errno == errno.EPIPE:
 527             utils.warn("[examine_package] Caught EPIPE; skipping.")
 528             pass
 529         else:
 530             raise
 531     except KeyboardInterrupt:
 532         utils.warn("[examine_package] Caught C-c; skipping.")
 533         pass
 534
 535 ################################################################################
 536
 537 ## FIXME: horribly Debian specific
 538
 539 def do_bxa_notification():
 540     files = Upload.pkg.files
 541     summary = ""
 542     for f in files.keys():
 543         if files[f]["type"] == "deb":
 544             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 545             summary += "\n"
 546             summary += "Package: %s\n" % (control.Find("Package"))
 547             summary += "Description: %s\n" % (control.Find("Description"))
 548     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 549     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 550     utils.send_mail(bxa_mail)
 551
 552 ################################################################################
 553
 554 def add_overrides (new):
 555     changes = Upload.pkg.changes
 556     files = Upload.pkg.files
 557     srcpkg = changes.get("source")
 558
 559     projectB.query("BEGIN WORK")
 560     for suite in changes["suite"].keys():
 561         suite_id = database.get_suite_id(suite)
 562         for pkg in new.keys():
 563             component_id = database.get_component_id(new[pkg]["component"])
 564             type_id = database.get_override_type_id(new[pkg]["type"])
 565             priority_id = new[pkg]["priority id"]
 566             section_id = new[pkg]["section id"]
 567             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 568             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 569             for f in new[pkg]["files"]:
 570                 if files[f].has_key("new"):
 571                     del files[f]["new"]
 572             del new[pkg]
 573
 574     projectB.query("COMMIT WORK")
 575
 576     if Cnf.FindB("Dinstall::BXANotify"):
 577         do_bxa_notification()
 578
 579 ################################################################################
 580
 581 def prod_maintainer (note):
 582     # Here we prepare an editor and get them ready to prod...
 583     (fd, temp_filename) = utils.temp_filename()
 584     temp_file = os.fdopen(fd, 'w')
 585     if len(note) > 0:
 586         for line in note:
 587             temp_file.write(line)
 588     temp_file.close()
 589     editor = os.environ.get("EDITOR","vi")
 590     answer = 'E'
 591     while answer == 'E':
 592         os.system("%s %s" % (editor, temp_filename))
 593         temp_fh = utils.open_file(temp_filename)
 594         prod_message = "".join(temp_fh.readlines())
 595         temp_fh.close()
 596         print "Prod message:"
 597         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 598         prompt = "[P]rod, Edit, Abandon, Quit ?"
 599         answer = "XXX"
 600         while prompt.find(answer) == -1:
 601             answer = utils.our_raw_input(prompt)
 602             m = re_default_answer.search(prompt)
 603             if answer == "":
 604                 answer = m.group(1)
 605             answer = answer[:1].upper()
 606     os.unlink(temp_filename)
 607     if answer == 'A':
 608         return
 609     elif answer == 'Q':
 610         end()
 611         sys.exit(0)
 612     # Otherwise, do the proding...
 613     user_email_address = utils.whoami() + " <%s>" % (
 614         Cnf["Dinstall::MyAdminAddress"])
 615
 616     Subst = Upload.Subst
 617
 618     Subst["__FROM_ADDRESS__"] = user_email_address
 619     Subst["__PROD_MESSAGE__"] = prod_message
 620     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 621
 622     prod_mail_message = utils.TemplateSubst(
 623         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 624
 625     # Send the prod mail if appropriate
 626     if not Cnf["Dinstall::Options::No-Mail"]:
 627         utils.send_mail(prod_mail_message)
 628
 629     print "Sent proding message"
 630
 631 ################################################################################
 632
 633 def do_new():
 634     print "NEW\n"
 635     files = Upload.pkg.files
 636     changes = Upload.pkg.changes
 637
 638     # Make a copy of distribution we can happily trample on
 639     changes["suite"] = copy.copy(changes["distribution"])
 640
 641     # Fix up the list of target suites
 642     for suite in changes["suite"].keys():
 643         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 644         if override:
 645             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 646               database.get_suite_id(override) == -1)
 647             if olderr or newerr:
 648                 (oinv, newinv) = ("", "")
 649                 if olderr: oinv = "invalid "
 650                 if newerr: ninv = "invalid "
 651                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 652                         oinv, suite, ninv, override)
 653             del changes["suite"][suite]
 654             changes["suite"][override] = 1
 655     # Validate suites
 656     for suite in changes["suite"].keys():
 657         suite_id = database.get_suite_id(suite)
 658         if suite_id == -1:
 659             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 660
 661     # The main NEW processing loop
 662     done = 0
 663     while not done:
 664         # Find out what's new
 665         new = queue.determine_new(changes, files, projectB)
 666
 667         if not new:
 668             break
 669
 670         answer = "XXX"
 671         if Options["No-Action"] or Options["Automatic"]:
 672             answer = 'S'
 673
 674         (broken, note) = print_new(new, 0)
 675         prompt = ""
 676
 677         if not broken and not note:
 678             prompt = "Add overrides, "
 679         if broken:
 680             print "W: [!] marked entries must be fixed before package can be processed."
 681         if note:
 682             print "W: note must be removed before package can be processed."
 683             prompt += "RemOve all notes, Remove note, "
 684
 685         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 686
 687         while prompt.find(answer) == -1:
 688             answer = utils.our_raw_input(prompt)
 689             m = re_default_answer.search(prompt)
 690             if answer == "":
 691                 answer = m.group(1)
 692             answer = answer[:1].upper()
 693
 694         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 695             utils.warn("Trainees can't do that")
 696             continue
 697
 698         if answer == 'A' and not Options["Trainee"]:
 699             try:
 700                 check_daily_lock()
 701                 done = add_overrides (new)
 702                 Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 703             except CantGetLockError:
 704                 print "Hello? Operator! Give me the number for 911!"
 705                 print "Dinstall in the locked area, cant process packages, come back later"
 706         elif answer == 'C':
 707             check_pkg()
 708         elif answer == 'E' and not Options["Trainee"]:
 709             new = edit_overrides (new)
 710         elif answer == 'M' and not Options["Trainee"]:
 711             aborted = Upload.do_reject(manual=1,
 712                                        reject_message=Options["Manual-Reject"],
 713                                        note=database.get_new_comments(changes.get("source", "")))
 714             if not aborted:
 715                 Logger.log([utils.getusername(), "NEW REJECT: %s" % (Upload.pkg.changes_file)])
 716                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 717                 done = 1
 718         elif answer == 'N':
 719             edit_note(database.get_new_comments(changes.get("source", "")))
 720         elif answer == 'P' and not Options["Trainee"]:
 721             prod_maintainer(database.get_new_comments(changes.get("source", "")))
 722             Logger.log([utils.getusername(), "NEW PROD: %s" % (Upload.pkg.changes_file)])
 723         elif answer == 'R' and not Options["Trainee"]:
 724             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 725             if confirm == "y":
 726                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 727         elif answer == 'O' and not Options["Trainee"]:
 728             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 729             if confirm == "y":
 730                 database.delete_all_new_comments(changes.get("source"))
 731         elif answer == 'S':
 732             done = 1
 733         elif answer == 'Q':
 734             end()
 735             sys.exit(0)
 736
 737 ################################################################################
 738 ################################################################################
 739 ################################################################################
 740
 741 def usage (exit_code=0):
 742     print """Usage: dak process-new [OPTION]... [CHANGES]...
 743   -a, --automatic           automatic run
 744   -h, --help                show this help and exit.
 745   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 746   -m, --manual-reject=MSG   manual reject with `msg'
 747   -n, --no-action           don't do anything
 748   -t, --trainee             FTP Trainee mode
 749   -V, --version             display the version number and exit"""
 750     sys.exit(exit_code)
 751
 752 ################################################################################
 753
 754 def init():
 755     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 756
 757     Cnf = utils.get_conf()
 758
 759     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 760                  ('h',"help","Process-New::Options::Help"),
 761                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 762                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 763                  ('t',"trainee","Process-New::Options::Trainee"),
 764                  ('n',"no-action","Process-New::Options::No-Action")]
 765
 766     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 767         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 768             Cnf["Process-New::Options::%s" % (i)] = ""
 769
 770     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 771     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 772         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 773
 774     Options = Cnf.SubTree("Process-New::Options")
 775
 776     if Options["Help"]:
 777         usage()
 778
 779     Upload = queue.Upload(Cnf)
 780
 781     if not Options["No-Action"]:
 782         try:
 783             Logger = Upload.Logger = daklog.Logger(Cnf, "process-new")
 784         except CantOpenError, e:
 785             Options["Trainee"] = "True"
 786
 787     projectB = Upload.projectB
 788
 789     Sections = Section_Completer()
 790     Priorities = Priority_Completer()
 791     readline.parse_and_bind("tab: complete")
 792
 793     return changes_files
 794
 795 ################################################################################
 796
 797 def do_byhand():
 798     done = 0
 799     while not done:
 800         files = Upload.pkg.files
 801         will_install = 1
 802         byhand = []
 803
 804         for f in files.keys():
 805             if files[f]["type"] == "byhand":
 806                 if os.path.exists(f):
 807                     print "W: %s still present; please process byhand components and try again." % (f)
 808                     will_install = 0
 809                 else:
 810                     byhand.append(f)
 811
 812         answer = "XXXX"
 813         if Options["No-Action"]:
 814             answer = "S"
 815         if will_install:
 816             if Options["Automatic"] and not Options["No-Action"]:
 817                 answer = 'A'
 818             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 819         else:
 820             prompt = "Manual reject, [S]kip, Quit ?"
 821
 822         while prompt.find(answer) == -1:
 823             answer = utils.our_raw_input(prompt)
 824             m = re_default_answer.search(prompt)
 825             if answer == "":
 826                 answer = m.group(1)
 827             answer = answer[:1].upper()
 828
 829         if answer == 'A':
 830             try:
 831                 check_daily_lock()
 832                 done = 1
 833                 for f in byhand:
 834                     del files[f]
 835                 Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (Upload.pkg.changes_file)])
 836             except CantGetLockError:
 837                 print "Hello? Operator! Give me the number for 911!"
 838                 print "Dinstall in the locked area, cant process packages, come back later"
 839         elif answer == 'M':
 840             Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (Upload.pkg.changes_file)])
 841             Upload.do_reject(1, Options["Manual-Reject"])
 842             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 843             done = 1
 844         elif answer == 'S':
 845             done = 1
 846         elif answer == 'Q':
 847             end()
 848             sys.exit(0)
 849
 850 ################################################################################
 851
 852 def check_daily_lock():
 853     """
 854     Raises CantGetLockError if the dinstall daily.lock exists.
 855     """
 856
 857     try:
 858         os.open(Cnf["Process-New::DinstallLockFile"],  os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 859     except OSError, e:
 860         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 861             raise CantGetLockError
 862
 863     os.unlink(Cnf["Process-New::DinstallLockFile"])
 864
 865
 866 @contextlib.contextmanager
 867 def lock_package(package):
 868     """
 869     Lock C{package} so that noone else jumps in processing it.
 870
 871     @type package: string
 872     @param package: source package name to lock
 873     """
 874
 875     path = os.path.join(Cnf["Process-New::LockDir"], package)
 876     try:
 877         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 878     except OSError, e:
 879         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 880             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 881             raise AlreadyLockedError, user
 882
 883     try:
 884         yield fd
 885     finally:
 886         os.unlink(path)
 887
 888 def move_to_dir (dest, perms=0660, changesperms=0664):
 889     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 890     file_keys = Upload.pkg.files.keys()
 891     for f in file_keys:
 892         utils.move (f, dest, perms=perms)
 893
 894 def is_source_in_queue_dir(qdir):
 895     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 896                 and x.endswith(".changes") ]
 897     for entry in entries:
 898         # read the .dak
 899         u = queue.Upload(Cnf)
 900         u.pkg.changes_file = os.path.join(qdir, entry)
 901         u.update_vars()
 902         if not u.pkg.changes["architecture"].has_key("source"):
 903             # another binary upload, ignore
 904             continue
 905         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 906             # another version, ignore
 907             continue
 908         # found it!
 909         return True
 910     return False
 911
 912 def move_to_holding(suite, queue_dir):
 913     print "Moving to %s holding area." % (suite.upper(),)
 914     if Options["No-Action"]:
 915         return
 916     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 917     Upload.dump_vars(queue_dir)
 918     move_to_dir(queue_dir, perms=0664)
 919     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 920
 921 def _accept():
 922     if Options["No-Action"]:
 923         return
 924     (summary, short_summary) = Upload.build_summaries()
 925     Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"])
 926     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 927
 928 def do_accept_stableupdate(suite, q):
 929     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 930     if not Upload.pkg.changes["architecture"].has_key("source"):
 931         # It is not a sourceful upload.  So its source may be either in p-u
 932         # holding, in new, in accepted or already installed.
 933         if is_source_in_queue_dir(queue_dir):
 934             # It's in p-u holding, so move it there.
 935             print "Binary-only upload, source in %s." % (q,)
 936             move_to_holding(suite, queue_dir)
 937         elif Upload.source_exists(Upload.pkg.changes["source"],
 938                 Upload.pkg.changes["version"]):
 939             # dak tells us that there is source available.  At time of
 940             # writing this means that it is installed, so put it into
 941             # accepted.
 942             print "Binary-only upload, source installed."
 943             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 944             _accept()
 945         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 946             # The source is in accepted, the binary cleared NEW: accept it.
 947             print "Binary-only upload, source in accepted."
 948             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 949             _accept()
 950         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 951             # It's in NEW.  We expect the source to land in p-u holding
 952             # pretty soon.
 953             print "Binary-only upload, source in new."
 954             move_to_holding(suite, queue_dir)
 955         elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
 956             # It's in newstage.  Accept into the holding area
 957             print "Binary-only upload, source in newstage."
 958             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
 959             _accept()
 960         else:
 961             # No case applicable.  Bail out.  Return will cause the upload
 962             # to be skipped.
 963             print "ERROR"
 964             print "Stable update failed.  Source not found."
 965             return
 966     else:
 967         # We are handling a sourceful upload.  Move to accepted if currently
 968         # in p-u holding and to p-u holding otherwise.
 969         if is_source_in_queue_dir(queue_dir):
 970             print "Sourceful upload in %s, accepting." % (q,)
 971             _accept()
 972         else:
 973             move_to_holding(suite, queue_dir)
 974
 975 def do_accept():
 976     print "ACCEPT"
 977     if not Options["No-Action"]:
 978         (summary, short_summary) = Upload.build_summaries()
 979     if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 980         Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 981         move_to_dir(Cnf["Dir::Queue::Embargoed"])
 982         Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 983         # Check for override disparities
 984         Upload.Subst["__SUMMARY__"] = summary
 985     else:
 986         # Stable updates need to be copied to proposed-updates holding
 987         # area instead of accepted.  Sourceful uploads need to go
 988         # to it directly, binaries only if the source has not yet been
 989         # accepted into p-u.
 990         for suite, q in [("proposed-updates", "ProposedUpdates"),
 991                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
 992             if not Upload.pkg.changes["distribution"].has_key(suite):
 993                 continue
 994             return do_accept_stableupdate(suite, q)
 995         # Just a normal upload, accept it...
 996         _accept()
 997
 998 def check_status(files):
 999     new = byhand = 0
1000     for f in files.keys():
1001         if files[f]["type"] == "byhand":
1002             byhand = 1
1003         elif files[f].has_key("new"):
1004             new = 1
1005     return (new, byhand)
1006
1007 def do_pkg(changes_file):
1008     Upload.pkg.changes_file = changes_file
1009     Upload.init_vars()
1010     Upload.update_vars()
1011     Upload.update_subst()
1012     files = Upload.pkg.files
1013
1014     try:
1015         with lock_package(Upload.pkg.changes["source"]):
1016             if not recheck():
1017                 return
1018
1019             (new, byhand) = check_status(files)
1020             if new or byhand:
1021                 if new:
1022                     do_new()
1023                 if byhand:
1024                     do_byhand()
1025                 (new, byhand) = check_status(files)
1026
1027             if not new and not byhand:
1028                 try:
1029                     check_daily_lock()
1030                     do_accept()
1031                 except CantGetLockError:
1032                     print "Hello? Operator! Give me the number for 911!"
1033                     print "Dinstall in the locked area, cant process packages, come back later"
1034     except AlreadyLockedError, e:
1035         print "Seems to be locked by %s already, skipping..." % (e)
1036
1037 ################################################################################
1038
1039 def end():
1040     accept_count = SummaryStats().accept_count
1041     accept_bytes = SummaryStats().accept_bytes
1042
1043     if accept_count:
1044         sets = "set"
1045         if accept_count > 1:
1046             sets = "sets"
1047         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1048         Logger.log([utils.getusername(), "total",accept_count,accept_bytes])
1049
1050     if not Options["No-Action"] and not Options["Trainee"]:
1051         Logger.close()
1052
1053 ################################################################################
1054
1055 def do_comments(dir, opref, npref, line, fn):
1056     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1057         lines = open("%s/%s" % (dir, comm)).readlines()
1058         if len(lines) == 0 or lines[0] != line + "\n": continue
1059         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1060                                 and x.endswith(".changes") ]
1061         changes_files = sort_changes(changes_files)
1062         for f in changes_files:
1063             f = utils.validate_changes_file_arg(f, 0)
1064             if not f: continue
1065             print "\n" + f
1066             fn(f, "".join(lines[1:]))
1067
1068         if opref != npref and not Options["No-Action"]:
1069             newcomm = npref + comm[len(opref):]
1070             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1071
1072 ################################################################################
1073
1074 def comment_accept(changes_file, comments):
1075     Upload.pkg.changes_file = changes_file
1076     Upload.init_vars()
1077     Upload.update_vars()
1078     Upload.update_subst()
1079     files = Upload.pkg.files
1080
1081     if not recheck():
1082         return # dak wants to REJECT, crap
1083
1084     (new, byhand) = check_status(files)
1085     if not new and not byhand:
1086         do_accept()
1087
1088 ################################################################################
1089
1090 def comment_reject(changes_file, comments):
1091     Upload.pkg.changes_file = changes_file
1092     Upload.init_vars()
1093     Upload.update_vars()
1094     Upload.update_subst()
1095
1096     if not recheck():
1097         pass # dak has its own reasons to reject as well, which is fine
1098
1099     reject(comments)
1100     print "REJECT\n" + reject_message,
1101     if not Options["No-Action"]:
1102         Upload.do_reject(0, reject_message)
1103         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1104
1105 ################################################################################
1106
1107 def main():
1108     changes_files = init()
1109     if len(changes_files) > 50:
1110         sys.stderr.write("Sorting changes...\n")
1111     changes_files = sort_changes(changes_files)
1112
1113     # Kill me now? **FIXME**
1114     Cnf["Dinstall::Options::No-Mail"] = ""
1115     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1116     if Cnf.has_key("Dinstall::Bcc"):
1117         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1118     else:
1119         Upload.Subst["__BCC__"] = bcc
1120
1121     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1122     if commentsdir:
1123         if changes_files != []:
1124             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1125             sys.exit(1)
1126         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1127         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1128     else:
1129         for changes_file in changes_files:
1130             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1131             if not changes_file:
1132                 continue
1133             print "\n" + changes_file
1134
1135             do_pkg (changes_file)
1136
1137     end()
1138
1139 ################################################################################
1140
1141 if __name__ == '__main__':
1142     main()