dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 from __future__ import with_statement
  45
  46 import copy
  47 import errno
  48 import os
  49 import readline
  50 import stat
  51 import sys
  52 import time
  53 import contextlib
  54 import pwd
  55 import apt_pkg, apt_inst
  56 import examine_package
  57
  58 from daklib import database
  59 from daklib import daklog
  60 from daklib import queue
  61 from daklib import utils
  62 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  63 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  64 from daklib.summarystats import SummaryStats
  65
  66 # Globals
  67 Cnf = None       #: Configuration, apt_pkg.Configuration
  68 Options = None
  69 Upload = None
  70 projectB = None  #: database connection, pgobject
  71 Logger = None
  72
  73 Priorities = None
  74 Sections = None
  75
  76 reject_message = ""
  77
  78 ################################################################################
  79 ################################################################################
  80 ################################################################################
  81
  82 def reject (str, prefix="Rejected: "):
  83     global reject_message
  84     if str:
  85         reject_message += prefix + str + "\n"
  86
  87 def recheck():
  88     global reject_message
  89     files = Upload.pkg.files
  90     reject_message = ""
  91
  92     for f in files.keys():
  93         # The .orig.tar.gz can disappear out from under us is it's a
  94         # duplicate of one in the archive.
  95         if not files.has_key(f):
  96             continue
  97         # Check that the source still exists
  98         if files[f]["type"] == "deb":
  99             source_version = files[f]["source version"]
 100             source_package = files[f]["source package"]
 101             if not Upload.pkg.changes["architecture"].has_key("source") \
 102                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
 103                 source_epochless_version = re_no_epoch.sub('', source_version)
 104                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 105                 found = 0
 106                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
 107                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 108                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 109                             found = 1
 110                 if not found:
 111                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 112
 113         # Version and file overwrite checks
 114         if files[f]["type"] == "deb":
 115             reject(Upload.check_binary_against_db(f), "")
 116         elif files[f]["type"] == "dsc":
 117             reject(Upload.check_source_against_db(f), "")
 118             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 119             reject(reject_msg, "")
 120
 121     if reject_message.find("Rejected") != -1:
 122         answer = "XXX"
 123         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 124             answer = 'S'
 125
 126         print "REJECT\n" + reject_message,
 127         prompt = "[R]eject, Skip, Quit ?"
 128
 129         while prompt.find(answer) == -1:
 130             answer = utils.our_raw_input(prompt)
 131             m = re_default_answer.match(prompt)
 132             if answer == "":
 133                 answer = m.group(1)
 134             answer = answer[:1].upper()
 135
 136         if answer == 'R':
 137             Upload.do_reject(0, reject_message)
 138             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 139             return 0
 140         elif answer == 'S':
 141             return 0
 142         elif answer == 'Q':
 143             end()
 144             sys.exit(0)
 145
 146     return 1
 147
 148 ################################################################################
 149
 150 def indiv_sg_compare (a, b):
 151     """Sort by source name, source, version, 'have source', and
 152        finally by filename."""
 153     # Sort by source version
 154     q = apt_pkg.VersionCompare(a["version"], b["version"])
 155     if q:
 156         return -q
 157
 158     # Sort by 'have source'
 159     a_has_source = a["architecture"].get("source")
 160     b_has_source = b["architecture"].get("source")
 161     if a_has_source and not b_has_source:
 162         return -1
 163     elif b_has_source and not a_has_source:
 164         return 1
 165
 166     return cmp(a["filename"], b["filename"])
 167
 168 ############################################################
 169
 170 def sg_compare (a, b):
 171     a = a[1]
 172     b = b[1]
 173     """Sort by have note, source already in database and time of oldest upload."""
 174     # Sort by have note
 175     a_note_state = a["note_state"]
 176     b_note_state = b["note_state"]
 177     if a_note_state < b_note_state:
 178         return -1
 179     elif a_note_state > b_note_state:
 180         return 1
 181     # Sort by source already in database (descending)
 182     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 183     if source_in_database:
 184         return -source_in_database
 185
 186     # Sort by time of oldest upload
 187     return cmp(a["oldest"], b["oldest"])
 188
 189 def sort_changes(changes_files):
 190     """Sort into source groups, then sort each source group by version,
 191     have source, filename.  Finally, sort the source groups by have
 192     note, time of oldest upload of each source upload."""
 193     if len(changes_files) == 1:
 194         return changes_files
 195
 196     sorted_list = []
 197     cache = {}
 198     # Read in all the .changes files
 199     for filename in changes_files:
 200         try:
 201             Upload.pkg.changes_file = filename
 202             Upload.init_vars()
 203             Upload.update_vars()
 204             cache[filename] = copy.copy(Upload.pkg.changes)
 205             cache[filename]["filename"] = filename
 206         except:
 207             sorted_list.append(filename)
 208             break
 209     # Divide the .changes into per-source groups
 210     per_source = {}
 211     for filename in cache.keys():
 212         source = cache[filename]["source"]
 213         if not per_source.has_key(source):
 214             per_source[source] = {}
 215             per_source[source]["list"] = []
 216         per_source[source]["list"].append(cache[filename])
 217     # Determine oldest time and have note status for each source group
 218     for source in per_source.keys():
 219         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 220         ql = q.getresult()
 221         per_source[source]["source_in_database"] = len(ql)>0
 222         source_list = per_source[source]["list"]
 223         first = source_list[0]
 224         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 225         have_note = 0
 226         for d in per_source[source]["list"]:
 227             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 228             if mtime < oldest:
 229                 oldest = mtime
 230             have_note += (database.has_new_comment(d["source"], d["version"], True))
 231         per_source[source]["oldest"] = oldest
 232         if not have_note:
 233             per_source[source]["note_state"] = 0; # none
 234         elif have_note < len(source_list):
 235             per_source[source]["note_state"] = 1; # some
 236         else:
 237             per_source[source]["note_state"] = 2; # all
 238         per_source[source]["list"].sort(indiv_sg_compare)
 239     per_source_items = per_source.items()
 240     per_source_items.sort(sg_compare)
 241     for i in per_source_items:
 242         for j in i[1]["list"]:
 243             sorted_list.append(j["filename"])
 244     return sorted_list
 245
 246 ################################################################################
 247
 248 class Section_Completer:
 249     def __init__ (self):
 250         self.sections = []
 251         self.matches = []
 252         q = projectB.query("SELECT section FROM section")
 253         for i in q.getresult():
 254             self.sections.append(i[0])
 255
 256     def complete(self, text, state):
 257         if state == 0:
 258             self.matches = []
 259             n = len(text)
 260             for word in self.sections:
 261                 if word[:n] == text:
 262                     self.matches.append(word)
 263         try:
 264             return self.matches[state]
 265         except IndexError:
 266             return None
 267
 268 ############################################################
 269
 270 class Priority_Completer:
 271     def __init__ (self):
 272         self.priorities = []
 273         self.matches = []
 274         q = projectB.query("SELECT priority FROM priority")
 275         for i in q.getresult():
 276             self.priorities.append(i[0])
 277
 278     def complete(self, text, state):
 279         if state == 0:
 280             self.matches = []
 281             n = len(text)
 282             for word in self.priorities:
 283                 if word[:n] == text:
 284                     self.matches.append(word)
 285         try:
 286             return self.matches[state]
 287         except IndexError:
 288             return None
 289
 290 ################################################################################
 291
 292 def print_new (new, indexed, file=sys.stdout):
 293     queue.check_valid(new)
 294     broken = 0
 295     index = 0
 296     for pkg in new.keys():
 297         index += 1
 298         section = new[pkg]["section"]
 299         priority = new[pkg]["priority"]
 300         if new[pkg]["section id"] == -1:
 301             section += "[!]"
 302             broken = 1
 303         if new[pkg]["priority id"] == -1:
 304             priority += "[!]"
 305             broken = 1
 306         if indexed:
 307             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 308         else:
 309             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 310         line = line.strip()+'\n'
 311         file.write(line)
 312     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 313     if len(note) > 0:
 314         for line in note:
 315             print line
 316     return broken, note
 317
 318 ################################################################################
 319
 320 def index_range (index):
 321     if index == 1:
 322         return "1"
 323     else:
 324         return "1-%s" % (index)
 325
 326 ################################################################################
 327 ################################################################################
 328
 329 def edit_new (new):
 330     # Write the current data to a temporary file
 331     (fd, temp_filename) = utils.temp_filename()
 332     temp_file = os.fdopen(fd, 'w')
 333     print_new (new, 0, temp_file)
 334     temp_file.close()
 335     # Spawn an editor on that file
 336     editor = os.environ.get("EDITOR","vi")
 337     result = os.system("%s %s" % (editor, temp_filename))
 338     if result != 0:
 339         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 340     # Read the edited data back in
 341     temp_file = utils.open_file(temp_filename)
 342     lines = temp_file.readlines()
 343     temp_file.close()
 344     os.unlink(temp_filename)
 345     # Parse the new data
 346     for line in lines:
 347         line = line.strip()
 348         if line == "":
 349             continue
 350         s = line.split()
 351         # Pad the list if necessary
 352         s[len(s):3] = [None] * (3-len(s))
 353         (pkg, priority, section) = s[:3]
 354         if not new.has_key(pkg):
 355             utils.warn("Ignoring unknown package '%s'" % (pkg))
 356         else:
 357             # Strip off any invalid markers, print_new will readd them.
 358             if section.endswith("[!]"):
 359                 section = section[:-3]
 360             if priority.endswith("[!]"):
 361                 priority = priority[:-3]
 362             for f in new[pkg]["files"]:
 363                 Upload.pkg.files[f]["section"] = section
 364                 Upload.pkg.files[f]["priority"] = priority
 365             new[pkg]["section"] = section
 366             new[pkg]["priority"] = priority
 367
 368 ################################################################################
 369
 370 def edit_index (new, index):
 371     priority = new[index]["priority"]
 372     section = new[index]["section"]
 373     ftype = new[index]["type"]
 374     done = 0
 375     while not done:
 376         print "\t".join([index, priority, section])
 377
 378         answer = "XXX"
 379         if ftype != "dsc":
 380             prompt = "[B]oth, Priority, Section, Done ? "
 381         else:
 382             prompt = "[S]ection, Done ? "
 383         edit_priority = edit_section = 0
 384
 385         while prompt.find(answer) == -1:
 386             answer = utils.our_raw_input(prompt)
 387             m = re_default_answer.match(prompt)
 388             if answer == "":
 389                 answer = m.group(1)
 390             answer = answer[:1].upper()
 391
 392         if answer == 'P':
 393             edit_priority = 1
 394         elif answer == 'S':
 395             edit_section = 1
 396         elif answer == 'B':
 397             edit_priority = edit_section = 1
 398         elif answer == 'D':
 399             done = 1
 400
 401         # Edit the priority
 402         if edit_priority:
 403             readline.set_completer(Priorities.complete)
 404             got_priority = 0
 405             while not got_priority:
 406                 new_priority = utils.our_raw_input("New priority: ").strip()
 407                 if new_priority not in Priorities.priorities:
 408                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 409                 else:
 410                     got_priority = 1
 411                     priority = new_priority
 412
 413         # Edit the section
 414         if edit_section:
 415             readline.set_completer(Sections.complete)
 416             got_section = 0
 417             while not got_section:
 418                 new_section = utils.our_raw_input("New section: ").strip()
 419                 if new_section not in Sections.sections:
 420                     print "E: '%s' is not a valid section, try again." % (new_section)
 421                 else:
 422                     got_section = 1
 423                     section = new_section
 424
 425         # Reset the readline completer
 426         readline.set_completer(None)
 427
 428     for f in new[index]["files"]:
 429         Upload.pkg.files[f]["section"] = section
 430         Upload.pkg.files[f]["priority"] = priority
 431     new[index]["priority"] = priority
 432     new[index]["section"] = section
 433     return new
 434
 435 ################################################################################
 436
 437 def edit_overrides (new):
 438     print
 439     done = 0
 440     while not done:
 441         print_new (new, 1)
 442         new_index = {}
 443         index = 0
 444         for i in new.keys():
 445             index += 1
 446             new_index[index] = i
 447
 448         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 449
 450         got_answer = 0
 451         while not got_answer:
 452             answer = utils.our_raw_input(prompt)
 453             if not answer.isdigit():
 454                 answer = answer[:1].upper()
 455             if answer == "E" or answer == "D":
 456                 got_answer = 1
 457             elif re_isanum.match (answer):
 458                 answer = int(answer)
 459                 if (answer < 1) or (answer > index):
 460                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 461                 else:
 462                     got_answer = 1
 463
 464         if answer == 'E':
 465             edit_new(new)
 466         elif answer == 'D':
 467             done = 1
 468         else:
 469             edit_index (new, new_index[answer])
 470
 471     return new
 472
 473 ################################################################################
 474
 475 def edit_note(note):
 476     # Write the current data to a temporary file
 477     (fd, temp_filename) = utils.temp_filename()
 478     editor = os.environ.get("EDITOR","vi")
 479     answer = 'E'
 480     while answer == 'E':
 481         os.system("%s %s" % (editor, temp_filename))
 482         temp_file = utils.open_file(temp_filename)
 483         newnote = temp_file.read().rstrip()
 484         temp_file.close()
 485         print "New Note:"
 486         print utils.prefix_multi_line_string(newnote,"  ")
 487         prompt = "[D]one, Edit, Abandon, Quit ?"
 488         answer = "XXX"
 489         while prompt.find(answer) == -1:
 490             answer = utils.our_raw_input(prompt)
 491             m = re_default_answer.search(prompt)
 492             if answer == "":
 493                 answer = m.group(1)
 494             answer = answer[:1].upper()
 495     os.unlink(temp_filename)
 496     if answer == 'A':
 497         return
 498     elif answer == 'Q':
 499         end()
 500         sys.exit(0)
 501
 502     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami(), bool(Options["Trainee"]))
 503
 504 ################################################################################
 505
 506 def check_pkg ():
 507     try:
 508         less_fd = os.popen("less -R -", 'w', 0)
 509         stdout_fd = sys.stdout
 510         try:
 511             sys.stdout = less_fd
 512             changes = utils.parse_changes (Upload.pkg.changes_file)
 513             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 514             files = Upload.pkg.files
 515             for f in files.keys():
 516                 if files[f].has_key("new"):
 517                     ftype = files[f]["type"]
 518                     if ftype == "deb":
 519                         examine_package.check_deb(changes['distribution'], f)
 520                     elif ftype == "dsc":
 521                         examine_package.check_dsc(changes['distribution'], f)
 522         finally:
 523             examine_package.output_package_relations()
 524             sys.stdout = stdout_fd
 525     except IOError, e:
 526         if e.errno == errno.EPIPE:
 527             utils.warn("[examine_package] Caught EPIPE; skipping.")
 528             pass
 529         else:
 530             raise
 531     except KeyboardInterrupt:
 532         utils.warn("[examine_package] Caught C-c; skipping.")
 533         pass
 534
 535 ################################################################################
 536
 537 ## FIXME: horribly Debian specific
 538
 539 def do_bxa_notification():
 540     files = Upload.pkg.files
 541     summary = ""
 542     for f in files.keys():
 543         if files[f]["type"] == "deb":
 544             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 545             summary += "\n"
 546             summary += "Package: %s\n" % (control.Find("Package"))
 547             summary += "Description: %s\n" % (control.Find("Description"))
 548     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 549     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 550     utils.send_mail(bxa_mail)
 551
 552 ################################################################################
 553
 554 def add_overrides (new):
 555     changes = Upload.pkg.changes
 556     files = Upload.pkg.files
 557
 558     projectB.query("BEGIN WORK")
 559     for suite in changes["suite"].keys():
 560         suite_id = database.get_suite_id(suite)
 561         for pkg in new.keys():
 562             component_id = database.get_component_id(new[pkg]["component"])
 563             type_id = database.get_override_type_id(new[pkg]["type"])
 564             priority_id = new[pkg]["priority id"]
 565             section_id = new[pkg]["section id"]
 566             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 567             for f in new[pkg]["files"]:
 568                 if files[f].has_key("new"):
 569                     del files[f]["new"]
 570             del new[pkg]
 571
 572     projectB.query("COMMIT WORK")
 573
 574     if Cnf.FindB("Dinstall::BXANotify"):
 575         do_bxa_notification()
 576
 577 ################################################################################
 578
 579 def prod_maintainer (note):
 580     # Here we prepare an editor and get them ready to prod...
 581     (fd, temp_filename) = utils.temp_filename()
 582     temp_file = os.fdopen(fd, 'w')
 583     if len(note) > 0:
 584         for line in note:
 585             temp_file.write(line)
 586     temp_file.close()
 587     editor = os.environ.get("EDITOR","vi")
 588     answer = 'E'
 589     while answer == 'E':
 590         os.system("%s %s" % (editor, temp_filename))
 591         temp_fh = utils.open_file(temp_filename)
 592         prod_message = "".join(temp_fh.readlines())
 593         temp_fh.close()
 594         print "Prod message:"
 595         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 596         prompt = "[P]rod, Edit, Abandon, Quit ?"
 597         answer = "XXX"
 598         while prompt.find(answer) == -1:
 599             answer = utils.our_raw_input(prompt)
 600             m = re_default_answer.search(prompt)
 601             if answer == "":
 602                 answer = m.group(1)
 603             answer = answer[:1].upper()
 604     os.unlink(temp_filename)
 605     if answer == 'A':
 606         return
 607     elif answer == 'Q':
 608         end()
 609         sys.exit(0)
 610     # Otherwise, do the proding...
 611     user_email_address = utils.whoami() + " <%s>" % (
 612         Cnf["Dinstall::MyAdminAddress"])
 613
 614     Subst = Upload.Subst
 615
 616     Subst["__FROM_ADDRESS__"] = user_email_address
 617     Subst["__PROD_MESSAGE__"] = prod_message
 618     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 619
 620     prod_mail_message = utils.TemplateSubst(
 621         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 622
 623     # Send the prod mail if appropriate
 624     if not Cnf["Dinstall::Options::No-Mail"]:
 625         utils.send_mail(prod_mail_message)
 626
 627     print "Sent proding message"
 628
 629 ################################################################################
 630
 631 def do_new():
 632     print "NEW\n"
 633     files = Upload.pkg.files
 634     changes = Upload.pkg.changes
 635
 636     # Make a copy of distribution we can happily trample on
 637     changes["suite"] = copy.copy(changes["distribution"])
 638
 639     # Fix up the list of target suites
 640     for suite in changes["suite"].keys():
 641         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 642         if override:
 643             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 644               database.get_suite_id(override) == -1)
 645             if olderr or newerr:
 646                 (oinv, newinv) = ("", "")
 647                 if olderr: oinv = "invalid "
 648                 if newerr: ninv = "invalid "
 649                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 650                         oinv, suite, ninv, override)
 651             del changes["suite"][suite]
 652             changes["suite"][override] = 1
 653     # Validate suites
 654     for suite in changes["suite"].keys():
 655         suite_id = database.get_suite_id(suite)
 656         if suite_id == -1:
 657             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 658
 659     # The main NEW processing loop
 660     done = 0
 661     while not done:
 662         # Find out what's new
 663         new = queue.determine_new(changes, files, projectB)
 664
 665         if not new:
 666             break
 667
 668         answer = "XXX"
 669         if Options["No-Action"] or Options["Automatic"]:
 670             answer = 'S'
 671
 672         (broken, note) = print_new(new, 0)
 673         prompt = ""
 674
 675         if not broken and not note:
 676             prompt = "Add overrides, "
 677         if broken:
 678             print "W: [!] marked entries must be fixed before package can be processed."
 679         if note:
 680             print "W: note must be removed before package can be processed."
 681             prompt += "RemOve all notes, Remove note, "
 682
 683         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 684
 685         while prompt.find(answer) == -1:
 686             answer = utils.our_raw_input(prompt)
 687             m = re_default_answer.search(prompt)
 688             if answer == "":
 689                 answer = m.group(1)
 690             answer = answer[:1].upper()
 691
 692         if answer == 'A' and not Options["Trainee"]:
 693             try:
 694                 check_daily_lock()
 695                 done = add_overrides (new)
 696             except CantGetLockError:
 697                 print "Hello? Operator! Give me the number for 911!"
 698                 print "Dinstall in the locked area, cant process packages, come back later"
 699         elif answer == 'C':
 700             check_pkg()
 701         elif answer == 'E' and not Options["Trainee"]:
 702             new = edit_overrides (new)
 703         elif answer == 'M' and not Options["Trainee"]:
 704             aborted = Upload.do_reject(manual=1,
 705                                        reject_message=Options["Manual-Reject"],
 706                                        note=database.get_new_comments(changes.get("source", "")))
 707             if not aborted:
 708                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 709                 done = 1
 710         elif answer == 'N':
 711             edit_note(database.get_new_comments(changes.get("source", "")))
 712         elif answer == 'P' and not Options["Trainee"]:
 713             prod_maintainer(database.get_new_comments(changes.get("source", "")))
 714         elif answer == 'R' and not Options["Trainee"]:
 715             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 716             if confirm == "y":
 717                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 718         elif answer == 'O' and not Options["Trainee"]:
 719             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 720             if confirm == "y":
 721                 database.delete_all_new_comments(changes.get("source"))
 722         elif answer == 'S':
 723             done = 1
 724         elif answer == 'Q':
 725             end()
 726             sys.exit(0)
 727
 728 ################################################################################
 729 ################################################################################
 730 ################################################################################
 731
 732 def usage (exit_code=0):
 733     print """Usage: dak process-new [OPTION]... [CHANGES]...
 734   -a, --automatic           automatic run
 735   -h, --help                show this help and exit.
 736   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 737   -m, --manual-reject=MSG   manual reject with `msg'
 738   -n, --no-action           don't do anything
 739   -t, --trainee             FTP Trainee mode
 740   -V, --version             display the version number and exit"""
 741     sys.exit(exit_code)
 742
 743 ################################################################################
 744
 745 def init():
 746     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 747
 748     Cnf = utils.get_conf()
 749
 750     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 751                  ('h',"help","Process-New::Options::Help"),
 752                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 753                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 754                  ('t',"trainee","Process-New::Options::Trainee"),
 755                  ('n',"no-action","Process-New::Options::No-Action")]
 756
 757     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 758         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 759             Cnf["Process-New::Options::%s" % (i)] = ""
 760
 761     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 762     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 763         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 764
 765     Options = Cnf.SubTree("Process-New::Options")
 766
 767     if Options["Help"]:
 768         usage()
 769
 770     Upload = queue.Upload(Cnf)
 771
 772     if not Options["No-Action"]:
 773         try:
 774             Logger = Upload.Logger = daklog.Logger(Cnf, "process-new")
 775         except CantOpenError, e:
 776             Options["Trainee"] = "True"
 777
 778     projectB = Upload.projectB
 779
 780     Sections = Section_Completer()
 781     Priorities = Priority_Completer()
 782     readline.parse_and_bind("tab: complete")
 783
 784     return changes_files
 785
 786 ################################################################################
 787
 788 def do_byhand():
 789     done = 0
 790     while not done:
 791         files = Upload.pkg.files
 792         will_install = 1
 793         byhand = []
 794
 795         for f in files.keys():
 796             if files[f]["type"] == "byhand":
 797                 if os.path.exists(f):
 798                     print "W: %s still present; please process byhand components and try again." % (f)
 799                     will_install = 0
 800                 else:
 801                     byhand.append(f)
 802
 803         answer = "XXXX"
 804         if Options["No-Action"]:
 805             answer = "S"
 806         if will_install:
 807             if Options["Automatic"] and not Options["No-Action"]:
 808                 answer = 'A'
 809             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 810         else:
 811             prompt = "Manual reject, [S]kip, Quit ?"
 812
 813         while prompt.find(answer) == -1:
 814             answer = utils.our_raw_input(prompt)
 815             m = re_default_answer.search(prompt)
 816             if answer == "":
 817                 answer = m.group(1)
 818             answer = answer[:1].upper()
 819
 820         if answer == 'A':
 821             try:
 822                 check_daily_lock()
 823                 done = 1
 824                 for f in byhand:
 825                     del files[f]
 826             except CantGetLockError:
 827                 print "Hello? Operator! Give me the number for 911!"
 828                 print "Dinstall in the locked area, cant process packages, come back later"
 829         elif answer == 'M':
 830             Upload.do_reject(1, Options["Manual-Reject"])
 831             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 832             done = 1
 833         elif answer == 'S':
 834             done = 1
 835         elif answer == 'Q':
 836             end()
 837             sys.exit(0)
 838
 839 ################################################################################
 840
 841 def check_daily_lock():
 842     """
 843     Raises CantGetLockError if the dinstall daily.lock exists.
 844     """
 845
 846     try:
 847         os.open(Cnf["Process-New::DinstallLockFile"],  os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 848     except OSError, e:
 849         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 850             raise CantGetLockError
 851
 852     os.unlink(Cnf["Process-New::DinstallLockFile"])
 853
 854
 855 @contextlib.contextmanager
 856 def lock_package(package):
 857     """
 858     Lock C{package} so that noone else jumps in processing it.
 859
 860     @type package: string
 861     @param package: source package name to lock
 862     """
 863
 864     path = os.path.join(Cnf["Process-New::LockDir"], package)
 865     try:
 866         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 867     except OSError, e:
 868         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 869             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 870             raise AlreadyLockedError, user
 871
 872     try:
 873         yield fd
 874     finally:
 875         os.unlink(path)
 876
 877 def move_to_dir (dest, perms=0660, changesperms=0664):
 878     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 879     file_keys = Upload.pkg.files.keys()
 880     for f in file_keys:
 881         utils.move (f, dest, perms=perms)
 882
 883 def is_source_in_queue_dir(qdir):
 884     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 885                 and x.endswith(".changes") ]
 886     for entry in entries:
 887         # read the .dak
 888         u = queue.Upload(Cnf)
 889         u.pkg.changes_file = os.path.join(qdir, entry)
 890         u.update_vars()
 891         if not u.pkg.changes["architecture"].has_key("source"):
 892             # another binary upload, ignore
 893             continue
 894         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 895             # another version, ignore
 896             continue
 897         # found it!
 898         return True
 899     return False
 900
 901 def move_to_holding(suite, queue_dir):
 902     print "Moving to %s holding area." % (suite.upper(),)
 903     if Options["No-Action"]:
 904         return
 905     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 906     Upload.dump_vars(queue_dir)
 907     move_to_dir(queue_dir, perms=0664)
 908     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 909
 910 def _accept():
 911     if Options["No-Action"]:
 912         return
 913     (summary, short_summary) = Upload.build_summaries()
 914     Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"])
 915     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 916
 917 def do_accept_stableupdate(suite, q):
 918     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 919     if not Upload.pkg.changes["architecture"].has_key("source"):
 920         # It is not a sourceful upload.  So its source may be either in p-u
 921         # holding, in new, in accepted or already installed.
 922         if is_source_in_queue_dir(queue_dir):
 923             # It's in p-u holding, so move it there.
 924             print "Binary-only upload, source in %s." % (q,)
 925             move_to_holding(suite, queue_dir)
 926         elif Upload.source_exists(Upload.pkg.changes["source"],
 927                 Upload.pkg.changes["version"]):
 928             # dak tells us that there is source available.  At time of
 929             # writing this means that it is installed, so put it into
 930             # accepted.
 931             print "Binary-only upload, source installed."
 932             _accept()
 933         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 934             # The source is in accepted, the binary cleared NEW: accept it.
 935             print "Binary-only upload, source in accepted."
 936             _accept()
 937         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 938             # It's in NEW.  We expect the source to land in p-u holding
 939             # pretty soon.
 940             print "Binary-only upload, source in new."
 941             move_to_holding(suite, queue_dir)
 942         elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
 943             # It's in newstage.  Accept into the holding area
 944             print "Binary-only upload, source in newstage."
 945             _accept()
 946         else:
 947             # No case applicable.  Bail out.  Return will cause the upload
 948             # to be skipped.
 949             print "ERROR"
 950             print "Stable update failed.  Source not found."
 951             return
 952     else:
 953         # We are handling a sourceful upload.  Move to accepted if currently
 954         # in p-u holding and to p-u holding otherwise.
 955         if is_source_in_queue_dir(queue_dir):
 956             print "Sourceful upload in %s, accepting." % (q,)
 957             _accept()
 958         else:
 959             move_to_holding(suite, queue_dir)
 960
 961 def do_accept():
 962     print "ACCEPT"
 963     if not Options["No-Action"]:
 964         (summary, short_summary) = Upload.build_summaries()
 965     if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 966         Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 967         move_to_dir(Cnf["Dir::Queue::Embargoed"])
 968         Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 969         # Check for override disparities
 970         Upload.Subst["__SUMMARY__"] = summary
 971     else:
 972         # Stable updates need to be copied to proposed-updates holding
 973         # area instead of accepted.  Sourceful uploads need to go
 974         # to it directly, binaries only if the source has not yet been
 975         # accepted into p-u.
 976         for suite, q in [("proposed-updates", "ProposedUpdates"),
 977                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
 978             if not Upload.pkg.changes["distribution"].has_key(suite):
 979                 continue
 980             return do_accept_stableupdate(suite, q)
 981         # Just a normal upload, accept it...
 982         _accept()
 983
 984 def check_status(files):
 985     new = byhand = 0
 986     for f in files.keys():
 987         if files[f]["type"] == "byhand":
 988             byhand = 1
 989         elif files[f].has_key("new"):
 990             new = 1
 991     return (new, byhand)
 992
 993 def do_pkg(changes_file):
 994     Upload.pkg.changes_file = changes_file
 995     Upload.init_vars()
 996     Upload.update_vars()
 997     Upload.update_subst()
 998     files = Upload.pkg.files
 999
1000     try:
1001         with lock_package(Upload.pkg.changes["source"]):
1002             if not recheck():
1003                 return
1004
1005             (new, byhand) = check_status(files)
1006             if new or byhand:
1007                 if new:
1008                     do_new()
1009                 if byhand:
1010                     do_byhand()
1011                 (new, byhand) = check_status(files)
1012
1013             if not new and not byhand:
1014                 try:
1015                     check_daily_lock()
1016                     do_accept()
1017                 except CantGetLockError:
1018                     print "Hello? Operator! Give me the number for 911!"
1019                     print "Dinstall in the locked area, cant process packages, come back later"
1020     except AlreadyLockedError, e:
1021         print "Seems to be locked by %s already, skipping..." % (e)
1022
1023 ################################################################################
1024
1025 def end():
1026     accept_count = SummaryStats().accept_count
1027     accept_bytes = SummaryStats().accept_bytes
1028
1029     if accept_count:
1030         sets = "set"
1031         if accept_count > 1:
1032             sets = "sets"
1033         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1034         Logger.log(["total",accept_count,accept_bytes])
1035
1036     if not Options["No-Action"] and not Options["Trainee"]:
1037         Logger.close()
1038
1039 ################################################################################
1040
1041 def do_comments(dir, opref, npref, line, fn):
1042     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1043         lines = open("%s/%s" % (dir, comm)).readlines()
1044         if len(lines) == 0 or lines[0] != line + "\n": continue
1045         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1046                                 and x.endswith(".changes") ]
1047         changes_files = sort_changes(changes_files)
1048         for f in changes_files:
1049             f = utils.validate_changes_file_arg(f, 0)
1050             if not f: continue
1051             print "\n" + f
1052             fn(f, "".join(lines[1:]))
1053
1054         if opref != npref and not Options["No-Action"]:
1055             newcomm = npref + comm[len(opref):]
1056             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1057
1058 ################################################################################
1059
1060 def comment_accept(changes_file, comments):
1061     Upload.pkg.changes_file = changes_file
1062     Upload.init_vars()
1063     Upload.update_vars()
1064     Upload.update_subst()
1065     files = Upload.pkg.files
1066
1067     if not recheck():
1068         return # dak wants to REJECT, crap
1069
1070     (new, byhand) = check_status(files)
1071     if not new and not byhand:
1072         do_accept()
1073
1074 ################################################################################
1075
1076 def comment_reject(changes_file, comments):
1077     Upload.pkg.changes_file = changes_file
1078     Upload.init_vars()
1079     Upload.update_vars()
1080     Upload.update_subst()
1081
1082     if not recheck():
1083         pass # dak has its own reasons to reject as well, which is fine
1084
1085     reject(comments)
1086     print "REJECT\n" + reject_message,
1087     if not Options["No-Action"]:
1088         Upload.do_reject(0, reject_message)
1089         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1090
1091 ################################################################################
1092
1093 def main():
1094     changes_files = init()
1095     if len(changes_files) > 50:
1096         sys.stderr.write("Sorting changes...\n")
1097     changes_files = sort_changes(changes_files)
1098
1099     # Kill me now? **FIXME**
1100     Cnf["Dinstall::Options::No-Mail"] = ""
1101     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1102     if Cnf.has_key("Dinstall::Bcc"):
1103         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1104     else:
1105         Upload.Subst["__BCC__"] = bcc
1106
1107     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1108     if commentsdir:
1109         if changes_files != []:
1110             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1111             sys.exit(1)
1112         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1113         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1114     else:
1115         for changes_file in changes_files:
1116             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1117             if not changes_file:
1118                 continue
1119             print "\n" + changes_file
1120
1121             do_pkg (changes_file)
1122
1123     end()
1124
1125 ################################################################################
1126
1127 if __name__ == '__main__':
1128     main()