dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # 23:12|<aj> I will not hush!
  28 # 23:12|<elmo> :>
  29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  30 # 23:13|<aj> I shall not be silenced!
  31 # 23:13|<aj> The world shall know!
  32 # 23:13|<aj> The world *must* know!
  33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  34 # 23:13|<aj> yay powerpuff girls!!
  35 # 23:13|<aj> buttercup's my favourite, who's yours?
  36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  37 # 23:14|<aj> *AREN'T YOU*?!
  38 # 23:15|<aj> I will not be treated like this.
  39 # 23:15|<aj> I shall have my revenge.
  40 # 23:15|<aj> I SHALL!!!
  41
  42 ################################################################################
  43
  44 import copy
  45 import errno
  46 import os
  47 import readline
  48 import stat
  49 import sys
  50 import time
  51 import contextlib
  52 import apt_pkg, apt_inst
  53 import examine_package
  54 from daklib import database
  55 from daklib import logging
  56 from daklib import queue
  57 from daklib import utils
  58 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  59 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError
  60
  61 # Globals
  62 Cnf = None       #: Configuration, apt_pkg.Configuration
  63 Options = None
  64 Upload = None
  65 projectB = None  #: database connection, pgobject
  66 Logger = None
  67
  68 Priorities = None
  69 Sections = None
  70
  71 reject_message = ""
  72
  73 ################################################################################
  74 ################################################################################
  75 ################################################################################
  76
  77 def reject (str, prefix="Rejected: "):
  78     global reject_message
  79     if str:
  80         reject_message += prefix + str + "\n"
  81
  82 def recheck():
  83     global reject_message
  84     files = Upload.pkg.files
  85     reject_message = ""
  86
  87     for f in files.keys():
  88         # The .orig.tar.gz can disappear out from under us is it's a
  89         # duplicate of one in the archive.
  90         if not files.has_key(f):
  91             continue
  92         # Check that the source still exists
  93         if files[f]["type"] == "deb":
  94             source_version = files[f]["source version"]
  95             source_package = files[f]["source package"]
  96             if not Upload.pkg.changes["architecture"].has_key("source") \
  97                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  98                 source_epochless_version = re_no_epoch.sub('', source_version)
  99                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 100                 found = 0
 101                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
 102                     if Cnf.has_key("Dir::Queue::%s" % (q)):
 103                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
 104                             found = 1
 105                 if not found:
 106                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
 107
 108         # Version and file overwrite checks
 109         if files[f]["type"] == "deb":
 110             reject(Upload.check_binary_against_db(f), "")
 111         elif files[f]["type"] == "dsc":
 112             reject(Upload.check_source_against_db(f), "")
 113             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 114             reject(reject_msg, "")
 115
 116     if reject_message.find("Rejected") != -1:
 117         answer = "XXX"
 118         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
 119             answer = 'S'
 120
 121         print "REJECT\n" + reject_message,
 122         prompt = "[R]eject, Skip, Quit ?"
 123
 124         while prompt.find(answer) == -1:
 125             answer = utils.our_raw_input(prompt)
 126             m = re_default_answer.match(prompt)
 127             if answer == "":
 128                 answer = m.group(1)
 129             answer = answer[:1].upper()
 130
 131         if answer == 'R':
 132             Upload.do_reject(0, reject_message)
 133             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 134             return 0
 135         elif answer == 'S':
 136             return 0
 137         elif answer == 'Q':
 138             end()
 139             sys.exit(0)
 140
 141     return 1
 142
 143 ################################################################################
 144
 145 def indiv_sg_compare (a, b):
 146     """Sort by source name, source, version, 'have source', and
 147        finally by filename."""
 148     # Sort by source version
 149     q = apt_pkg.VersionCompare(a["version"], b["version"])
 150     if q:
 151         return -q
 152
 153     # Sort by 'have source'
 154     a_has_source = a["architecture"].get("source")
 155     b_has_source = b["architecture"].get("source")
 156     if a_has_source and not b_has_source:
 157         return -1
 158     elif b_has_source and not a_has_source:
 159         return 1
 160
 161     return cmp(a["filename"], b["filename"])
 162
 163 ############################################################
 164
 165 def sg_compare (a, b):
 166     a = a[1]
 167     b = b[1]
 168     """Sort by have note, source already in database and time of oldest upload."""
 169     # Sort by have note
 170     a_note_state = a["note_state"]
 171     b_note_state = b["note_state"]
 172     if a_note_state < b_note_state:
 173         return -1
 174     elif a_note_state > b_note_state:
 175         return 1
 176     # Sort by source already in database (descending)
 177     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 178     if source_in_database:
 179         return -source_in_database
 180
 181     # Sort by time of oldest upload
 182     return cmp(a["oldest"], b["oldest"])
 183
 184 def sort_changes(changes_files):
 185     """Sort into source groups, then sort each source group by version,
 186     have source, filename.  Finally, sort the source groups by have
 187     note, time of oldest upload of each source upload."""
 188     if len(changes_files) == 1:
 189         return changes_files
 190
 191     sorted_list = []
 192     cache = {}
 193     # Read in all the .changes files
 194     for filename in changes_files:
 195         try:
 196             Upload.pkg.changes_file = filename
 197             Upload.init_vars()
 198             Upload.update_vars()
 199             cache[filename] = copy.copy(Upload.pkg.changes)
 200             cache[filename]["filename"] = filename
 201         except:
 202             sorted_list.append(filename)
 203             break
 204     # Divide the .changes into per-source groups
 205     per_source = {}
 206     for filename in cache.keys():
 207         source = cache[filename]["source"]
 208         if not per_source.has_key(source):
 209             per_source[source] = {}
 210             per_source[source]["list"] = []
 211         per_source[source]["list"].append(cache[filename])
 212     # Determine oldest time and have note status for each source group
 213     for source in per_source.keys():
 214         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 215         ql = q.getresult()
 216         per_source[source]["source_in_database"] = len(ql)>0
 217         source_list = per_source[source]["list"]
 218         first = source_list[0]
 219         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 220         have_note = 0
 221         for d in per_source[source]["list"]:
 222             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 223             if mtime < oldest:
 224                 oldest = mtime
 225             have_note += (database.has_new_comment(d["source"], d["version"]))
 226         per_source[source]["oldest"] = oldest
 227         if not have_note:
 228             per_source[source]["note_state"] = 0; # none
 229         elif have_note < len(source_list):
 230             per_source[source]["note_state"] = 1; # some
 231         else:
 232             per_source[source]["note_state"] = 2; # all
 233         per_source[source]["list"].sort(indiv_sg_compare)
 234     per_source_items = per_source.items()
 235     per_source_items.sort(sg_compare)
 236     for i in per_source_items:
 237         for j in i[1]["list"]:
 238             sorted_list.append(j["filename"])
 239     return sorted_list
 240
 241 ################################################################################
 242
 243 class Section_Completer:
 244     def __init__ (self):
 245         self.sections = []
 246         self.matches = []
 247         q = projectB.query("SELECT section FROM section")
 248         for i in q.getresult():
 249             self.sections.append(i[0])
 250
 251     def complete(self, text, state):
 252         if state == 0:
 253             self.matches = []
 254             n = len(text)
 255             for word in self.sections:
 256                 if word[:n] == text:
 257                     self.matches.append(word)
 258         try:
 259             return self.matches[state]
 260         except IndexError:
 261             return None
 262
 263 ############################################################
 264
 265 class Priority_Completer:
 266     def __init__ (self):
 267         self.priorities = []
 268         self.matches = []
 269         q = projectB.query("SELECT priority FROM priority")
 270         for i in q.getresult():
 271             self.priorities.append(i[0])
 272
 273     def complete(self, text, state):
 274         if state == 0:
 275             self.matches = []
 276             n = len(text)
 277             for word in self.priorities:
 278                 if word[:n] == text:
 279                     self.matches.append(word)
 280         try:
 281             return self.matches[state]
 282         except IndexError:
 283             return None
 284
 285 ################################################################################
 286
 287 def print_new (new, indexed, file=sys.stdout):
 288     queue.check_valid(new)
 289     broken = 0
 290     index = 0
 291     for pkg in new.keys():
 292         index += 1
 293         section = new[pkg]["section"]
 294         priority = new[pkg]["priority"]
 295         if new[pkg]["section id"] == -1:
 296             section += "[!]"
 297             broken = 1
 298         if new[pkg]["priority id"] == -1:
 299             priority += "[!]"
 300             broken = 1
 301         if indexed:
 302             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 303         else:
 304             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 305         line = line.strip()+'\n'
 306         file.write(line)
 307     note = database.get_new_comments(Upload.pkg.changes.get("source"))
 308     if len(note) > 0:
 309         for line in note:
 310             print line
 311     return broken, note
 312
 313 ################################################################################
 314
 315 def index_range (index):
 316     if index == 1:
 317         return "1"
 318     else:
 319         return "1-%s" % (index)
 320
 321 ################################################################################
 322 ################################################################################
 323
 324 def edit_new (new):
 325     # Write the current data to a temporary file
 326     (fd, temp_filename) = utils.temp_filename()
 327     temp_file = os.fdopen(fd, 'w')
 328     print_new (new, 0, temp_file)
 329     temp_file.close()
 330     # Spawn an editor on that file
 331     editor = os.environ.get("EDITOR","vi")
 332     result = os.system("%s %s" % (editor, temp_filename))
 333     if result != 0:
 334         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 335     # Read the edited data back in
 336     temp_file = utils.open_file(temp_filename)
 337     lines = temp_file.readlines()
 338     temp_file.close()
 339     os.unlink(temp_filename)
 340     # Parse the new data
 341     for line in lines:
 342         line = line.strip()
 343         if line == "":
 344             continue
 345         s = line.split()
 346         # Pad the list if necessary
 347         s[len(s):3] = [None] * (3-len(s))
 348         (pkg, priority, section) = s[:3]
 349         if not new.has_key(pkg):
 350             utils.warn("Ignoring unknown package '%s'" % (pkg))
 351         else:
 352             # Strip off any invalid markers, print_new will readd them.
 353             if section.endswith("[!]"):
 354                 section = section[:-3]
 355             if priority.endswith("[!]"):
 356                 priority = priority[:-3]
 357             for f in new[pkg]["files"]:
 358                 Upload.pkg.files[f]["section"] = section
 359                 Upload.pkg.files[f]["priority"] = priority
 360             new[pkg]["section"] = section
 361             new[pkg]["priority"] = priority
 362
 363 ################################################################################
 364
 365 def edit_index (new, index):
 366     priority = new[index]["priority"]
 367     section = new[index]["section"]
 368     ftype = new[index]["type"]
 369     done = 0
 370     while not done:
 371         print "\t".join([index, priority, section])
 372
 373         answer = "XXX"
 374         if ftype != "dsc":
 375             prompt = "[B]oth, Priority, Section, Done ? "
 376         else:
 377             prompt = "[S]ection, Done ? "
 378         edit_priority = edit_section = 0
 379
 380         while prompt.find(answer) == -1:
 381             answer = utils.our_raw_input(prompt)
 382             m = re_default_answer.match(prompt)
 383             if answer == "":
 384                 answer = m.group(1)
 385             answer = answer[:1].upper()
 386
 387         if answer == 'P':
 388             edit_priority = 1
 389         elif answer == 'S':
 390             edit_section = 1
 391         elif answer == 'B':
 392             edit_priority = edit_section = 1
 393         elif answer == 'D':
 394             done = 1
 395
 396         # Edit the priority
 397         if edit_priority:
 398             readline.set_completer(Priorities.complete)
 399             got_priority = 0
 400             while not got_priority:
 401                 new_priority = utils.our_raw_input("New priority: ").strip()
 402                 if new_priority not in Priorities.priorities:
 403                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 404                 else:
 405                     got_priority = 1
 406                     priority = new_priority
 407
 408         # Edit the section
 409         if edit_section:
 410             readline.set_completer(Sections.complete)
 411             got_section = 0
 412             while not got_section:
 413                 new_section = utils.our_raw_input("New section: ").strip()
 414                 if new_section not in Sections.sections:
 415                     print "E: '%s' is not a valid section, try again." % (new_section)
 416                 else:
 417                     got_section = 1
 418                     section = new_section
 419
 420         # Reset the readline completer
 421         readline.set_completer(None)
 422
 423     for f in new[index]["files"]:
 424         Upload.pkg.files[f]["section"] = section
 425         Upload.pkg.files[f]["priority"] = priority
 426     new[index]["priority"] = priority
 427     new[index]["section"] = section
 428     return new
 429
 430 ################################################################################
 431
 432 def edit_overrides (new):
 433     print
 434     done = 0
 435     while not done:
 436         print_new (new, 1)
 437         new_index = {}
 438         index = 0
 439         for i in new.keys():
 440             index += 1
 441             new_index[index] = i
 442
 443         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 444
 445         got_answer = 0
 446         while not got_answer:
 447             answer = utils.our_raw_input(prompt)
 448             if not answer.isdigit():
 449                 answer = answer[:1].upper()
 450             if answer == "E" or answer == "D":
 451                 got_answer = 1
 452             elif re_isanum.match (answer):
 453                 answer = int(answer)
 454                 if (answer < 1) or (answer > index):
 455                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 456                 else:
 457                     got_answer = 1
 458
 459         if answer == 'E':
 460             edit_new(new)
 461         elif answer == 'D':
 462             done = 1
 463         else:
 464             edit_index (new, new_index[answer])
 465
 466     return new
 467
 468 ################################################################################
 469
 470 def edit_note(note):
 471     # Write the current data to a temporary file
 472     (fd, temp_filename) = utils.temp_filename()
 473     editor = os.environ.get("EDITOR","vi")
 474     answer = 'E'
 475     while answer == 'E':
 476         os.system("%s %s" % (editor, temp_filename))
 477         temp_file = utils.open_file(temp_filename)
 478         newnote = temp_file.read().rstrip()
 479         temp_file.close()
 480         print "New Note:"
 481         print utils.prefix_multi_line_string(newnote,"  ")
 482         prompt = "[D]one, Edit, Abandon, Quit ?"
 483         answer = "XXX"
 484         while prompt.find(answer) == -1:
 485             answer = utils.our_raw_input(prompt)
 486             m = re_default_answer.search(prompt)
 487             if answer == "":
 488                 answer = m.group(1)
 489             answer = answer[:1].upper()
 490     os.unlink(temp_filename)
 491     if answer == 'A':
 492         return
 493     elif answer == 'Q':
 494         end()
 495         sys.exit(0)
 496     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami())
 497
 498 ################################################################################
 499
 500 def check_pkg ():
 501     try:
 502         less_fd = os.popen("less -R -", 'w', 0)
 503         stdout_fd = sys.stdout
 504         try:
 505             sys.stdout = less_fd
 506             changes = utils.parse_changes (Upload.pkg.changes_file)
 507             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 508             files = Upload.pkg.files
 509             for f in files.keys():
 510                 if files[f].has_key("new"):
 511                     ftype = files[f]["type"]
 512                     if ftype == "deb":
 513                         examine_package.check_deb(changes['distribution'], f)
 514                     elif ftype == "dsc":
 515                         examine_package.check_dsc(changes['distribution'], f)
 516         finally:
 517             examine_package.output_package_relations()
 518             sys.stdout = stdout_fd
 519     except IOError, e:
 520         if e.errno == errno.EPIPE:
 521             utils.warn("[examine_package] Caught EPIPE; skipping.")
 522             pass
 523         else:
 524             raise
 525     except KeyboardInterrupt:
 526         utils.warn("[examine_package] Caught C-c; skipping.")
 527         pass
 528
 529 ################################################################################
 530
 531 ## FIXME: horribly Debian specific
 532
 533 def do_bxa_notification():
 534     files = Upload.pkg.files
 535     summary = ""
 536     for f in files.keys():
 537         if files[f]["type"] == "deb":
 538             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 539             summary += "\n"
 540             summary += "Package: %s\n" % (control.Find("Package"))
 541             summary += "Description: %s\n" % (control.Find("Description"))
 542     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 543     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 544     utils.send_mail(bxa_mail)
 545
 546 ################################################################################
 547
 548 def add_overrides (new):
 549     changes = Upload.pkg.changes
 550     files = Upload.pkg.files
 551
 552     projectB.query("BEGIN WORK")
 553     for suite in changes["suite"].keys():
 554         suite_id = database.get_suite_id(suite)
 555         for pkg in new.keys():
 556             component_id = database.get_component_id(new[pkg]["component"])
 557             type_id = database.get_override_type_id(new[pkg]["type"])
 558             priority_id = new[pkg]["priority id"]
 559             section_id = new[pkg]["section id"]
 560             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 561             for f in new[pkg]["files"]:
 562                 if files[f].has_key("new"):
 563                     del files[f]["new"]
 564             del new[pkg]
 565
 566     projectB.query("COMMIT WORK")
 567
 568     if Cnf.FindB("Dinstall::BXANotify"):
 569         do_bxa_notification()
 570
 571 ################################################################################
 572
 573 def prod_maintainer (note):
 574     # Here we prepare an editor and get them ready to prod...
 575     (fd, temp_filename) = utils.temp_filename()
 576     temp_file = os.fdopen(fd, 'w')
 577     if len(note) > 0:
 578         for line in note:
 579             temp_file.write(line)
 580     temp_file.close()
 581     editor = os.environ.get("EDITOR","vi")
 582     answer = 'E'
 583     while answer == 'E':
 584         os.system("%s %s" % (editor, temp_filename))
 585         temp_fh = utils.open_file(temp_filename)
 586         prod_message = "".join(temp_fh.readlines())
 587         temp_fh.close()
 588         print "Prod message:"
 589         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 590         prompt = "[P]rod, Edit, Abandon, Quit ?"
 591         answer = "XXX"
 592         while prompt.find(answer) == -1:
 593             answer = utils.our_raw_input(prompt)
 594             m = re_default_answer.search(prompt)
 595             if answer == "":
 596                 answer = m.group(1)
 597             answer = answer[:1].upper()
 598     os.unlink(temp_filename)
 599     if answer == 'A':
 600         return
 601     elif answer == 'Q':
 602         end()
 603         sys.exit(0)
 604     # Otherwise, do the proding...
 605     user_email_address = utils.whoami() + " <%s>" % (
 606         Cnf["Dinstall::MyAdminAddress"])
 607
 608     Subst = Upload.Subst
 609
 610     Subst["__FROM_ADDRESS__"] = user_email_address
 611     Subst["__PROD_MESSAGE__"] = prod_message
 612     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 613
 614     prod_mail_message = utils.TemplateSubst(
 615         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 616
 617     # Send the prod mail if appropriate
 618     if not Cnf["Dinstall::Options::No-Mail"]:
 619         utils.send_mail(prod_mail_message)
 620
 621     print "Sent proding message"
 622
 623 ################################################################################
 624
 625 def do_new():
 626     print "NEW\n"
 627     files = Upload.pkg.files
 628     changes = Upload.pkg.changes
 629
 630     # Make a copy of distribution we can happily trample on
 631     changes["suite"] = copy.copy(changes["distribution"])
 632
 633     # Fix up the list of target suites
 634     for suite in changes["suite"].keys():
 635         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 636         if override:
 637             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 638               database.get_suite_id(override) == -1)
 639             if olderr or newerr:
 640                 (oinv, newinv) = ("", "")
 641                 if olderr: oinv = "invalid "
 642                 if newerr: ninv = "invalid "
 643                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 644                         oinv, suite, ninv, override)
 645             del changes["suite"][suite]
 646             changes["suite"][override] = 1
 647     # Validate suites
 648     for suite in changes["suite"].keys():
 649         suite_id = database.get_suite_id(suite)
 650         if suite_id == -1:
 651             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 652
 653     # The main NEW processing loop
 654     done = 0
 655     while not done:
 656         # Find out what's new
 657         new = queue.determine_new(changes, files, projectB)
 658
 659         if not new:
 660             break
 661
 662         answer = "XXX"
 663         if Options["No-Action"] or Options["Automatic"]:
 664             answer = 'S'
 665
 666         (broken, note) = print_new(new, 0)
 667         prompt = ""
 668
 669         if not broken and not note:
 670             prompt = "Add overrides, "
 671         if broken:
 672             print "W: [!] marked entries must be fixed before package can be processed."
 673         if note:
 674             print "W: note must be removed before package can be processed."
 675             prompt += "Remove note, "
 676
 677         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 678
 679         while prompt.find(answer) == -1:
 680             answer = utils.our_raw_input(prompt)
 681             m = re_default_answer.search(prompt)
 682             if answer == "":
 683                 answer = m.group(1)
 684             answer = answer[:1].upper()
 685
 686         if answer == 'A' and not Options["Trainee"]:
 687             done = add_overrides (new)
 688         elif answer == 'C':
 689             check_pkg()
 690         elif answer == 'E' and not Options["Trainee"]:
 691             new = edit_overrides (new)
 692         elif answer == 'M' and not Options["Trainee"]:
 693             aborted = Upload.do_reject(manual=1,
 694                                        reject_message=Options["Manual-Reject"],
 695                                        note=database.get_new_comments(changes.get("source", "")))
 696             if not aborted:
 697                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 698                 done = 1
 699         elif answer == 'N':
 700             edit_note(database.get_new_comments(changes.get("source", "")))
 701         elif answer == 'P' and not Options["Trainee"]:
 702             prod_maintainer(database.get_new_comments(changes.get("source", "")))
 703         elif answer == 'R':
 704             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 705             if confirm == "y":
 706                 database.delete_new_comments(changes.get("source"), changes.get("version"))
 707         elif answer == 'S':
 708             done = 1
 709         elif answer == 'Q':
 710             end()
 711             sys.exit(0)
 712
 713 ################################################################################
 714 ################################################################################
 715 ################################################################################
 716
 717 def usage (exit_code=0):
 718     print """Usage: dak process-new [OPTION]... [CHANGES]...
 719   -a, --automatic           automatic run
 720   -h, --help                show this help and exit.
 721   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 722   -m, --manual-reject=MSG   manual reject with `msg'
 723   -n, --no-action           don't do anything
 724   -t, --trainee             FTP Trainee mode
 725   -V, --version             display the version number and exit"""
 726     sys.exit(exit_code)
 727
 728 ################################################################################
 729
 730 def init():
 731     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 732
 733     Cnf = utils.get_conf()
 734
 735     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 736                  ('h',"help","Process-New::Options::Help"),
 737                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 738                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 739                  ('t',"trainee","Process-New::Options::Trainee"),
 740                  ('n',"no-action","Process-New::Options::No-Action")]
 741
 742     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
 743         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 744             Cnf["Process-New::Options::%s" % (i)] = ""
 745
 746     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 747     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
 748         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
 749
 750     Options = Cnf.SubTree("Process-New::Options")
 751
 752     if Options["Help"]:
 753         usage()
 754
 755     Upload = queue.Upload(Cnf)
 756
 757     if not Options["No-Action"]:
 758         try:
 759             Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 760         except CantOpenError, e:
 761             Options["Trainee"] = "Oh yes"
 762
 763     projectB = Upload.projectB
 764
 765     Sections = Section_Completer()
 766     Priorities = Priority_Completer()
 767     readline.parse_and_bind("tab: complete")
 768
 769     return changes_files
 770
 771 ################################################################################
 772
 773 def do_byhand():
 774     done = 0
 775     while not done:
 776         files = Upload.pkg.files
 777         will_install = 1
 778         byhand = []
 779
 780         for f in files.keys():
 781             if files[f]["type"] == "byhand":
 782                 if os.path.exists(f):
 783                     print "W: %s still present; please process byhand components and try again." % (f)
 784                     will_install = 0
 785                 else:
 786                     byhand.append(f)
 787
 788         answer = "XXXX"
 789         if Options["No-Action"]:
 790             answer = "S"
 791         if will_install:
 792             if Options["Automatic"] and not Options["No-Action"]:
 793                 answer = 'A'
 794             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 795         else:
 796             prompt = "Manual reject, [S]kip, Quit ?"
 797
 798         while prompt.find(answer) == -1:
 799             answer = utils.our_raw_input(prompt)
 800             m = re_default_answer.search(prompt)
 801             if answer == "":
 802                 answer = m.group(1)
 803             answer = answer[:1].upper()
 804
 805         if answer == 'A':
 806             done = 1
 807             for f in byhand:
 808                 del files[f]
 809         elif answer == 'M':
 810             Upload.do_reject(1, Options["Manual-Reject"])
 811             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 812             done = 1
 813         elif answer == 'S':
 814             done = 1
 815         elif answer == 'Q':
 816             end()
 817             sys.exit(0)
 818
 819 ################################################################################
 820
 821 def get_accept_lock():
 822     retry = 0
 823     while retry < 10:
 824         try:
 825             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 826             retry = 10
 827         except OSError, e:
 828             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 829                 retry += 1
 830                 if (retry >= 10):
 831                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 832                 else:
 833                     print("Unable to get accepted lock (try %d of 10)" % retry)
 834                 time.sleep(60)
 835             else:
 836                 raise
 837
 838
 839 @contextlib.contextmanager
 840 def lock_package(package):
 841     """
 842     Lock C{package} so that noone else jumps in processing it.
 843
 844     @type package: string
 845     @param package: source package name to lock
 846     """
 847
 848     path = os.path.join(Cnf["Process-New::LockDir"], package)
 849     try:
 850         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 851     except OSError, e:
 852         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 853             raise AlreadyLockedError, e.errno
 854
 855     try:
 856         yield fd
 857     finally:
 858         os.unlink(path)
 859
 860 def move_to_dir (dest, perms=0660, changesperms=0664):
 861     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 862     file_keys = Upload.pkg.files.keys()
 863     for f in file_keys:
 864         utils.move (f, dest, perms=perms)
 865
 866 def is_source_in_queue_dir(qdir):
 867     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 868                 and x.endswith(".changes") ]
 869     for entry in entries:
 870         # read the .dak
 871         u = queue.Upload(Cnf)
 872         u.pkg.changes_file = os.path.join(qdir, entry)
 873         u.update_vars()
 874         if not u.pkg.changes["architecture"].has_key("source"):
 875             # another binary upload, ignore
 876             continue
 877         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 878             # another version, ignore
 879             continue
 880         # found it!
 881         return True
 882     return False
 883
 884 def move_to_holding(suite, queue_dir):
 885     print "Moving to %s holding area." % (suite.upper(),)
 886     if Options["No-Action"]:
 887         return
 888     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 889     Upload.dump_vars(queue_dir)
 890     move_to_dir(queue_dir, perms=0664)
 891     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 892
 893 def _accept():
 894     if Options["No-Action"]:
 895         return
 896     (summary, short_summary) = Upload.build_summaries()
 897     Upload.accept(summary, short_summary)
 898     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 899
 900 def do_accept_stableupdate(suite, q):
 901     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 902     if not Upload.pkg.changes["architecture"].has_key("source"):
 903         # It is not a sourceful upload.  So its source may be either in p-u
 904         # holding, in new, in accepted or already installed.
 905         if is_source_in_queue_dir(queue_dir):
 906             # It's in p-u holding, so move it there.
 907             print "Binary-only upload, source in %s." % (q,)
 908             move_to_holding(suite, queue_dir)
 909         elif Upload.source_exists(Upload.pkg.changes["source"],
 910                 Upload.pkg.changes["version"]):
 911             # dak tells us that there is source available.  At time of
 912             # writing this means that it is installed, so put it into
 913             # accepted.
 914             print "Binary-only upload, source installed."
 915             _accept()
 916         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 917             # The source is in accepted, the binary cleared NEW: accept it.
 918             print "Binary-only upload, source in accepted."
 919             _accept()
 920         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 921             # It's in NEW.  We expect the source to land in p-u holding
 922             # pretty soon.
 923             print "Binary-only upload, source in new."
 924             move_to_holding(suite, queue_dir)
 925         else:
 926             # No case applicable.  Bail out.  Return will cause the upload
 927             # to be skipped.
 928             print "ERROR"
 929             print "Stable update failed.  Source not found."
 930             return
 931     else:
 932         # We are handling a sourceful upload.  Move to accepted if currently
 933         # in p-u holding and to p-u holding otherwise.
 934         if is_source_in_queue_dir(queue_dir):
 935             print "Sourceful upload in %s, accepting." % (q,)
 936             _accept()
 937         else:
 938             move_to_holding(suite, queue_dir)
 939
 940 def do_accept():
 941     print "ACCEPT"
 942     if not Options["No-Action"]:
 943         get_accept_lock()
 944         (summary, short_summary) = Upload.build_summaries()
 945     try:
 946         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 947             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 948             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 949             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 950             # Check for override disparities
 951             Upload.Subst["__SUMMARY__"] = summary
 952         else:
 953             # Stable updates need to be copied to proposed-updates holding
 954             # area instead of accepted.  Sourceful uploads need to go
 955             # to it directly, binaries only if the source has not yet been
 956             # accepted into p-u.
 957             for suite, q in [("proposed-updates", "ProposedUpdates"),
 958                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 959                 if not Upload.pkg.changes["distribution"].has_key(suite):
 960                     continue
 961                 return do_accept_stableupdate(suite, q)
 962             # Just a normal upload, accept it...
 963             _accept()
 964     finally:
 965         if not Options["No-Action"]:
 966             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 967
 968 def check_status(files):
 969     new = byhand = 0
 970     for f in files.keys():
 971         if files[f]["type"] == "byhand":
 972             byhand = 1
 973         elif files[f].has_key("new"):
 974             new = 1
 975     return (new, byhand)
 976
 977 def do_pkg(changes_file):
 978     Upload.pkg.changes_file = changes_file
 979     Upload.init_vars()
 980     Upload.update_vars()
 981     Upload.update_subst()
 982     files = Upload.pkg.files
 983
 984     try:
 985         with lock_package(Upload.pkg.changes["source"]):
 986             if not recheck():
 987                 return
 988
 989             (new, byhand) = check_status(files)
 990             if new or byhand:
 991                 if new:
 992                     do_new()
 993                 if byhand:
 994                     do_byhand()
 995                 (new, byhand) = check_status(files)
 996
 997             if not new and not byhand:
 998                 do_accept()
 999     except AlreadyLockedError, e:
1000         print "Seems to be locked already, skipping..."
1001
1002 ################################################################################
1003
1004 def end():
1005     accept_count = Upload.accept_count
1006     accept_bytes = Upload.accept_bytes
1007
1008     if accept_count:
1009         sets = "set"
1010         if accept_count > 1:
1011             sets = "sets"
1012         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1013         Logger.log(["total",accept_count,accept_bytes])
1014
1015     if not Options["No-Action"] and not Options["Trainee"]:
1016         Logger.close()
1017
1018 ################################################################################
1019
1020 def do_comments(dir, opref, npref, line, fn):
1021     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1022         lines = open("%s/%s" % (dir, comm)).readlines()
1023         if len(lines) == 0 or lines[0] != line + "\n": continue
1024         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1025                                 and x.endswith(".changes") ]
1026         changes_files = sort_changes(changes_files)
1027         for f in changes_files:
1028             f = utils.validate_changes_file_arg(f, 0)
1029             if not f: continue
1030             print "\n" + f
1031             fn(f, "".join(lines[1:]))
1032
1033         if opref != npref and not Options["No-Action"]:
1034             newcomm = npref + comm[len(opref):]
1035             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1036
1037 ################################################################################
1038
1039 def comment_accept(changes_file, comments):
1040     Upload.pkg.changes_file = changes_file
1041     Upload.init_vars()
1042     Upload.update_vars()
1043     Upload.update_subst()
1044     files = Upload.pkg.files
1045
1046     if not recheck():
1047         return # dak wants to REJECT, crap
1048
1049     (new, byhand) = check_status(files)
1050     if not new and not byhand:
1051         do_accept()
1052
1053 ################################################################################
1054
1055 def comment_reject(changes_file, comments):
1056     Upload.pkg.changes_file = changes_file
1057     Upload.init_vars()
1058     Upload.update_vars()
1059     Upload.update_subst()
1060
1061     if not recheck():
1062         pass # dak has its own reasons to reject as well, which is fine
1063
1064     reject(comments)
1065     print "REJECT\n" + reject_message,
1066     if not Options["No-Action"]:
1067         Upload.do_reject(0, reject_message)
1068         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1069
1070 ################################################################################
1071
1072 def main():
1073     changes_files = init()
1074     if len(changes_files) > 50:
1075         sys.stderr.write("Sorting changes...\n")
1076     changes_files = sort_changes(changes_files)
1077
1078     # Kill me now? **FIXME**
1079     Cnf["Dinstall::Options::No-Mail"] = ""
1080     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1081     if Cnf.has_key("Dinstall::Bcc"):
1082         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1083     else:
1084         Upload.Subst["__BCC__"] = bcc
1085
1086     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1087     if commentsdir:
1088         if changes_files != []:
1089             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1090             sys.exit(1)
1091         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1092         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1093     else:
1094         for changes_file in changes_files:
1095             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1096             if not changes_file:
1097                 continue
1098             print "\n" + changes_file
1099             do_pkg (changes_file)
1100
1101     end()
1102
1103 ################################################################################
1104
1105 if __name__ == '__main__':
1106     main()