dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 from __future__ import with_statement
  46
  47 import copy
  48 import errno
  49 import os
  50 import readline
  51 import stat
  52 import sys
  53 import time
  54 import contextlib
  55 import pwd
  56 import apt_pkg, apt_inst
  57 import examine_package
  58
  59 from daklib.dbconn import *
  60 from daklib.queue import *
  61 from daklib import daklog
  62 from daklib import utils
  63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
  64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  65 from daklib.summarystats import SummaryStats
  66 from daklib.config import Config
  67
  68 # Globals
  69 Options = None
  70 Logger = None
  71
  72 Priorities = None
  73 Sections = None
  74
  75 ################################################################################
  76 ################################################################################
  77 ################################################################################
  78
  79 def recheck(upload, session):
  80 # STU: I'm not sure, but I don't thin kthis is necessary any longer:    upload.recheck(session)
  81     if len(upload.rejects) > 0:
  82         answer = "XXX"
  83         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
  84             answer = 'S'
  85
  86         print "REJECT\n%s" % '\n'.join(upload.rejects)
  87         prompt = "[R]eject, Skip, Quit ?"
  88
  89         while prompt.find(answer) == -1:
  90             answer = utils.our_raw_input(prompt)
  91             m = re_default_answer.match(prompt)
  92             if answer == "":
  93                 answer = m.group(1)
  94             answer = answer[:1].upper()
  95
  96         if answer == 'R':
  97             upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
  98             upload.pkg.remove_known_changes(session=session)
  99             return 0
 100         elif answer == 'S':
 101             return 0
 102         elif answer == 'Q':
 103             end()
 104             sys.exit(0)
 105
 106     return 1
 107
 108 ################################################################################
 109
 110 def indiv_sg_compare (a, b):
 111     """Sort by source name, source, version, 'have source', and
 112        finally by filename."""
 113     # Sort by source version
 114     q = apt_pkg.VersionCompare(a["version"], b["version"])
 115     if q:
 116         return -q
 117
 118     # Sort by 'have source'
 119     a_has_source = a["architecture"].get("source")
 120     b_has_source = b["architecture"].get("source")
 121     if a_has_source and not b_has_source:
 122         return -1
 123     elif b_has_source and not a_has_source:
 124         return 1
 125
 126     return cmp(a["filename"], b["filename"])
 127
 128 ############################################################
 129
 130 def sg_compare (a, b):
 131     a = a[1]
 132     b = b[1]
 133     """Sort by have note, source already in database and time of oldest upload."""
 134     # Sort by have note
 135     a_note_state = a["note_state"]
 136     b_note_state = b["note_state"]
 137     if a_note_state < b_note_state:
 138         return -1
 139     elif a_note_state > b_note_state:
 140         return 1
 141     # Sort by source already in database (descending)
 142     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 143     if source_in_database:
 144         return -source_in_database
 145
 146     # Sort by time of oldest upload
 147     return cmp(a["oldest"], b["oldest"])
 148
 149 def sort_changes(changes_files, session):
 150     """Sort into source groups, then sort each source group by version,
 151     have source, filename.  Finally, sort the source groups by have
 152     note, time of oldest upload of each source upload."""
 153     if len(changes_files) == 1:
 154         return changes_files
 155
 156     sorted_list = []
 157     cache = {}
 158     # Read in all the .changes files
 159     for filename in changes_files:
 160         u = Upload()
 161         try:
 162             u.pkg.changes_file = filename
 163             u.load_changes(filename)
 164             u.update_subst()
 165             cache[filename] = copy.copy(u.pkg.changes)
 166             cache[filename]["filename"] = filename
 167         except:
 168             sorted_list.append(filename)
 169             break
 170     # Divide the .changes into per-source groups
 171     per_source = {}
 172     for filename in cache.keys():
 173         source = cache[filename]["source"]
 174         if not per_source.has_key(source):
 175             per_source[source] = {}
 176             per_source[source]["list"] = []
 177         per_source[source]["list"].append(cache[filename])
 178     # Determine oldest time and have note status for each source group
 179     for source in per_source.keys():
 180         q = session.query(DBSource).filter_by(source = source).all()
 181         per_source[source]["source_in_database"] = len(q)>0
 182         source_list = per_source[source]["list"]
 183         first = source_list[0]
 184         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 185         have_note = 0
 186         for d in per_source[source]["list"]:
 187             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 188             if mtime < oldest:
 189                 oldest = mtime
 190             have_note += has_new_comment(d["source"], d["version"], session)
 191         per_source[source]["oldest"] = oldest
 192         if not have_note:
 193             per_source[source]["note_state"] = 0; # none
 194         elif have_note < len(source_list):
 195             per_source[source]["note_state"] = 1; # some
 196         else:
 197             per_source[source]["note_state"] = 2; # all
 198         per_source[source]["list"].sort(indiv_sg_compare)
 199     per_source_items = per_source.items()
 200     per_source_items.sort(sg_compare)
 201     for i in per_source_items:
 202         for j in i[1]["list"]:
 203             sorted_list.append(j["filename"])
 204     return sorted_list
 205
 206 ################################################################################
 207
 208 class Section_Completer:
 209     def __init__ (self, session):
 210         self.sections = []
 211         self.matches = []
 212         for s, in session.query(Section.section):
 213             self.sections.append(s)
 214
 215     def complete(self, text, state):
 216         if state == 0:
 217             self.matches = []
 218             n = len(text)
 219             for word in self.sections:
 220                 if word[:n] == text:
 221                     self.matches.append(word)
 222         try:
 223             return self.matches[state]
 224         except IndexError:
 225             return None
 226
 227 ############################################################
 228
 229 class Priority_Completer:
 230     def __init__ (self, session):
 231         self.priorities = []
 232         self.matches = []
 233         for p, in session.query(Priority.priority):
 234             self.priorities.append(p)
 235
 236     def complete(self, text, state):
 237         if state == 0:
 238             self.matches = []
 239             n = len(text)
 240             for word in self.priorities:
 241                 if word[:n] == text:
 242                     self.matches.append(word)
 243         try:
 244             return self.matches[state]
 245         except IndexError:
 246             return None
 247
 248 ################################################################################
 249
 250 def print_new (new, upload, indexed, file=sys.stdout):
 251     check_valid(new)
 252     broken = False
 253     index = 0
 254     for pkg in new.keys():
 255         index += 1
 256         section = new[pkg]["section"]
 257         priority = new[pkg]["priority"]
 258         if new[pkg]["section id"] == -1:
 259             section += "[!]"
 260             broken = True
 261         if new[pkg]["priority id"] == -1:
 262             priority += "[!]"
 263             broken = True
 264         if indexed:
 265             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 266         else:
 267             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 268         line = line.strip()+'\n'
 269         file.write(line)
 270     notes = get_new_comments(upload.pkg.changes.get("source"))
 271     for note in notes:
 272         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 273               % (note.author, note.version, note.notedate, note.comment)
 274         print "-" * 72
 275     return broken, len(notes) > 0
 276
 277 ################################################################################
 278
 279 def index_range (index):
 280     if index == 1:
 281         return "1"
 282     else:
 283         return "1-%s" % (index)
 284
 285 ################################################################################
 286 ################################################################################
 287
 288 def edit_new (new, upload):
 289     # Write the current data to a temporary file
 290     (fd, temp_filename) = utils.temp_filename()
 291     temp_file = os.fdopen(fd, 'w')
 292     print_new (new, upload, indexed=0, file=temp_file)
 293     temp_file.close()
 294     # Spawn an editor on that file
 295     editor = os.environ.get("EDITOR","vi")
 296     result = os.system("%s %s" % (editor, temp_filename))
 297     if result != 0:
 298         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 299     # Read the edited data back in
 300     temp_file = utils.open_file(temp_filename)
 301     lines = temp_file.readlines()
 302     temp_file.close()
 303     os.unlink(temp_filename)
 304     # Parse the new data
 305     for line in lines:
 306         line = line.strip()
 307         if line == "":
 308             continue
 309         s = line.split()
 310         # Pad the list if necessary
 311         s[len(s):3] = [None] * (3-len(s))
 312         (pkg, priority, section) = s[:3]
 313         if not new.has_key(pkg):
 314             utils.warn("Ignoring unknown package '%s'" % (pkg))
 315         else:
 316             # Strip off any invalid markers, print_new will readd them.
 317             if section.endswith("[!]"):
 318                 section = section[:-3]
 319             if priority.endswith("[!]"):
 320                 priority = priority[:-3]
 321             for f in new[pkg]["files"]:
 322                 upload.pkg.files[f]["section"] = section
 323                 upload.pkg.files[f]["priority"] = priority
 324             new[pkg]["section"] = section
 325             new[pkg]["priority"] = priority
 326
 327 ################################################################################
 328
 329 def edit_index (new, upload, index):
 330     priority = new[index]["priority"]
 331     section = new[index]["section"]
 332     ftype = new[index]["type"]
 333     done = 0
 334     while not done:
 335         print "\t".join([index, priority, section])
 336
 337         answer = "XXX"
 338         if ftype != "dsc":
 339             prompt = "[B]oth, Priority, Section, Done ? "
 340         else:
 341             prompt = "[S]ection, Done ? "
 342         edit_priority = edit_section = 0
 343
 344         while prompt.find(answer) == -1:
 345             answer = utils.our_raw_input(prompt)
 346             m = re_default_answer.match(prompt)
 347             if answer == "":
 348                 answer = m.group(1)
 349             answer = answer[:1].upper()
 350
 351         if answer == 'P':
 352             edit_priority = 1
 353         elif answer == 'S':
 354             edit_section = 1
 355         elif answer == 'B':
 356             edit_priority = edit_section = 1
 357         elif answer == 'D':
 358             done = 1
 359
 360         # Edit the priority
 361         if edit_priority:
 362             readline.set_completer(Priorities.complete)
 363             got_priority = 0
 364             while not got_priority:
 365                 new_priority = utils.our_raw_input("New priority: ").strip()
 366                 if new_priority not in Priorities.priorities:
 367                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 368                 else:
 369                     got_priority = 1
 370                     priority = new_priority
 371
 372         # Edit the section
 373         if edit_section:
 374             readline.set_completer(Sections.complete)
 375             got_section = 0
 376             while not got_section:
 377                 new_section = utils.our_raw_input("New section: ").strip()
 378                 if new_section not in Sections.sections:
 379                     print "E: '%s' is not a valid section, try again." % (new_section)
 380                 else:
 381                     got_section = 1
 382                     section = new_section
 383
 384         # Reset the readline completer
 385         readline.set_completer(None)
 386
 387     for f in new[index]["files"]:
 388         upload.pkg.files[f]["section"] = section
 389         upload.pkg.files[f]["priority"] = priority
 390     new[index]["priority"] = priority
 391     new[index]["section"] = section
 392     return new
 393
 394 ################################################################################
 395
 396 def edit_overrides (new, upload, session):
 397     print
 398     done = 0
 399     while not done:
 400         print_new (new, upload, indexed=1)
 401         new_index = {}
 402         index = 0
 403         for i in new.keys():
 404             index += 1
 405             new_index[index] = i
 406
 407         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 408
 409         got_answer = 0
 410         while not got_answer:
 411             answer = utils.our_raw_input(prompt)
 412             if not answer.isdigit():
 413                 answer = answer[:1].upper()
 414             if answer == "E" or answer == "D":
 415                 got_answer = 1
 416             elif re_isanum.match (answer):
 417                 answer = int(answer)
 418                 if (answer < 1) or (answer > index):
 419                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 420                 else:
 421                     got_answer = 1
 422
 423         if answer == 'E':
 424             edit_new(new, upload)
 425         elif answer == 'D':
 426             done = 1
 427         else:
 428             edit_index (new, upload, new_index[answer])
 429
 430     return new
 431
 432 ################################################################################
 433
 434 def edit_note(note, upload, session):
 435     # Write the current data to a temporary file
 436     (fd, temp_filename) = utils.temp_filename()
 437     editor = os.environ.get("EDITOR","vi")
 438     answer = 'E'
 439     while answer == 'E':
 440         os.system("%s %s" % (editor, temp_filename))
 441         temp_file = utils.open_file(temp_filename)
 442         newnote = temp_file.read().rstrip()
 443         temp_file.close()
 444         print "New Note:"
 445         print utils.prefix_multi_line_string(newnote,"  ")
 446         prompt = "[D]one, Edit, Abandon, Quit ?"
 447         answer = "XXX"
 448         while prompt.find(answer) == -1:
 449             answer = utils.our_raw_input(prompt)
 450             m = re_default_answer.search(prompt)
 451             if answer == "":
 452                 answer = m.group(1)
 453             answer = answer[:1].upper()
 454     os.unlink(temp_filename)
 455     if answer == 'A':
 456         return
 457     elif answer == 'Q':
 458         end()
 459         sys.exit(0)
 460
 461     comment = NewComment()
 462     comment.package = upload.pkg.changes["source"]
 463     comment.version = upload.pkg.changes["version"]
 464     comment.comment = newnote
 465     comment.author  = utils.whoami()
 466     comment.trainee = bool(Options["Trainee"])
 467     session.add(comment)
 468     session.commit()
 469
 470 ################################################################################
 471
 472 def check_pkg (upload):
 473     try:
 474         less_fd = os.popen("less -R -", 'w', 0)
 475         stdout_fd = sys.stdout
 476         try:
 477             sys.stdout = less_fd
 478             changes = utils.parse_changes (upload.pkg.changes_file)
 479             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
 480             files = upload.pkg.files
 481             for f in files.keys():
 482                 if files[f].has_key("new"):
 483                     ftype = files[f]["type"]
 484                     if ftype == "deb":
 485                         examine_package.check_deb(changes['distribution'], f)
 486                     elif ftype == "dsc":
 487                         examine_package.check_dsc(changes['distribution'], f)
 488         finally:
 489             examine_package.output_package_relations()
 490             sys.stdout = stdout_fd
 491     except IOError, e:
 492         if e.errno == errno.EPIPE:
 493             utils.warn("[examine_package] Caught EPIPE; skipping.")
 494             pass
 495         else:
 496             raise
 497     except KeyboardInterrupt:
 498         utils.warn("[examine_package] Caught C-c; skipping.")
 499         pass
 500
 501 ################################################################################
 502
 503 ## FIXME: horribly Debian specific
 504
 505 def do_bxa_notification(upload):
 506     files = upload.pkg.files
 507     summary = ""
 508     for f in files.keys():
 509         if files[f]["type"] == "deb":
 510             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 511             summary += "\n"
 512             summary += "Package: %s\n" % (control.Find("Package"))
 513             summary += "Description: %s\n" % (control.Find("Description"))
 514     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 515     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
 516     utils.send_mail(bxa_mail)
 517
 518 ################################################################################
 519
 520 def add_overrides (new, upload, session):
 521     changes = upload.pkg.changes
 522     files = upload.pkg.files
 523     srcpkg = changes.get("source")
 524
 525     for suite in changes["suite"].keys():
 526         suite_id = get_suite(suite).suite_id
 527         for pkg in new.keys():
 528             component_id = get_component(new[pkg]["component"]).component_id
 529             type_id = get_override_type(new[pkg]["type"]).overridetype_id
 530             priority_id = new[pkg]["priority id"]
 531             section_id = new[pkg]["section id"]
 532             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 533             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
 534                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
 535             for f in new[pkg]["files"]:
 536                 if files[f].has_key("new"):
 537                     del files[f]["new"]
 538             del new[pkg]
 539
 540     session.commit()
 541
 542     if Config().FindB("Dinstall::BXANotify"):
 543         do_bxa_notification(upload)
 544
 545 ################################################################################
 546
 547 def prod_maintainer (note, upload):
 548     cnf = Config()
 549     # Here we prepare an editor and get them ready to prod...
 550     (fd, temp_filename) = utils.temp_filename()
 551     temp_file = os.fdopen(fd, 'w')
 552     if len(note) > 0:
 553         for line in note:
 554             temp_file.write(line)
 555     temp_file.close()
 556     editor = os.environ.get("EDITOR","vi")
 557     answer = 'E'
 558     while answer == 'E':
 559         os.system("%s %s" % (editor, temp_filename))
 560         temp_fh = utils.open_file(temp_filename)
 561         prod_message = "".join(temp_fh.readlines())
 562         temp_fh.close()
 563         print "Prod message:"
 564         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 565         prompt = "[P]rod, Edit, Abandon, Quit ?"
 566         answer = "XXX"
 567         while prompt.find(answer) == -1:
 568             answer = utils.our_raw_input(prompt)
 569             m = re_default_answer.search(prompt)
 570             if answer == "":
 571                 answer = m.group(1)
 572             answer = answer[:1].upper()
 573     os.unlink(temp_filename)
 574     if answer == 'A':
 575         return
 576     elif answer == 'Q':
 577         end()
 578         sys.exit(0)
 579     # Otherwise, do the proding...
 580     user_email_address = utils.whoami() + " <%s>" % (
 581         cnf["Dinstall::MyAdminAddress"])
 582
 583     Subst = upload.Subst
 584
 585     Subst["__FROM_ADDRESS__"] = user_email_address
 586     Subst["__PROD_MESSAGE__"] = prod_message
 587     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
 588
 589     prod_mail_message = utils.TemplateSubst(
 590         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 591
 592     # Send the prod mail if appropriate
 593     if not cnf["Dinstall::Options::No-Mail"]:
 594         utils.send_mail(prod_mail_message)
 595
 596     print "Sent proding message"
 597
 598 ################################################################################
 599
 600 def do_new(upload, session):
 601     print "NEW\n"
 602     files = upload.pkg.files
 603     upload.check_files(not Options["No-Action"])
 604     changes = upload.pkg.changes
 605     cnf = Config()
 606
 607     # Check for a valid distribution
 608     upload.check_distributions()
 609
 610     # Make a copy of distribution we can happily trample on
 611     changes["suite"] = copy.copy(changes["distribution"])
 612
 613     # Fix up the list of target suites
 614     for suite in changes["suite"].keys():
 615         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
 616         if override:
 617             (olderr, newerr) = (get_suite(suite, session) == None,
 618                                 get_suite(override, session) == None)
 619             if olderr or newerr:
 620                 (oinv, newinv) = ("", "")
 621                 if olderr: oinv = "invalid "
 622                 if newerr: ninv = "invalid "
 623                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 624                         oinv, suite, ninv, override)
 625             del changes["suite"][suite]
 626             changes["suite"][override] = 1
 627
 628     # The main NEW processing loop
 629     done = 0
 630     while not done:
 631         # Find out what's new
 632         new = determine_new(changes, files)
 633
 634         if not new:
 635             break
 636
 637         answer = "XXX"
 638         if Options["No-Action"] or Options["Automatic"]:
 639             answer = 'S'
 640
 641         (broken, note) = print_new(new, upload, indexed=0)
 642         prompt = ""
 643
 644         if not broken and not note:
 645             prompt = "Add overrides, "
 646         if broken:
 647             print "W: [!] marked entries must be fixed before package can be processed."
 648         if note:
 649             print "W: note must be removed before package can be processed."
 650             prompt += "RemOve all notes, Remove note, "
 651
 652         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 653
 654         while prompt.find(answer) == -1:
 655             answer = utils.our_raw_input(prompt)
 656             m = re_default_answer.search(prompt)
 657             if answer == "":
 658                 answer = m.group(1)
 659             answer = answer[:1].upper()
 660
 661         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 662             utils.warn("Trainees can't do that")
 663             continue
 664
 665         if answer == 'A' and not Options["Trainee"]:
 666             try:
 667                 check_daily_lock()
 668                 done = add_overrides (new, upload, session)
 669                 do_accept(upload, session)
 670                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
 671             except CantGetLockError:
 672                 print "Hello? Operator! Give me the number for 911!"
 673                 print "Dinstall in the locked area, cant process packages, come back later"
 674         elif answer == 'C':
 675             check_pkg(upload)
 676         elif answer == 'E' and not Options["Trainee"]:
 677             new = edit_overrides (new, upload, session)
 678         elif answer == 'M' and not Options["Trainee"]:
 679             aborted = upload.do_reject(manual=1,
 680                                        reject_message=Options["Manual-Reject"],
 681                                        note=get_new_comments(changes.get("source", ""), session=session))
 682             if not aborted:
 683                 upload.pkg.remove_known_changes(session=session)
 684                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
 685                 done = 1
 686         elif answer == 'N':
 687             edit_note(get_new_comments(changes.get("source", ""), session=session),
 688                       upload, session)
 689         elif answer == 'P' and not Options["Trainee"]:
 690             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
 691                             upload)
 692             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
 693         elif answer == 'R' and not Options["Trainee"]:
 694             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 695             if confirm == "y":
 696                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
 697                     session.delete(c)
 698                 session.commit()
 699         elif answer == 'O' and not Options["Trainee"]:
 700             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 701             if confirm == "y":
 702                 for c in get_new_comments(changes.get("source", ""), session=session):
 703                     session.delete(c)
 704                 session.commit()
 705
 706         elif answer == 'S':
 707             done = 1
 708         elif answer == 'Q':
 709             end()
 710             sys.exit(0)
 711
 712 ################################################################################
 713 ################################################################################
 714 ################################################################################
 715
 716 def usage (exit_code=0):
 717     print """Usage: dak process-new [OPTION]... [CHANGES]...
 718   -a, --automatic           automatic run
 719   -h, --help                show this help and exit.
 720   -m, --manual-reject=MSG   manual reject with `msg'
 721   -n, --no-action           don't do anything
 722   -t, --trainee             FTP Trainee mode
 723   -V, --version             display the version number and exit"""
 724     sys.exit(exit_code)
 725
 726 ################################################################################
 727
 728 def do_byhand(upload, session):
 729     done = 0
 730     while not done:
 731         files = upload.pkg.files
 732         will_install = 1
 733         byhand = []
 734
 735         for f in files.keys():
 736             if files[f]["type"] == "byhand":
 737                 if os.path.exists(f):
 738                     print "W: %s still present; please process byhand components and try again." % (f)
 739                     will_install = 0
 740                 else:
 741                     byhand.append(f)
 742
 743         answer = "XXXX"
 744         if Options["No-Action"]:
 745             answer = "S"
 746         if will_install:
 747             if Options["Automatic"] and not Options["No-Action"]:
 748                 answer = 'A'
 749             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 750         else:
 751             prompt = "Manual reject, [S]kip, Quit ?"
 752
 753         while prompt.find(answer) == -1:
 754             answer = utils.our_raw_input(prompt)
 755             m = re_default_answer.search(prompt)
 756             if answer == "":
 757                 answer = m.group(1)
 758             answer = answer[:1].upper()
 759
 760         if answer == 'A':
 761             try:
 762                 check_daily_lock()
 763                 done = 1
 764                 for f in byhand:
 765                     del files[f]
 766                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
 767             except CantGetLockError:
 768                 print "Hello? Operator! Give me the number for 911!"
 769                 print "Dinstall in the locked area, cant process packages, come back later"
 770         elif answer == 'M':
 771             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
 772             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
 773             upload.pkg.remove_known_changes(session=session)
 774             done = 1
 775         elif answer == 'S':
 776             done = 1
 777         elif answer == 'Q':
 778             end()
 779             sys.exit(0)
 780
 781 ################################################################################
 782
 783 def check_daily_lock():
 784     """
 785     Raises CantGetLockError if the dinstall daily.lock exists.
 786     """
 787
 788     cnf = Config()
 789     try:
 790         os.open(cnf["Process-New::DinstallLockFile"],
 791                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 792     except OSError, e:
 793         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 794             raise CantGetLockError
 795
 796     os.unlink(cnf["Process-New::DinstallLockFile"])
 797
 798
 799 @contextlib.contextmanager
 800 def lock_package(package):
 801     """
 802     Lock C{package} so that noone else jumps in processing it.
 803
 804     @type package: string
 805     @param package: source package name to lock
 806     """
 807
 808     path = os.path.join(Config()["Process-New::LockDir"], package)
 809     try:
 810         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 811     except OSError, e:
 812         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 813             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 814             raise AlreadyLockedError, user
 815
 816     try:
 817         yield fd
 818     finally:
 819         os.unlink(path)
 820
 821 class clean_holding(object):
 822     def __init__(self,pkg):
 823         self.pkg = pkg
 824
 825     def __enter__(self):
 826         pass
 827
 828     def __exit__(self, type, value, traceback):
 829         h = Holding()
 830
 831         for f in self.pkg.files.keys():
 832             if os.path.exists(os.path.join(h.holding_dir, f)):
 833                 os.unlink(os.path.join(h.holding_dir, f))
 834
 835
 836
 837 def changes_to_newstage(upload, session):
 838     """move a changes file to newstage"""
 839     new = get_policy_queue('new', session );
 840     newstage = get_policy_queue('newstage', session );
 841
 842     chg = session.query(DBChange).filter_by(changesname=os.path.basename(upload.pkg.changes_file)).one()
 843     chg.approved_for = newstage.policy_queue_id
 844
 845     for f in chg.files:
 846         # update the changes_pending_files row
 847         f.queue = newstage
 848         utils.move(os.path.join(new.path, f.filename), newstage.path, perms=int(newstage.perms, 8))
 849
 850     utils.move(os.path.join(new.path, upload.pkg.changes_file), newstage.path, perms=int(newstage.perms, 8))
 851     chg.in_queue = newstage
 852     session.commit()
 853
 854 def _accept(upload, session):
 855     if Options["No-Action"]:
 856         return
 857     (summary, short_summary) = upload.build_summaries()
 858     # upload.accept(summary, short_summary, targetqueue)
 859
 860     changes_to_newstage(upload, session)
 861
 862 def do_accept(upload, session):
 863     print "ACCEPT"
 864     cnf = Config()
 865     if not Options["No-Action"]:
 866         (summary, short_summary) = upload.build_summaries()
 867
 868         if cnf.FindB("Dinstall::SecurityQueueHandling"):
 869             upload.dump_vars(cnf["Dir::Queue::Embargoed"])
 870             upload.move_to_queue(get_policy_queue('embargoed'))
 871             upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
 872             # Check for override disparities
 873             upload.Subst["__SUMMARY__"] = summary
 874         else:
 875             # Just a normal upload, accept it...
 876             _accept(upload, session)
 877
 878 def do_pkg(changes_file, session):
 879     new_queue = get_policy_queue('new', session );
 880     u = Upload()
 881     u.pkg.changes_file = changes_file
 882     (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
 883     u.load_changes(changes_file)
 884     u.pkg.directory = new_queue.path
 885     u.update_subst()
 886     u.logger = Logger
 887     origchanges = os.path.abspath(u.pkg.changes_file)
 888
 889     cnf = Config()
 890     bcc = "X-DAK: dak process-new"
 891     if cnf.has_key("Dinstall::Bcc"):
 892         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 893     else:
 894         u.Subst["__BCC__"] = bcc
 895
 896     files = u.pkg.files
 897     for deb_filename, f in files.items():
 898         if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
 899             u.binary_file_checks(deb_filename, session)
 900             u.check_binary_against_db(deb_filename, session)
 901         else:
 902             u.source_file_checks(deb_filename, session)
 903             u.check_source_against_db(deb_filename, session)
 904
 905         u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
 906
 907     try:
 908         with lock_package(u.pkg.changes["source"]):
 909             with clean_holding(u.pkg):
 910                 if not recheck(u, session):
 911                     return
 912
 913                 # FIXME: This does need byhand checks added!
 914                 new = determine_new(u.pkg.changes, files)
 915                 if new:
 916                     do_new(u, session)
 917                 else:
 918                     try:
 919                         check_daily_lock()
 920                         do_accept(u, session)
 921                     except CantGetLockError:
 922                         print "Hello? Operator! Give me the number for 911!"
 923                         print "Dinstall in the locked area, cant process packages, come back later"
 924 #             (new, byhand) = check_status(files)
 925 #             if new or byhand:
 926 #                 if new:
 927 #                     do_new(u, session)
 928 #                 if byhand:
 929 #                     do_byhand(u, session)
 930 #                 (new, byhand) = check_status(files)
 931
 932 #             if not new and not byhand:
 933 #                 try:
 934 #                     check_daily_lock()
 935 #                     do_accept(u)
 936 #                 except CantGetLockError:
 937 #                     print "Hello? Operator! Give me the number for 911!"
 938 #                     print "Dinstall in the locked area, cant process packages, come back later"
 939     except AlreadyLockedError, e:
 940         print "Seems to be locked by %s already, skipping..." % (e)
 941
 942 ################################################################################
 943
 944 def end():
 945     accept_count = SummaryStats().accept_count
 946     accept_bytes = SummaryStats().accept_bytes
 947
 948     if accept_count:
 949         sets = "set"
 950         if accept_count > 1:
 951             sets = "sets"
 952         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 953         Logger.log(["total",accept_count,accept_bytes])
 954
 955     if not Options["No-Action"] and not Options["Trainee"]:
 956         Logger.close()
 957
 958 ################################################################################
 959
 960 def main():
 961     global Options, Logger, Sections, Priorities
 962
 963     cnf = Config()
 964     session = DBConn().session()
 965
 966     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 967                  ('h',"help","Process-New::Options::Help"),
 968                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 969                  ('t',"trainee","Process-New::Options::Trainee"),
 970                  ('n',"no-action","Process-New::Options::No-Action")]
 971
 972     for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
 973         if not cnf.has_key("Process-New::Options::%s" % (i)):
 974             cnf["Process-New::Options::%s" % (i)] = ""
 975
 976     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
 977     if len(changes_files) == 0:
 978         new_queue = get_policy_queue('new', session );
 979         changes_files = utils.get_changes_files(new_queue.path)
 980
 981     Options = cnf.SubTree("Process-New::Options")
 982
 983     if Options["Help"]:
 984         usage()
 985
 986     if not Options["No-Action"]:
 987         try:
 988             Logger = daklog.Logger(cnf, "process-new")
 989         except CantOpenError, e:
 990             Options["Trainee"] = "True"
 991
 992     Sections = Section_Completer(session)
 993     Priorities = Priority_Completer(session)
 994     readline.parse_and_bind("tab: complete")
 995
 996     if len(changes_files) > 1:
 997         sys.stderr.write("Sorting changes...\n")
 998     changes_files = sort_changes(changes_files, session)
 999
1000     # Kill me now? **FIXME**
1001     cnf["Dinstall::Options::No-Mail"] = ""
1002
1003     for changes_file in changes_files:
1004         changes_file = utils.validate_changes_file_arg(changes_file, 0)
1005         if not changes_file:
1006             continue
1007         print "\n" + changes_file
1008
1009         do_pkg (changes_file, session)
1010
1011     end()
1012
1013 ################################################################################
1014
1015 if __name__ == '__main__':
1016     main()