dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 from __future__ import with_statement
  46
  47 import copy
  48 import errno
  49 import os
  50 import readline
  51 import stat
  52 import sys
  53 import time
  54 import contextlib
  55 import pwd
  56 import apt_pkg, apt_inst
  57 import examine_package
  58
  59 from daklib.dbconn import *
  60 from daklib.queue import *
  61 from daklib import daklog
  62 from daklib import utils
  63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
  64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  65 from daklib.summarystats import SummaryStats
  66 from daklib.config import Config
  67
  68 # Globals
  69 Options = None
  70 Logger = None
  71
  72 Priorities = None
  73 Sections = None
  74
  75 ################################################################################
  76 ################################################################################
  77 ################################################################################
  78
  79 def recheck(upload, session):
  80 # STU: I'm not sure, but I don't thin kthis is necessary any longer:    upload.recheck(session)
  81     if len(upload.rejects) > 0:
  82         answer = "XXX"
  83         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
  84             answer = 'S'
  85
  86         print "REJECT\n%s" % '\n'.join(upload.rejects)
  87         prompt = "[R]eject, Skip, Quit ?"
  88
  89         while prompt.find(answer) == -1:
  90             answer = utils.our_raw_input(prompt)
  91             m = re_default_answer.match(prompt)
  92             if answer == "":
  93                 answer = m.group(1)
  94             answer = answer[:1].upper()
  95
  96         if answer == 'R':
  97             upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
  98             upload.pkg.remove_known_changes(session=session)
  99             session.commit()
 100             return 0
 101         elif answer == 'S':
 102             return 0
 103         elif answer == 'Q':
 104             end()
 105             sys.exit(0)
 106
 107     return 1
 108
 109 ################################################################################
 110
 111 def indiv_sg_compare (a, b):
 112     """Sort by source name, source, version, 'have source', and
 113        finally by filename."""
 114     # Sort by source version
 115     q = apt_pkg.VersionCompare(a["version"], b["version"])
 116     if q:
 117         return -q
 118
 119     # Sort by 'have source'
 120     a_has_source = a["architecture"].get("source")
 121     b_has_source = b["architecture"].get("source")
 122     if a_has_source and not b_has_source:
 123         return -1
 124     elif b_has_source and not a_has_source:
 125         return 1
 126
 127     return cmp(a["filename"], b["filename"])
 128
 129 ############################################################
 130
 131 def sg_compare (a, b):
 132     a = a[1]
 133     b = b[1]
 134     """Sort by have note, source already in database and time of oldest upload."""
 135     # Sort by have note
 136     a_note_state = a["note_state"]
 137     b_note_state = b["note_state"]
 138     if a_note_state < b_note_state:
 139         return -1
 140     elif a_note_state > b_note_state:
 141         return 1
 142     # Sort by source already in database (descending)
 143     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 144     if source_in_database:
 145         return -source_in_database
 146
 147     # Sort by time of oldest upload
 148     return cmp(a["oldest"], b["oldest"])
 149
 150 def sort_changes(changes_files, session):
 151     """Sort into source groups, then sort each source group by version,
 152     have source, filename.  Finally, sort the source groups by have
 153     note, time of oldest upload of each source upload."""
 154     if len(changes_files) == 1:
 155         return changes_files
 156
 157     sorted_list = []
 158     cache = {}
 159     # Read in all the .changes files
 160     for filename in changes_files:
 161         u = Upload()
 162         try:
 163             u.pkg.changes_file = filename
 164             u.load_changes(filename)
 165             u.update_subst()
 166             cache[filename] = copy.copy(u.pkg.changes)
 167             cache[filename]["filename"] = filename
 168         except:
 169             sorted_list.append(filename)
 170             break
 171     # Divide the .changes into per-source groups
 172     per_source = {}
 173     for filename in cache.keys():
 174         source = cache[filename]["source"]
 175         if not per_source.has_key(source):
 176             per_source[source] = {}
 177             per_source[source]["list"] = []
 178         per_source[source]["list"].append(cache[filename])
 179     # Determine oldest time and have note status for each source group
 180     for source in per_source.keys():
 181         q = session.query(DBSource).filter_by(source = source).all()
 182         per_source[source]["source_in_database"] = len(q)>0
 183         source_list = per_source[source]["list"]
 184         first = source_list[0]
 185         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 186         have_note = 0
 187         for d in per_source[source]["list"]:
 188             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 189             if mtime < oldest:
 190                 oldest = mtime
 191             have_note += has_new_comment(d["source"], d["version"], session)
 192         per_source[source]["oldest"] = oldest
 193         if not have_note:
 194             per_source[source]["note_state"] = 0; # none
 195         elif have_note < len(source_list):
 196             per_source[source]["note_state"] = 1; # some
 197         else:
 198             per_source[source]["note_state"] = 2; # all
 199         per_source[source]["list"].sort(indiv_sg_compare)
 200     per_source_items = per_source.items()
 201     per_source_items.sort(sg_compare)
 202     for i in per_source_items:
 203         for j in i[1]["list"]:
 204             sorted_list.append(j["filename"])
 205     return sorted_list
 206
 207 ################################################################################
 208
 209 class Section_Completer:
 210     def __init__ (self, session):
 211         self.sections = []
 212         self.matches = []
 213         for s, in session.query(Section.section):
 214             self.sections.append(s)
 215
 216     def complete(self, text, state):
 217         if state == 0:
 218             self.matches = []
 219             n = len(text)
 220             for word in self.sections:
 221                 if word[:n] == text:
 222                     self.matches.append(word)
 223         try:
 224             return self.matches[state]
 225         except IndexError:
 226             return None
 227
 228 ############################################################
 229
 230 class Priority_Completer:
 231     def __init__ (self, session):
 232         self.priorities = []
 233         self.matches = []
 234         for p, in session.query(Priority.priority):
 235             self.priorities.append(p)
 236
 237     def complete(self, text, state):
 238         if state == 0:
 239             self.matches = []
 240             n = len(text)
 241             for word in self.priorities:
 242                 if word[:n] == text:
 243                     self.matches.append(word)
 244         try:
 245             return self.matches[state]
 246         except IndexError:
 247             return None
 248
 249 ################################################################################
 250
 251 def print_new (new, upload, indexed, file=sys.stdout):
 252     check_valid(new)
 253     broken = False
 254     index = 0
 255     for pkg in new.keys():
 256         index += 1
 257         section = new[pkg]["section"]
 258         priority = new[pkg]["priority"]
 259         if new[pkg]["section id"] == -1:
 260             section += "[!]"
 261             broken = True
 262         if new[pkg]["priority id"] == -1:
 263             priority += "[!]"
 264             broken = True
 265         if indexed:
 266             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 267         else:
 268             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 269         line = line.strip()+'\n'
 270         file.write(line)
 271     notes = get_new_comments(upload.pkg.changes.get("source"))
 272     for note in notes:
 273         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 274               % (note.author, note.version, note.notedate, note.comment)
 275         print "-" * 72
 276     return broken, len(notes) > 0
 277
 278 ################################################################################
 279
 280 def index_range (index):
 281     if index == 1:
 282         return "1"
 283     else:
 284         return "1-%s" % (index)
 285
 286 ################################################################################
 287 ################################################################################
 288
 289 def edit_new (new, upload):
 290     # Write the current data to a temporary file
 291     (fd, temp_filename) = utils.temp_filename()
 292     temp_file = os.fdopen(fd, 'w')
 293     print_new (new, upload, indexed=0, file=temp_file)
 294     temp_file.close()
 295     # Spawn an editor on that file
 296     editor = os.environ.get("EDITOR","vi")
 297     result = os.system("%s %s" % (editor, temp_filename))
 298     if result != 0:
 299         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 300     # Read the edited data back in
 301     temp_file = utils.open_file(temp_filename)
 302     lines = temp_file.readlines()
 303     temp_file.close()
 304     os.unlink(temp_filename)
 305     # Parse the new data
 306     for line in lines:
 307         line = line.strip()
 308         if line == "":
 309             continue
 310         s = line.split()
 311         # Pad the list if necessary
 312         s[len(s):3] = [None] * (3-len(s))
 313         (pkg, priority, section) = s[:3]
 314         if not new.has_key(pkg):
 315             utils.warn("Ignoring unknown package '%s'" % (pkg))
 316         else:
 317             # Strip off any invalid markers, print_new will readd them.
 318             if section.endswith("[!]"):
 319                 section = section[:-3]
 320             if priority.endswith("[!]"):
 321                 priority = priority[:-3]
 322             for f in new[pkg]["files"]:
 323                 upload.pkg.files[f]["section"] = section
 324                 upload.pkg.files[f]["priority"] = priority
 325             new[pkg]["section"] = section
 326             new[pkg]["priority"] = priority
 327
 328 ################################################################################
 329
 330 def edit_index (new, upload, index):
 331     priority = new[index]["priority"]
 332     section = new[index]["section"]
 333     ftype = new[index]["type"]
 334     done = 0
 335     while not done:
 336         print "\t".join([index, priority, section])
 337
 338         answer = "XXX"
 339         if ftype != "dsc":
 340             prompt = "[B]oth, Priority, Section, Done ? "
 341         else:
 342             prompt = "[S]ection, Done ? "
 343         edit_priority = edit_section = 0
 344
 345         while prompt.find(answer) == -1:
 346             answer = utils.our_raw_input(prompt)
 347             m = re_default_answer.match(prompt)
 348             if answer == "":
 349                 answer = m.group(1)
 350             answer = answer[:1].upper()
 351
 352         if answer == 'P':
 353             edit_priority = 1
 354         elif answer == 'S':
 355             edit_section = 1
 356         elif answer == 'B':
 357             edit_priority = edit_section = 1
 358         elif answer == 'D':
 359             done = 1
 360
 361         # Edit the priority
 362         if edit_priority:
 363             readline.set_completer(Priorities.complete)
 364             got_priority = 0
 365             while not got_priority:
 366                 new_priority = utils.our_raw_input("New priority: ").strip()
 367                 if new_priority not in Priorities.priorities:
 368                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 369                 else:
 370                     got_priority = 1
 371                     priority = new_priority
 372
 373         # Edit the section
 374         if edit_section:
 375             readline.set_completer(Sections.complete)
 376             got_section = 0
 377             while not got_section:
 378                 new_section = utils.our_raw_input("New section: ").strip()
 379                 if new_section not in Sections.sections:
 380                     print "E: '%s' is not a valid section, try again." % (new_section)
 381                 else:
 382                     got_section = 1
 383                     section = new_section
 384
 385         # Reset the readline completer
 386         readline.set_completer(None)
 387
 388     for f in new[index]["files"]:
 389         upload.pkg.files[f]["section"] = section
 390         upload.pkg.files[f]["priority"] = priority
 391     new[index]["priority"] = priority
 392     new[index]["section"] = section
 393     return new
 394
 395 ################################################################################
 396
 397 def edit_overrides (new, upload, session):
 398     print
 399     done = 0
 400     while not done:
 401         print_new (new, upload, indexed=1)
 402         new_index = {}
 403         index = 0
 404         for i in new.keys():
 405             index += 1
 406             new_index[index] = i
 407
 408         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 409
 410         got_answer = 0
 411         while not got_answer:
 412             answer = utils.our_raw_input(prompt)
 413             if not answer.isdigit():
 414                 answer = answer[:1].upper()
 415             if answer == "E" or answer == "D":
 416                 got_answer = 1
 417             elif re_isanum.match (answer):
 418                 answer = int(answer)
 419                 if (answer < 1) or (answer > index):
 420                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 421                 else:
 422                     got_answer = 1
 423
 424         if answer == 'E':
 425             edit_new(new, upload)
 426         elif answer == 'D':
 427             done = 1
 428         else:
 429             edit_index (new, upload, new_index[answer])
 430
 431     return new
 432
 433 ################################################################################
 434
 435 def edit_note(note, upload, session):
 436     # Write the current data to a temporary file
 437     (fd, temp_filename) = utils.temp_filename()
 438     editor = os.environ.get("EDITOR","vi")
 439     answer = 'E'
 440     while answer == 'E':
 441         os.system("%s %s" % (editor, temp_filename))
 442         temp_file = utils.open_file(temp_filename)
 443         newnote = temp_file.read().rstrip()
 444         temp_file.close()
 445         print "New Note:"
 446         print utils.prefix_multi_line_string(newnote,"  ")
 447         prompt = "[D]one, Edit, Abandon, Quit ?"
 448         answer = "XXX"
 449         while prompt.find(answer) == -1:
 450             answer = utils.our_raw_input(prompt)
 451             m = re_default_answer.search(prompt)
 452             if answer == "":
 453                 answer = m.group(1)
 454             answer = answer[:1].upper()
 455     os.unlink(temp_filename)
 456     if answer == 'A':
 457         return
 458     elif answer == 'Q':
 459         end()
 460         sys.exit(0)
 461
 462     comment = NewComment()
 463     comment.package = upload.pkg.changes["source"]
 464     comment.version = upload.pkg.changes["version"]
 465     comment.comment = newnote
 466     comment.author  = utils.whoami()
 467     comment.trainee = bool(Options["Trainee"])
 468     session.add(comment)
 469     session.commit()
 470
 471 ################################################################################
 472
 473 def check_pkg (upload):
 474     try:
 475         less_fd = os.popen("less -R -", 'w', 0)
 476         stdout_fd = sys.stdout
 477         try:
 478             sys.stdout = less_fd
 479             changes = utils.parse_changes (upload.pkg.changes_file)
 480             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
 481             files = upload.pkg.files
 482             for f in files.keys():
 483                 if files[f].has_key("new"):
 484                     ftype = files[f]["type"]
 485                     if ftype == "deb":
 486                         examine_package.check_deb(changes['distribution'], f)
 487                     elif ftype == "dsc":
 488                         examine_package.check_dsc(changes['distribution'], f)
 489         finally:
 490             examine_package.output_package_relations()
 491             sys.stdout = stdout_fd
 492     except IOError, e:
 493         if e.errno == errno.EPIPE:
 494             utils.warn("[examine_package] Caught EPIPE; skipping.")
 495             pass
 496         else:
 497             raise
 498     except KeyboardInterrupt:
 499         utils.warn("[examine_package] Caught C-c; skipping.")
 500         pass
 501
 502 ################################################################################
 503
 504 ## FIXME: horribly Debian specific
 505
 506 def do_bxa_notification(upload):
 507     files = upload.pkg.files
 508     summary = ""
 509     for f in files.keys():
 510         if files[f]["type"] == "deb":
 511             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 512             summary += "\n"
 513             summary += "Package: %s\n" % (control.Find("Package"))
 514             summary += "Description: %s\n" % (control.Find("Description"))
 515     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 516     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
 517     utils.send_mail(bxa_mail)
 518
 519 ################################################################################
 520
 521 def add_overrides (new, upload, session):
 522     changes = upload.pkg.changes
 523     files = upload.pkg.files
 524     srcpkg = changes.get("source")
 525
 526     for suite in changes["suite"].keys():
 527         suite_id = get_suite(suite).suite_id
 528         for pkg in new.keys():
 529             component_id = get_component(new[pkg]["component"]).component_id
 530             type_id = get_override_type(new[pkg]["type"]).overridetype_id
 531             priority_id = new[pkg]["priority id"]
 532             section_id = new[pkg]["section id"]
 533             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 534             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
 535                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
 536             for f in new[pkg]["files"]:
 537                 if files[f].has_key("new"):
 538                     del files[f]["new"]
 539             del new[pkg]
 540
 541     session.commit()
 542
 543     if Config().FindB("Dinstall::BXANotify"):
 544         do_bxa_notification(upload)
 545
 546 ################################################################################
 547
 548 def prod_maintainer (note, upload):
 549     cnf = Config()
 550     # Here we prepare an editor and get them ready to prod...
 551     (fd, temp_filename) = utils.temp_filename()
 552     temp_file = os.fdopen(fd, 'w')
 553     if len(note) > 0:
 554         for line in note:
 555             temp_file.write(line)
 556     temp_file.close()
 557     editor = os.environ.get("EDITOR","vi")
 558     answer = 'E'
 559     while answer == 'E':
 560         os.system("%s %s" % (editor, temp_filename))
 561         temp_fh = utils.open_file(temp_filename)
 562         prod_message = "".join(temp_fh.readlines())
 563         temp_fh.close()
 564         print "Prod message:"
 565         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 566         prompt = "[P]rod, Edit, Abandon, Quit ?"
 567         answer = "XXX"
 568         while prompt.find(answer) == -1:
 569             answer = utils.our_raw_input(prompt)
 570             m = re_default_answer.search(prompt)
 571             if answer == "":
 572                 answer = m.group(1)
 573             answer = answer[:1].upper()
 574     os.unlink(temp_filename)
 575     if answer == 'A':
 576         return
 577     elif answer == 'Q':
 578         end()
 579         sys.exit(0)
 580     # Otherwise, do the proding...
 581     user_email_address = utils.whoami() + " <%s>" % (
 582         cnf["Dinstall::MyAdminAddress"])
 583
 584     Subst = upload.Subst
 585
 586     Subst["__FROM_ADDRESS__"] = user_email_address
 587     Subst["__PROD_MESSAGE__"] = prod_message
 588     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
 589
 590     prod_mail_message = utils.TemplateSubst(
 591         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 592
 593     # Send the prod mail if appropriate
 594     if not cnf["Dinstall::Options::No-Mail"]:
 595         utils.send_mail(prod_mail_message)
 596
 597     print "Sent proding message"
 598
 599 ################################################################################
 600
 601 def do_new(upload, session):
 602     print "NEW\n"
 603     files = upload.pkg.files
 604     upload.check_files(not Options["No-Action"])
 605     changes = upload.pkg.changes
 606     cnf = Config()
 607
 608     # Check for a valid distribution
 609     upload.check_distributions()
 610
 611     # Make a copy of distribution we can happily trample on
 612     changes["suite"] = copy.copy(changes["distribution"])
 613
 614     # The main NEW processing loop
 615     done = 0
 616     while not done:
 617         # Find out what's new
 618         new = determine_new(changes, files)
 619
 620         if not new:
 621             break
 622
 623         answer = "XXX"
 624         if Options["No-Action"] or Options["Automatic"]:
 625             answer = 'S'
 626
 627         (broken, note) = print_new(new, upload, indexed=0)
 628         prompt = ""
 629
 630         if not broken and not note:
 631             prompt = "Add overrides, "
 632         if broken:
 633             print "W: [!] marked entries must be fixed before package can be processed."
 634         if note:
 635             print "W: note must be removed before package can be processed."
 636             prompt += "RemOve all notes, Remove note, "
 637
 638         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 639
 640         while prompt.find(answer) == -1:
 641             answer = utils.our_raw_input(prompt)
 642             m = re_default_answer.search(prompt)
 643             if answer == "":
 644                 answer = m.group(1)
 645             answer = answer[:1].upper()
 646
 647         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 648             utils.warn("Trainees can't do that")
 649             continue
 650
 651         if answer == 'A' and not Options["Trainee"]:
 652             try:
 653                 check_daily_lock()
 654                 done = add_overrides (new, upload, session)
 655                 do_accept(upload, session)
 656                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
 657             except CantGetLockError:
 658                 print "Hello? Operator! Give me the number for 911!"
 659                 print "Dinstall in the locked area, cant process packages, come back later"
 660         elif answer == 'C':
 661             check_pkg(upload)
 662         elif answer == 'E' and not Options["Trainee"]:
 663             new = edit_overrides (new, upload, session)
 664         elif answer == 'M' and not Options["Trainee"]:
 665             aborted = upload.do_reject(manual=1,
 666                                        reject_message=Options["Manual-Reject"],
 667                                        notes=get_new_comments(changes.get("source", ""), session=session))
 668             if not aborted:
 669                 upload.pkg.remove_known_changes(session=session)
 670                 session.commit()
 671                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
 672                 done = 1
 673         elif answer == 'N':
 674             edit_note(get_new_comments(changes.get("source", ""), session=session),
 675                       upload, session)
 676         elif answer == 'P' and not Options["Trainee"]:
 677             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
 678                             upload)
 679             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
 680         elif answer == 'R' and not Options["Trainee"]:
 681             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 682             if confirm == "y":
 683                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
 684                     session.delete(c)
 685                 session.commit()
 686         elif answer == 'O' and not Options["Trainee"]:
 687             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 688             if confirm == "y":
 689                 for c in get_new_comments(changes.get("source", ""), session=session):
 690                     session.delete(c)
 691                 session.commit()
 692
 693         elif answer == 'S':
 694             done = 1
 695         elif answer == 'Q':
 696             end()
 697             sys.exit(0)
 698
 699 ################################################################################
 700 ################################################################################
 701 ################################################################################
 702
 703 def usage (exit_code=0):
 704     print """Usage: dak process-new [OPTION]... [CHANGES]...
 705   -a, --automatic           automatic run
 706   -h, --help                show this help and exit.
 707   -m, --manual-reject=MSG   manual reject with `msg'
 708   -n, --no-action           don't do anything
 709   -t, --trainee             FTP Trainee mode
 710   -V, --version             display the version number and exit"""
 711     sys.exit(exit_code)
 712
 713 ################################################################################
 714
 715 def do_byhand(upload, session):
 716     done = 0
 717     while not done:
 718         files = upload.pkg.files
 719         will_install = 1
 720         byhand = []
 721
 722         for f in files.keys():
 723             if files[f]["type"] == "byhand":
 724                 if os.path.exists(f):
 725                     print "W: %s still present; please process byhand components and try again." % (f)
 726                     will_install = 0
 727                 else:
 728                     byhand.append(f)
 729
 730         answer = "XXXX"
 731         if Options["No-Action"]:
 732             answer = "S"
 733         if will_install:
 734             if Options["Automatic"] and not Options["No-Action"]:
 735                 answer = 'A'
 736             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 737         else:
 738             prompt = "Manual reject, [S]kip, Quit ?"
 739
 740         while prompt.find(answer) == -1:
 741             answer = utils.our_raw_input(prompt)
 742             m = re_default_answer.search(prompt)
 743             if answer == "":
 744                 answer = m.group(1)
 745             answer = answer[:1].upper()
 746
 747         if answer == 'A':
 748             try:
 749                 check_daily_lock()
 750                 done = 1
 751                 for f in byhand:
 752                     del files[f]
 753                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
 754             except CantGetLockError:
 755                 print "Hello? Operator! Give me the number for 911!"
 756                 print "Dinstall in the locked area, cant process packages, come back later"
 757         elif answer == 'M':
 758             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
 759             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
 760             upload.pkg.remove_known_changes(session=session)
 761             session.commit()
 762             done = 1
 763         elif answer == 'S':
 764             done = 1
 765         elif answer == 'Q':
 766             end()
 767             sys.exit(0)
 768
 769 ################################################################################
 770
 771 def check_daily_lock():
 772     """
 773     Raises CantGetLockError if the dinstall daily.lock exists.
 774     """
 775
 776     cnf = Config()
 777     try:
 778         os.open(cnf["Process-New::DinstallLockFile"],
 779                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 780     except OSError, e:
 781         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 782             raise CantGetLockError
 783
 784     os.unlink(cnf["Process-New::DinstallLockFile"])
 785
 786
 787 @contextlib.contextmanager
 788 def lock_package(package):
 789     """
 790     Lock C{package} so that noone else jumps in processing it.
 791
 792     @type package: string
 793     @param package: source package name to lock
 794     """
 795
 796     path = os.path.join(Config()["Process-New::LockDir"], package)
 797     try:
 798         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 799     except OSError, e:
 800         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 801             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 802             raise AlreadyLockedError, user
 803
 804     try:
 805         yield fd
 806     finally:
 807         os.unlink(path)
 808
 809 class clean_holding(object):
 810     def __init__(self,pkg):
 811         self.pkg = pkg
 812
 813     def __enter__(self):
 814         pass
 815
 816     def __exit__(self, type, value, traceback):
 817         h = Holding()
 818
 819         for f in self.pkg.files.keys():
 820             if os.path.exists(os.path.join(h.holding_dir, f)):
 821                 os.unlink(os.path.join(h.holding_dir, f))
 822
 823
 824
 825 def changes_to_newstage(upload, session):
 826     """move a changes file to newstage"""
 827     new = get_policy_queue('new', session );
 828     newstage = get_policy_queue('newstage', session );
 829
 830     chg = session.query(DBChange).filter_by(changesname=os.path.basename(upload.pkg.changes_file)).one()
 831     chg.approved_for = newstage.policy_queue_id
 832
 833     for f in chg.files:
 834         # update the changes_pending_files row
 835         f.queue = newstage
 836         utils.move(os.path.join(new.path, f.filename), newstage.path, perms=int(newstage.perms, 8))
 837
 838     utils.move(os.path.join(new.path, upload.pkg.changes_file), newstage.path, perms=int(newstage.perms, 8))
 839     chg.in_queue = newstage
 840     session.commit()
 841
 842 def _accept(upload, session):
 843     if Options["No-Action"]:
 844         return
 845     (summary, short_summary) = upload.build_summaries()
 846     # upload.accept(summary, short_summary, targetqueue)
 847
 848     changes_to_newstage(upload, session)
 849
 850 def do_accept(upload, session):
 851     print "ACCEPT"
 852     cnf = Config()
 853     if not Options["No-Action"]:
 854         (summary, short_summary) = upload.build_summaries()
 855
 856         if cnf.FindB("Dinstall::SecurityQueueHandling"):
 857             upload.dump_vars(cnf["Dir::Queue::Embargoed"])
 858             upload.move_to_queue(get_policy_queue('embargoed'))
 859             upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
 860             # Check for override disparities
 861             upload.Subst["__SUMMARY__"] = summary
 862         else:
 863             # Just a normal upload, accept it...
 864             _accept(upload, session)
 865
 866 def do_pkg(changes_file, session):
 867     new_queue = get_policy_queue('new', session );
 868     u = Upload()
 869     u.pkg.changes_file = changes_file
 870     (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
 871     u.load_changes(changes_file)
 872     u.pkg.directory = new_queue.path
 873     u.update_subst()
 874     u.logger = Logger
 875     origchanges = os.path.abspath(u.pkg.changes_file)
 876
 877     cnf = Config()
 878     bcc = "X-DAK: dak process-new"
 879     if cnf.has_key("Dinstall::Bcc"):
 880         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 881     else:
 882         u.Subst["__BCC__"] = bcc
 883
 884     files = u.pkg.files
 885     for deb_filename, f in files.items():
 886         if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
 887             u.binary_file_checks(deb_filename, session)
 888             u.check_binary_against_db(deb_filename, session)
 889         else:
 890             u.source_file_checks(deb_filename, session)
 891             u.check_source_against_db(deb_filename, session)
 892
 893         u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
 894
 895     try:
 896         with lock_package(u.pkg.changes["source"]):
 897             with clean_holding(u.pkg):
 898                 if not recheck(u, session):
 899                     return
 900
 901                 # FIXME: This does need byhand checks added!
 902                 new = determine_new(u.pkg.changes, files)
 903                 if new:
 904                     do_new(u, session)
 905                 else:
 906                     try:
 907                         check_daily_lock()
 908                         do_accept(u, session)
 909                     except CantGetLockError:
 910                         print "Hello? Operator! Give me the number for 911!"
 911                         print "Dinstall in the locked area, cant process packages, come back later"
 912 #             (new, byhand) = check_status(files)
 913 #             if new or byhand:
 914 #                 if new:
 915 #                     do_new(u, session)
 916 #                 if byhand:
 917 #                     do_byhand(u, session)
 918 #                 (new, byhand) = check_status(files)
 919
 920 #             if not new and not byhand:
 921 #                 try:
 922 #                     check_daily_lock()
 923 #                     do_accept(u)
 924 #                 except CantGetLockError:
 925 #                     print "Hello? Operator! Give me the number for 911!"
 926 #                     print "Dinstall in the locked area, cant process packages, come back later"
 927     except AlreadyLockedError, e:
 928         print "Seems to be locked by %s already, skipping..." % (e)
 929
 930 ################################################################################
 931
 932 def end():
 933     accept_count = SummaryStats().accept_count
 934     accept_bytes = SummaryStats().accept_bytes
 935
 936     if accept_count:
 937         sets = "set"
 938         if accept_count > 1:
 939             sets = "sets"
 940         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 941         Logger.log(["total",accept_count,accept_bytes])
 942
 943     if not Options["No-Action"] and not Options["Trainee"]:
 944         Logger.close()
 945
 946 ################################################################################
 947
 948 def main():
 949     global Options, Logger, Sections, Priorities
 950
 951     cnf = Config()
 952     session = DBConn().session()
 953
 954     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 955                  ('h',"help","Process-New::Options::Help"),
 956                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 957                  ('t',"trainee","Process-New::Options::Trainee"),
 958                  ('n',"no-action","Process-New::Options::No-Action")]
 959
 960     for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
 961         if not cnf.has_key("Process-New::Options::%s" % (i)):
 962             cnf["Process-New::Options::%s" % (i)] = ""
 963
 964     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
 965     if len(changes_files) == 0:
 966         new_queue = get_policy_queue('new', session );
 967         changes_files = utils.get_changes_files(new_queue.path)
 968
 969     Options = cnf.SubTree("Process-New::Options")
 970
 971     if Options["Help"]:
 972         usage()
 973
 974     if not Options["No-Action"]:
 975         try:
 976             Logger = daklog.Logger(cnf, "process-new")
 977         except CantOpenError, e:
 978             Options["Trainee"] = "True"
 979
 980     Sections = Section_Completer(session)
 981     Priorities = Priority_Completer(session)
 982     readline.parse_and_bind("tab: complete")
 983
 984     if len(changes_files) > 1:
 985         sys.stderr.write("Sorting changes...\n")
 986     changes_files = sort_changes(changes_files, session)
 987
 988     # Kill me now? **FIXME**
 989     cnf["Dinstall::Options::No-Mail"] = ""
 990
 991     for changes_file in changes_files:
 992         changes_file = utils.validate_changes_file_arg(changes_file, 0)
 993         if not changes_file:
 994             continue
 995         print "\n" + changes_file
 996
 997         do_pkg (changes_file, session)
 998
 999     end()
1000
1001 ################################################################################
1002
1003 if __name__ == '__main__':
1004     main()