dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 """ Handles NEW and BYHAND packages
   5
   6 @contact: Debian FTP Master <ftpmaster@debian.org>
   7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 # 23:12|<aj> I will not hush!
  29 # 23:12|<elmo> :>
  30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  31 # 23:13|<aj> I shall not be silenced!
  32 # 23:13|<aj> The world shall know!
  33 # 23:13|<aj> The world *must* know!
  34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  35 # 23:13|<aj> yay powerpuff girls!!
  36 # 23:13|<aj> buttercup's my favourite, who's yours?
  37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  38 # 23:14|<aj> *AREN'T YOU*?!
  39 # 23:15|<aj> I will not be treated like this.
  40 # 23:15|<aj> I shall have my revenge.
  41 # 23:15|<aj> I SHALL!!!
  42
  43 ################################################################################
  44
  45 from __future__ import with_statement
  46
  47 import copy
  48 import errno
  49 import os
  50 import readline
  51 import stat
  52 import sys
  53 import time
  54 import contextlib
  55 import pwd
  56 import apt_pkg, apt_inst
  57 import examine_package
  58
  59 from daklib.dbconn import *
  60 from daklib.queue import *
  61 from daklib import daklog
  62 from daklib import utils
  63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
  64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
  65 from daklib.summarystats import SummaryStats
  66 from daklib.config import Config
  67
  68 # Globals
  69 Options = None
  70 Logger = None
  71
  72 Priorities = None
  73 Sections = None
  74
  75 ################################################################################
  76 ################################################################################
  77 ################################################################################
  78
  79 def recheck(upload, session):
  80 # STU: I'm not sure, but I don't thin kthis is necessary any longer:    upload.recheck(session)
  81     if len(upload.rejects) > 0:
  82         answer = "XXX"
  83         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
  84             answer = 'S'
  85
  86         print "REJECT\n%s" % '\n'.join(upload.rejects)
  87         prompt = "[R]eject, Skip, Quit ?"
  88
  89         while prompt.find(answer) == -1:
  90             answer = utils.our_raw_input(prompt)
  91             m = re_default_answer.match(prompt)
  92             if answer == "":
  93                 answer = m.group(1)
  94             answer = answer[:1].upper()
  95
  96         if answer == 'R':
  97             upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
  98             upload.pkg.remove_known_changes(session=session)
  99             session.commit()
 100             return 0
 101         elif answer == 'S':
 102             return 0
 103         elif answer == 'Q':
 104             end()
 105             sys.exit(0)
 106
 107     return 1
 108
 109 ################################################################################
 110
 111 def indiv_sg_compare (a, b):
 112     """Sort by source name, source, version, 'have source', and
 113        finally by filename."""
 114     # Sort by source version
 115     q = apt_pkg.VersionCompare(a["version"], b["version"])
 116     if q:
 117         return -q
 118
 119     # Sort by 'have source'
 120     a_has_source = a["architecture"].get("source")
 121     b_has_source = b["architecture"].get("source")
 122     if a_has_source and not b_has_source:
 123         return -1
 124     elif b_has_source and not a_has_source:
 125         return 1
 126
 127     return cmp(a["filename"], b["filename"])
 128
 129 ############################################################
 130
 131 def sg_compare (a, b):
 132     a = a[1]
 133     b = b[1]
 134     """Sort by have note, source already in database and time of oldest upload."""
 135     # Sort by have note
 136     a_note_state = a["note_state"]
 137     b_note_state = b["note_state"]
 138     if a_note_state < b_note_state:
 139         return -1
 140     elif a_note_state > b_note_state:
 141         return 1
 142     # Sort by source already in database (descending)
 143     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 144     if source_in_database:
 145         return -source_in_database
 146
 147     # Sort by time of oldest upload
 148     return cmp(a["oldest"], b["oldest"])
 149
 150 def sort_changes(changes_files, session):
 151     """Sort into source groups, then sort each source group by version,
 152     have source, filename.  Finally, sort the source groups by have
 153     note, time of oldest upload of each source upload."""
 154     if len(changes_files) == 1:
 155         return changes_files
 156
 157     sorted_list = []
 158     cache = {}
 159     # Read in all the .changes files
 160     for filename in changes_files:
 161         u = Upload()
 162         try:
 163             u.pkg.changes_file = filename
 164             u.load_changes(filename)
 165             u.update_subst()
 166             cache[filename] = copy.copy(u.pkg.changes)
 167             cache[filename]["filename"] = filename
 168         except:
 169             sorted_list.append(filename)
 170             break
 171     # Divide the .changes into per-source groups
 172     per_source = {}
 173     for filename in cache.keys():
 174         source = cache[filename]["source"]
 175         if not per_source.has_key(source):
 176             per_source[source] = {}
 177             per_source[source]["list"] = []
 178         per_source[source]["list"].append(cache[filename])
 179     # Determine oldest time and have note status for each source group
 180     for source in per_source.keys():
 181         q = session.query(DBSource).filter_by(source = source).all()
 182         per_source[source]["source_in_database"] = len(q)>0
 183         source_list = per_source[source]["list"]
 184         first = source_list[0]
 185         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 186         have_note = 0
 187         for d in per_source[source]["list"]:
 188             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 189             if mtime < oldest:
 190                 oldest = mtime
 191             have_note += has_new_comment(d["source"], d["version"], session)
 192         per_source[source]["oldest"] = oldest
 193         if not have_note:
 194             per_source[source]["note_state"] = 0; # none
 195         elif have_note < len(source_list):
 196             per_source[source]["note_state"] = 1; # some
 197         else:
 198             per_source[source]["note_state"] = 2; # all
 199         per_source[source]["list"].sort(indiv_sg_compare)
 200     per_source_items = per_source.items()
 201     per_source_items.sort(sg_compare)
 202     for i in per_source_items:
 203         for j in i[1]["list"]:
 204             sorted_list.append(j["filename"])
 205     return sorted_list
 206
 207 ################################################################################
 208
 209 class Section_Completer:
 210     def __init__ (self, session):
 211         self.sections = []
 212         self.matches = []
 213         for s, in session.query(Section.section):
 214             self.sections.append(s)
 215
 216     def complete(self, text, state):
 217         if state == 0:
 218             self.matches = []
 219             n = len(text)
 220             for word in self.sections:
 221                 if word[:n] == text:
 222                     self.matches.append(word)
 223         try:
 224             return self.matches[state]
 225         except IndexError:
 226             return None
 227
 228 ############################################################
 229
 230 class Priority_Completer:
 231     def __init__ (self, session):
 232         self.priorities = []
 233         self.matches = []
 234         for p, in session.query(Priority.priority):
 235             self.priorities.append(p)
 236
 237     def complete(self, text, state):
 238         if state == 0:
 239             self.matches = []
 240             n = len(text)
 241             for word in self.priorities:
 242                 if word[:n] == text:
 243                     self.matches.append(word)
 244         try:
 245             return self.matches[state]
 246         except IndexError:
 247             return None
 248
 249 ################################################################################
 250
 251 def print_new (new, upload, indexed, file=sys.stdout):
 252     check_valid(new)
 253     broken = False
 254     index = 0
 255     for pkg in new.keys():
 256         index += 1
 257         section = new[pkg]["section"]
 258         priority = new[pkg]["priority"]
 259         if new[pkg]["section id"] == -1:
 260             section += "[!]"
 261             broken = True
 262         if new[pkg]["priority id"] == -1:
 263             priority += "[!]"
 264             broken = True
 265         if indexed:
 266             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 267         else:
 268             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 269         line = line.strip()+'\n'
 270         file.write(line)
 271     notes = get_new_comments(upload.pkg.changes.get("source"))
 272     for note in notes:
 273         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
 274               % (note.author, note.version, note.notedate, note.comment)
 275         print "-" * 72
 276     return broken, len(notes) > 0
 277
 278 ################################################################################
 279
 280 def index_range (index):
 281     if index == 1:
 282         return "1"
 283     else:
 284         return "1-%s" % (index)
 285
 286 ################################################################################
 287 ################################################################################
 288
 289 def edit_new (new, upload):
 290     # Write the current data to a temporary file
 291     (fd, temp_filename) = utils.temp_filename()
 292     temp_file = os.fdopen(fd, 'w')
 293     print_new (new, upload, indexed=0, file=temp_file)
 294     temp_file.close()
 295     # Spawn an editor on that file
 296     editor = os.environ.get("EDITOR","vi")
 297     result = os.system("%s %s" % (editor, temp_filename))
 298     if result != 0:
 299         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 300     # Read the edited data back in
 301     temp_file = utils.open_file(temp_filename)
 302     lines = temp_file.readlines()
 303     temp_file.close()
 304     os.unlink(temp_filename)
 305     # Parse the new data
 306     for line in lines:
 307         line = line.strip()
 308         if line == "":
 309             continue
 310         s = line.split()
 311         # Pad the list if necessary
 312         s[len(s):3] = [None] * (3-len(s))
 313         (pkg, priority, section) = s[:3]
 314         if not new.has_key(pkg):
 315             utils.warn("Ignoring unknown package '%s'" % (pkg))
 316         else:
 317             # Strip off any invalid markers, print_new will readd them.
 318             if section.endswith("[!]"):
 319                 section = section[:-3]
 320             if priority.endswith("[!]"):
 321                 priority = priority[:-3]
 322             for f in new[pkg]["files"]:
 323                 upload.pkg.files[f]["section"] = section
 324                 upload.pkg.files[f]["priority"] = priority
 325             new[pkg]["section"] = section
 326             new[pkg]["priority"] = priority
 327
 328 ################################################################################
 329
 330 def edit_index (new, upload, index):
 331     priority = new[index]["priority"]
 332     section = new[index]["section"]
 333     ftype = new[index]["type"]
 334     done = 0
 335     while not done:
 336         print "\t".join([index, priority, section])
 337
 338         answer = "XXX"
 339         if ftype != "dsc":
 340             prompt = "[B]oth, Priority, Section, Done ? "
 341         else:
 342             prompt = "[S]ection, Done ? "
 343         edit_priority = edit_section = 0
 344
 345         while prompt.find(answer) == -1:
 346             answer = utils.our_raw_input(prompt)
 347             m = re_default_answer.match(prompt)
 348             if answer == "":
 349                 answer = m.group(1)
 350             answer = answer[:1].upper()
 351
 352         if answer == 'P':
 353             edit_priority = 1
 354         elif answer == 'S':
 355             edit_section = 1
 356         elif answer == 'B':
 357             edit_priority = edit_section = 1
 358         elif answer == 'D':
 359             done = 1
 360
 361         # Edit the priority
 362         if edit_priority:
 363             readline.set_completer(Priorities.complete)
 364             got_priority = 0
 365             while not got_priority:
 366                 new_priority = utils.our_raw_input("New priority: ").strip()
 367                 if new_priority not in Priorities.priorities:
 368                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 369                 else:
 370                     got_priority = 1
 371                     priority = new_priority
 372
 373         # Edit the section
 374         if edit_section:
 375             readline.set_completer(Sections.complete)
 376             got_section = 0
 377             while not got_section:
 378                 new_section = utils.our_raw_input("New section: ").strip()
 379                 if new_section not in Sections.sections:
 380                     print "E: '%s' is not a valid section, try again." % (new_section)
 381                 else:
 382                     got_section = 1
 383                     section = new_section
 384
 385         # Reset the readline completer
 386         readline.set_completer(None)
 387
 388     for f in new[index]["files"]:
 389         upload.pkg.files[f]["section"] = section
 390         upload.pkg.files[f]["priority"] = priority
 391     new[index]["priority"] = priority
 392     new[index]["section"] = section
 393     return new
 394
 395 ################################################################################
 396
 397 def edit_overrides (new, upload, session):
 398     print
 399     done = 0
 400     while not done:
 401         print_new (new, upload, indexed=1)
 402         new_index = {}
 403         index = 0
 404         for i in new.keys():
 405             index += 1
 406             new_index[index] = i
 407
 408         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 409
 410         got_answer = 0
 411         while not got_answer:
 412             answer = utils.our_raw_input(prompt)
 413             if not answer.isdigit():
 414                 answer = answer[:1].upper()
 415             if answer == "E" or answer == "D":
 416                 got_answer = 1
 417             elif re_isanum.match (answer):
 418                 answer = int(answer)
 419                 if (answer < 1) or (answer > index):
 420                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 421                 else:
 422                     got_answer = 1
 423
 424         if answer == 'E':
 425             edit_new(new, upload)
 426         elif answer == 'D':
 427             done = 1
 428         else:
 429             edit_index (new, upload, new_index[answer])
 430
 431     return new
 432
 433 ################################################################################
 434
 435 def edit_note(note, upload, session):
 436     # Write the current data to a temporary file
 437     (fd, temp_filename) = utils.temp_filename()
 438     editor = os.environ.get("EDITOR","vi")
 439     answer = 'E'
 440     while answer == 'E':
 441         os.system("%s %s" % (editor, temp_filename))
 442         temp_file = utils.open_file(temp_filename)
 443         newnote = temp_file.read().rstrip()
 444         temp_file.close()
 445         print "New Note:"
 446         print utils.prefix_multi_line_string(newnote,"  ")
 447         prompt = "[D]one, Edit, Abandon, Quit ?"
 448         answer = "XXX"
 449         while prompt.find(answer) == -1:
 450             answer = utils.our_raw_input(prompt)
 451             m = re_default_answer.search(prompt)
 452             if answer == "":
 453                 answer = m.group(1)
 454             answer = answer[:1].upper()
 455     os.unlink(temp_filename)
 456     if answer == 'A':
 457         return
 458     elif answer == 'Q':
 459         end()
 460         sys.exit(0)
 461
 462     comment = NewComment()
 463     comment.package = upload.pkg.changes["source"]
 464     comment.version = upload.pkg.changes["version"]
 465     comment.comment = newnote
 466     comment.author  = utils.whoami()
 467     comment.trainee = bool(Options["Trainee"])
 468     session.add(comment)
 469     session.commit()
 470
 471 ################################################################################
 472
 473 def check_pkg (upload):
 474     try:
 475         less_fd = os.popen("less -R -", 'w', 0)
 476         stdout_fd = sys.stdout
 477         try:
 478             sys.stdout = less_fd
 479             changes = utils.parse_changes (upload.pkg.changes_file)
 480             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
 481             files = upload.pkg.files
 482             for f in files.keys():
 483                 if files[f].has_key("new"):
 484                     ftype = files[f]["type"]
 485                     if ftype == "deb":
 486                         examine_package.check_deb(changes['distribution'], f)
 487                     elif ftype == "dsc":
 488                         examine_package.check_dsc(changes['distribution'], f)
 489         finally:
 490             examine_package.output_package_relations()
 491             sys.stdout = stdout_fd
 492     except IOError, e:
 493         if e.errno == errno.EPIPE:
 494             utils.warn("[examine_package] Caught EPIPE; skipping.")
 495             pass
 496         else:
 497             raise
 498     except KeyboardInterrupt:
 499         utils.warn("[examine_package] Caught C-c; skipping.")
 500         pass
 501
 502 ################################################################################
 503
 504 ## FIXME: horribly Debian specific
 505
 506 def do_bxa_notification(upload):
 507     files = upload.pkg.files
 508     summary = ""
 509     for f in files.keys():
 510         if files[f]["type"] == "deb":
 511             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 512             summary += "\n"
 513             summary += "Package: %s\n" % (control.Find("Package"))
 514             summary += "Description: %s\n" % (control.Find("Description"))
 515     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 516     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
 517     utils.send_mail(bxa_mail)
 518
 519 ################################################################################
 520
 521 def add_overrides (new, upload, session):
 522     changes = upload.pkg.changes
 523     files = upload.pkg.files
 524     srcpkg = changes.get("source")
 525
 526     for suite in changes["suite"].keys():
 527         suite_id = get_suite(suite).suite_id
 528         for pkg in new.keys():
 529             component_id = get_component(new[pkg]["component"]).component_id
 530             type_id = get_override_type(new[pkg]["type"]).overridetype_id
 531             priority_id = new[pkg]["priority id"]
 532             section_id = new[pkg]["section id"]
 533             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
 534             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
 535                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
 536             for f in new[pkg]["files"]:
 537                 if files[f].has_key("new"):
 538                     del files[f]["new"]
 539             del new[pkg]
 540
 541     session.commit()
 542
 543     if Config().FindB("Dinstall::BXANotify"):
 544         do_bxa_notification(upload)
 545
 546 ################################################################################
 547
 548 def prod_maintainer (note, upload):
 549     cnf = Config()
 550     # Here we prepare an editor and get them ready to prod...
 551     (fd, temp_filename) = utils.temp_filename()
 552     temp_file = os.fdopen(fd, 'w')
 553     if len(note) > 0:
 554         for line in note:
 555             temp_file.write(line)
 556     temp_file.close()
 557     editor = os.environ.get("EDITOR","vi")
 558     answer = 'E'
 559     while answer == 'E':
 560         os.system("%s %s" % (editor, temp_filename))
 561         temp_fh = utils.open_file(temp_filename)
 562         prod_message = "".join(temp_fh.readlines())
 563         temp_fh.close()
 564         print "Prod message:"
 565         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 566         prompt = "[P]rod, Edit, Abandon, Quit ?"
 567         answer = "XXX"
 568         while prompt.find(answer) == -1:
 569             answer = utils.our_raw_input(prompt)
 570             m = re_default_answer.search(prompt)
 571             if answer == "":
 572                 answer = m.group(1)
 573             answer = answer[:1].upper()
 574     os.unlink(temp_filename)
 575     if answer == 'A':
 576         return
 577     elif answer == 'Q':
 578         end()
 579         sys.exit(0)
 580     # Otherwise, do the proding...
 581     user_email_address = utils.whoami() + " <%s>" % (
 582         cnf["Dinstall::MyAdminAddress"])
 583
 584     Subst = upload.Subst
 585
 586     Subst["__FROM_ADDRESS__"] = user_email_address
 587     Subst["__PROD_MESSAGE__"] = prod_message
 588     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
 589
 590     prod_mail_message = utils.TemplateSubst(
 591         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 592
 593     # Send the prod mail if appropriate
 594     if not cnf["Dinstall::Options::No-Mail"]:
 595         utils.send_mail(prod_mail_message)
 596
 597     print "Sent proding message"
 598
 599 ################################################################################
 600
 601 def do_new(upload, session):
 602     print "NEW\n"
 603     files = upload.pkg.files
 604     upload.check_files(not Options["No-Action"])
 605     changes = upload.pkg.changes
 606     cnf = Config()
 607
 608     # Check for a valid distribution
 609     upload.check_distributions()
 610
 611     # Make a copy of distribution we can happily trample on
 612     changes["suite"] = copy.copy(changes["distribution"])
 613
 614     # Fix up the list of target suites
 615     for suite in changes["suite"].keys():
 616         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
 617         if override:
 618             (olderr, newerr) = (get_suite(suite, session) == None,
 619                                 get_suite(override, session) == None)
 620             if olderr or newerr:
 621                 (oinv, newinv) = ("", "")
 622                 if olderr: oinv = "invalid "
 623                 if newerr: ninv = "invalid "
 624                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 625                         oinv, suite, ninv, override)
 626             del changes["suite"][suite]
 627             changes["suite"][override] = 1
 628
 629     # The main NEW processing loop
 630     done = 0
 631     while not done:
 632         # Find out what's new
 633         new = determine_new(changes, files)
 634
 635         if not new:
 636             break
 637
 638         answer = "XXX"
 639         if Options["No-Action"] or Options["Automatic"]:
 640             answer = 'S'
 641
 642         (broken, note) = print_new(new, upload, indexed=0)
 643         prompt = ""
 644
 645         if not broken and not note:
 646             prompt = "Add overrides, "
 647         if broken:
 648             print "W: [!] marked entries must be fixed before package can be processed."
 649         if note:
 650             print "W: note must be removed before package can be processed."
 651             prompt += "RemOve all notes, Remove note, "
 652
 653         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 654
 655         while prompt.find(answer) == -1:
 656             answer = utils.our_raw_input(prompt)
 657             m = re_default_answer.search(prompt)
 658             if answer == "":
 659                 answer = m.group(1)
 660             answer = answer[:1].upper()
 661
 662         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
 663             utils.warn("Trainees can't do that")
 664             continue
 665
 666         if answer == 'A' and not Options["Trainee"]:
 667             try:
 668                 check_daily_lock()
 669                 done = add_overrides (new, upload, session)
 670                 do_accept(upload, session)
 671                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
 672             except CantGetLockError:
 673                 print "Hello? Operator! Give me the number for 911!"
 674                 print "Dinstall in the locked area, cant process packages, come back later"
 675         elif answer == 'C':
 676             check_pkg(upload)
 677         elif answer == 'E' and not Options["Trainee"]:
 678             new = edit_overrides (new, upload, session)
 679         elif answer == 'M' and not Options["Trainee"]:
 680             aborted = upload.do_reject(manual=1,
 681                                        reject_message=Options["Manual-Reject"],
 682                                        note=get_new_comments(changes.get("source", ""), session=session))
 683             if not aborted:
 684                 upload.pkg.remove_known_changes(session=session)
 685                 session.commit()
 686                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
 687                 done = 1
 688         elif answer == 'N':
 689             edit_note(get_new_comments(changes.get("source", ""), session=session),
 690                       upload, session)
 691         elif answer == 'P' and not Options["Trainee"]:
 692             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
 693                             upload)
 694             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
 695         elif answer == 'R' and not Options["Trainee"]:
 696             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 697             if confirm == "y":
 698                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
 699                     session.delete(c)
 700                 session.commit()
 701         elif answer == 'O' and not Options["Trainee"]:
 702             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
 703             if confirm == "y":
 704                 for c in get_new_comments(changes.get("source", ""), session=session):
 705                     session.delete(c)
 706                 session.commit()
 707
 708         elif answer == 'S':
 709             done = 1
 710         elif answer == 'Q':
 711             end()
 712             sys.exit(0)
 713
 714 ################################################################################
 715 ################################################################################
 716 ################################################################################
 717
 718 def usage (exit_code=0):
 719     print """Usage: dak process-new [OPTION]... [CHANGES]...
 720   -a, --automatic           automatic run
 721   -h, --help                show this help and exit.
 722   -m, --manual-reject=MSG   manual reject with `msg'
 723   -n, --no-action           don't do anything
 724   -t, --trainee             FTP Trainee mode
 725   -V, --version             display the version number and exit"""
 726     sys.exit(exit_code)
 727
 728 ################################################################################
 729
 730 def do_byhand(upload, session):
 731     done = 0
 732     while not done:
 733         files = upload.pkg.files
 734         will_install = 1
 735         byhand = []
 736
 737         for f in files.keys():
 738             if files[f]["type"] == "byhand":
 739                 if os.path.exists(f):
 740                     print "W: %s still present; please process byhand components and try again." % (f)
 741                     will_install = 0
 742                 else:
 743                     byhand.append(f)
 744
 745         answer = "XXXX"
 746         if Options["No-Action"]:
 747             answer = "S"
 748         if will_install:
 749             if Options["Automatic"] and not Options["No-Action"]:
 750                 answer = 'A'
 751             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 752         else:
 753             prompt = "Manual reject, [S]kip, Quit ?"
 754
 755         while prompt.find(answer) == -1:
 756             answer = utils.our_raw_input(prompt)
 757             m = re_default_answer.search(prompt)
 758             if answer == "":
 759                 answer = m.group(1)
 760             answer = answer[:1].upper()
 761
 762         if answer == 'A':
 763             try:
 764                 check_daily_lock()
 765                 done = 1
 766                 for f in byhand:
 767                     del files[f]
 768                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
 769             except CantGetLockError:
 770                 print "Hello? Operator! Give me the number for 911!"
 771                 print "Dinstall in the locked area, cant process packages, come back later"
 772         elif answer == 'M':
 773             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
 774             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
 775             upload.pkg.remove_known_changes(session=session)
 776             session.commit()
 777             done = 1
 778         elif answer == 'S':
 779             done = 1
 780         elif answer == 'Q':
 781             end()
 782             sys.exit(0)
 783
 784 ################################################################################
 785
 786 def check_daily_lock():
 787     """
 788     Raises CantGetLockError if the dinstall daily.lock exists.
 789     """
 790
 791     cnf = Config()
 792     try:
 793         os.open(cnf["Process-New::DinstallLockFile"],
 794                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 795     except OSError, e:
 796         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 797             raise CantGetLockError
 798
 799     os.unlink(cnf["Process-New::DinstallLockFile"])
 800
 801
 802 @contextlib.contextmanager
 803 def lock_package(package):
 804     """
 805     Lock C{package} so that noone else jumps in processing it.
 806
 807     @type package: string
 808     @param package: source package name to lock
 809     """
 810
 811     path = os.path.join(Config()["Process-New::LockDir"], package)
 812     try:
 813         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
 814     except OSError, e:
 815         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
 816             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
 817             raise AlreadyLockedError, user
 818
 819     try:
 820         yield fd
 821     finally:
 822         os.unlink(path)
 823
 824 class clean_holding(object):
 825     def __init__(self,pkg):
 826         self.pkg = pkg
 827
 828     def __enter__(self):
 829         pass
 830
 831     def __exit__(self, type, value, traceback):
 832         h = Holding()
 833
 834         for f in self.pkg.files.keys():
 835             if os.path.exists(os.path.join(h.holding_dir, f)):
 836                 os.unlink(os.path.join(h.holding_dir, f))
 837
 838
 839
 840 def changes_to_newstage(upload, session):
 841     """move a changes file to newstage"""
 842     new = get_policy_queue('new', session );
 843     newstage = get_policy_queue('newstage', session );
 844
 845     chg = session.query(DBChange).filter_by(changesname=os.path.basename(upload.pkg.changes_file)).one()
 846     chg.approved_for = newstage.policy_queue_id
 847
 848     for f in chg.files:
 849         # update the changes_pending_files row
 850         f.queue = newstage
 851         utils.move(os.path.join(new.path, f.filename), newstage.path, perms=int(newstage.perms, 8))
 852
 853     utils.move(os.path.join(new.path, upload.pkg.changes_file), newstage.path, perms=int(newstage.perms, 8))
 854     chg.in_queue = newstage
 855     session.commit()
 856
 857 def _accept(upload, session):
 858     if Options["No-Action"]:
 859         return
 860     (summary, short_summary) = upload.build_summaries()
 861     # upload.accept(summary, short_summary, targetqueue)
 862
 863     changes_to_newstage(upload, session)
 864
 865 def do_accept(upload, session):
 866     print "ACCEPT"
 867     cnf = Config()
 868     if not Options["No-Action"]:
 869         (summary, short_summary) = upload.build_summaries()
 870
 871         if cnf.FindB("Dinstall::SecurityQueueHandling"):
 872             upload.dump_vars(cnf["Dir::Queue::Embargoed"])
 873             upload.move_to_queue(get_policy_queue('embargoed'))
 874             upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
 875             # Check for override disparities
 876             upload.Subst["__SUMMARY__"] = summary
 877         else:
 878             # Just a normal upload, accept it...
 879             _accept(upload, session)
 880
 881 def do_pkg(changes_file, session):
 882     new_queue = get_policy_queue('new', session );
 883     u = Upload()
 884     u.pkg.changes_file = changes_file
 885     (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
 886     u.load_changes(changes_file)
 887     u.pkg.directory = new_queue.path
 888     u.update_subst()
 889     u.logger = Logger
 890     origchanges = os.path.abspath(u.pkg.changes_file)
 891
 892     cnf = Config()
 893     bcc = "X-DAK: dak process-new"
 894     if cnf.has_key("Dinstall::Bcc"):
 895         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
 896     else:
 897         u.Subst["__BCC__"] = bcc
 898
 899     files = u.pkg.files
 900     for deb_filename, f in files.items():
 901         if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
 902             u.binary_file_checks(deb_filename, session)
 903             u.check_binary_against_db(deb_filename, session)
 904         else:
 905             u.source_file_checks(deb_filename, session)
 906             u.check_source_against_db(deb_filename, session)
 907
 908         u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
 909
 910     try:
 911         with lock_package(u.pkg.changes["source"]):
 912             with clean_holding(u.pkg):
 913                 if not recheck(u, session):
 914                     return
 915
 916                 # FIXME: This does need byhand checks added!
 917                 new = determine_new(u.pkg.changes, files)
 918                 if new:
 919                     do_new(u, session)
 920                 else:
 921                     try:
 922                         check_daily_lock()
 923                         do_accept(u, session)
 924                     except CantGetLockError:
 925                         print "Hello? Operator! Give me the number for 911!"
 926                         print "Dinstall in the locked area, cant process packages, come back later"
 927 #             (new, byhand) = check_status(files)
 928 #             if new or byhand:
 929 #                 if new:
 930 #                     do_new(u, session)
 931 #                 if byhand:
 932 #                     do_byhand(u, session)
 933 #                 (new, byhand) = check_status(files)
 934
 935 #             if not new and not byhand:
 936 #                 try:
 937 #                     check_daily_lock()
 938 #                     do_accept(u)
 939 #                 except CantGetLockError:
 940 #                     print "Hello? Operator! Give me the number for 911!"
 941 #                     print "Dinstall in the locked area, cant process packages, come back later"
 942     except AlreadyLockedError, e:
 943         print "Seems to be locked by %s already, skipping..." % (e)
 944
 945 ################################################################################
 946
 947 def end():
 948     accept_count = SummaryStats().accept_count
 949     accept_bytes = SummaryStats().accept_bytes
 950
 951     if accept_count:
 952         sets = "set"
 953         if accept_count > 1:
 954             sets = "sets"
 955         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 956         Logger.log(["total",accept_count,accept_bytes])
 957
 958     if not Options["No-Action"] and not Options["Trainee"]:
 959         Logger.close()
 960
 961 ################################################################################
 962
 963 def main():
 964     global Options, Logger, Sections, Priorities
 965
 966     cnf = Config()
 967     session = DBConn().session()
 968
 969     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 970                  ('h',"help","Process-New::Options::Help"),
 971                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 972                  ('t',"trainee","Process-New::Options::Trainee"),
 973                  ('n',"no-action","Process-New::Options::No-Action")]
 974
 975     for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
 976         if not cnf.has_key("Process-New::Options::%s" % (i)):
 977             cnf["Process-New::Options::%s" % (i)] = ""
 978
 979     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
 980     if len(changes_files) == 0:
 981         new_queue = get_policy_queue('new', session );
 982         changes_files = utils.get_changes_files(new_queue.path)
 983
 984     Options = cnf.SubTree("Process-New::Options")
 985
 986     if Options["Help"]:
 987         usage()
 988
 989     if not Options["No-Action"]:
 990         try:
 991             Logger = daklog.Logger(cnf, "process-new")
 992         except CantOpenError, e:
 993             Options["Trainee"] = "True"
 994
 995     Sections = Section_Completer(session)
 996     Priorities = Priority_Completer(session)
 997     readline.parse_and_bind("tab: complete")
 998
 999     if len(changes_files) > 1:
1000         sys.stderr.write("Sorting changes...\n")
1001     changes_files = sort_changes(changes_files, session)
1002
1003     # Kill me now? **FIXME**
1004     cnf["Dinstall::Options::No-Mail"] = ""
1005
1006     for changes_file in changes_files:
1007         changes_file = utils.validate_changes_file_arg(changes_file, 0)
1008         if not changes_file:
1009             continue
1010         print "\n" + changes_file
1011
1012         do_pkg (changes_file, session)
1013
1014     end()
1015
1016 ################################################################################
1017
1018 if __name__ == '__main__':
1019     main()