X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_new.py;h=aaae0581ca818fd1b86e06895cc3fbc679de3d3b;hb=52b14d883272923a4296ac02921ccf350815db83;hp=3a1074c49202e7a5ce87373461fabcf83f6e6fb9;hpb=567c91b57f1e6af8da3e9e0ab5eeffa14796a256;p=dak.git diff --git a/dak/process_new.py b/dak/process_new.py index 3a1074c4..77f6e5af 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -1,8 +1,14 @@ #!/usr/bin/env python +# vim:set et ts=4 sw=4: -# Handles NEW and BYHAND packages -# Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 James Troup +""" Handles NEW and BYHAND packages +@contact: Debian FTP Master +@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup +@copyright: 2009 Joerg Jaspert +@copyright: 2009 Frank Lichtenheld +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -36,157 +42,71 @@ ################################################################################ -import copy, errno, os, readline, stat, sys, time +from __future__ import with_statement + +import copy +import errno +import os +import readline +import stat +import sys +import time +import contextlib +import pwd import apt_pkg, apt_inst import examine_package -import daklib.database -import daklib.logging -import daklib.queue -import daklib.utils + +from daklib.dbconn import * +from daklib.queue import * +from daklib import daklog +from daklib import utils +from daklib.regexes import re_no_epoch, re_default_answer, re_isanum +from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError +from daklib.summarystats import SummaryStats +from daklib.config import Config # Globals -Cnf = None Options = None -Upload = None -projectB = None Logger = None Priorities = None Sections = None -reject_message = "" - ################################################################################ ################################################################################ ################################################################################ -def reject (str, prefix="Rejected: "): - global reject_message - if str: - reject_message += prefix + str + "\n" - -def recheck(): - global reject_message - files = Upload.pkg.files - reject_message = "" - - for file in files.keys(): - # The .orig.tar.gz can disappear out from under us is it's a - # duplicate of one in the archive. - if not files.has_key(file): - continue - # Check that the source still exists - if files[file]["type"] == "deb": - source_version = files[file]["source version"] - source_package = files[file]["source package"] - if not Upload.pkg.changes["architecture"].has_key("source") \ - and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()): - source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version) - dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) - if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename): - reject("no source found for %s %s (%s)." % (source_package, source_version, file)) - - # Version and file overwrite checks - if files[file]["type"] == "deb": - reject(Upload.check_binary_against_db(file)) - elif files[file]["type"] == "dsc": - reject(Upload.check_source_against_db(file)) - (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file) - reject(reject_msg) - - if reject_message: +def recheck(upload, session): + upload.recheck() + if len(upload.rejects) > 0: answer = "XXX" - if Options["No-Action"] or Options["Automatic"]: + if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]: answer = 'S' - print "REJECT\n" + reject_message, + print "REJECT\n" + upload.rejects.join("\n"), prompt = "[R]eject, Skip, Quit ?" while prompt.find(answer) == -1: - answer = daklib.utils.our_raw_input(prompt) - m = daklib.queue.re_default_answer.match(prompt) + answer = utils.our_raw_input(prompt) + m = re_default_answer.match(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() if answer == 'R': - Upload.do_reject(0, reject_message) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + upload.do_reject(manual=0, reject_message=upload.rejects.join("\n")) + os.unlink(upload.pkg.changes_file[:-8]+".dak") return 0 elif answer == 'S': return 0 elif answer == 'Q': + end() sys.exit(0) return 1 ################################################################################ -def determine_new (changes, files): - new = {} - - # Build up a list of potentially new things - for file in files.keys(): - f = files[file] - # Skip byhand elements - if f["type"] == "byhand": - continue - pkg = f["package"] - priority = f["priority"] - section = f["section"] - # FIXME: unhardcode - if section == "non-US/main": - section = "non-US" - type = get_type(f) - component = f["component"] - - if type == "dsc": - priority = "source" - if not new.has_key(pkg): - new[pkg] = {} - new[pkg]["priority"] = priority - new[pkg]["section"] = section - new[pkg]["type"] = type - new[pkg]["component"] = component - new[pkg]["files"] = [] - else: - old_type = new[pkg]["type"] - if old_type != type: - # source gets trumped by deb or udeb - if old_type == "dsc": - new[pkg]["priority"] = priority - new[pkg]["section"] = section - new[pkg]["type"] = type - new[pkg]["component"] = component - new[pkg]["files"].append(file) - if f.has_key("othercomponents"): - new[pkg]["othercomponents"] = f["othercomponents"] - - for suite in changes["suite"].keys(): - suite_id = daklib.database.get_suite_id(suite) - for pkg in new.keys(): - component_id = daklib.database.get_component_id(new[pkg]["component"]) - type_id = daklib.database.get_override_type_id(new[pkg]["type"]) - q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id)) - ql = q.getresult() - if ql: - for file in new[pkg]["files"]: - if files[file].has_key("new"): - del files[file]["new"] - del new[pkg] - - if changes["suite"].has_key("stable"): - print "WARNING: overrides will be added for stable!" - if changes["suite"].has_key("oldstable"): - print "WARNING: overrides will be added for OLDstable!" - for pkg in new.keys(): - if new[pkg].has_key("othercomponents"): - print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]) - - return new - -################################################################################ - def indiv_sg_compare (a, b): """Sort by source name, source, version, 'have source', and finally by filename.""" @@ -210,7 +130,7 @@ def indiv_sg_compare (a, b): def sg_compare (a, b): a = a[1] b = b[1] - """Sort by have note, time of oldest upload.""" + """Sort by have note, source already in database and time of oldest upload.""" # Sort by have note a_note_state = a["note_state"] b_note_state = b["note_state"] @@ -218,11 +138,15 @@ def sg_compare (a, b): return -1 elif a_note_state > b_note_state: return 1 + # Sort by source already in database (descending) + source_in_database = cmp(a["source_in_database"], b["source_in_database"]) + if source_in_database: + return -source_in_database # Sort by time of oldest upload return cmp(a["oldest"], b["oldest"]) -def sort_changes(changes_files): +def sort_changes(changes_files, session): """Sort into source groups, then sort each source group by version, have source, filename. Finally, sort the source groups by have note, time of oldest upload of each source upload.""" @@ -233,11 +157,11 @@ def sort_changes(changes_files): cache = {} # Read in all the .changes files for filename in changes_files: + u = Upload() try: - Upload.pkg.changes_file = filename - Upload.init_vars() - Upload.update_vars() - cache[filename] = copy.copy(Upload.pkg.changes) + u.pkg.load_dot_dak(filename) + u.update_subst() + cache[filename] = copy.copy(u.pkg.changes) cache[filename]["filename"] = filename except: sorted_list.append(filename) @@ -252,6 +176,8 @@ def sort_changes(changes_files): per_source[source]["list"].append(cache[filename]) # Determine oldest time and have note status for each source group for source in per_source.keys(): + q = session.query(DBSource).filter_by(source = source).all() + per_source[source]["source_in_database"] = len(q)>0 source_list = per_source[source]["list"] first = source_list[0] oldest = os.stat(first["filename"])[stat.ST_MTIME] @@ -260,7 +186,7 @@ def sort_changes(changes_files): mtime = os.stat(d["filename"])[stat.ST_MTIME] if mtime < oldest: oldest = mtime - have_note += (d.has_key("process-new note")) + have_note += has_new_comment(d["source"], d["version"], session) per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none @@ -279,11 +205,11 @@ def sort_changes(changes_files): ################################################################################ class Section_Completer: - def __init__ (self): + def __init__ (self, session): self.sections = [] - q = projectB.query("SELECT section FROM section") - for i in q.getresult(): - self.sections.append(i[0]) + self.matches = [] + for s, in session.query(Section.section): + self.sections.append(s) def complete(self, text, state): if state == 0: @@ -300,11 +226,11 @@ class Section_Completer: ############################################################ class Priority_Completer: - def __init__ (self): + def __init__ (self, session): self.priorities = [] - q = projectB.query("SELECT priority FROM priority") - for i in q.getresult(): - self.priorities.append(i[0]) + self.matches = [] + for p, in session.query(Priority.priority): + self.priorities.append(p) def complete(self, text, state): if state == 0: @@ -320,26 +246,9 @@ class Priority_Completer: ################################################################################ -def check_valid (new): - for pkg in new.keys(): - section = new[pkg]["section"] - priority = new[pkg]["priority"] - type = new[pkg]["type"] - new[pkg]["section id"] = daklib.database.get_section_id(section) - new[pkg]["priority id"] = daklib.database.get_priority_id(new[pkg]["priority"]) - # Sanity checks - if (section == "debian-installer" and type != "udeb") or \ - (section != "debian-installer" and type == "udeb"): - new[pkg]["section id"] = -1 - if (priority == "source" and type != "dsc") or \ - (priority != "source" and type == "dsc"): - new[pkg]["priority id"] = -1 - -################################################################################ - -def print_new (new, indexed, file=sys.stdout): +def print_new (new, upload, indexed, file=sys.stdout): check_valid(new) - broken = 0 + broken = False index = 0 for pkg in new.keys(): index += 1 @@ -347,40 +256,22 @@ def print_new (new, indexed, file=sys.stdout): priority = new[pkg]["priority"] if new[pkg]["section id"] == -1: section += "[!]" - broken = 1 + broken = True if new[pkg]["priority id"] == -1: priority += "[!]" - broken = 1 + broken = True if indexed: line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section) else: line = "%-20s %-20s %-20s" % (pkg, priority, section) line = line.strip()+'\n' file.write(line) - note = Upload.pkg.changes.get("process-new note") - if note: - print "*"*75 - print note - print "*"*75 - return broken, note - -################################################################################ - -def get_type (f): - # Determine the type - if f.has_key("dbtype"): - type = f["dbtype"] - elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc": - type = "dsc" - else: - daklib.utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type)) - - # Validate the override type - type_id = daklib.database.get_override_type_id(type) - if type_id == -1: - daklib.utils.fubar("invalid type (%s) for new. Say wha?" % (type)) - - return type + notes = get_new_comments(upload.pkg.changes.get("source")) + for note in notes: + print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \ + % (note.author, note.version, note.notedate, note.comment) + print "-" * 72 + return broken, len(notes) > 0 ################################################################################ @@ -393,19 +284,19 @@ def index_range (index): ################################################################################ ################################################################################ -def edit_new (new): +def edit_new (new, upload): # Write the current data to a temporary file - temp_filename = daklib.utils.temp_filename() - temp_file = daklib.utils.open_file(temp_filename, 'w') - print_new (new, 0, temp_file) + (fd, temp_filename) = utils.temp_filename() + temp_file = os.fdopen(fd, 'w') + print_new (new, upload, indexed=0, file=temp_file) temp_file.close() # Spawn an editor on that file editor = os.environ.get("EDITOR","vi") result = os.system("%s %s" % (editor, temp_filename)) if result != 0: - daklib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result) + utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result) # Read the edited data back in - temp_file = daklib.utils.open_file(temp_filename) + temp_file = utils.open_file(temp_filename) lines = temp_file.readlines() temp_file.close() os.unlink(temp_filename) @@ -419,39 +310,39 @@ def edit_new (new): s[len(s):3] = [None] * (3-len(s)) (pkg, priority, section) = s[:3] if not new.has_key(pkg): - daklib.utils.warn("Ignoring unknown package '%s'" % (pkg)) + utils.warn("Ignoring unknown package '%s'" % (pkg)) else: # Strip off any invalid markers, print_new will readd them. if section.endswith("[!]"): section = section[:-3] if priority.endswith("[!]"): priority = priority[:-3] - for file in new[pkg]["files"]: - Upload.pkg.files[file]["section"] = section - Upload.pkg.files[file]["priority"] = priority + for f in new[pkg]["files"]: + upload.pkg.files[f]["section"] = section + upload.pkg.files[f]["priority"] = priority new[pkg]["section"] = section new[pkg]["priority"] = priority ################################################################################ -def edit_index (new, index): +def edit_index (new, upload, index): priority = new[index]["priority"] section = new[index]["section"] - type = new[index]["type"] + ftype = new[index]["type"] done = 0 while not done: print "\t".join([index, priority, section]) answer = "XXX" - if type != "dsc": + if ftype != "dsc": prompt = "[B]oth, Priority, Section, Done ? " else: prompt = "[S]ection, Done ? " edit_priority = edit_section = 0 while prompt.find(answer) == -1: - answer = daklib.utils.our_raw_input(prompt) - m = daklib.queue.re_default_answer.match(prompt) + answer = utils.our_raw_input(prompt) + m = re_default_answer.match(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() @@ -470,7 +361,7 @@ def edit_index (new, index): readline.set_completer(Priorities.complete) got_priority = 0 while not got_priority: - new_priority = daklib.utils.our_raw_input("New priority: ").strip() + new_priority = utils.our_raw_input("New priority: ").strip() if new_priority not in Priorities.priorities: print "E: '%s' is not a valid priority, try again." % (new_priority) else: @@ -482,7 +373,7 @@ def edit_index (new, index): readline.set_completer(Sections.complete) got_section = 0 while not got_section: - new_section = daklib.utils.our_raw_input("New section: ").strip() + new_section = utils.our_raw_input("New section: ").strip() if new_section not in Sections.sections: print "E: '%s' is not a valid section, try again." % (new_section) else: @@ -492,20 +383,20 @@ def edit_index (new, index): # Reset the readline completer readline.set_completer(None) - for file in new[index]["files"]: - Upload.pkg.files[file]["section"] = section - Upload.pkg.files[file]["priority"] = priority + for f in new[index]["files"]: + upload.pkg.files[f]["section"] = section + upload.pkg.files[f]["priority"] = priority new[index]["priority"] = priority new[index]["section"] = section return new ################################################################################ -def edit_overrides (new): +def edit_overrides (new, upload, session): print done = 0 while not done: - print_new (new, 1) + print_new (new, upload, indexed=1) new_index = {} index = 0 for i in new.keys(): @@ -516,12 +407,12 @@ def edit_overrides (new): got_answer = 0 while not got_answer: - answer = daklib.utils.our_raw_input(prompt) + answer = utils.our_raw_input(prompt) if not answer.isdigit(): answer = answer[:1].upper() if answer == "E" or answer == "D": got_answer = 1 - elif daklib.queue.re_isanum.match (answer): + elif re_isanum.match (answer): answer = int(answer) if (answer < 1) or (answer > index): print "%s is not a valid index (%s). Please retry." % (answer, index_range(index)) @@ -529,36 +420,33 @@ def edit_overrides (new): got_answer = 1 if answer == 'E': - edit_new(new) + edit_new(new, upload) elif answer == 'D': done = 1 else: - edit_index (new, new_index[answer]) + edit_index (new, upload, new_index[answer]) return new ################################################################################ -def edit_note(note): +def edit_note(note, upload, session): # Write the current data to a temporary file - temp_filename = daklib.utils.temp_filename() - temp_file = daklib.utils.open_file(temp_filename, 'w') - temp_file.write(note) - temp_file.close() + (fd, temp_filename) = utils.temp_filename() editor = os.environ.get("EDITOR","vi") answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) - temp_file = daklib.utils.open_file(temp_filename) - note = temp_file.read().rstrip() + temp_file = utils.open_file(temp_filename) + newnote = temp_file.read().rstrip() temp_file.close() - print "Note:" - print daklib.utils.prefix_multi_line_string(note," ") + print "New Note:" + print utils.prefix_multi_line_string(newnote," ") prompt = "[D]one, Edit, Abandon, Quit ?" answer = "XXX" while prompt.find(answer) == -1: - answer = daklib.utils.our_raw_input(prompt) - m = daklib.queue.re_default_answer.search(prompt) + answer = utils.our_raw_input(prompt) + m = re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() @@ -566,156 +454,175 @@ def edit_note(note): if answer == 'A': return elif answer == 'Q': + end() sys.exit(0) - Upload.pkg.changes["process-new note"] = note - Upload.dump_vars(Cnf["Dir::Queue::New"]) + + comment = NewComment() + comment.package = upload.pkg.changes["source"] + comment.version = upload.pkg.changes["version"] + comment.comment = newnote + comment.author = utils.whoami() + comment.trainee = bool(Options["Trainee"]) + session.add(comment) + session.commit() ################################################################################ -def check_pkg (): +def check_pkg (upload): try: less_fd = os.popen("less -R -", 'w', 0) stdout_fd = sys.stdout try: sys.stdout = less_fd - examine_package.display_changes(Upload.pkg.changes_file) - files = Upload.pkg.files - for file in files.keys(): - if files[file].has_key("new"): - type = files[file]["type"] - if type == "deb": - examine_package.check_deb(file) - elif type == "dsc": - examine_package.check_dsc(file) + changes = utils.parse_changes (upload.pkg.changes_file) + examine_package.display_changes(changes['distribution'], upload.pkg.changes_file) + files = upload.pkg.files + for f in files.keys(): + if files[f].has_key("new"): + ftype = files[f]["type"] + if ftype == "deb": + examine_package.check_deb(changes['distribution'], f) + elif ftype == "dsc": + examine_package.check_dsc(changes['distribution'], f) finally: + examine_package.output_package_relations() sys.stdout = stdout_fd except IOError, e: - if errno.errorcode[e.errno] == 'EPIPE': - daklib.utils.warn("[examine_package] Caught EPIPE; skipping.") + if e.errno == errno.EPIPE: + utils.warn("[examine_package] Caught EPIPE; skipping.") pass else: raise except KeyboardInterrupt: - daklib.utils.warn("[examine_package] Caught C-c; skipping.") + utils.warn("[examine_package] Caught C-c; skipping.") pass ################################################################################ ## FIXME: horribly Debian specific -def do_bxa_notification(): - files = Upload.pkg.files +def do_bxa_notification(upload): + files = upload.pkg.files summary = "" - for file in files.keys(): - if files[file]["type"] == "deb": - control = apt_pkg.ParseSection(apt_inst.debExtractControl(daklib.utils.open_file(file))) + for f in files.keys(): + if files[f]["type"] == "deb": + control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f))) summary += "\n" summary += "Package: %s\n" % (control.Find("Package")) summary += "Description: %s\n" % (control.Find("Description")) - Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary - bxa_mail = daklib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification") - daklib.utils.send_mail(bxa_mail) + upload.Subst["__BINARY_DESCRIPTIONS__"] = summary + bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification") + utils.send_mail(bxa_mail) ################################################################################ -def add_overrides (new): - changes = Upload.pkg.changes - files = Upload.pkg.files +def add_overrides (new, upload, session): + changes = upload.pkg.changes + files = upload.pkg.files + srcpkg = changes.get("source") - projectB.query("BEGIN WORK") for suite in changes["suite"].keys(): - suite_id = daklib.database.get_suite_id(suite) + suite_id = get_suite(suite).suite_id for pkg in new.keys(): - component_id = daklib.database.get_component_id(new[pkg]["component"]) - type_id = daklib.database.get_override_type_id(new[pkg]["type"]) + component_id = get_component(new[pkg]["component"]).component_id + type_id = get_override_type(new[pkg]["type"]).overridetype_id priority_id = new[pkg]["priority id"] section_id = new[pkg]["section id"] - projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id)) - for file in new[pkg]["files"]: - if files[file].has_key("new"): - del files[file]["new"] + Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]]) + session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')", + { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id}) + for f in new[pkg]["files"]: + if files[f].has_key("new"): + del files[f]["new"] del new[pkg] - projectB.query("COMMIT WORK") + session.commit() - if Cnf.FindB("Dinstall::BXANotify"): - do_bxa_notification() + if Config().FindB("Dinstall::BXANotify"): + do_bxa_notification(upload) ################################################################################ -def prod_maintainer (): +def prod_maintainer (note, upload): + cnf = Config() # Here we prepare an editor and get them ready to prod... - temp_filename = daklib.utils.temp_filename() + (fd, temp_filename) = utils.temp_filename() + temp_file = os.fdopen(fd, 'w') + if len(note) > 0: + for line in note: + temp_file.write(line) + temp_file.close() editor = os.environ.get("EDITOR","vi") answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) - file = daklib.utils.open_file(temp_filename) - prod_message = "".join(file.readlines()) - file.close() + temp_fh = utils.open_file(temp_filename) + prod_message = "".join(temp_fh.readlines()) + temp_fh.close() print "Prod message:" - print daklib.utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1) + print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1) prompt = "[P]rod, Edit, Abandon, Quit ?" answer = "XXX" while prompt.find(answer) == -1: - answer = daklib.utils.our_raw_input(prompt) - m = daklib.queue.re_default_answer.search(prompt) + answer = utils.our_raw_input(prompt) + m = re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() - os.unlink(temp_filename) - if answer == 'A': - return - elif answer == 'Q': - sys.exit(0) + os.unlink(temp_filename) + if answer == 'A': + return + elif answer == 'Q': + end() + sys.exit(0) # Otherwise, do the proding... - user_email_address = daklib.utils.whoami() + " <%s>" % ( - Cnf["Dinstall::MyAdminAddress"]) + user_email_address = utils.whoami() + " <%s>" % ( + cnf["Dinstall::MyAdminAddress"]) - Subst = Upload.Subst + Subst = upload.Subst Subst["__FROM_ADDRESS__"] = user_email_address Subst["__PROD_MESSAGE__"] = prod_message - Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"] + Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"] - prod_mail_message = daklib.utils.TemplateSubst( - Subst,Cnf["Dir::Templates"]+"/process-new.prod") + prod_mail_message = utils.TemplateSubst( + Subst,cnf["Dir::Templates"]+"/process-new.prod") # Send the prod mail if appropriate - if not Cnf["Dinstall::Options::No-Mail"]: - daklib.utils.send_mail(prod_mail_message) + if not cnf["Dinstall::Options::No-Mail"]: + utils.send_mail(prod_mail_message) print "Sent proding message" ################################################################################ -def do_new(): +def do_new(upload, session): print "NEW\n" - files = Upload.pkg.files - changes = Upload.pkg.changes + files = upload.pkg.files + changes = upload.pkg.changes + cnf = Config() # Make a copy of distribution we can happily trample on changes["suite"] = copy.copy(changes["distribution"]) # Fix up the list of target suites for suite in changes["suite"].keys(): - override = Cnf.Find("Suite::%s::OverrideSuite" % (suite)) + override = cnf.Find("Suite::%s::OverrideSuite" % (suite)) if override: - (olderr, newerr) = (daklib.database.get_suite_id(suite) == -1, - daklib.database.get_suite_id(override) == -1) - if olderr or newerr: - (oinv, newinv) = ("", "") - if olderr: oinv = "invalid " - if newerr: ninv = "invalid " - print "warning: overriding %ssuite %s to %ssuite %s" % ( - oinv, suite, ninv, override) + (olderr, newerr) = (get_suite(suite, session) == None, + get_suite(override, session) == None) + if olderr or newerr: + (oinv, newinv) = ("", "") + if olderr: oinv = "invalid " + if newerr: ninv = "invalid " + print "warning: overriding %ssuite %s to %ssuite %s" % ( + oinv, suite, ninv, override) del changes["suite"][suite] changes["suite"][override] = 1 # Validate suites for suite in changes["suite"].keys(): - suite_id = daklib.database.get_suite_id(suite) - if suite_id == -1: - daklib.utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite)) + if get_suite(suite, session) is None: + utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite)) # The main NEW processing loop done = 0 @@ -730,7 +637,7 @@ def do_new(): if Options["No-Action"] or Options["Automatic"]: answer = 'S' - (broken, note) = print_new(new, 0) + (broken, note) = print_new(new, upload, indexed=0) prompt = "" if not broken and not note: @@ -739,39 +646,65 @@ def do_new(): print "W: [!] marked entries must be fixed before package can be processed." if note: print "W: note must be removed before package can be processed." - prompt += "Remove note, " + prompt += "RemOve all notes, Remove note, " prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?" while prompt.find(answer) == -1: - answer = daklib.utils.our_raw_input(prompt) - m = daklib.queue.re_default_answer.search(prompt) + answer = utils.our_raw_input(prompt) + m = re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() - if answer == 'A': - done = add_overrides (new) + if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]: + utils.warn("Trainees can't do that") + continue + + if answer == 'A' and not Options["Trainee"]: + try: + check_daily_lock() + done = add_overrides (new, upload, session) + Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (upload.pkg.changes_file)]) + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'C': - check_pkg() - elif answer == 'E': - new = edit_overrides (new) - elif answer == 'M': - aborted = Upload.do_reject(1, Options["Manual-Reject"]) + check_pkg(upload) + elif answer == 'E' and not Options["Trainee"]: + new = edit_overrides (new, upload, session) + elif answer == 'M' and not Options["Trainee"]: + aborted = upload.do_reject(manual=1, + reject_message=Options["Manual-Reject"], + note=get_new_comments(changes.get("source", ""), session=session)) if not aborted: - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + Logger.log([utils.getusername(), "NEW REJECT: %s" % (upload.pkg.changes_file)]) + os.unlink(upload.pkg.changes_file[:-8]+".dak") done = 1 elif answer == 'N': - edit_note(changes.get("process-new note", "")) - elif answer == 'P': - prod_maintainer() - elif answer == 'R': - confirm = daklib.utils.our_raw_input("Really clear note (y/N)? ").lower() + edit_note(get_new_comments(changes.get("source", ""), session=session), + upload, session) + elif answer == 'P' and not Options["Trainee"]: + prod_maintainer(get_new_comments(changes.get("source", ""), session=session), + upload) + Logger.log([utils.getusername(), "NEW PROD: %s" % (upload.pkg.changes_file)]) + elif answer == 'R' and not Options["Trainee"]: + confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() + if confirm == "y": + for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session): + session.delete(c) + session.commit() + elif answer == 'O' and not Options["Trainee"]: + confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower() if confirm == "y": - del changes["process-new note"] + for c in get_new_comments(changes.get("source", ""), session=session): + session.delete(c) + session.commit() + elif answer == 'S': done = 1 elif answer == 'Q': + end() sys.exit(0) ################################################################################ @@ -784,60 +717,26 @@ def usage (exit_code=0): -h, --help show this help and exit. -m, --manual-reject=MSG manual reject with `msg' -n, --no-action don't do anything + -t, --trainee FTP Trainee mode -V, --version display the version number and exit""" sys.exit(exit_code) ################################################################################ -def init(): - global Cnf, Options, Logger, Upload, projectB, Sections, Priorities - - Cnf = daklib.utils.get_conf() - - Arguments = [('a',"automatic","Process-New::Options::Automatic"), - ('h',"help","Process-New::Options::Help"), - ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), - ('n',"no-action","Process-New::Options::No-Action")] - - for i in ["automatic", "help", "manual-reject", "no-action", "version"]: - if not Cnf.has_key("Process-New::Options::%s" % (i)): - Cnf["Process-New::Options::%s" % (i)] = "" - - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) - Options = Cnf.SubTree("Process-New::Options") - - if Options["Help"]: - usage() - - Upload = daklib.queue.Upload(Cnf) - - if not Options["No-Action"]: - Logger = Upload.Logger = daklib.logging.Logger(Cnf, "process-new") - - projectB = Upload.projectB - - Sections = Section_Completer() - Priorities = Priority_Completer() - readline.parse_and_bind("tab: complete") - - return changes_files - -################################################################################ - -def do_byhand(): +def do_byhand(upload, session): done = 0 while not done: - files = Upload.pkg.files + files = upload.pkg.files will_install = 1 byhand = [] - for file in files.keys(): - if files[file]["type"] == "byhand": - if os.path.exists(file): - print "W: %s still present; please process byhand components and try again." % (file) + for f in files.keys(): + if files[f]["type"] == "byhand": + if os.path.exists(f): + print "W: %s still present; please process byhand components and try again." % (f) will_install = 0 else: - byhand.append(file) + byhand.append(f) answer = "XXXX" if Options["No-Action"]: @@ -850,118 +749,200 @@ def do_byhand(): prompt = "Manual reject, [S]kip, Quit ?" while prompt.find(answer) == -1: - answer = daklib.utils.our_raw_input(prompt) - m = daklib.queue.re_default_answer.search(prompt) + answer = utils.our_raw_input(prompt) + m = re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() if answer == 'A': - done = 1 - for file in byhand: - del files[file] + try: + check_daily_lock() + done = 1 + for f in byhand: + del files[f] + Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'M': - Upload.do_reject(1, Options["Manual-Reject"]) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (upload.pkg.changes_file)]) + upload.do_reject(manual=1, reject_message=Options["Manual-Reject"]) + os.unlink(upload.pkg.changes_file[:-8]+".dak") done = 1 elif answer == 'S': done = 1 elif answer == 'Q': + end() sys.exit(0) ################################################################################ -def do_accept(): +def check_daily_lock(): + """ + Raises CantGetLockError if the dinstall daily.lock exists. + """ + + cnf = Config() + try: + os.open(cnf["Process-New::DinstallLockFile"], + os.O_RDONLY | os.O_CREAT | os.O_EXCL) + except OSError, e: + if e.errno == errno.EEXIST or e.errno == errno.EACCES: + raise CantGetLockError + + os.unlink(cnf["Process-New::DinstallLockFile"]) + + +@contextlib.contextmanager +def lock_package(package): + """ + Lock C{package} so that noone else jumps in processing it. + + @type package: string + @param package: source package name to lock + """ + + path = os.path.join(Config()["Process-New::LockDir"], package) + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) + except OSError, e: + if e.errno == errno.EEXIST or e.errno == errno.EACCES: + user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '') + raise AlreadyLockedError, user + + try: + yield fd + finally: + os.unlink(path) + +def _accept(upload): + if Options["No-Action"]: + return + (summary, short_summary) = upload.build_summaries() + upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"]) + os.unlink(upload.pkg.changes_file[:-8]+".dak") + +def do_accept(upload): print "ACCEPT" + cnf = Config() if not Options["No-Action"]: - retry = 0 - while retry < 10: - try: - lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) - retry = 10 - except OSError, e: - if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST': - retry += 1 - if (retry >= 10): - daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.") - else: - print("Unable to get accepted lock (try %d of 10)" % retry) - time.sleep(60) - else: - raise - (summary, short_summary) = Upload.build_summaries() - Upload.accept(summary, short_summary) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") - os.unlink(Cnf["Process-New::AcceptedLockFile"]) - -def check_status(files): - new = byhand = 0 - for file in files.keys(): - if files[file]["type"] == "byhand": - byhand = 1 - elif files[file].has_key("new"): - new = 1 - return (new, byhand) - -def do_pkg(changes_file): - Upload.pkg.changes_file = changes_file - Upload.init_vars() - Upload.update_vars() - Upload.update_subst() - files = Upload.pkg.files - - if not recheck(): - return + (summary, short_summary) = upload.build_summaries() + + if cnf.FindB("Dinstall::SecurityQueueHandling"): + upload.dump_vars(cnf["Dir::Queue::Embargoed"]) + upload.move_to_dir(cnf["Dir::Queue::Embargoed"]) + upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"]) + # Check for override disparities + upload.Subst["__SUMMARY__"] = summary + else: + # Just a normal upload, accept it... + _accept(upload) + +def do_pkg(changes_file, session): + u = Upload() + u.pkg.load_dot_dak(changes_file) + u.update_subst() + + cnf = Config() + bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $" + if cnf.has_key("Dinstall::Bcc"): + u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) + else: + u.Subst["__BCC__"] = bcc - (new, byhand) = check_status(files) - if new or byhand: - if new: - do_new() - if byhand: - do_byhand() - (new, byhand) = check_status(files) + files = u.pkg.files - if not new and not byhand: - do_accept() + try: + with lock_package(u.pkg.changes["source"]): + if not recheck(u, session): + return + + (new, byhand) = check_status(files) + if new or byhand: + if new: + do_new(u, session) + if byhand: + do_byhand(u, session) + (new, byhand) = check_status(files) + + if not new and not byhand: + try: + check_daily_lock() + do_accept(u) + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" + except AlreadyLockedError, e: + print "Seems to be locked by %s already, skipping..." % (e) ################################################################################ def end(): - accept_count = Upload.accept_count - accept_bytes = Upload.accept_bytes + accept_count = SummaryStats().accept_count + accept_bytes = SummaryStats().accept_bytes if accept_count: sets = "set" if accept_count > 1: sets = "sets" - sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, daklib.utils.size_type(int(accept_bytes)))) - Logger.log(["total",accept_count,accept_bytes]) + sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes)))) + Logger.log([utils.getusername(), "total",accept_count,accept_bytes]) - if not Options["No-Action"]: + if not Options["No-Action"] and not Options["Trainee"]: Logger.close() ################################################################################ def main(): - changes_files = init() - if len(changes_files) > 50: + global Options, Logger, Sections, Priorities + + cnf = Config() + session = DBConn().session() + + Arguments = [('a',"automatic","Process-New::Options::Automatic"), + ('h',"help","Process-New::Options::Help"), + ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), + ('t',"trainee","Process-New::Options::Trainee"), + ('n',"no-action","Process-New::Options::No-Action")] + + for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]: + if not cnf.has_key("Process-New::Options::%s" % (i)): + cnf["Process-New::Options::%s" % (i)] = "" + + changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv) + if len(changes_files) == 0: + changes_files = utils.get_changes_files(cnf["Dir::Queue::New"]) + + Options = cnf.SubTree("Process-New::Options") + + if Options["Help"]: + usage() + + if not Options["No-Action"]: + try: + Logger = daklog.Logger(cnf, "process-new") + except CantOpenError, e: + Options["Trainee"] = "True" + + Sections = Section_Completer(session) + Priorities = Priority_Completer(session) + readline.parse_and_bind("tab: complete") + + if len(changes_files) > 1: sys.stderr.write("Sorting changes...\n") - changes_files = sort_changes(changes_files) + changes_files = sort_changes(changes_files, session) # Kill me now? **FIXME** - Cnf["Dinstall::Options::No-Mail"] = "" - bcc = "X-DAK: dak process-new\nX-Katie: this header is obsolete" - if Cnf.has_key("Dinstall::Bcc"): - Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]) - else: - Upload.Subst["__BCC__"] = bcc + cnf["Dinstall::Options::No-Mail"] = "" for changes_file in changes_files: - changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0) + changes_file = utils.validate_changes_file_arg(changes_file, 0) if not changes_file: continue print "\n" + changes_file - do_pkg (changes_file) + + do_pkg (changes_file, session) end()