From: Mark Hymers Date: Tue, 27 Oct 2009 10:04:35 +0000 (+0000) Subject: Merge commit 'djpig/djpig' X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=832b7e7007fae655b4850f55dd35ef95a92a5373;hp=0a77b3e40ef9bcc95555fb9a791023efa3777261;p=dak.git Merge commit 'djpig/djpig' Conflicts: daklib/dbconn.py Signed-off-by: Mark Hymers --- diff --git a/dak/process_new.py b/dak/process_new.py index f15a5600..f01f4b91 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -6,6 +6,7 @@ @contact: Debian FTP Master @copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup @copyright: 2009 Joerg Jaspert +@copyright: 2009 Frank Lichtenheld @license: GNU General Public License version 2 or later """ # This program is free software; you can redistribute it and/or modify @@ -55,40 +56,30 @@ import pwd import apt_pkg, apt_inst import examine_package -from daklib import database +from daklib.dbconn import * +from daklib.queue import * from daklib import daklog -from daklib import queue from daklib import utils from daklib.regexes import re_no_epoch, re_default_answer, re_isanum from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError from daklib.summarystats import SummaryStats +from daklib.config import Config # Globals -Cnf = None #: Configuration, apt_pkg.Configuration Options = None -Upload = None -projectB = None #: database connection, pgobject Logger = None Priorities = None Sections = None -reject_message = "" - ################################################################################ ################################################################################ ################################################################################ -def reject (str, prefix="Rejected: "): - global reject_message - if str: - reject_message += prefix + str + "\n" - -def recheck(): - global reject_message - files = Upload.pkg.files - reject_message = "" +def recheck(upload, session): + files = upload.pkg.files + cnf = Config() for f in files.keys(): # The .orig.tar.gz can disappear out from under us is it's a # duplicate of one in the archive. @@ -98,32 +89,31 @@ def recheck(): if files[f]["type"] == "deb": source_version = files[f]["source version"] source_package = files[f]["source package"] - if not Upload.pkg.changes["architecture"].has_key("source") \ - and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()): + if not upload.pkg.changes["architecture"].has_key("source") \ + and not upload.source_exists(source_package, source_version, upload.pkg.changes["distribution"].keys()): source_epochless_version = re_no_epoch.sub('', source_version) dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) found = 0 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]: - if Cnf.has_key("Dir::Queue::%s" % (q)): - if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename): + if cnf.has_key("Dir::Queue::%s" % (q)): + if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename): found = 1 if not found: - reject("no source found for %s %s (%s)." % (source_package, source_version, f)) + upload.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f)) # Version and file overwrite checks if files[f]["type"] == "deb": - reject(Upload.check_binary_against_db(f), "") + upload.check_binary_against_db(f, session) elif files[f]["type"] == "dsc": - reject(Upload.check_source_against_db(f), "") - (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f) - reject(reject_msg, "") + upload.check_source_against_db(f, session) + upload.check_dsc_against_db(f, session) - if reject_message.find("Rejected") != -1: + if len(upload.rejects) > 0: answer = "XXX" if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]: answer = 'S' - print "REJECT\n" + reject_message, + print "REJECT\n" + upload.rejects.join("\n"), prompt = "[R]eject, Skip, Quit ?" while prompt.find(answer) == -1: @@ -134,8 +124,8 @@ def recheck(): answer = answer[:1].upper() if answer == 'R': - Upload.do_reject(0, reject_message) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + upload.do_reject(manual=0, reject_message=upload.rejects.join("\n")) + os.unlink(upload.pkg.changes_file[:-8]+".dak") return 0 elif answer == 'S': return 0 @@ -186,7 +176,7 @@ def sg_compare (a, b): # Sort by time of oldest upload return cmp(a["oldest"], b["oldest"]) -def sort_changes(changes_files): +def sort_changes(changes_files, session): """Sort into source groups, then sort each source group by version, have source, filename. Finally, sort the source groups by have note, time of oldest upload of each source upload.""" @@ -197,11 +187,11 @@ def sort_changes(changes_files): cache = {} # Read in all the .changes files for filename in changes_files: + u = Upload() try: - Upload.pkg.changes_file = filename - Upload.init_vars() - Upload.update_vars() - cache[filename] = copy.copy(Upload.pkg.changes) + u.pkg.load_dot_dak(filename) + u.update_subst() + cache[filename] = copy.copy(u.pkg.changes) cache[filename]["filename"] = filename except: sorted_list.append(filename) @@ -216,9 +206,8 @@ def sort_changes(changes_files): per_source[source]["list"].append(cache[filename]) # Determine oldest time and have note status for each source group for source in per_source.keys(): - q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source) - ql = q.getresult() - per_source[source]["source_in_database"] = len(ql)>0 + q = session.query(DBSource).filter_by(source = source).all() + per_source[source]["source_in_database"] = len(q)>0 source_list = per_source[source]["list"] first = source_list[0] oldest = os.stat(first["filename"])[stat.ST_MTIME] @@ -227,7 +216,7 @@ def sort_changes(changes_files): mtime = os.stat(d["filename"])[stat.ST_MTIME] if mtime < oldest: oldest = mtime - have_note += (database.has_new_comment(d["source"], d["version"], True)) + have_note += has_new_comment(d["source"], d["version"], session) per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none @@ -246,12 +235,11 @@ def sort_changes(changes_files): ################################################################################ class Section_Completer: - def __init__ (self): + def __init__ (self, session): self.sections = [] self.matches = [] - q = projectB.query("SELECT section FROM section") - for i in q.getresult(): - self.sections.append(i[0]) + for s, in session.query(Section.section): + self.sections.append(s) def complete(self, text, state): if state == 0: @@ -268,12 +256,11 @@ class Section_Completer: ############################################################ class Priority_Completer: - def __init__ (self): + def __init__ (self, session): self.priorities = [] self.matches = [] - q = projectB.query("SELECT priority FROM priority") - for i in q.getresult(): - self.priorities.append(i[0]) + for p, in session.query(Priority.priority): + self.priorities.append(p) def complete(self, text, state): if state == 0: @@ -289,9 +276,9 @@ class Priority_Completer: ################################################################################ -def print_new (new, indexed, file=sys.stdout): - queue.check_valid(new) - broken = 0 +def print_new (new, upload, indexed, file=sys.stdout): + check_valid(new) + broken = False index = 0 for pkg in new.keys(): index += 1 @@ -299,21 +286,22 @@ def print_new (new, indexed, file=sys.stdout): priority = new[pkg]["priority"] if new[pkg]["section id"] == -1: section += "[!]" - broken = 1 + broken = True if new[pkg]["priority id"] == -1: priority += "[!]" - broken = 1 + broken = True if indexed: line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section) else: line = "%-20s %-20s %-20s" % (pkg, priority, section) line = line.strip()+'\n' file.write(line) - note = database.get_new_comments(Upload.pkg.changes.get("source")) - if len(note) > 0: - for line in note: - print line - return broken, note + notes = get_new_comments(upload.pkg.changes.get("source")) + for note in notes: + print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \ + % (note.author, note.version, note.notedate, note.comment) + print "-" * 72 + return broken, len(notes) > 0 ################################################################################ @@ -326,11 +314,11 @@ def index_range (index): ################################################################################ ################################################################################ -def edit_new (new): +def edit_new (new, upload): # Write the current data to a temporary file (fd, temp_filename) = utils.temp_filename() temp_file = os.fdopen(fd, 'w') - print_new (new, 0, temp_file) + print_new (new, upload, indexed=0, file=temp_file) temp_file.close() # Spawn an editor on that file editor = os.environ.get("EDITOR","vi") @@ -360,14 +348,14 @@ def edit_new (new): if priority.endswith("[!]"): priority = priority[:-3] for f in new[pkg]["files"]: - Upload.pkg.files[f]["section"] = section - Upload.pkg.files[f]["priority"] = priority + upload.pkg.files[f]["section"] = section + upload.pkg.files[f]["priority"] = priority new[pkg]["section"] = section new[pkg]["priority"] = priority ################################################################################ -def edit_index (new, index): +def edit_index (new, upload, index): priority = new[index]["priority"] section = new[index]["section"] ftype = new[index]["type"] @@ -426,19 +414,19 @@ def edit_index (new, index): readline.set_completer(None) for f in new[index]["files"]: - Upload.pkg.files[f]["section"] = section - Upload.pkg.files[f]["priority"] = priority + upload.pkg.files[f]["section"] = section + upload.pkg.files[f]["priority"] = priority new[index]["priority"] = priority new[index]["section"] = section return new ################################################################################ -def edit_overrides (new): +def edit_overrides (new, upload, session): print done = 0 while not done: - print_new (new, 1) + print_new (new, upload, indexed=1) new_index = {} index = 0 for i in new.keys(): @@ -462,17 +450,17 @@ def edit_overrides (new): got_answer = 1 if answer == 'E': - edit_new(new) + edit_new(new, upload) elif answer == 'D': done = 1 else: - edit_index (new, new_index[answer]) + edit_index (new, upload, new_index[answer]) return new ################################################################################ -def edit_note(note): +def edit_note(note, upload, session): # Write the current data to a temporary file (fd, temp_filename) = utils.temp_filename() editor = os.environ.get("EDITOR","vi") @@ -499,19 +487,26 @@ def edit_note(note): end() sys.exit(0) - database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami(), bool(Options["Trainee"])) + comment = NewComment() + comment.package = upload.pkg.changes["source"] + comment.version = upload.pkg.changes["version"] + comment.comment = newnote + comment.author = utils.whoami() + comment.trainee = bool(Options["Trainee"]) + session.add(comment) + session.commit() ################################################################################ -def check_pkg (): +def check_pkg (upload): try: less_fd = os.popen("less -R -", 'w', 0) stdout_fd = sys.stdout try: sys.stdout = less_fd - changes = utils.parse_changes (Upload.pkg.changes_file) - examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file) - files = Upload.pkg.files + changes = utils.parse_changes (upload.pkg.changes_file) + examine_package.display_changes(changes['distribution'], upload.pkg.changes_file) + files = upload.pkg.files for f in files.keys(): if files[f].has_key("new"): ftype = files[f]["type"] @@ -536,8 +531,8 @@ def check_pkg (): ## FIXME: horribly Debian specific -def do_bxa_notification(): - files = Upload.pkg.files +def do_bxa_notification(upload): + files = upload.pkg.files summary = "" for f in files.keys(): if files[f]["type"] == "deb": @@ -545,40 +540,41 @@ def do_bxa_notification(): summary += "\n" summary += "Package: %s\n" % (control.Find("Package")) summary += "Description: %s\n" % (control.Find("Description")) - Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary - bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification") + upload.Subst["__BINARY_DESCRIPTIONS__"] = summary + bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification") utils.send_mail(bxa_mail) ################################################################################ -def add_overrides (new): - changes = Upload.pkg.changes - files = Upload.pkg.files +def add_overrides (new, upload, session): + changes = upload.pkg.changes + files = upload.pkg.files srcpkg = changes.get("source") - projectB.query("BEGIN WORK") for suite in changes["suite"].keys(): - suite_id = database.get_suite_id(suite) + suite_id = get_suite(suite).suite_id for pkg in new.keys(): - component_id = database.get_component_id(new[pkg]["component"]) - type_id = database.get_override_type_id(new[pkg]["type"]) + component_id = get_component(new[pkg]["component"]).component_id + type_id = get_override_type(new[pkg]["type"]).overridetype_id priority_id = new[pkg]["priority id"] section_id = new[pkg]["section id"] Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]]) - projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id)) + session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')", + { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id}) for f in new[pkg]["files"]: if files[f].has_key("new"): del files[f]["new"] del new[pkg] - projectB.query("COMMIT WORK") + session.commit() - if Cnf.FindB("Dinstall::BXANotify"): - do_bxa_notification() + if Config().FindB("Dinstall::BXANotify"): + do_bxa_notification(upload) ################################################################################ -def prod_maintainer (note): +def prod_maintainer (note, upload): + cnf = Config() # Here we prepare an editor and get them ready to prod... (fd, temp_filename) = utils.temp_filename() temp_file = os.fdopen(fd, 'w') @@ -611,39 +607,40 @@ def prod_maintainer (note): sys.exit(0) # Otherwise, do the proding... user_email_address = utils.whoami() + " <%s>" % ( - Cnf["Dinstall::MyAdminAddress"]) + cnf["Dinstall::MyAdminAddress"]) - Subst = Upload.Subst + Subst = upload.Subst Subst["__FROM_ADDRESS__"] = user_email_address Subst["__PROD_MESSAGE__"] = prod_message - Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"] + Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"] prod_mail_message = utils.TemplateSubst( - Subst,Cnf["Dir::Templates"]+"/process-new.prod") + Subst,cnf["Dir::Templates"]+"/process-new.prod") # Send the prod mail if appropriate - if not Cnf["Dinstall::Options::No-Mail"]: + if not cnf["Dinstall::Options::No-Mail"]: utils.send_mail(prod_mail_message) print "Sent proding message" ################################################################################ -def do_new(): +def do_new(upload, session): print "NEW\n" - files = Upload.pkg.files - changes = Upload.pkg.changes + files = upload.pkg.files + changes = upload.pkg.changes + cnf = Config() # Make a copy of distribution we can happily trample on changes["suite"] = copy.copy(changes["distribution"]) # Fix up the list of target suites for suite in changes["suite"].keys(): - override = Cnf.Find("Suite::%s::OverrideSuite" % (suite)) + override = cnf.Find("Suite::%s::OverrideSuite" % (suite)) if override: - (olderr, newerr) = (database.get_suite_id(suite) == -1, - database.get_suite_id(override) == -1) + (olderr, newerr) = (get_suite(suite, session) == None, + get_suite(override, session) == None) if olderr or newerr: (oinv, newinv) = ("", "") if olderr: oinv = "invalid " @@ -654,15 +651,14 @@ def do_new(): changes["suite"][override] = 1 # Validate suites for suite in changes["suite"].keys(): - suite_id = database.get_suite_id(suite) - if suite_id == -1: + if get_suite(suite, session) is None: utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite)) # The main NEW processing loop done = 0 while not done: # Find out what's new - new = queue.determine_new(changes, files, projectB) + new = determine_new(changes, files) if not new: break @@ -671,7 +667,7 @@ def do_new(): if Options["No-Action"] or Options["Automatic"]: answer = 'S' - (broken, note) = print_new(new, 0) + (broken, note) = print_new(new, upload, indexed=0) prompt = "" if not broken and not note: @@ -691,39 +687,50 @@ def do_new(): answer = m.group(1) answer = answer[:1].upper() + if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]: + utils.warn("Trainees can't do that") + continue + if answer == 'A' and not Options["Trainee"]: try: check_daily_lock() - done = add_overrides (new) - Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (Upload.pkg.changes_file)]) + done = add_overrides (new, upload, session) + Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (upload.pkg.changes_file)]) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'C': - check_pkg() + check_pkg(upload) elif answer == 'E' and not Options["Trainee"]: - new = edit_overrides (new) + new = edit_overrides (new, upload, session) elif answer == 'M' and not Options["Trainee"]: - aborted = Upload.do_reject(manual=1, + aborted = upload.do_reject(manual=1, reject_message=Options["Manual-Reject"], - note=database.get_new_comments(changes.get("source", ""))) + note=get_new_comments(changes.get("source", ""), session=session)) if not aborted: - Logger.log([utils.getusername(), "NEW REJECT: %s" % (Upload.pkg.changes_file)]) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + Logger.log([utils.getusername(), "NEW REJECT: %s" % (upload.pkg.changes_file)]) + os.unlink(upload.pkg.changes_file[:-8]+".dak") done = 1 elif answer == 'N': - edit_note(database.get_new_comments(changes.get("source", ""))) + edit_note(get_new_comments(changes.get("source", ""), session=session), + upload, session) elif answer == 'P' and not Options["Trainee"]: - prod_maintainer(database.get_new_comments(changes.get("source", ""))) - Logger.log([utils.getusername(), "NEW PROD: %s" % (Upload.pkg.changes_file)]) + prod_maintainer(get_new_comments(changes.get("source", ""), session=session), + upload) + Logger.log([utils.getusername(), "NEW PROD: %s" % (upload.pkg.changes_file)]) elif answer == 'R' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() if confirm == "y": - database.delete_new_comments(changes.get("source"), changes.get("version")) + for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session): + session.delete(c) + session.commit() elif answer == 'O' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower() if confirm == "y": - database.delete_all_new_comments(changes.get("source")) + for c in get_new_comments(changes.get("source", ""), session=session): + session.delete(c) + session.commit() + elif answer == 'S': done = 1 elif answer == 'Q': @@ -747,53 +754,10 @@ def usage (exit_code=0): ################################################################################ -def init(): - global Cnf, Options, Logger, Upload, projectB, Sections, Priorities - - Cnf = utils.get_conf() - - Arguments = [('a',"automatic","Process-New::Options::Automatic"), - ('h',"help","Process-New::Options::Help"), - ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"), - ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), - ('t',"trainee","Process-New::Options::Trainee"), - ('n',"no-action","Process-New::Options::No-Action")] - - for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]: - if not Cnf.has_key("Process-New::Options::%s" % (i)): - Cnf["Process-New::Options::%s" % (i)] = "" - - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) - if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""): - changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"]) - - Options = Cnf.SubTree("Process-New::Options") - - if Options["Help"]: - usage() - - Upload = queue.Upload(Cnf) - - if not Options["No-Action"]: - try: - Logger = Upload.Logger = daklog.Logger(Cnf, "process-new") - except CantOpenError, e: - Options["Trainee"] = "True" - - projectB = Upload.projectB - - Sections = Section_Completer() - Priorities = Priority_Completer() - readline.parse_and_bind("tab: complete") - - return changes_files - -################################################################################ - -def do_byhand(): +def do_byhand(upload, session): done = 0 while not done: - files = Upload.pkg.files + files = upload.pkg.files will_install = 1 byhand = [] @@ -828,14 +792,14 @@ def do_byhand(): done = 1 for f in byhand: del files[f] - Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (Upload.pkg.changes_file)]) + Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'M': - Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (Upload.pkg.changes_file)]) - Upload.do_reject(1, Options["Manual-Reject"]) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (upload.pkg.changes_file)]) + upload.do_reject(manual=1, reject_message=Options["Manual-Reject"]) + os.unlink(upload.pkg.changes_file[:-8]+".dak") done = 1 elif answer == 'S': done = 1 @@ -850,13 +814,15 @@ def check_daily_lock(): Raises CantGetLockError if the dinstall daily.lock exists. """ + cnf = Config() try: - os.open(Cnf["Process-New::DinstallLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) + os.open(cnf["Process-New::DinstallLockFile"], + os.O_RDONLY | os.O_CREAT | os.O_EXCL) except OSError, e: if e.errno == errno.EEXIST or e.errno == errno.EACCES: raise CantGetLockError - os.unlink(Cnf["Process-New::DinstallLockFile"]) + os.unlink(cnf["Process-New::DinstallLockFile"]) @contextlib.contextmanager @@ -868,7 +834,7 @@ def lock_package(package): @param package: source package name to lock """ - path = os.path.join(Cnf["Process-New::LockDir"], package) + path = os.path.join(Config()["Process-New::LockDir"], package) try: fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) except OSError, e: @@ -881,115 +847,118 @@ def lock_package(package): finally: os.unlink(path) -def move_to_dir (dest, perms=0660, changesperms=0664): - utils.move (Upload.pkg.changes_file, dest, perms=changesperms) - file_keys = Upload.pkg.files.keys() - for f in file_keys: - utils.move (f, dest, perms=perms) - -def is_source_in_queue_dir(qdir): - entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"]) - and x.endswith(".changes") ] - for entry in entries: - # read the .dak - u = queue.Upload(Cnf) - u.pkg.changes_file = os.path.join(qdir, entry) - u.update_vars() - if not u.pkg.changes["architecture"].has_key("source"): - # another binary upload, ignore - continue - if Upload.pkg.changes["version"] != u.pkg.changes["version"]: - # another version, ignore - continue - # found it! - return True - return False - -def move_to_holding(suite, queue_dir): - print "Moving to %s holding area." % (suite.upper(),) - if Options["No-Action"]: - return - Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file]) - Upload.dump_vars(queue_dir) - move_to_dir(queue_dir, perms=0664) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") - -def _accept(): +# def move_to_dir (upload, dest, perms=0660, changesperms=0664): +# utils.move (upload.pkg.changes_file, dest, perms=changesperms) +# file_keys = upload.pkg.files.keys() +# for f in file_keys: +# utils.move (f, dest, perms=perms) + +# def is_source_in_queue_dir(qdir): +# entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"]) +# and x.endswith(".changes") ] +# for entry in entries: +# # read the .dak +# u = queue.Upload(Cnf) +# u.pkg.changes_file = os.path.join(qdir, entry) +# u.update_vars() +# if not u.pkg.changes["architecture"].has_key("source"): +# # another binary upload, ignore +# continue +# if Upload.pkg.changes["version"] != u.pkg.changes["version"]: +# # another version, ignore +# continue +# # found it! +# return True +# return False + +# def move_to_holding(suite, queue_dir): +# print "Moving to %s holding area." % (suite.upper(),) +# if Options["No-Action"]: +# return +# Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file]) +# Upload.dump_vars(queue_dir) +# move_to_dir(queue_dir, perms=0664) +# os.unlink(Upload.pkg.changes_file[:-8]+".dak") + +def _accept(upload): if Options["No-Action"]: return - (summary, short_summary) = Upload.build_summaries() - Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"]) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") - -def do_accept_stableupdate(suite, q): - queue_dir = Cnf["Dir::Queue::%s" % (q,)] - if not Upload.pkg.changes["architecture"].has_key("source"): - # It is not a sourceful upload. So its source may be either in p-u - # holding, in new, in accepted or already installed. - if is_source_in_queue_dir(queue_dir): - # It's in p-u holding, so move it there. - print "Binary-only upload, source in %s." % (q,) - move_to_holding(suite, queue_dir) - elif Upload.source_exists(Upload.pkg.changes["source"], - Upload.pkg.changes["version"]): - # dak tells us that there is source available. At time of - # writing this means that it is installed, so put it into - # accepted. - print "Binary-only upload, source installed." - Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) - _accept() - elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]): - # The source is in accepted, the binary cleared NEW: accept it. - print "Binary-only upload, source in accepted." - Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) - _accept() - elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]): - # It's in NEW. We expect the source to land in p-u holding - # pretty soon. - print "Binary-only upload, source in new." - move_to_holding(suite, queue_dir) - elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]): - # It's in newstage. Accept into the holding area - print "Binary-only upload, source in newstage." - Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) - _accept() - else: - # No case applicable. Bail out. Return will cause the upload - # to be skipped. - print "ERROR" - print "Stable update failed. Source not found." - return - else: - # We are handling a sourceful upload. Move to accepted if currently - # in p-u holding and to p-u holding otherwise. - if is_source_in_queue_dir(queue_dir): - print "Sourceful upload in %s, accepting." % (q,) - _accept() - else: - move_to_holding(suite, queue_dir) - -def do_accept(): + (summary, short_summary) = upload.build_summaries() + upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"]) + os.unlink(upload.pkg.changes_file[:-8]+".dak") + +# def do_accept_stableupdate(upload,suite, q): +# cnf = Config() +# queue_dir = cnf["Dir::Queue::%s" % (q,)] +# if not upload.pkg.changes["architecture"].has_key("source"): +# # It is not a sourceful upload. So its source may be either in p-u +# # holding, in new, in accepted or already installed. +# if is_source_in_queue_dir(queue_dir): +# # It's in p-u holding, so move it there. +# print "Binary-only upload, source in %s." % (q,) +# move_to_holding(suite, queue_dir) +# elif Upload.source_exists(Upload.pkg.changes["source"], +# Upload.pkg.changes["version"]): +# # dak tells us that there is source available. At time of +# # writing this means that it is installed, so put it into +# # accepted. +# print "Binary-only upload, source installed." +# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) +# _accept() +# elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]): +# # The source is in accepted, the binary cleared NEW: accept it. +# print "Binary-only upload, source in accepted." +# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) +# _accept() +# elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]): +# # It's in NEW. We expect the source to land in p-u holding +# # pretty soon. +# print "Binary-only upload, source in new." +# move_to_holding(suite, queue_dir) +# elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]): +# # It's in newstage. Accept into the holding area +# print "Binary-only upload, source in newstage." +# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) +# _accept() +# else: +# # No case applicable. Bail out. Return will cause the upload +# # to be skipped. +# print "ERROR" +# print "Stable update failed. Source not found." +# return +# else: +# # We are handling a sourceful upload. Move to accepted if currently +# # in p-u holding and to p-u holding otherwise. +# if is_source_in_queue_dir(queue_dir): +# print "Sourceful upload in %s, accepting." % (q,) +# _accept() +# else: +# move_to_holding(suite, queue_dir) + +def do_accept(upload): print "ACCEPT" + cnf = Config() if not Options["No-Action"]: - (summary, short_summary) = Upload.build_summaries() - if Cnf.FindB("Dinstall::SecurityQueueHandling"): - Upload.dump_vars(Cnf["Dir::Queue::Embargoed"]) - move_to_dir(Cnf["Dir::Queue::Embargoed"]) - Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"]) - # Check for override disparities - Upload.Subst["__SUMMARY__"] = summary - else: + (summary, short_summary) = upload.build_summaries() +# if cnf.FindB("Dinstall::SecurityQueueHandling"): +# upload.dump_vars(cnf["Dir::Queue::Embargoed"]) +# move_to_dir(cnf["Dir::Queue::Embargoed"]) +# upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"]) +# # Check for override disparities +# upload.Subst["__SUMMARY__"] = summary +# else: # Stable updates need to be copied to proposed-updates holding # area instead of accepted. Sourceful uploads need to go # to it directly, binaries only if the source has not yet been # accepted into p-u. for suite, q in [("proposed-updates", "ProposedUpdates"), ("oldstable-proposed-updates", "OldProposedUpdates")]: - if not Upload.pkg.changes["distribution"].has_key(suite): + if not upload.pkg.changes["distribution"].has_key(suite): continue - return do_accept_stableupdate(suite, q) + utils.fubar("stable accept not supported yet") +# return do_accept_stableupdate(suite, q) # Just a normal upload, accept it... - _accept() + _accept(upload) def check_status(files): new = byhand = 0 @@ -1000,30 +969,37 @@ def check_status(files): new = 1 return (new, byhand) -def do_pkg(changes_file): - Upload.pkg.changes_file = changes_file - Upload.init_vars() - Upload.update_vars() - Upload.update_subst() - files = Upload.pkg.files +def do_pkg(changes_file, session): + u = Upload() + u.pkg.load_dot_dak(changes_file) + u.update_subst() + + cnf = Config() + bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $" + if cnf.has_key("Dinstall::Bcc"): + u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) + else: + u.Subst["__BCC__"] = bcc + + files = u.pkg.files try: - with lock_package(Upload.pkg.changes["source"]): - if not recheck(): + with lock_package(u.pkg.changes["source"]): + if not recheck(u, session): return (new, byhand) = check_status(files) if new or byhand: if new: - do_new() + do_new(u, session) if byhand: - do_byhand() + do_byhand(u, session) (new, byhand) = check_status(files) if not new and not byhand: try: check_daily_lock() - do_accept() + do_accept(u) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" print "Dinstall in the locked area, cant process packages, come back later" @@ -1048,87 +1024,117 @@ def end(): ################################################################################ -def do_comments(dir, opref, npref, line, fn): - for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]: - lines = open("%s/%s" % (dir, comm)).readlines() - if len(lines) == 0 or lines[0] != line + "\n": continue - changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_") - and x.endswith(".changes") ] - changes_files = sort_changes(changes_files) - for f in changes_files: - f = utils.validate_changes_file_arg(f, 0) - if not f: continue - print "\n" + f - fn(f, "".join(lines[1:])) - - if opref != npref and not Options["No-Action"]: - newcomm = npref + comm[len(opref):] - os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm)) +# def do_comments(dir, opref, npref, line, fn): +# for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]: +# lines = open("%s/%s" % (dir, comm)).readlines() +# if len(lines) == 0 or lines[0] != line + "\n": continue +# changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_") +# and x.endswith(".changes") ] +# changes_files = sort_changes(changes_files) +# for f in changes_files: +# f = utils.validate_changes_file_arg(f, 0) +# if not f: continue +# print "\n" + f +# fn(f, "".join(lines[1:])) + +# if opref != npref and not Options["No-Action"]: +# newcomm = npref + comm[len(opref):] +# os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm)) + +# ################################################################################ + +# def comment_accept(changes_file, comments): +# Upload.pkg.changes_file = changes_file +# Upload.init_vars() +# Upload.update_vars() +# Upload.update_subst() +# files = Upload.pkg.files + +# if not recheck(): +# return # dak wants to REJECT, crap + +# (new, byhand) = check_status(files) +# if not new and not byhand: +# do_accept() + +# ################################################################################ + +# def comment_reject(changes_file, comments): +# Upload.pkg.changes_file = changes_file +# Upload.init_vars() +# Upload.update_vars() +# Upload.update_subst() + +# if not recheck(): +# pass # dak has its own reasons to reject as well, which is fine + +# reject(comments) +# print "REJECT\n" + reject_message, +# if not Options["No-Action"]: +# Upload.do_reject(0, reject_message) +# os.unlink(Upload.pkg.changes_file[:-8]+".dak") ################################################################################ -def comment_accept(changes_file, comments): - Upload.pkg.changes_file = changes_file - Upload.init_vars() - Upload.update_vars() - Upload.update_subst() - files = Upload.pkg.files +def main(): + global Options, Logger, Sections, Priorities - if not recheck(): - return # dak wants to REJECT, crap + cnf = Config() + session = DBConn().session() - (new, byhand) = check_status(files) - if not new and not byhand: - do_accept() + Arguments = [('a',"automatic","Process-New::Options::Automatic"), + ('h',"help","Process-New::Options::Help"), + ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"), + ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), + ('t',"trainee","Process-New::Options::Trainee"), + ('n',"no-action","Process-New::Options::No-Action")] -################################################################################ + for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]: + if not cnf.has_key("Process-New::Options::%s" % (i)): + cnf["Process-New::Options::%s" % (i)] = "" -def comment_reject(changes_file, comments): - Upload.pkg.changes_file = changes_file - Upload.init_vars() - Upload.update_vars() - Upload.update_subst() + changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv) + if len(changes_files) == 0 and not cnf.get("Process-New::Options::Comments-Dir",""): + changes_files = utils.get_changes_files(cnf["Dir::Queue::New"]) - if not recheck(): - pass # dak has its own reasons to reject as well, which is fine + Options = cnf.SubTree("Process-New::Options") + + if Options["Help"]: + usage() - reject(comments) - print "REJECT\n" + reject_message, if not Options["No-Action"]: - Upload.do_reject(0, reject_message) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + try: + Logger = daklog.Logger(cnf, "process-new") + except CantOpenError, e: + Options["Trainee"] = "True" -################################################################################ + Sections = Section_Completer(session) + Priorities = Priority_Completer(session) + readline.parse_and_bind("tab: complete") -def main(): - changes_files = init() - if len(changes_files) > 50: + if len(changes_files) > 1: sys.stderr.write("Sorting changes...\n") - changes_files = sort_changes(changes_files) + changes_files = sort_changes(changes_files, session) # Kill me now? **FIXME** - Cnf["Dinstall::Options::No-Mail"] = "" - bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $" - if Cnf.has_key("Dinstall::Bcc"): - Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]) - else: - Upload.Subst["__BCC__"] = bcc - - commentsdir = Cnf.get("Process-New::Options::Comments-Dir","") - if commentsdir: - if changes_files != []: - sys.stderr.write("Can't specify any changes files if working with comments-dir") - sys.exit(1) - do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept) - do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject) - else: + cnf["Dinstall::Options::No-Mail"] = "" + +# commentsdir = cnf.get("Process-New::Options::Comments-Dir","") +# if commentsdir: +# if changes_files != []: +# sys.stderr.write("Can't specify any changes files if working with comments-dir") +# sys.exit(1) +# do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept) +# do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject) +# else: + if True: for changes_file in changes_files: changes_file = utils.validate_changes_file_arg(changes_file, 0) if not changes_file: continue print "\n" + changes_file - do_pkg (changes_file) + do_pkg (changes_file, session) end() diff --git a/daklib/dbconn.py b/daklib/dbconn.py index f5a19c12..8ee90768 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -168,7 +168,7 @@ __all__.append('Archive') @session_wrapper def get_archive(archive, session=None): """ - returns database id for given c{archive}. + returns database id for given C{archive}. @type archive: string @param archive: the name of the arhive @@ -1904,7 +1904,7 @@ def get_suite(suite, session=None): generated if not supplied) @rtype: Suite - @return: Suite object for the requested suite name (None if not presenT) + @return: Suite object for the requested suite name (None if not present) """ q = session.query(Suite).filter_by(suite_name=suite) diff --git a/daklib/queue.py b/daklib/queue.py index aca1b8a0..b1383be0 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -72,7 +72,7 @@ def get_type(f, session): """ # Determine the type if f.has_key("dbtype"): - file_type = file["dbtype"] + file_type = f["dbtype"] elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]: file_type = "dsc" else: @@ -353,7 +353,7 @@ class Upload(object): ########################################################################### def load_changes(self, filename): """ - @rtype boolean + @rtype: boolean @rvalue: whether the changes file was valid or not. We may want to reject even if this is True (see what gets put in self.rejects). This is simply to prevent us even trying things later which will @@ -1849,7 +1849,7 @@ distribution.""" user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"]) self.Subst["__REJECTOR_ADDRESS__"] = user_email_address self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message - self.Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"] + self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"] reject_mail_message = utils.TemplateSubst(self.Subst, rej_template) # Write the rejection email out as the .reason file os.write(reason_fd, reject_mail_message)