X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_new.py;h=55397c7dc79a4e2df577397d9262a6addb047df1;hb=245c6549dbacaeab5ee36ec74372b1df8675b477;hp=e052af49a1909ee1d7f249251dffab4afa9d3be2;hpb=9a14adff2ca4e7a5346c0c3b5723d9959022291a;p=dak.git diff --git a/dak/process_new.py b/dak/process_new.py index e052af49..55397c7d 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -42,8 +42,6 @@ ################################################################################ -from __future__ import with_statement - import copy import errno import os @@ -55,15 +53,19 @@ import contextlib import pwd import apt_pkg, apt_inst import examine_package +import subprocess +import daklib.daksubprocess +from sqlalchemy import or_ from daklib.dbconn import * from daklib.queue import * from daklib import daklog from daklib import utils -from daklib.regexes import re_no_epoch, re_default_answer, re_isanum +from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError from daklib.summarystats import SummaryStats from daklib.config import Config +from daklib.policy import UploadCopy, PolicyQueueUploadHandler # Globals Options = None @@ -76,134 +78,6 @@ Sections = None ################################################################################ ################################################################################ -def recheck(upload, session): - upload.recheck() - if len(upload.rejects) > 0: - answer = "XXX" - if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]: - answer = 'S' - - print "REJECT\n%s" % '\n'.join(upload.rejects) - prompt = "[R]eject, Skip, Quit ?" - - while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt) - m = re_default_answer.match(prompt) - if answer == "": - answer = m.group(1) - answer = answer[:1].upper() - - if answer == 'R': - upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects)) - os.unlink(upload.pkg.changes_file[:-8]+".dak") - return 0 - elif answer == 'S': - return 0 - elif answer == 'Q': - end() - sys.exit(0) - - return 1 - -################################################################################ - -def indiv_sg_compare (a, b): - """Sort by source name, source, version, 'have source', and - finally by filename.""" - # Sort by source version - q = apt_pkg.VersionCompare(a["version"], b["version"]) - if q: - return -q - - # Sort by 'have source' - a_has_source = a["architecture"].get("source") - b_has_source = b["architecture"].get("source") - if a_has_source and not b_has_source: - return -1 - elif b_has_source and not a_has_source: - return 1 - - return cmp(a["filename"], b["filename"]) - -############################################################ - -def sg_compare (a, b): - a = a[1] - b = b[1] - """Sort by have note, source already in database and time of oldest upload.""" - # Sort by have note - a_note_state = a["note_state"] - b_note_state = b["note_state"] - if a_note_state < b_note_state: - return -1 - elif a_note_state > b_note_state: - return 1 - # Sort by source already in database (descending) - source_in_database = cmp(a["source_in_database"], b["source_in_database"]) - if source_in_database: - return -source_in_database - - # Sort by time of oldest upload - return cmp(a["oldest"], b["oldest"]) - -def sort_changes(changes_files, session): - """Sort into source groups, then sort each source group by version, - have source, filename. Finally, sort the source groups by have - note, time of oldest upload of each source upload.""" - if len(changes_files) == 1: - return changes_files - - sorted_list = [] - cache = {} - # Read in all the .changes files - for filename in changes_files: - u = Upload() - try: - u.pkg.load_dot_dak(filename) - u.update_subst() - cache[filename] = copy.copy(u.pkg.changes) - cache[filename]["filename"] = filename - except: - sorted_list.append(filename) - break - # Divide the .changes into per-source groups - per_source = {} - for filename in cache.keys(): - source = cache[filename]["source"] - if not per_source.has_key(source): - per_source[source] = {} - per_source[source]["list"] = [] - per_source[source]["list"].append(cache[filename]) - # Determine oldest time and have note status for each source group - for source in per_source.keys(): - q = session.query(DBSource).filter_by(source = source).all() - per_source[source]["source_in_database"] = len(q)>0 - source_list = per_source[source]["list"] - first = source_list[0] - oldest = os.stat(first["filename"])[stat.ST_MTIME] - have_note = 0 - for d in per_source[source]["list"]: - mtime = os.stat(d["filename"])[stat.ST_MTIME] - if mtime < oldest: - oldest = mtime - have_note += has_new_comment(d["source"], d["version"], session) - per_source[source]["oldest"] = oldest - if not have_note: - per_source[source]["note_state"] = 0; # none - elif have_note < len(source_list): - per_source[source]["note_state"] = 1; # some - else: - per_source[source]["note_state"] = 2; # all - per_source[source]["list"].sort(indiv_sg_compare) - per_source_items = per_source.items() - per_source_items.sort(sg_compare) - for i in per_source_items: - for j in i[1]["list"]: - sorted_list.append(j["filename"]) - return sorted_list - -################################################################################ - class Section_Completer: def __init__ (self, session): self.sections = [] @@ -246,32 +120,61 @@ class Priority_Completer: ################################################################################ -def print_new (new, upload, indexed, file=sys.stdout): - check_valid(new) - broken = False +def takenover_binaries(upload, missing, session): + rows = [] + binaries = set([x.package for x in upload.binaries]) + for m in missing: + if m['type'] != 'dsc': + binaries.discard(m['package']) + if binaries: + source = upload.binaries[0].source.source + suite = upload.target_suite.overridesuite or \ + upload.target_suite.suite_name + suites = [s[0] for s in session.query(Suite.suite_name).filter \ + (or_(Suite.suite_name == suite, + Suite.overridesuite == suite)).all()] + rows = session.query(DBSource.source, DBBinary.package).distinct(). \ + filter(DBBinary.package.in_(binaries)). \ + join(DBBinary.source). \ + filter(DBSource.source != source). \ + join(DBBinary.suites). \ + filter(Suite.suite_name.in_(suites)). \ + order_by(DBSource.source, DBBinary.package).all() + return rows + +################################################################################ + +def print_new (upload, missing, indexed, session, file=sys.stdout): + check_valid(missing, session) index = 0 - for pkg in new.keys(): + for m in missing: index += 1 - section = new[pkg]["section"] - priority = new[pkg]["priority"] - if new[pkg]["section id"] == -1: - section += "[!]" - broken = True - if new[pkg]["priority id"] == -1: - priority += "[!]" - broken = True + if m['type'] != 'deb': + package = '{0}:{1}'.format(m['type'], m['package']) + else: + package = m['package'] + section = m['section'] + priority = m['priority'] + included = "" if m['included'] else "NOT UPLOADED" if indexed: - line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section) + line = "(%s): %-20s %-20s %-20s %s" % (index, package, priority, section, included) else: - line = "%-20s %-20s %-20s" % (pkg, priority, section) - line = line.strip()+'\n' - file.write(line) - notes = get_new_comments(upload.pkg.changes.get("source")) + line = "%-20s %-20s %-20s %s" % (package, priority, section, included) + line = line.strip() + if not m['valid']: + line = line + ' [!]' + print >>file, line + takenover = takenover_binaries(upload, missing, session) + if takenover: + print '\n\nBINARIES TAKEN OVER\n' + for t in takenover: + print '%s: %s' % (t[0], t[1]) + notes = get_new_comments(upload.policy_queue, upload.changes.source) for note in notes: print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \ % (note.author, note.version, note.notedate, note.comment) print "-" * 72 - return broken, len(notes) > 0 + return len(notes) > 0 ################################################################################ @@ -284,11 +187,11 @@ def index_range (index): ################################################################################ ################################################################################ -def edit_new (new, upload): +def edit_new (overrides, upload, session): # Write the current data to a temporary file (fd, temp_filename) = utils.temp_filename() temp_file = os.fdopen(fd, 'w') - print_new (new, upload, indexed=0, file=temp_file) + print_new (upload, overrides, indexed=0, session=session, file=temp_file) temp_file.close() # Spawn an editor on that file editor = os.environ.get("EDITOR","vi") @@ -300,38 +203,50 @@ def edit_new (new, upload): lines = temp_file.readlines() temp_file.close() os.unlink(temp_filename) + + overrides_map = dict([ ((o['type'], o['package']), o) for o in overrides ]) + new_overrides = [] # Parse the new data for line in lines: line = line.strip() - if line == "": + if line == "" or line[0] == '#': continue s = line.split() # Pad the list if necessary s[len(s):3] = [None] * (3-len(s)) (pkg, priority, section) = s[:3] - if not new.has_key(pkg): + if pkg.find(':') != -1: + type, pkg = pkg.split(':', 1) + else: + type = 'deb' + o = overrides_map.get((type, pkg), None) + if o is None: utils.warn("Ignoring unknown package '%s'" % (pkg)) else: - # Strip off any invalid markers, print_new will readd them. - if section.endswith("[!]"): - section = section[:-3] - if priority.endswith("[!]"): - priority = priority[:-3] - for f in new[pkg]["files"]: - upload.pkg.files[f]["section"] = section - upload.pkg.files[f]["priority"] = priority - new[pkg]["section"] = section - new[pkg]["priority"] = priority + if section.find('/') != -1: + component = section.split('/', 1)[0] + else: + component = 'main' + new_overrides.append(dict( + package=pkg, + type=type, + section=section, + component=component, + priority=priority, + included=o['included'], + )) + return new_overrides ################################################################################ def edit_index (new, upload, index): + package = new[index]['package'] priority = new[index]["priority"] section = new[index]["section"] ftype = new[index]["type"] done = 0 while not done: - print "\t".join([index, priority, section]) + print "\t".join([package, priority, section]) answer = "XXX" if ftype != "dsc": @@ -383,11 +298,14 @@ def edit_index (new, upload, index): # Reset the readline completer readline.set_completer(None) - for f in new[index]["files"]: - upload.pkg.files[f]["section"] = section - upload.pkg.files[f]["priority"] = priority new[index]["priority"] = priority new[index]["section"] = section + if section.find('/') != -1: + component = section.split('/', 1)[0] + else: + component = 'main' + new[index]['component'] = component + return new ################################################################################ @@ -396,14 +314,8 @@ def edit_overrides (new, upload, session): print done = 0 while not done: - print_new (new, upload, indexed=1) - new_index = {} - index = 0 - for i in new.keys(): - index += 1 - new_index[index] = i - - prompt = "(%s) edit override , Editor, Done ? " % (index_range(index)) + print_new (upload, new, indexed=1, session=session) + prompt = "edit override , Editor, Done ? " got_answer = 0 while not got_answer: @@ -414,154 +326,158 @@ def edit_overrides (new, upload, session): got_answer = 1 elif re_isanum.match (answer): answer = int(answer) - if (answer < 1) or (answer > index): - print "%s is not a valid index (%s). Please retry." % (answer, index_range(index)) + if answer < 1 or answer > len(new): + print "{0} is not a valid index. Please retry.".format(answer) else: got_answer = 1 if answer == 'E': - edit_new(new, upload) + new = edit_new(new, upload, session) elif answer == 'D': done = 1 else: - edit_index (new, upload, new_index[answer]) + edit_index (new, upload, answer - 1) return new -################################################################################ - -def edit_note(note, upload, session): - # Write the current data to a temporary file - (fd, temp_filename) = utils.temp_filename() - editor = os.environ.get("EDITOR","vi") - answer = 'E' - while answer == 'E': - os.system("%s %s" % (editor, temp_filename)) - temp_file = utils.open_file(temp_filename) - newnote = temp_file.read().rstrip() - temp_file.close() - print "New Note:" - print utils.prefix_multi_line_string(newnote," ") - prompt = "[D]one, Edit, Abandon, Quit ?" - answer = "XXX" - while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt) - m = re_default_answer.search(prompt) - if answer == "": - answer = m.group(1) - answer = answer[:1].upper() - os.unlink(temp_filename) - if answer == 'A': - return - elif answer == 'Q': - end() - sys.exit(0) - - comment = NewComment() - comment.package = upload.pkg.changes["source"] - comment.version = upload.pkg.changes["version"] - comment.comment = newnote - comment.author = utils.whoami() - comment.trainee = bool(Options["Trainee"]) - session.add(comment) - session.commit() ################################################################################ -def check_pkg (upload): +def check_pkg (upload, upload_copy, session): + missing = [] + save_stdout = sys.stdout + changes = os.path.join(upload_copy.directory, upload.changes.changesname) + suite_name = upload.target_suite.suite_name + handler = PolicyQueueUploadHandler(upload, session) + missing = [(m['type'], m["package"]) for m in handler.missing_overrides(hints=missing)] + + less_cmd = ("less", "-R", "-") + less_process = daklib.daksubprocess.Popen(less_cmd, bufsize=0, stdin=subprocess.PIPE) try: - less_fd = os.popen("less -R -", 'w', 0) - stdout_fd = sys.stdout - try: - sys.stdout = less_fd - changes = utils.parse_changes (upload.pkg.changes_file) - examine_package.display_changes(changes['distribution'], upload.pkg.changes_file) - files = upload.pkg.files - for f in files.keys(): - if files[f].has_key("new"): - ftype = files[f]["type"] - if ftype == "deb": - examine_package.check_deb(changes['distribution'], f) - elif ftype == "dsc": - examine_package.check_dsc(changes['distribution'], f) - finally: - examine_package.output_package_relations() - sys.stdout = stdout_fd - except IOError, e: + sys.stdout = less_process.stdin + print examine_package.display_changes(suite_name, changes) + + source = upload.source + if source is not None: + source_file = os.path.join(upload_copy.directory, os.path.basename(source.poolfile.filename)) + print examine_package.check_dsc(suite_name, source_file) + + for binary in upload.binaries: + binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename)) + examined = examine_package.check_deb(suite_name, binary_file) + # We always need to call check_deb to display package relations for every binary, + # but we print its output only if new overrides are being added. + if ("deb", binary.package) in missing: + print examined + + print examine_package.output_package_relations() + less_process.stdin.close() + except IOError as e: if e.errno == errno.EPIPE: utils.warn("[examine_package] Caught EPIPE; skipping.") - pass else: raise except KeyboardInterrupt: utils.warn("[examine_package] Caught C-c; skipping.") - pass + finally: + less_process.wait() + sys.stdout = save_stdout ################################################################################ ## FIXME: horribly Debian specific -def do_bxa_notification(upload): - files = upload.pkg.files +def do_bxa_notification(new, upload, session): + cnf = Config() + + new = set([ o['package'] for o in new if o['type'] == 'deb' ]) + if len(new) == 0: + return + + key = session.query(MetadataKey).filter_by(key='Description').one() summary = "" - for f in files.keys(): - if files[f]["type"] == "deb": - control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f))) - summary += "\n" - summary += "Package: %s\n" % (control.Find("Package")) - summary += "Description: %s\n" % (control.Find("Description")) - upload.Subst["__BINARY_DESCRIPTIONS__"] = summary - bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification") + for binary in upload.binaries: + if binary.package not in new: + continue + description = session.query(BinaryMetadata).filter_by(binary=binary, key=key).one().value + summary += "\n" + summary += "Package: {0}\n".format(binary.package) + summary += "Description: {0}\n".format(description) + + subst = { + '__DISTRO__': cnf['Dinstall::MyDistribution'], + '__BCC__': 'X-DAK: dak process-new', + '__BINARY_DESCRIPTIONS__': summary, + } + + bxa_mail = utils.TemplateSubst(subst,os.path.join(cnf["Dir::Templates"], "process-new.bxa_notification")) utils.send_mail(bxa_mail) ################################################################################ -def add_overrides (new, upload, session): - changes = upload.pkg.changes - files = upload.pkg.files - srcpkg = changes.get("source") - - for suite in changes["suite"].keys(): - suite_id = get_suite(suite).suite_id - for pkg in new.keys(): - component_id = get_component(new[pkg]["component"]).component_id - type_id = get_override_type(new[pkg]["type"]).overridetype_id - priority_id = new[pkg]["priority id"] - section_id = new[pkg]["section id"] - Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]]) - session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')", - { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id}) - for f in new[pkg]["files"]: - if files[f].has_key("new"): - del files[f]["new"] - del new[pkg] +def add_overrides (new_overrides, suite, session): + if suite.overridesuite is not None: + suite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one() + + for override in new_overrides: + package = override['package'] + priority = session.query(Priority).filter_by(priority=override['priority']).first() + section = session.query(Section).filter_by(section=override['section']).first() + component = get_mapped_component(override['component'], session) + overridetype = session.query(OverrideType).filter_by(overridetype=override['type']).one() + + if priority is None: + raise Exception('Invalid priority {0} for package {1}'.format(priority, package)) + if section is None: + raise Exception('Invalid section {0} for package {1}'.format(section, package)) + if component is None: + raise Exception('Invalid component {0} for package {1}'.format(component, package)) + + o = Override(package=package, suite=suite, component=component, priority=priority, section=section, overridetype=overridetype) + session.add(o) session.commit() - if Config().FindB("Dinstall::BXANotify"): - do_bxa_notification(upload) +################################################################################ + +def run_user_inspect_command(upload, upload_copy): + command = os.environ.get('DAK_INSPECT_UPLOAD') + if command is None: + return + + directory = upload_copy.directory + if upload.source: + dsc = os.path.basename(upload.source.poolfile.filename) + else: + dsc = '' + changes = upload.changes.changesname + + shell_command = command.format( + directory=directory, + dsc=dsc, + changes=changes, + ) + + daklib.daksubprocess.check_call(shell_command, shell=True) ################################################################################ -def prod_maintainer (note, upload): - cnf = Config() - # Here we prepare an editor and get them ready to prod... - (fd, temp_filename) = utils.temp_filename() - temp_file = os.fdopen(fd, 'w') - if len(note) > 0: - for line in note: - temp_file.write(line) - temp_file.close() - editor = os.environ.get("EDITOR","vi") +def get_reject_reason(reason=''): + """get reason for rejection + + @rtype: str + @return: string giving the reason for the rejection or C{None} if the + rejection should be cancelled + """ answer = 'E' + if Options['Automatic']: + answer = 'R' + while answer == 'E': - os.system("%s %s" % (editor, temp_filename)) - temp_fh = utils.open_file(temp_filename) - prod_message = "".join(temp_fh.readlines()) - temp_fh.close() - print "Prod message:" - print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1) - prompt = "[P]rod, Edit, Abandon, Quit ?" + reason = utils.call_editor(reason) + print "Reject message:" + print utils.prefix_multi_line_string(reason, " ", include_blank_lines=1) + prompt = "[R]eject, Edit, Abandon, Quit ?" answer = "XXX" while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt) @@ -569,79 +485,67 @@ def prod_maintainer (note, upload): if answer == "": answer = m.group(1) answer = answer[:1].upper() - os.unlink(temp_filename) - if answer == 'A': - return - elif answer == 'Q': - end() - sys.exit(0) - # Otherwise, do the proding... - user_email_address = utils.whoami() + " <%s>" % ( - cnf["Dinstall::MyAdminAddress"]) - Subst = upload.Subst - - Subst["__FROM_ADDRESS__"] = user_email_address - Subst["__PROD_MESSAGE__"] = prod_message - Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"] - - prod_mail_message = utils.TemplateSubst( - Subst,cnf["Dir::Templates"]+"/process-new.prod") - - # Send the prod mail if appropriate - if not cnf["Dinstall::Options::No-Mail"]: - utils.send_mail(prod_mail_message) + if answer == 'Q': + sys.exit(0) - print "Sent proding message" + if answer == 'R': + return reason + return None ################################################################################ -def do_new(upload, session): - print "NEW\n" - files = upload.pkg.files - changes = upload.pkg.changes +def do_new(upload, upload_copy, handler, session): cnf = Config() - # Make a copy of distribution we can happily trample on - changes["suite"] = copy.copy(changes["distribution"]) - - # Fix up the list of target suites - for suite in changes["suite"].keys(): - override = cnf.Find("Suite::%s::OverrideSuite" % (suite)) - if override: - (olderr, newerr) = (get_suite(suite, session) == None, - get_suite(override, session) == None) - if olderr or newerr: - (oinv, newinv) = ("", "") - if olderr: oinv = "invalid " - if newerr: ninv = "invalid " - print "warning: overriding %ssuite %s to %ssuite %s" % ( - oinv, suite, ninv, override) - del changes["suite"][suite] - changes["suite"][override] = 1 - # Validate suites - for suite in changes["suite"].keys(): - if get_suite(suite, session) is None: - utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite)) + run_user_inspect_command(upload, upload_copy) # The main NEW processing loop - done = 0 + done = False + missing = [] while not done: - # Find out what's new - new = determine_new(changes, files) + queuedir = upload.policy_queue.path + byhand = upload.byhand - if not new: - break + missing = handler.missing_overrides(hints=missing) + broken = not check_valid(missing, session) + + changesname = os.path.basename(upload.changes.changesname) + + print + print changesname + print "-" * len(changesname) + print + print " Target: {0}".format(upload.target_suite.suite_name) + print " Changed-By: {0}".format(upload.changes.changedby) + print + + #if len(byhand) == 0 and len(missing) == 0: + # break + + if missing: + print "NEW\n" answer = "XXX" if Options["No-Action"] or Options["Automatic"]: answer = 'S' - (broken, note) = print_new(new, upload, indexed=0) + note = print_new(upload, missing, indexed=0, session=session) prompt = "" - if not broken and not note: - prompt = "Add overrides, " + has_unprocessed_byhand = False + for f in byhand: + path = os.path.join(queuedir, f.filename) + if not f.processed and os.path.exists(path): + print "W: {0} still present; please process byhand components and try again".format(f.filename) + has_unprocessed_byhand = True + + if not has_unprocessed_byhand and not broken and not note: + if len(missing) == 0: + prompt = "Accept, " + answer = 'A' + else: + prompt = "Add overrides, " if broken: print "W: [!] marked entries must be fixed before package can be processed." if note: @@ -662,52 +566,57 @@ def do_new(upload, session): continue if answer == 'A' and not Options["Trainee"]: - try: - check_daily_lock() - done = add_overrides (new, upload, session) - Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)]) - except CantGetLockError: - print "Hello? Operator! Give me the number for 911!" - print "Dinstall in the locked area, cant process packages, come back later" + add_overrides(missing, upload.target_suite, session) + if Config().find_b("Dinstall::BXANotify"): + do_bxa_notification(missing, upload, session) + handler.accept() + done = True + Logger.log(["NEW ACCEPT", upload.changes.changesname]) elif answer == 'C': - check_pkg(upload) + check_pkg(upload, upload_copy, session) elif answer == 'E' and not Options["Trainee"]: - new = edit_overrides (new, upload, session) + missing = edit_overrides (missing, upload, session) elif answer == 'M' and not Options["Trainee"]: - upload.pkg.remove_known_changes() - aborted = upload.do_reject(manual=1, - reject_message=Options["Manual-Reject"], - note=get_new_comments(changes.get("source", ""), session=session)) - if not aborted: - Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)]) - os.unlink(upload.pkg.changes_file[:-8]+".dak") - done = 1 + reason = Options.get('Manual-Reject', '') + "\n" + reason = reason + "\n\n=====\n\n".join([n.comment for n in get_new_comments(upload.policy_queue, upload.changes.source, session=session)]) + reason = get_reject_reason(reason) + if reason is not None: + Logger.log(["NEW REJECT", upload.changes.changesname]) + handler.reject(reason) + done = True elif answer == 'N': - edit_note(get_new_comments(changes.get("source", ""), session=session), - upload, session) + if edit_note(get_new_comments(upload.policy_queue, upload.changes.source, session=session), + upload, session, bool(Options["Trainee"])) == 0: + end() + sys.exit(0) elif answer == 'P' and not Options["Trainee"]: - prod_maintainer(get_new_comments(changes.get("source", ""), session=session), - upload) - Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)]) + if prod_maintainer(get_new_comments(upload.policy_queue, upload.changes.source, session=session), + upload) == 0: + end() + sys.exit(0) + Logger.log(["NEW PROD", upload.changes.changesname]) elif answer == 'R' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() if confirm == "y": - for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session): + for c in get_new_comments(upload.policy_queue, upload.changes.source, upload.changes.version, session=session): session.delete(c) session.commit() elif answer == 'O' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower() if confirm == "y": - for c in get_new_comments(changes.get("source", ""), session=session): + for c in get_new_comments(upload.policy_queue, upload.changes.source, session=session): session.delete(c) session.commit() elif answer == 'S': - done = 1 + done = True elif answer == 'Q': end() sys.exit(0) + if handler.get_action(): + print "PENDING %s\n" % handler.get_action() + ################################################################################ ################################################################################ ################################################################################ @@ -715,85 +624,40 @@ def do_new(upload, session): def usage (exit_code=0): print """Usage: dak process-new [OPTION]... [CHANGES]... -a, --automatic automatic run + -b, --no-binaries do not sort binary-NEW packages first + -c, --comments show NEW comments -h, --help show this help and exit. -m, --manual-reject=MSG manual reject with `msg' -n, --no-action don't do anything + -q, --queue=QUEUE operate on a different queue -t, --trainee FTP Trainee mode - -V, --version display the version number and exit""" - sys.exit(exit_code) + -V, --version display the version number and exit -################################################################################ +ENVIRONMENT VARIABLES -def do_byhand(upload, session): - done = 0 - while not done: - files = upload.pkg.files - will_install = 1 - byhand = [] - - for f in files.keys(): - if files[f]["type"] == "byhand": - if os.path.exists(f): - print "W: %s still present; please process byhand components and try again." % (f) - will_install = 0 - else: - byhand.append(f) + DAK_INSPECT_UPLOAD: shell command to run to inspect a package + The command is automatically run in a shell when an upload + is checked. The following substitutions are available: - answer = "XXXX" - if Options["No-Action"]: - answer = "S" - if will_install: - if Options["Automatic"] and not Options["No-Action"]: - answer = 'A' - prompt = "[A]ccept, Manual reject, Skip, Quit ?" - else: - prompt = "Manual reject, [S]kip, Quit ?" + {directory}: directory the upload is contained in + {dsc}: name of the included dsc or the empty string + {changes}: name of the changes file - while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt) - m = re_default_answer.search(prompt) - if answer == "": - answer = m.group(1) - answer = answer[:1].upper() + Note that Python's 'format' method is used to format the command. - if answer == 'A': - try: - check_daily_lock() - done = 1 - for f in byhand: - del files[f] - Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) - except CantGetLockError: - print "Hello? Operator! Give me the number for 911!" - print "Dinstall in the locked area, cant process packages, come back later" - elif answer == 'M': - Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)]) - upload.do_reject(manual=1, reject_message=Options["Manual-Reject"]) - os.unlink(upload.pkg.changes_file[:-8]+".dak") - done = 1 - elif answer == 'S': - done = 1 - elif answer == 'Q': - end() - sys.exit(0) + Example: run mc in a tmux session to inspect the upload -################################################################################ + export DAK_INSPECT_UPLOAD='tmux new-session -d -s process-new 2>/dev/null; tmux new-window -n "{changes}" -t process-new:0 -k "cd {directory}; mc"' -def check_daily_lock(): - """ - Raises CantGetLockError if the dinstall daily.lock exists. - """ + and run - cnf = Config() - try: - os.open(cnf["Process-New::DinstallLockFile"], - os.O_RDONLY | os.O_CREAT | os.O_EXCL) - except OSError, e: - if e.errno == errno.EEXIST or e.errno == errno.EACCES: - raise CantGetLockError + tmux attach -t process-new - os.unlink(cnf["Process-New::DinstallLockFile"]) + in a separate terminal session. +""" + sys.exit(exit_code) +################################################################################ @contextlib.contextmanager def lock_package(package): @@ -804,79 +668,101 @@ def lock_package(package): @param package: source package name to lock """ - path = os.path.join(Config()["Process-New::LockDir"], package) + cnf = Config() + + path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package) + try: fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST or e.errno == errno.EACCES: user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '') - raise AlreadyLockedError, user + raise AlreadyLockedError(user) try: yield fd finally: os.unlink(path) -def _accept(upload): - if Options["No-Action"]: - return - (summary, short_summary) = upload.build_summaries() - upload.accept(summary, short_summary, targetqueue) - os.unlink(upload.pkg.changes_file[:-8]+".dak") +def do_pkg(upload, session): + # Try to get an included dsc + dsc = upload.source -def do_accept(upload): - print "ACCEPT" cnf = Config() - if not Options["No-Action"]: - (summary, short_summary) = upload.build_summaries() - - if cnf.FindB("Dinstall::SecurityQueueHandling"): - upload.dump_vars(cnf["Dir::Queue::Embargoed"]) - upload.move_to_queue(get_policy_queue('embargoed')) - upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"]) - # Check for override disparities - upload.Subst["__SUMMARY__"] = summary - else: - # Just a normal upload, accept it... - _accept(upload) + group = cnf.get('Dinstall::UnprivGroup') or None -def do_pkg(changes_file, session): - u = Upload() - u.pkg.load_dot_dak(changes_file) - u.update_subst() - - cnf = Config() - bcc = "X-DAK: dak process-new" - if cnf.has_key("Dinstall::Bcc"): - u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) - else: - u.Subst["__BCC__"] = bcc - - files = u.pkg.files + #bcc = "X-DAK: dak process-new" + #if cnf.has_key("Dinstall::Bcc"): + # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) + #else: + # u.Subst["__BCC__"] = bcc try: - with lock_package(u.pkg.changes["source"]): - if not recheck(u, session): - return - - (new, byhand) = check_status(files) - if new or byhand: - if new: - do_new(u, session) - if byhand: - do_byhand(u, session) - (new, byhand) = check_status(files) - - if not new and not byhand: - try: - check_daily_lock() - do_accept(u) - except CantGetLockError: - print "Hello? Operator! Give me the number for 911!" - print "Dinstall in the locked area, cant process packages, come back later" - except AlreadyLockedError, e: + with lock_package(upload.changes.source): + with UploadCopy(upload, group=group) as upload_copy: + handler = PolicyQueueUploadHandler(upload, session) + if handler.get_action() is not None: + print "PENDING %s\n" % handler.get_action() + return + + do_new(upload, upload_copy, handler, session) + except AlreadyLockedError as e: print "Seems to be locked by %s already, skipping..." % (e) +def show_new_comments(uploads, session): + sources = [ upload.changes.source for upload in uploads ] + if len(sources) == 0: + return + + query = """SELECT package, version, comment, author + FROM new_comments + WHERE package IN :sources + ORDER BY package, version""" + + r = session.execute(query, params=dict(sources=tuple(sources))) + + for i in r: + print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3]) + + session.rollback() + +################################################################################ + +def sort_uploads(new_queue, uploads, session, nobinaries=False): + sources = {} + sorteduploads = [] + suitesrc = [s.source for s in session.query(DBSource.source). \ + filter(DBSource.suites.any(Suite.suite_name.in_(['unstable', 'experimental'])))] + comments = [p.package for p in session.query(NewComment.package). \ + filter_by(trainee=False, policy_queue=new_queue).distinct()] + for upload in uploads: + source = upload.changes.source + if not source in sources: + sources[source] = [] + sources[source].append({'upload': upload, + 'date': upload.changes.created, + 'stack': 1, + 'binary': True if source in suitesrc else False, + 'comments': True if source in comments else False}) + for src in sources: + if len(sources[src]) > 1: + changes = sources[src] + firstseen = sorted(changes, key=lambda k: (k['date']))[0]['date'] + changes.sort(key=lambda item:item['date']) + for i in range (0, len(changes)): + changes[i]['date'] = firstseen + changes[i]['stack'] = i + 1 + sorteduploads += sources[src] + if nobinaries: + sorteduploads = [u["upload"] for u in sorted(sorteduploads, + key=lambda k: (k["comments"], k["binary"], + k["date"], -k["stack"]))] + else: + sorteduploads = [u["upload"] for u in sorted(sorteduploads, + key=lambda k: (k["comments"], -k["binary"], + k["date"], -k["stack"]))] + return sorteduploads + ################################################################################ def end(): @@ -898,56 +784,56 @@ def end(): def main(): global Options, Logger, Sections, Priorities - print "NO NEW PROCESSING CURRENTLY AVAILABLE" - print "(Go and do something more interesting)" - sys.exit(0) - cnf = Config() session = DBConn().session() Arguments = [('a',"automatic","Process-New::Options::Automatic"), + ('b',"no-binaries","Process-New::Options::No-Binaries"), + ('c',"comments","Process-New::Options::Comments"), ('h',"help","Process-New::Options::Help"), ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), ('t',"trainee","Process-New::Options::Trainee"), + ('q','queue','Process-New::Options::Queue', 'HasArg'), ('n',"no-action","Process-New::Options::No-Action")] - for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]: + changes_files = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv) + + for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]: if not cnf.has_key("Process-New::Options::%s" % (i)): cnf["Process-New::Options::%s" % (i)] = "" - changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv) + queue_name = cnf.get('Process-New::Options::Queue', 'new') + new_queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).one() if len(changes_files) == 0: - changes_files = utils.get_changes_files(cnf["Dir::Queue::New"]) + uploads = new_queue.uploads + else: + uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \ + .join(DBChange).filter(DBChange.changesname.in_(changes_files)).all() - Options = cnf.SubTree("Process-New::Options") + Options = cnf.subtree("Process-New::Options") if Options["Help"]: usage() if not Options["No-Action"]: try: - Logger = daklog.Logger(cnf, "process-new") - except CantOpenError, e: + Logger = daklog.Logger("process-new") + except CantOpenError as e: Options["Trainee"] = "True" Sections = Section_Completer(session) Priorities = Priority_Completer(session) readline.parse_and_bind("tab: complete") - if len(changes_files) > 1: + if len(uploads) > 1: sys.stderr.write("Sorting changes...\n") - changes_files = sort_changes(changes_files, session) + uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"]) - # Kill me now? **FIXME** - cnf["Dinstall::Options::No-Mail"] = "" - - for changes_file in changes_files: - changes_file = utils.validate_changes_file_arg(changes_file, 0) - if not changes_file: - continue - print "\n" + changes_file - - do_pkg (changes_file, session) + if Options["Comments"]: + show_new_comments(uploads, session) + else: + for upload in uploads: + do_pkg (upload, session) end()