X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_new.py;h=55397c7dc79a4e2df577397d9262a6addb047df1;hb=026bdb9580e6d18bf504d2f44d46890df5d89f1a;hp=d98c0df683c77735d3fe47c512c5209abfb52f2e;hpb=59fd5aa2a8be3b76dbc968429c457f096adfa472;p=dak.git diff --git a/dak/process_new.py b/dak/process_new.py index d98c0df6..55397c7d 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -1,9 +1,14 @@ #!/usr/bin/env python +# vim:set et ts=4 sw=4: -# Handles NEW and BYHAND packages -# Copyright (C) 2001, 2002, 2003, 2004, 2005 James Troup -# $Id: lisa,v 1.31 2005-11-15 09:50:32 ajt Exp $ +""" Handles NEW and BYHAND packages +@contact: Debian FTP Master +@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup +@copyright: 2009 Joerg Jaspert +@copyright: 2009 Frank Lichtenheld +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -37,260 +42,56 @@ ################################################################################ -import copy, errno, os, readline, stat, sys, time; -import apt_pkg, apt_inst; -import db_access, fernanda, katie, logging, utils; +import copy +import errno +import os +import readline +import stat +import sys +import time +import contextlib +import pwd +import apt_pkg, apt_inst +import examine_package +import subprocess +import daklib.daksubprocess +from sqlalchemy import or_ + +from daklib.dbconn import * +from daklib.queue import * +from daklib import daklog +from daklib import utils +from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package +from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError +from daklib.summarystats import SummaryStats +from daklib.config import Config +from daklib.policy import UploadCopy, PolicyQueueUploadHandler # Globals -lisa_version = "$Revision: 1.31 $"; +Options = None +Logger = None -Cnf = None; -Options = None; -Katie = None; -projectB = None; -Logger = None; - -Priorities = None; -Sections = None; - -reject_message = ""; - -################################################################################ -################################################################################ -################################################################################ - -def reject (str, prefix="Rejected: "): - global reject_message; - if str: - reject_message += prefix + str + "\n"; - -def recheck(): - global reject_message; - files = Katie.pkg.files; - reject_message = ""; - - for file in files.keys(): - # The .orig.tar.gz can disappear out from under us is it's a - # duplicate of one in the archive. - if not files.has_key(file): - continue; - # Check that the source still exists - if files[file]["type"] == "deb": - source_version = files[file]["source version"]; - source_package = files[file]["source package"]; - if not Katie.pkg.changes["architecture"].has_key("source") \ - and not Katie.source_exists(source_package, source_version, Katie.pkg.changes["distribution"].keys()): - source_epochless_version = utils.re_no_epoch.sub('', source_version); - dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version); - if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename): - reject("no source found for %s %s (%s)." % (source_package, source_version, file)); - - # Version and file overwrite checks - if files[file]["type"] == "deb": - reject(Katie.check_binary_against_db(file)); - elif files[file]["type"] == "dsc": - reject(Katie.check_source_against_db(file)); - (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file); - reject(reject_msg); - - if reject_message: - answer = "XXX"; - if Options["No-Action"] or Options["Automatic"]: - answer = 'S' - - print "REJECT\n" + reject_message,; - prompt = "[R]eject, Skip, Quit ?"; - - while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.match(prompt); - if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); - - if answer == 'R': - Katie.do_reject(0, reject_message); - os.unlink(Katie.pkg.changes_file[:-8]+".katie"); - return 0; - elif answer == 'S': - return 0; - elif answer == 'Q': - sys.exit(0); - - return 1; +Priorities = None +Sections = None ################################################################################ - -def determine_new (changes, files): - new = {}; - - # Build up a list of potentially new things - for file in files.keys(): - f = files[file]; - # Skip byhand elements - if f["type"] == "byhand": - continue; - pkg = f["package"]; - priority = f["priority"]; - section = f["section"]; - # FIXME: unhardcode - if section == "non-US/main": - section = "non-US"; - type = get_type(f); - component = f["component"]; - - if type == "dsc": - priority = "source"; - if not new.has_key(pkg): - new[pkg] = {}; - new[pkg]["priority"] = priority; - new[pkg]["section"] = section; - new[pkg]["type"] = type; - new[pkg]["component"] = component; - new[pkg]["files"] = []; - else: - old_type = new[pkg]["type"]; - if old_type != type: - # source gets trumped by deb or udeb - if old_type == "dsc": - new[pkg]["priority"] = priority; - new[pkg]["section"] = section; - new[pkg]["type"] = type; - new[pkg]["component"] = component; - new[pkg]["files"].append(file); - if f.has_key("othercomponents"): - new[pkg]["othercomponents"] = f["othercomponents"]; - - for suite in changes["suite"].keys(): - suite_id = db_access.get_suite_id(suite); - for pkg in new.keys(): - component_id = db_access.get_component_id(new[pkg]["component"]); - type_id = db_access.get_override_type_id(new[pkg]["type"]); - q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id)); - ql = q.getresult(); - if ql: - for file in new[pkg]["files"]: - if files[file].has_key("new"): - del files[file]["new"]; - del new[pkg]; - - if changes["suite"].has_key("stable"): - print "WARNING: overrides will be added for stable!"; - if changes["suite"].has_key("oldstable"): - print "WARNING: overrides will be added for OLDstable!"; - for pkg in new.keys(): - if new[pkg].has_key("othercomponents"): - print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]); - - return new; - ################################################################################ - -def indiv_sg_compare (a, b): - """Sort by source name, source, version, 'have source', and - finally by filename.""" - # Sort by source version - q = apt_pkg.VersionCompare(a["version"], b["version"]); - if q: - return -q; - - # Sort by 'have source' - a_has_source = a["architecture"].get("source"); - b_has_source = b["architecture"].get("source"); - if a_has_source and not b_has_source: - return -1; - elif b_has_source and not a_has_source: - return 1; - - return cmp(a["filename"], b["filename"]); - -############################################################ - -def sg_compare (a, b): - a = a[1]; - b = b[1]; - """Sort by have note, time of oldest upload.""" - # Sort by have note - a_note_state = a["note_state"]; - b_note_state = b["note_state"]; - if a_note_state < b_note_state: - return -1; - elif a_note_state > b_note_state: - return 1; - - # Sort by time of oldest upload - return cmp(a["oldest"], b["oldest"]); - -def sort_changes(changes_files): - """Sort into source groups, then sort each source group by version, - have source, filename. Finally, sort the source groups by have - note, time of oldest upload of each source upload.""" - if len(changes_files) == 1: - return changes_files; - - sorted_list = []; - cache = {}; - # Read in all the .changes files - for filename in changes_files: - try: - Katie.pkg.changes_file = filename; - Katie.init_vars(); - Katie.update_vars(); - cache[filename] = copy.copy(Katie.pkg.changes); - cache[filename]["filename"] = filename; - except: - sorted_list.append(filename); - break; - # Divide the .changes into per-source groups - per_source = {}; - for filename in cache.keys(): - source = cache[filename]["source"]; - if not per_source.has_key(source): - per_source[source] = {}; - per_source[source]["list"] = []; - per_source[source]["list"].append(cache[filename]); - # Determine oldest time and have note status for each source group - for source in per_source.keys(): - source_list = per_source[source]["list"]; - first = source_list[0]; - oldest = os.stat(first["filename"])[stat.ST_MTIME]; - have_note = 0; - for d in per_source[source]["list"]: - mtime = os.stat(d["filename"])[stat.ST_MTIME]; - if mtime < oldest: - oldest = mtime; - have_note += (d.has_key("lisa note")); - per_source[source]["oldest"] = oldest; - if not have_note: - per_source[source]["note_state"] = 0; # none - elif have_note < len(source_list): - per_source[source]["note_state"] = 1; # some - else: - per_source[source]["note_state"] = 2; # all - per_source[source]["list"].sort(indiv_sg_compare); - per_source_items = per_source.items(); - per_source_items.sort(sg_compare); - for i in per_source_items: - for j in i[1]["list"]: - sorted_list.append(j["filename"]); - return sorted_list; - ################################################################################ class Section_Completer: - def __init__ (self): - self.sections = []; - q = projectB.query("SELECT section FROM section"); - for i in q.getresult(): - self.sections.append(i[0]); + def __init__ (self, session): + self.sections = [] + self.matches = [] + for s, in session.query(Section.section): + self.sections.append(s) def complete(self, text, state): if state == 0: - self.matches = []; - n = len(text); + self.matches = [] + n = len(text) for word in self.sections: if word[:n] == text: - self.matches.append(word); + self.matches.append(word) try: return self.matches[state] except IndexError: @@ -299,19 +100,19 @@ class Section_Completer: ############################################################ class Priority_Completer: - def __init__ (self): - self.priorities = []; - q = projectB.query("SELECT priority FROM priority"); - for i in q.getresult(): - self.priorities.append(i[0]); + def __init__ (self, session): + self.priorities = [] + self.matches = [] + for p, in session.query(Priority.priority): + self.priorities.append(p) def complete(self, text, state): if state == 0: - self.matches = []; - n = len(text); + self.matches = [] + n = len(text) for word in self.priorities: if word[:n] == text: - self.matches.append(word); + self.matches.append(word) try: return self.matches[state] except IndexError: @@ -319,647 +120,722 @@ class Priority_Completer: ################################################################################ -def check_valid (new): - for pkg in new.keys(): - section = new[pkg]["section"]; - priority = new[pkg]["priority"]; - type = new[pkg]["type"]; - new[pkg]["section id"] = db_access.get_section_id(section); - new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]); - # Sanity checks - if (section == "debian-installer" and type != "udeb") or \ - (section != "debian-installer" and type == "udeb"): - new[pkg]["section id"] = -1; - if (priority == "source" and type != "dsc") or \ - (priority != "source" and type == "dsc"): - new[pkg]["priority id"] = -1; +def takenover_binaries(upload, missing, session): + rows = [] + binaries = set([x.package for x in upload.binaries]) + for m in missing: + if m['type'] != 'dsc': + binaries.discard(m['package']) + if binaries: + source = upload.binaries[0].source.source + suite = upload.target_suite.overridesuite or \ + upload.target_suite.suite_name + suites = [s[0] for s in session.query(Suite.suite_name).filter \ + (or_(Suite.suite_name == suite, + Suite.overridesuite == suite)).all()] + rows = session.query(DBSource.source, DBBinary.package).distinct(). \ + filter(DBBinary.package.in_(binaries)). \ + join(DBBinary.source). \ + filter(DBSource.source != source). \ + join(DBBinary.suites). \ + filter(Suite.suite_name.in_(suites)). \ + order_by(DBSource.source, DBBinary.package).all() + return rows ################################################################################ -def print_new (new, indexed, file=sys.stdout): - check_valid(new); - broken = 0; - index = 0; - for pkg in new.keys(): - index += 1; - section = new[pkg]["section"]; - priority = new[pkg]["priority"]; - if new[pkg]["section id"] == -1: - section += "[!]"; - broken = 1; - if new[pkg]["priority id"] == -1: - priority += "[!]"; - broken = 1; +def print_new (upload, missing, indexed, session, file=sys.stdout): + check_valid(missing, session) + index = 0 + for m in missing: + index += 1 + if m['type'] != 'deb': + package = '{0}:{1}'.format(m['type'], m['package']) + else: + package = m['package'] + section = m['section'] + priority = m['priority'] + included = "" if m['included'] else "NOT UPLOADED" if indexed: - line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section); + line = "(%s): %-20s %-20s %-20s %s" % (index, package, priority, section, included) else: - line = "%-20s %-20s %-20s" % (pkg, priority, section); - line = line.strip()+'\n'; - file.write(line); - note = Katie.pkg.changes.get("lisa note"); - if note: - print "*"*75; - print note; - print "*"*75; - return broken, note; - -################################################################################ - -def get_type (f): - # Determine the type - if f.has_key("dbtype"): - type = f["dbtype"]; - elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc": - type = "dsc"; - else: - utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type)); - - # Validate the override type - type_id = db_access.get_override_type_id(type); - if type_id == -1: - utils.fubar("invalid type (%s) for new. Say wha?" % (type)); - - return type; + line = "%-20s %-20s %-20s %s" % (package, priority, section, included) + line = line.strip() + if not m['valid']: + line = line + ' [!]' + print >>file, line + takenover = takenover_binaries(upload, missing, session) + if takenover: + print '\n\nBINARIES TAKEN OVER\n' + for t in takenover: + print '%s: %s' % (t[0], t[1]) + notes = get_new_comments(upload.policy_queue, upload.changes.source) + for note in notes: + print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \ + % (note.author, note.version, note.notedate, note.comment) + print "-" * 72 + return len(notes) > 0 ################################################################################ def index_range (index): if index == 1: - return "1"; + return "1" else: - return "1-%s" % (index); + return "1-%s" % (index) ################################################################################ ################################################################################ -def edit_new (new): +def edit_new (overrides, upload, session): # Write the current data to a temporary file - temp_filename = utils.temp_filename(); - temp_file = utils.open_file(temp_filename, 'w'); - print_new (new, 0, temp_file); - temp_file.close(); + (fd, temp_filename) = utils.temp_filename() + temp_file = os.fdopen(fd, 'w') + print_new (upload, overrides, indexed=0, session=session, file=temp_file) + temp_file.close() # Spawn an editor on that file editor = os.environ.get("EDITOR","vi") result = os.system("%s %s" % (editor, temp_filename)) if result != 0: utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result) # Read the edited data back in - temp_file = utils.open_file(temp_filename); - lines = temp_file.readlines(); - temp_file.close(); - os.unlink(temp_filename); + temp_file = utils.open_file(temp_filename) + lines = temp_file.readlines() + temp_file.close() + os.unlink(temp_filename) + + overrides_map = dict([ ((o['type'], o['package']), o) for o in overrides ]) + new_overrides = [] # Parse the new data for line in lines: - line = line.strip(); - if line == "": - continue; - s = line.split(); + line = line.strip() + if line == "" or line[0] == '#': + continue + s = line.split() # Pad the list if necessary - s[len(s):3] = [None] * (3-len(s)); - (pkg, priority, section) = s[:3]; - if not new.has_key(pkg): - utils.warn("Ignoring unknown package '%s'" % (pkg)); + s[len(s):3] = [None] * (3-len(s)) + (pkg, priority, section) = s[:3] + if pkg.find(':') != -1: + type, pkg = pkg.split(':', 1) + else: + type = 'deb' + o = overrides_map.get((type, pkg), None) + if o is None: + utils.warn("Ignoring unknown package '%s'" % (pkg)) else: - # Strip off any invalid markers, print_new will readd them. - if section.endswith("[!]"): - section = section[:-3]; - if priority.endswith("[!]"): - priority = priority[:-3]; - for file in new[pkg]["files"]: - Katie.pkg.files[file]["section"] = section; - Katie.pkg.files[file]["priority"] = priority; - new[pkg]["section"] = section; - new[pkg]["priority"] = priority; + if section.find('/') != -1: + component = section.split('/', 1)[0] + else: + component = 'main' + new_overrides.append(dict( + package=pkg, + type=type, + section=section, + component=component, + priority=priority, + included=o['included'], + )) + return new_overrides ################################################################################ -def edit_index (new, index): +def edit_index (new, upload, index): + package = new[index]['package'] priority = new[index]["priority"] section = new[index]["section"] - type = new[index]["type"]; + ftype = new[index]["type"] done = 0 while not done: - print "\t".join([index, priority, section]); + print "\t".join([package, priority, section]) - answer = "XXX"; - if type != "dsc": - prompt = "[B]oth, Priority, Section, Done ? "; + answer = "XXX" + if ftype != "dsc": + prompt = "[B]oth, Priority, Section, Done ? " else: - prompt = "[S]ection, Done ? "; - edit_priority = edit_section = 0; + prompt = "[S]ection, Done ? " + edit_priority = edit_section = 0 while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.match(prompt) + answer = utils.our_raw_input(prompt) + m = re_default_answer.match(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() if answer == 'P': - edit_priority = 1; + edit_priority = 1 elif answer == 'S': - edit_section = 1; + edit_section = 1 elif answer == 'B': - edit_priority = edit_section = 1; + edit_priority = edit_section = 1 elif answer == 'D': - done = 1; + done = 1 # Edit the priority if edit_priority: - readline.set_completer(Priorities.complete); - got_priority = 0; + readline.set_completer(Priorities.complete) + got_priority = 0 while not got_priority: - new_priority = utils.our_raw_input("New priority: ").strip(); + new_priority = utils.our_raw_input("New priority: ").strip() if new_priority not in Priorities.priorities: - print "E: '%s' is not a valid priority, try again." % (new_priority); + print "E: '%s' is not a valid priority, try again." % (new_priority) else: - got_priority = 1; - priority = new_priority; + got_priority = 1 + priority = new_priority # Edit the section if edit_section: - readline.set_completer(Sections.complete); - got_section = 0; + readline.set_completer(Sections.complete) + got_section = 0 while not got_section: - new_section = utils.our_raw_input("New section: ").strip(); + new_section = utils.our_raw_input("New section: ").strip() if new_section not in Sections.sections: - print "E: '%s' is not a valid section, try again." % (new_section); + print "E: '%s' is not a valid section, try again." % (new_section) else: - got_section = 1; - section = new_section; + got_section = 1 + section = new_section # Reset the readline completer - readline.set_completer(None); + readline.set_completer(None) - for file in new[index]["files"]: - Katie.pkg.files[file]["section"] = section; - Katie.pkg.files[file]["priority"] = priority; - new[index]["priority"] = priority; - new[index]["section"] = section; - return new; + new[index]["priority"] = priority + new[index]["section"] = section + if section.find('/') != -1: + component = section.split('/', 1)[0] + else: + component = 'main' + new[index]['component'] = component + + return new ################################################################################ -def edit_overrides (new): - print; +def edit_overrides (new, upload, session): + print done = 0 while not done: - print_new (new, 1); - new_index = {}; - index = 0; - for i in new.keys(): - index += 1; - new_index[index] = i; - - prompt = "(%s) edit override , Editor, Done ? " % (index_range(index)); + print_new (upload, new, indexed=1, session=session) + prompt = "edit override , Editor, Done ? " got_answer = 0 while not got_answer: - answer = utils.our_raw_input(prompt); - if not utils.str_isnum(answer): - answer = answer[:1].upper(); + answer = utils.our_raw_input(prompt) + if not answer.isdigit(): + answer = answer[:1].upper() if answer == "E" or answer == "D": - got_answer = 1; - elif katie.re_isanum.match (answer): - answer = int(answer); - if (answer < 1) or (answer > index): - print "%s is not a valid index (%s). Please retry." % (answer, index_range(index)); + got_answer = 1 + elif re_isanum.match (answer): + answer = int(answer) + if answer < 1 or answer > len(new): + print "{0} is not a valid index. Please retry.".format(answer) else: - got_answer = 1; + got_answer = 1 if answer == 'E': - edit_new(new); + new = edit_new(new, upload, session) elif answer == 'D': - done = 1; + done = 1 else: - edit_index (new, new_index[answer]); + edit_index (new, upload, answer - 1) - return new; + return new -################################################################################ - -def edit_note(note): - # Write the current data to a temporary file - temp_filename = utils.temp_filename(); - temp_file = utils.open_file(temp_filename, 'w'); - temp_file.write(note); - temp_file.close(); - editor = os.environ.get("EDITOR","vi") - answer = 'E'; - while answer == 'E': - os.system("%s %s" % (editor, temp_filename)) - temp_file = utils.open_file(temp_filename); - note = temp_file.read().rstrip(); - temp_file.close(); - print "Note:"; - print utils.prefix_multi_line_string(note," "); - prompt = "[D]one, Edit, Abandon, Quit ?" - answer = "XXX"; - while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); - if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); - os.unlink(temp_filename); - if answer == 'A': - return; - elif answer == 'Q': - sys.exit(0); - Katie.pkg.changes["lisa note"] = note; - Katie.dump_vars(Cnf["Dir::Queue::New"]); ################################################################################ -def check_pkg (): +def check_pkg (upload, upload_copy, session): + missing = [] + save_stdout = sys.stdout + changes = os.path.join(upload_copy.directory, upload.changes.changesname) + suite_name = upload.target_suite.suite_name + handler = PolicyQueueUploadHandler(upload, session) + missing = [(m['type'], m["package"]) for m in handler.missing_overrides(hints=missing)] + + less_cmd = ("less", "-R", "-") + less_process = daklib.daksubprocess.Popen(less_cmd, bufsize=0, stdin=subprocess.PIPE) try: - less_fd = os.popen("less -R -", 'w', 0); - stdout_fd = sys.stdout; - try: - sys.stdout = less_fd; - fernanda.display_changes(Katie.pkg.changes_file); - files = Katie.pkg.files; - for file in files.keys(): - if files[file].has_key("new"): - type = files[file]["type"]; - if type == "deb": - fernanda.check_deb(file); - elif type == "dsc": - fernanda.check_dsc(file); - finally: - sys.stdout = stdout_fd; - except IOError, e: - if errno.errorcode[e.errno] == 'EPIPE': - utils.warn("[fernanda] Caught EPIPE; skipping."); - pass; + sys.stdout = less_process.stdin + print examine_package.display_changes(suite_name, changes) + + source = upload.source + if source is not None: + source_file = os.path.join(upload_copy.directory, os.path.basename(source.poolfile.filename)) + print examine_package.check_dsc(suite_name, source_file) + + for binary in upload.binaries: + binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename)) + examined = examine_package.check_deb(suite_name, binary_file) + # We always need to call check_deb to display package relations for every binary, + # but we print its output only if new overrides are being added. + if ("deb", binary.package) in missing: + print examined + + print examine_package.output_package_relations() + less_process.stdin.close() + except IOError as e: + if e.errno == errno.EPIPE: + utils.warn("[examine_package] Caught EPIPE; skipping.") else: - raise; + raise except KeyboardInterrupt: - utils.warn("[fernanda] Caught C-c; skipping."); - pass; + utils.warn("[examine_package] Caught C-c; skipping.") + finally: + less_process.wait() + sys.stdout = save_stdout ################################################################################ ## FIXME: horribly Debian specific -def do_bxa_notification(): - files = Katie.pkg.files; - summary = ""; - for file in files.keys(): - if files[file]["type"] == "deb": - control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file))); - summary += "\n"; - summary += "Package: %s\n" % (control.Find("Package")); - summary += "Description: %s\n" % (control.Find("Description")); - Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary; - bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification"); - utils.send_mail(bxa_mail); +def do_bxa_notification(new, upload, session): + cnf = Config() + + new = set([ o['package'] for o in new if o['type'] == 'deb' ]) + if len(new) == 0: + return + + key = session.query(MetadataKey).filter_by(key='Description').one() + summary = "" + for binary in upload.binaries: + if binary.package not in new: + continue + description = session.query(BinaryMetadata).filter_by(binary=binary, key=key).one().value + summary += "\n" + summary += "Package: {0}\n".format(binary.package) + summary += "Description: {0}\n".format(description) + + subst = { + '__DISTRO__': cnf['Dinstall::MyDistribution'], + '__BCC__': 'X-DAK: dak process-new', + '__BINARY_DESCRIPTIONS__': summary, + } + + bxa_mail = utils.TemplateSubst(subst,os.path.join(cnf["Dir::Templates"], "process-new.bxa_notification")) + utils.send_mail(bxa_mail) ################################################################################ -def add_overrides (new): - changes = Katie.pkg.changes; - files = Katie.pkg.files; - - projectB.query("BEGIN WORK"); - for suite in changes["suite"].keys(): - suite_id = db_access.get_suite_id(suite); - for pkg in new.keys(): - component_id = db_access.get_component_id(new[pkg]["component"]); - type_id = db_access.get_override_type_id(new[pkg]["type"]); - priority_id = new[pkg]["priority id"]; - section_id = new[pkg]["section id"]; - projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id)); - for file in new[pkg]["files"]: - if files[file].has_key("new"): - del files[file]["new"]; - del new[pkg]; - - projectB.query("COMMIT WORK"); - - if Cnf.FindB("Dinstall::BXANotify"): - do_bxa_notification(); +def add_overrides (new_overrides, suite, session): + if suite.overridesuite is not None: + suite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one() + + for override in new_overrides: + package = override['package'] + priority = session.query(Priority).filter_by(priority=override['priority']).first() + section = session.query(Section).filter_by(section=override['section']).first() + component = get_mapped_component(override['component'], session) + overridetype = session.query(OverrideType).filter_by(overridetype=override['type']).one() + + if priority is None: + raise Exception('Invalid priority {0} for package {1}'.format(priority, package)) + if section is None: + raise Exception('Invalid section {0} for package {1}'.format(section, package)) + if component is None: + raise Exception('Invalid component {0} for package {1}'.format(component, package)) + + o = Override(package=package, suite=suite, component=component, priority=priority, section=section, overridetype=overridetype) + session.add(o) + + session.commit() ################################################################################ -def prod_maintainer (): - # Here we prepare an editor and get them ready to prod... - temp_filename = utils.temp_filename(); - editor = os.environ.get("EDITOR","vi") - answer = 'E'; - while answer == 'E': - os.system("%s %s" % (editor, temp_filename)) - file = utils.open_file(temp_filename); - prod_message = "".join(file.readlines()); - file.close(); - print "Prod message:"; - print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1); - prompt = "[P]rod, Edit, Abandon, Quit ?" - answer = "XXX"; - while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); - if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); - os.unlink(temp_filename); - if answer == 'A': - return; - elif answer == 'Q': - sys.exit(0); - # Otherwise, do the proding... - user_email_address = utils.whoami() + " <%s>" % ( - Cnf["Dinstall::MyAdminAddress"]); +def run_user_inspect_command(upload, upload_copy): + command = os.environ.get('DAK_INSPECT_UPLOAD') + if command is None: + return + + directory = upload_copy.directory + if upload.source: + dsc = os.path.basename(upload.source.poolfile.filename) + else: + dsc = '' + changes = upload.changes.changesname - Subst = Katie.Subst; + shell_command = command.format( + directory=directory, + dsc=dsc, + changes=changes, + ) - Subst["__FROM_ADDRESS__"] = user_email_address; - Subst["__PROD_MESSAGE__"] = prod_message; - Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; + daklib.daksubprocess.check_call(shell_command, shell=True) - prod_mail_message = utils.TemplateSubst( - Subst,Cnf["Dir::Templates"]+"/lisa.prod"); +################################################################################ - # Send the prod mail if appropriate - if not Cnf["Dinstall::Options::No-Mail"]: - utils.send_mail(prod_mail_message); +def get_reject_reason(reason=''): + """get reason for rejection - print "Sent proding message"; + @rtype: str + @return: string giving the reason for the rejection or C{None} if the + rejection should be cancelled + """ + answer = 'E' + if Options['Automatic']: + answer = 'R' + + while answer == 'E': + reason = utils.call_editor(reason) + print "Reject message:" + print utils.prefix_multi_line_string(reason, " ", include_blank_lines=1) + prompt = "[R]eject, Edit, Abandon, Quit ?" + answer = "XXX" + while prompt.find(answer) == -1: + answer = utils.our_raw_input(prompt) + m = re_default_answer.search(prompt) + if answer == "": + answer = m.group(1) + answer = answer[:1].upper() + + if answer == 'Q': + sys.exit(0) + + if answer == 'R': + return reason + return None ################################################################################ -def do_new(): - print "NEW\n"; - files = Katie.pkg.files; - changes = Katie.pkg.changes; - - # Make a copy of distribution we can happily trample on - changes["suite"] = copy.copy(changes["distribution"]); - - # Fix up the list of target suites - for suite in changes["suite"].keys(): - override = Cnf.Find("Suite::%s::OverrideSuite" % (suite)); - if override: - del changes["suite"][suite]; - changes["suite"][override] = 1; - # Validate suites - for suite in changes["suite"].keys(): - suite_id = db_access.get_suite_id(suite); - if suite_id == -1: - utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite)); +def do_new(upload, upload_copy, handler, session): + cnf = Config() + + run_user_inspect_command(upload, upload_copy) # The main NEW processing loop - done = 0; + done = False + missing = [] while not done: - # Find out what's new - new = determine_new(changes, files); + queuedir = upload.policy_queue.path + byhand = upload.byhand - if not new: - break; + missing = handler.missing_overrides(hints=missing) + broken = not check_valid(missing, session) - answer = "XXX"; - if Options["No-Action"] or Options["Automatic"]: - answer = 'S'; + changesname = os.path.basename(upload.changes.changesname) + + print + print changesname + print "-" * len(changesname) + print + print " Target: {0}".format(upload.target_suite.suite_name) + print " Changed-By: {0}".format(upload.changes.changedby) + print + + #if len(byhand) == 0 and len(missing) == 0: + # break + + if missing: + print "NEW\n" - (broken, note) = print_new(new, 0); - prompt = ""; + answer = "XXX" + if Options["No-Action"] or Options["Automatic"]: + answer = 'S' - if not broken and not note: - prompt = "Add overrides, "; + note = print_new(upload, missing, indexed=0, session=session) + prompt = "" + + has_unprocessed_byhand = False + for f in byhand: + path = os.path.join(queuedir, f.filename) + if not f.processed and os.path.exists(path): + print "W: {0} still present; please process byhand components and try again".format(f.filename) + has_unprocessed_byhand = True + + if not has_unprocessed_byhand and not broken and not note: + if len(missing) == 0: + prompt = "Accept, " + answer = 'A' + else: + prompt = "Add overrides, " if broken: - print "W: [!] marked entries must be fixed before package can be processed."; + print "W: [!] marked entries must be fixed before package can be processed." if note: - print "W: note must be removed before package can be processed."; - prompt += "Remove note, "; + print "W: note must be removed before package can be processed." + prompt += "RemOve all notes, Remove note, " - prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"; + prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?" while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); + answer = utils.our_raw_input(prompt) + m = re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() - if answer == 'A': - done = add_overrides (new); + if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]: + utils.warn("Trainees can't do that") + continue + + if answer == 'A' and not Options["Trainee"]: + add_overrides(missing, upload.target_suite, session) + if Config().find_b("Dinstall::BXANotify"): + do_bxa_notification(missing, upload, session) + handler.accept() + done = True + Logger.log(["NEW ACCEPT", upload.changes.changesname]) elif answer == 'C': - check_pkg(); - elif answer == 'E': - new = edit_overrides (new); - elif answer == 'M': - aborted = Katie.do_reject(1, Options["Manual-Reject"]); - if not aborted: - os.unlink(Katie.pkg.changes_file[:-8]+".katie"); - done = 1; + check_pkg(upload, upload_copy, session) + elif answer == 'E' and not Options["Trainee"]: + missing = edit_overrides (missing, upload, session) + elif answer == 'M' and not Options["Trainee"]: + reason = Options.get('Manual-Reject', '') + "\n" + reason = reason + "\n\n=====\n\n".join([n.comment for n in get_new_comments(upload.policy_queue, upload.changes.source, session=session)]) + reason = get_reject_reason(reason) + if reason is not None: + Logger.log(["NEW REJECT", upload.changes.changesname]) + handler.reject(reason) + done = True elif answer == 'N': - edit_note(changes.get("lisa note", "")); - elif answer == 'P': - prod_maintainer(); - elif answer == 'R': - confirm = utils.our_raw_input("Really clear note (y/N)? ").lower(); + if edit_note(get_new_comments(upload.policy_queue, upload.changes.source, session=session), + upload, session, bool(Options["Trainee"])) == 0: + end() + sys.exit(0) + elif answer == 'P' and not Options["Trainee"]: + if prod_maintainer(get_new_comments(upload.policy_queue, upload.changes.source, session=session), + upload) == 0: + end() + sys.exit(0) + Logger.log(["NEW PROD", upload.changes.changesname]) + elif answer == 'R' and not Options["Trainee"]: + confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() if confirm == "y": - del changes["lisa note"]; + for c in get_new_comments(upload.policy_queue, upload.changes.source, upload.changes.version, session=session): + session.delete(c) + session.commit() + elif answer == 'O' and not Options["Trainee"]: + confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower() + if confirm == "y": + for c in get_new_comments(upload.policy_queue, upload.changes.source, session=session): + session.delete(c) + session.commit() + elif answer == 'S': - done = 1; + done = True elif answer == 'Q': + end() sys.exit(0) + if handler.get_action(): + print "PENDING %s\n" % handler.get_action() + ################################################################################ ################################################################################ ################################################################################ def usage (exit_code=0): - print """Usage: lisa [OPTION]... [CHANGES]... + print """Usage: dak process-new [OPTION]... [CHANGES]... -a, --automatic automatic run + -b, --no-binaries do not sort binary-NEW packages first + -c, --comments show NEW comments -h, --help show this help and exit. -m, --manual-reject=MSG manual reject with `msg' -n, --no-action don't do anything - -V, --version display the version number and exit""" + -q, --queue=QUEUE operate on a different queue + -t, --trainee FTP Trainee mode + -V, --version display the version number and exit + +ENVIRONMENT VARIABLES + + DAK_INSPECT_UPLOAD: shell command to run to inspect a package + The command is automatically run in a shell when an upload + is checked. The following substitutions are available: + + {directory}: directory the upload is contained in + {dsc}: name of the included dsc or the empty string + {changes}: name of the changes file + + Note that Python's 'format' method is used to format the command. + + Example: run mc in a tmux session to inspect the upload + + export DAK_INSPECT_UPLOAD='tmux new-session -d -s process-new 2>/dev/null; tmux new-window -n "{changes}" -t process-new:0 -k "cd {directory}; mc"' + + and run + + tmux attach -t process-new + + in a separate terminal session. +""" sys.exit(exit_code) ################################################################################ -def init(): - global Cnf, Options, Logger, Katie, projectB, Sections, Priorities; +@contextlib.contextmanager +def lock_package(package): + """ + Lock C{package} so that noone else jumps in processing it. - Cnf = utils.get_conf(); + @type package: string + @param package: source package name to lock + """ - Arguments = [('a',"automatic","Lisa::Options::Automatic"), - ('h',"help","Lisa::Options::Help"), - ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"), - ('n',"no-action","Lisa::Options::No-Action"), - ('V',"version","Lisa::Options::Version")]; + cnf = Config() - for i in ["automatic", "help", "manual-reject", "no-action", "version"]: - if not Cnf.has_key("Lisa::Options::%s" % (i)): - Cnf["Lisa::Options::%s" % (i)] = ""; + path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package) - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - Options = Cnf.SubTree("Lisa::Options") + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) + except OSError as e: + if e.errno == errno.EEXIST or e.errno == errno.EACCES: + user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '') + raise AlreadyLockedError(user) - if Options["Help"]: - usage(); + try: + yield fd + finally: + os.unlink(path) - if Options["Version"]: - print "lisa %s" % (lisa_version); - sys.exit(0); +def do_pkg(upload, session): + # Try to get an included dsc + dsc = upload.source - Katie = katie.Katie(Cnf); + cnf = Config() + group = cnf.get('Dinstall::UnprivGroup') or None - if not Options["No-Action"]: - Logger = Katie.Logger = logging.Logger(Cnf, "lisa"); + #bcc = "X-DAK: dak process-new" + #if cnf.has_key("Dinstall::Bcc"): + # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) + #else: + # u.Subst["__BCC__"] = bcc - projectB = Katie.projectB; + try: + with lock_package(upload.changes.source): + with UploadCopy(upload, group=group) as upload_copy: + handler = PolicyQueueUploadHandler(upload, session) + if handler.get_action() is not None: + print "PENDING %s\n" % handler.get_action() + return - Sections = Section_Completer(); - Priorities = Priority_Completer(); - readline.parse_and_bind("tab: complete"); + do_new(upload, upload_copy, handler, session) + except AlreadyLockedError as e: + print "Seems to be locked by %s already, skipping..." % (e) - return changes_files; +def show_new_comments(uploads, session): + sources = [ upload.changes.source for upload in uploads ] + if len(sources) == 0: + return -################################################################################ + query = """SELECT package, version, comment, author + FROM new_comments + WHERE package IN :sources + ORDER BY package, version""" -def do_byhand(): - done = 0; - while not done: - files = Katie.pkg.files; - will_install = 1; - byhand = []; - - for file in files.keys(): - if files[file]["type"] == "byhand": - if os.path.exists(file): - print "W: %s still present; please process byhand components and try again." % (file); - will_install = 0; - else: - byhand.append(file); - - answer = "XXXX"; - if Options["No-Action"]: - answer = "S"; - if will_install: - if Options["Automatic"] and not Options["No-Action"]: - answer = 'A'; - prompt = "[A]ccept, Manual reject, Skip, Quit ?"; - else: - prompt = "Manual reject, [S]kip, Quit ?"; + r = session.execute(query, params=dict(sources=tuple(sources))) - while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); - if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); - - if answer == 'A': - done = 1; - for file in byhand: - del files[file]; - elif answer == 'M': - Katie.do_reject(1, Options["Manual-Reject"]); - os.unlink(Katie.pkg.changes_file[:-8]+".katie"); - done = 1; - elif answer == 'S': - done = 1; - elif answer == 'Q': - sys.exit(0); + for i in r: + print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3]) + + session.rollback() ################################################################################ -def do_accept(): - print "ACCEPT"; - if not Options["No-Action"]: - retry = 0; - while retry < 10: - try: - lock_fd = os.open(Cnf["Lisa::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL); - retry = 10; - except OSError, e: - if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST': - retry += 1; - if (retry >= 10): - utils.fubar("Couldn't obtain lock; assuming jennifer is already running."); - else: - print("Unable to get accepted lock (try %d of 10)" % retry); - time.sleep(60); - else: - raise; - (summary, short_summary) = Katie.build_summaries(); - Katie.accept(summary, short_summary); - os.unlink(Katie.pkg.changes_file[:-8]+".katie"); - os.unlink(Cnf["Lisa::AcceptedLockFile"]); - -def check_status(files): - new = byhand = 0; - for file in files.keys(): - if files[file]["type"] == "byhand": - byhand = 1; - elif files[file].has_key("new"): - new = 1; - return (new, byhand); - -def do_pkg(changes_file): - Katie.pkg.changes_file = changes_file; - Katie.init_vars(); - Katie.update_vars(); - Katie.update_subst(); - files = Katie.pkg.files; - - if not recheck(): - return; - - (new, byhand) = check_status(files); - if new or byhand: - if new: - do_new(); - if byhand: - do_byhand(); - (new, byhand) = check_status(files); - - if not new and not byhand: - do_accept(); +def sort_uploads(new_queue, uploads, session, nobinaries=False): + sources = {} + sorteduploads = [] + suitesrc = [s.source for s in session.query(DBSource.source). \ + filter(DBSource.suites.any(Suite.suite_name.in_(['unstable', 'experimental'])))] + comments = [p.package for p in session.query(NewComment.package). \ + filter_by(trainee=False, policy_queue=new_queue).distinct()] + for upload in uploads: + source = upload.changes.source + if not source in sources: + sources[source] = [] + sources[source].append({'upload': upload, + 'date': upload.changes.created, + 'stack': 1, + 'binary': True if source in suitesrc else False, + 'comments': True if source in comments else False}) + for src in sources: + if len(sources[src]) > 1: + changes = sources[src] + firstseen = sorted(changes, key=lambda k: (k['date']))[0]['date'] + changes.sort(key=lambda item:item['date']) + for i in range (0, len(changes)): + changes[i]['date'] = firstseen + changes[i]['stack'] = i + 1 + sorteduploads += sources[src] + if nobinaries: + sorteduploads = [u["upload"] for u in sorted(sorteduploads, + key=lambda k: (k["comments"], k["binary"], + k["date"], -k["stack"]))] + else: + sorteduploads = [u["upload"] for u in sorted(sorteduploads, + key=lambda k: (k["comments"], -k["binary"], + k["date"], -k["stack"]))] + return sorteduploads ################################################################################ def end(): - accept_count = Katie.accept_count; - accept_bytes = Katie.accept_bytes; + accept_count = SummaryStats().accept_count + accept_bytes = SummaryStats().accept_bytes if accept_count: sets = "set" if accept_count > 1: sets = "sets" - sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes)))); - Logger.log(["total",accept_count,accept_bytes]); + sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes)))) + Logger.log(["total",accept_count,accept_bytes]) - if not Options["No-Action"]: - Logger.close(); + if not Options["No-Action"] and not Options["Trainee"]: + Logger.close() ################################################################################ def main(): - changes_files = init(); - if len(changes_files) > 50: - sys.stderr.write("Sorting changes...\n"); - changes_files = sort_changes(changes_files); - - # Kill me now? **FIXME** - Cnf["Dinstall::Options::No-Mail"] = ""; - bcc = "X-Katie: lisa %s" % (lisa_version); - if Cnf.has_key("Dinstall::Bcc"): - Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); + global Options, Logger, Sections, Priorities + + cnf = Config() + session = DBConn().session() + + Arguments = [('a',"automatic","Process-New::Options::Automatic"), + ('b',"no-binaries","Process-New::Options::No-Binaries"), + ('c',"comments","Process-New::Options::Comments"), + ('h',"help","Process-New::Options::Help"), + ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), + ('t',"trainee","Process-New::Options::Trainee"), + ('q','queue','Process-New::Options::Queue', 'HasArg'), + ('n',"no-action","Process-New::Options::No-Action")] + + changes_files = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv) + + for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]: + if not cnf.has_key("Process-New::Options::%s" % (i)): + cnf["Process-New::Options::%s" % (i)] = "" + + queue_name = cnf.get('Process-New::Options::Queue', 'new') + new_queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).one() + if len(changes_files) == 0: + uploads = new_queue.uploads else: - Katie.Subst["__BCC__"] = bcc; + uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \ + .join(DBChange).filter(DBChange.changesname.in_(changes_files)).all() + + Options = cnf.subtree("Process-New::Options") + + if Options["Help"]: + usage() + + if not Options["No-Action"]: + try: + Logger = daklog.Logger("process-new") + except CantOpenError as e: + Options["Trainee"] = "True" - for changes_file in changes_files: - changes_file = utils.validate_changes_file_arg(changes_file, 0); - if not changes_file: - continue; - print "\n" + changes_file; - do_pkg (changes_file); + Sections = Section_Completer(session) + Priorities = Priority_Completer(session) + readline.parse_and_bind("tab: complete") + + if len(uploads) > 1: + sys.stderr.write("Sorting changes...\n") + uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"]) + + if Options["Comments"]: + show_new_comments(uploads, session) + else: + for upload in uploads: + do_pkg (upload, session) - end(); + end() ################################################################################