X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=lisa;h=d98c0df683c77735d3fe47c512c5209abfb52f2e;hb=9540d873fa78598454af57f5f8a4875969ed0439;hp=044912860e558667b80d47490abee2ad2d73c036;hpb=8ecf66b7d933929ca5f504a32fb54542b0b87b71;p=dak.git diff --git a/lisa b/lisa index 04491286..d98c0df6 100755 --- a/lisa +++ b/lisa @@ -1,8 +1,8 @@ #!/usr/bin/env python # Handles NEW and BYHAND packages -# Copyright (C) 2001, 2002 James Troup -# $Id: lisa,v 1.9 2002-05-08 11:13:02 troup Exp $ +# Copyright (C) 2001, 2002, 2003, 2004, 2005 James Troup +# $Id: lisa,v 1.31 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,28 +37,12 @@ ################################################################################ -# TODO -# ---- - -# We don't check error codes very thoroughly; the old 'trust jennifer' -# chess nut... db_access calls in particular - -# Possible TODO -# ------------- - -# Handle multiple different section/priorities (?) -# hardcoded debianness (debian-installer, source priority etc.) (?) -# Slang/ncurses interface (?) -# write changed sections/priority back to katie for later processing (?) - -################################################################################ - -import copy, errno, os, readline, string, stat, sys, tempfile; +import copy, errno, os, readline, stat, sys, time; import apt_pkg, apt_inst; import db_access, fernanda, katie, logging, utils; # Globals -lisa_version = "$Revision: 1.9 $"; +lisa_version = "$Revision: 1.31 $"; Cnf = None; Options = None; @@ -69,10 +53,74 @@ Logger = None; Priorities = None; Sections = None; +reject_message = ""; + ################################################################################ ################################################################################ ################################################################################ +def reject (str, prefix="Rejected: "): + global reject_message; + if str: + reject_message += prefix + str + "\n"; + +def recheck(): + global reject_message; + files = Katie.pkg.files; + reject_message = ""; + + for file in files.keys(): + # The .orig.tar.gz can disappear out from under us is it's a + # duplicate of one in the archive. + if not files.has_key(file): + continue; + # Check that the source still exists + if files[file]["type"] == "deb": + source_version = files[file]["source version"]; + source_package = files[file]["source package"]; + if not Katie.pkg.changes["architecture"].has_key("source") \ + and not Katie.source_exists(source_package, source_version, Katie.pkg.changes["distribution"].keys()): + source_epochless_version = utils.re_no_epoch.sub('', source_version); + dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version); + if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename): + reject("no source found for %s %s (%s)." % (source_package, source_version, file)); + + # Version and file overwrite checks + if files[file]["type"] == "deb": + reject(Katie.check_binary_against_db(file)); + elif files[file]["type"] == "dsc": + reject(Katie.check_source_against_db(file)); + (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file); + reject(reject_msg); + + if reject_message: + answer = "XXX"; + if Options["No-Action"] or Options["Automatic"]: + answer = 'S' + + print "REJECT\n" + reject_message,; + prompt = "[R]eject, Skip, Quit ?"; + + while prompt.find(answer) == -1: + answer = utils.our_raw_input(prompt); + m = katie.re_default_answer.match(prompt); + if answer == "": + answer = m.group(1); + answer = answer[:1].upper(); + + if answer == 'R': + Katie.do_reject(0, reject_message); + os.unlink(Katie.pkg.changes_file[:-8]+".katie"); + return 0; + elif answer == 'S': + return 0; + elif answer == 'Q': + sys.exit(0); + + return 1; + +################################################################################ + def determine_new (changes, files): new = {}; @@ -128,6 +176,8 @@ def determine_new (changes, files): if changes["suite"].has_key("stable"): print "WARNING: overrides will be added for stable!"; + if changes["suite"].has_key("oldstable"): + print "WARNING: overrides will be added for OLDstable!"; for pkg in new.keys(): if new[pkg].has_key("othercomponents"): print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]); @@ -136,57 +186,94 @@ def determine_new (changes, files): ################################################################################ -# Sort by 'have source', by ctime, by source name, by source version number, by filename - -def changes_compare_by_time (a, b): - try: - a_changes = utils.parse_changes(a, 0) - except: - return 1; - - try: - b_changes = utils.parse_changes(b, 0) - except: - return -1; - - utils.cc_fix_changes (a_changes); - utils.cc_fix_changes (b_changes); +def indiv_sg_compare (a, b): + """Sort by source name, source, version, 'have source', and + finally by filename.""" + # Sort by source version + q = apt_pkg.VersionCompare(a["version"], b["version"]); + if q: + return -q; # Sort by 'have source' - - a_has_source = a_changes["architecture"].get("source") - b_has_source = b_changes["architecture"].get("source") + a_has_source = a["architecture"].get("source"); + b_has_source = b["architecture"].get("source"); if a_has_source and not b_has_source: return -1; elif b_has_source and not a_has_source: return 1; - # Sort by ctime - a_ctime = os.stat(a)[stat.ST_CTIME]; - b_ctime = os.stat(b)[stat.ST_CTIME]; - q = cmp (a_ctime, b_ctime); - if q: - return q; + return cmp(a["filename"], b["filename"]); - # Sort by source name - - a_source = a_changes.get("source"); - b_source = b_changes.get("source"); - q = cmp (a_source, b_source); - if q: - return q; +############################################################ - # Sort by source version +def sg_compare (a, b): + a = a[1]; + b = b[1]; + """Sort by have note, time of oldest upload.""" + # Sort by have note + a_note_state = a["note_state"]; + b_note_state = b["note_state"]; + if a_note_state < b_note_state: + return -1; + elif a_note_state > b_note_state: + return 1; - a_version = a_changes.get("version"); - b_version = b_changes.get("version"); - q = apt_pkg.VersionCompare(a_version, b_version); - if q: - return q + # Sort by time of oldest upload + return cmp(a["oldest"], b["oldest"]); - # Fall back to sort by filename +def sort_changes(changes_files): + """Sort into source groups, then sort each source group by version, + have source, filename. Finally, sort the source groups by have + note, time of oldest upload of each source upload.""" + if len(changes_files) == 1: + return changes_files; - return cmp(a, b); + sorted_list = []; + cache = {}; + # Read in all the .changes files + for filename in changes_files: + try: + Katie.pkg.changes_file = filename; + Katie.init_vars(); + Katie.update_vars(); + cache[filename] = copy.copy(Katie.pkg.changes); + cache[filename]["filename"] = filename; + except: + sorted_list.append(filename); + break; + # Divide the .changes into per-source groups + per_source = {}; + for filename in cache.keys(): + source = cache[filename]["source"]; + if not per_source.has_key(source): + per_source[source] = {}; + per_source[source]["list"] = []; + per_source[source]["list"].append(cache[filename]); + # Determine oldest time and have note status for each source group + for source in per_source.keys(): + source_list = per_source[source]["list"]; + first = source_list[0]; + oldest = os.stat(first["filename"])[stat.ST_MTIME]; + have_note = 0; + for d in per_source[source]["list"]: + mtime = os.stat(d["filename"])[stat.ST_MTIME]; + if mtime < oldest: + oldest = mtime; + have_note += (d.has_key("lisa note")); + per_source[source]["oldest"] = oldest; + if not have_note: + per_source[source]["note_state"] = 0; # none + elif have_note < len(source_list): + per_source[source]["note_state"] = 1; # some + else: + per_source[source]["note_state"] = 2; # all + per_source[source]["list"].sort(indiv_sg_compare); + per_source_items = per_source.items(); + per_source_items.sort(sg_compare); + for i in per_source_items: + for j in i[1]["list"]: + sorted_list.append(j["filename"]); + return sorted_list; ################################################################################ @@ -251,25 +338,30 @@ def check_valid (new): def print_new (new, indexed, file=sys.stdout): check_valid(new); - ret_code = 0; + broken = 0; index = 0; for pkg in new.keys(): - index = index + 1; + index += 1; section = new[pkg]["section"]; priority = new[pkg]["priority"]; if new[pkg]["section id"] == -1: - section = section + "[!]"; - ret_code = 1; + section += "[!]"; + broken = 1; if new[pkg]["priority id"] == -1: - priority = priority + "[!]"; - ret_code = 1; + priority += "[!]"; + broken = 1; if indexed: line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section); else: line = "%-20s %-20s %-20s" % (pkg, priority, section); - line = string.strip(line)+'\n'; + line = line.strip()+'\n'; file.write(line); - return ret_code; + note = Katie.pkg.changes.get("lisa note"); + if note: + print "*"*75; + print note; + print "*"*75; + return broken, note; ################################################################################ @@ -300,11 +392,9 @@ def index_range (index): ################################################################################ ################################################################################ -def spawn_editor (new): +def edit_new (new): # Write the current data to a temporary file - temp_filename = tempfile.mktemp(); - fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700); - os.close(fd); + temp_filename = utils.temp_filename(); temp_file = utils.open_file(temp_filename, 'w'); print_new (new, 0, temp_file); temp_file.close(); @@ -312,14 +402,18 @@ def spawn_editor (new): editor = os.environ.get("EDITOR","vi") result = os.system("%s %s" % (editor, temp_filename)) if result != 0: - utils.fubar ("vi invocation failed for `%s'!" % (temp_filename), result) - # Read the (edited) data back in - file = utils.open_file(temp_filename); - for line in file.readlines(): - line = string.strip(line[:-1]); + utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result) + # Read the edited data back in + temp_file = utils.open_file(temp_filename); + lines = temp_file.readlines(); + temp_file.close(); + os.unlink(temp_filename); + # Parse the new data + for line in lines: + line = line.strip(); if line == "": continue; - s = string.split(line); + s = line.split(); # Pad the list if necessary s[len(s):3] = [None] * (3-len(s)); (pkg, priority, section) = s[:3]; @@ -327,16 +421,15 @@ def spawn_editor (new): utils.warn("Ignoring unknown package '%s'" % (pkg)); else: # Strip off any invalid markers, print_new will readd them. - if section[-3:] == "[!]": + if section.endswith("[!]"): section = section[:-3]; - if priority[-3:] == "[!]": + if priority.endswith("[!]"): priority = priority[:-3]; for file in new[pkg]["files"]: Katie.pkg.files[file]["section"] = section; Katie.pkg.files[file]["priority"] = priority; new[pkg]["section"] = section; new[pkg]["priority"] = priority; - os.unlink(temp_filename); ################################################################################ @@ -346,7 +439,7 @@ def edit_index (new, index): type = new[index]["type"]; done = 0 while not done: - print string.join([index, priority, section], '\t'); + print "\t".join([index, priority, section]); answer = "XXX"; if type != "dsc": @@ -355,12 +448,12 @@ def edit_index (new, index): prompt = "[S]ection, Done ? "; edit_priority = edit_section = 0; - while string.find(prompt, answer) == -1: + while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt); m = katie.re_default_answer.match(prompt) if answer == "": answer = m.group(1) - answer = string.upper(answer[:1]) + answer = answer[:1].upper() if answer == 'P': edit_priority = 1; @@ -376,8 +469,8 @@ def edit_index (new, index): readline.set_completer(Priorities.complete); got_priority = 0; while not got_priority: - new_priority = string.strip(utils.our_raw_input("New priority: ")); - if Priorities.priorities.count(new_priority) == 0: + new_priority = utils.our_raw_input("New priority: ").strip(); + if new_priority not in Priorities.priorities: print "E: '%s' is not a valid priority, try again." % (new_priority); else: got_priority = 1; @@ -388,8 +481,8 @@ def edit_index (new, index): readline.set_completer(Sections.complete); got_section = 0; while not got_section: - new_section = string.strip(utils.our_raw_input("New section: ")); - if Sections.sections.count(new_section) == 0: + new_section = utils.our_raw_input("New section: ").strip(); + if new_section not in Sections.sections: print "E: '%s' is not a valid section, try again." % (new_section); else: got_section = 1; @@ -415,7 +508,7 @@ def edit_overrides (new): new_index = {}; index = 0; for i in new.keys(): - index = index + 1; + index += 1; new_index[index] = i; prompt = "(%s) edit override , Editor, Done ? " % (index_range(index)); @@ -423,18 +516,19 @@ def edit_overrides (new): got_answer = 0 while not got_answer: answer = utils.our_raw_input(prompt); - answer = string.upper(answer[:1]); + if not utils.str_isnum(answer): + answer = answer[:1].upper(); if answer == "E" or answer == "D": got_answer = 1; elif katie.re_isanum.match (answer): answer = int(answer); if (answer < 1) or (answer > index): - print "%s is not a valid index (%s). Please retry." % (index_range(index), answer); + print "%s is not a valid index (%s). Please retry." % (answer, index_range(index)); else: got_answer = 1; if answer == 'E': - spawn_editor(new); + edit_new(new); elif answer == 'D': done = 1; else: @@ -444,9 +538,42 @@ def edit_overrides (new): ################################################################################ +def edit_note(note): + # Write the current data to a temporary file + temp_filename = utils.temp_filename(); + temp_file = utils.open_file(temp_filename, 'w'); + temp_file.write(note); + temp_file.close(); + editor = os.environ.get("EDITOR","vi") + answer = 'E'; + while answer == 'E': + os.system("%s %s" % (editor, temp_filename)) + temp_file = utils.open_file(temp_filename); + note = temp_file.read().rstrip(); + temp_file.close(); + print "Note:"; + print utils.prefix_multi_line_string(note," "); + prompt = "[D]one, Edit, Abandon, Quit ?" + answer = "XXX"; + while prompt.find(answer) == -1: + answer = utils.our_raw_input(prompt); + m = katie.re_default_answer.search(prompt); + if answer == "": + answer = m.group(1); + answer = answer[:1].upper(); + os.unlink(temp_filename); + if answer == 'A': + return; + elif answer == 'Q': + sys.exit(0); + Katie.pkg.changes["lisa note"] = note; + Katie.dump_vars(Cnf["Dir::Queue::New"]); + +################################################################################ + def check_pkg (): try: - less_fd = os.popen("less -", 'w', 0); + less_fd = os.popen("less -R -", 'w', 0); stdout_fd = sys.stdout; try: sys.stdout = less_fd; @@ -481,12 +608,12 @@ def do_bxa_notification(): for file in files.keys(): if files[file]["type"] == "deb": control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file))); - summary = summary + "\n"; - summary = summary + "Package: %s\n" % (control.Find("Package")); - summary = summary + "Description: %s\n" % (control.Find("Description")); + summary += "\n"; + summary += "Package: %s\n" % (control.Find("Package")); + summary += "Description: %s\n" % (control.Find("Description")); Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary; bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification"); - utils.send_mail(bxa_mail,""); + utils.send_mail(bxa_mail); ################################################################################ @@ -502,7 +629,7 @@ def add_overrides (new): type_id = db_access.get_override_type_id(new[pkg]["type"]); priority_id = new[pkg]["priority id"]; section_id = new[pkg]["section id"]; - projectB.query("INSERT INTO override (suite, component, type, package, priority, section) VALUES (%s, %s, %s, '%s', %s, %s)" % (suite_id, component_id, type_id, pkg, priority_id, section_id)); + projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id)); for file in new[pkg]["files"]: if files[file].has_key("new"): del files[file]["new"]; @@ -515,6 +642,52 @@ def add_overrides (new): ################################################################################ +def prod_maintainer (): + # Here we prepare an editor and get them ready to prod... + temp_filename = utils.temp_filename(); + editor = os.environ.get("EDITOR","vi") + answer = 'E'; + while answer == 'E': + os.system("%s %s" % (editor, temp_filename)) + file = utils.open_file(temp_filename); + prod_message = "".join(file.readlines()); + file.close(); + print "Prod message:"; + print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1); + prompt = "[P]rod, Edit, Abandon, Quit ?" + answer = "XXX"; + while prompt.find(answer) == -1: + answer = utils.our_raw_input(prompt); + m = katie.re_default_answer.search(prompt); + if answer == "": + answer = m.group(1); + answer = answer[:1].upper(); + os.unlink(temp_filename); + if answer == 'A': + return; + elif answer == 'Q': + sys.exit(0); + # Otherwise, do the proding... + user_email_address = utils.whoami() + " <%s>" % ( + Cnf["Dinstall::MyAdminAddress"]); + + Subst = Katie.Subst; + + Subst["__FROM_ADDRESS__"] = user_email_address; + Subst["__PROD_MESSAGE__"] = prod_message; + Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; + + prod_mail_message = utils.TemplateSubst( + Subst,Cnf["Dir::Templates"]+"/lisa.prod"); + + # Send the prod mail if appropriate + if not Cnf["Dinstall::Options::No-Mail"]: + utils.send_mail(prod_mail_message); + + print "Sent proding message"; + +################################################################################ + def do_new(): print "NEW\n"; files = Katie.pkg.files; @@ -548,22 +721,25 @@ def do_new(): if Options["No-Action"] or Options["Automatic"]: answer = 'S'; - broken = print_new(new, 0); + (broken, note) = print_new(new, 0); prompt = ""; - if not broken: + + if not broken and not note: prompt = "Add overrides, "; - else: + if broken: print "W: [!] marked entries must be fixed before package can be processed."; - if answer == 'A': - answer = 'E'; - prompt = prompt + "Edit overrides, Check, Manual reject, [S]kip, Quit ?"; + if note: + print "W: note must be removed before package can be processed."; + prompt += "Remove note, "; - while string.find(prompt, answer) == -1: + prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"; + + while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt); m = katie.re_default_answer.search(prompt); if answer == "": answer = m.group(1) - answer = string.upper(answer[:1]) + answer = answer[:1].upper() if answer == 'A': done = add_overrides (new); @@ -576,6 +752,14 @@ def do_new(): if not aborted: os.unlink(Katie.pkg.changes_file[:-8]+".katie"); done = 1; + elif answer == 'N': + edit_note(changes.get("lisa note", "")); + elif answer == 'P': + prod_maintainer(); + elif answer == 'R': + confirm = utils.our_raw_input("Really clear note (y/N)? ").lower(); + if confirm == "y": + del changes["lisa note"]; elif answer == 'S': done = 1; elif answer == 'Q': @@ -591,7 +775,6 @@ def usage (exit_code=0): -h, --help show this help and exit. -m, --manual-reject=MSG manual reject with `msg' -n, --no-action don't do anything - -s, --sort=TYPE sort type ('time' or 'normal') -V, --version display the version number and exit""" sys.exit(exit_code) @@ -606,14 +789,11 @@ def init(): ('h',"help","Lisa::Options::Help"), ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"), ('n',"no-action","Lisa::Options::No-Action"), - ('s',"sort","Lisa::Options::Sort","HasArg"), ('V',"version","Lisa::Options::Version")]; for i in ["automatic", "help", "manual-reject", "no-action", "version"]: if not Cnf.has_key("Lisa::Options::%s" % (i)): Cnf["Lisa::Options::%s" % (i)] = ""; - if not Cnf.has_key("Lisa::Options::Sort"): - Cnf["Lisa::Options::Sort"] = "time"; changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); Options = Cnf.SubTree("Lisa::Options") @@ -625,9 +805,6 @@ def init(): print "lisa %s" % (lisa_version); sys.exit(0); - if Options["Sort"] != "time" and Options["Sort"] != "normal": - utils.fubar("Unrecognised sort type '%s'. (Recognised sort types are: time and normal)" % (Options["Sort"])); - Katie = katie.Katie(Cnf); if not Options["No-Action"]: @@ -668,12 +845,12 @@ def do_byhand(): else: prompt = "Manual reject, [S]kip, Quit ?"; - while string.find(prompt, answer) == -1: + while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt); m = katie.re_default_answer.search(prompt); if answer == "": answer = m.group(1); - answer = string.upper(answer[:1]); + answer = answer[:1].upper(); if answer == 'A': done = 1; @@ -693,9 +870,25 @@ def do_byhand(): def do_accept(): print "ACCEPT"; if not Options["No-Action"]: + retry = 0; + while retry < 10: + try: + lock_fd = os.open(Cnf["Lisa::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL); + retry = 10; + except OSError, e: + if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST': + retry += 1; + if (retry >= 10): + utils.fubar("Couldn't obtain lock; assuming jennifer is already running."); + else: + print("Unable to get accepted lock (try %d of 10)" % retry); + time.sleep(60); + else: + raise; (summary, short_summary) = Katie.build_summaries(); Katie.accept(summary, short_summary); os.unlink(Katie.pkg.changes_file[:-8]+".katie"); + os.unlink(Cnf["Lisa::AcceptedLockFile"]); def check_status(files): new = byhand = 0; @@ -713,6 +906,9 @@ def do_pkg(changes_file): Katie.update_subst(); files = Katie.pkg.files; + if not recheck(): + return; + (new, byhand) = check_status(files); if new or byhand: if new: @@ -744,24 +940,21 @@ def end(): def main(): changes_files = init(); - - # Sort the changes files - if Options["Sort"] == "time": - changes_files.sort(changes_compare_by_time); - else: - changes_files.sort(utils.changes_compare); + if len(changes_files) > 50: + sys.stderr.write("Sorting changes...\n"); + changes_files = sort_changes(changes_files); # Kill me now? **FIXME** Cnf["Dinstall::Options::No-Mail"] = ""; - bcc = "X-Katie: %s" % (lisa_version); + bcc = "X-Katie: lisa %s" % (lisa_version); if Cnf.has_key("Dinstall::Bcc"): Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); else: Katie.Subst["__BCC__"] = bcc; for changes_file in changes_files: - if not os.path.exists(changes_file): - print "\nSkipping %s - file does not exist." % (changes_file); + changes_file = utils.validate_changes_file_arg(changes_file, 0); + if not changes_file: continue; print "\n" + changes_file; do_pkg (changes_file);