#!/usr/bin/env python
# Handles NEW and BYHAND packages
-# Copyright (C) 2001, 2002 James Troup <james@nocrew.org>
-# $Id: lisa,v 1.9 2002-05-08 11:13:02 troup Exp $
+# Copyright (C) 2001, 2002, 2003, 2004, 2005 James Troup <james@nocrew.org>
+# $Id: lisa,v 1.31 2005-11-15 09:50:32 ajt Exp $
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
################################################################################
-# TODO
-# ----
-
-# We don't check error codes very thoroughly; the old 'trust jennifer'
-# chess nut... db_access calls in particular
-
-# Possible TODO
-# -------------
-
-# Handle multiple different section/priorities (?)
-# hardcoded debianness (debian-installer, source priority etc.) (?)
-# Slang/ncurses interface (?)
-# write changed sections/priority back to katie for later processing (?)
-
-################################################################################
-
-import copy, errno, os, readline, string, stat, sys, tempfile;
+import copy, errno, os, readline, stat, sys, time;
import apt_pkg, apt_inst;
import db_access, fernanda, katie, logging, utils;
# Globals
-lisa_version = "$Revision: 1.9 $";
+lisa_version = "$Revision: 1.31 $";
Cnf = None;
Options = None;
Priorities = None;
Sections = None;
+reject_message = "";
+
################################################################################
################################################################################
################################################################################
+def reject (str, prefix="Rejected: "):
+ global reject_message;
+ if str:
+ reject_message += prefix + str + "\n";
+
+def recheck():
+ global reject_message;
+ files = Katie.pkg.files;
+ reject_message = "";
+
+ for file in files.keys():
+ # The .orig.tar.gz can disappear out from under us is it's a
+ # duplicate of one in the archive.
+ if not files.has_key(file):
+ continue;
+ # Check that the source still exists
+ if files[file]["type"] == "deb":
+ source_version = files[file]["source version"];
+ source_package = files[file]["source package"];
+ if not Katie.pkg.changes["architecture"].has_key("source") \
+ and not Katie.source_exists(source_package, source_version, Katie.pkg.changes["distribution"].keys()):
+ source_epochless_version = utils.re_no_epoch.sub('', source_version);
+ dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version);
+ if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
+ reject("no source found for %s %s (%s)." % (source_package, source_version, file));
+
+ # Version and file overwrite checks
+ if files[file]["type"] == "deb":
+ reject(Katie.check_binary_against_db(file));
+ elif files[file]["type"] == "dsc":
+ reject(Katie.check_source_against_db(file));
+ (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
+ reject(reject_msg);
+
+ if reject_message:
+ answer = "XXX";
+ if Options["No-Action"] or Options["Automatic"]:
+ answer = 'S'
+
+ print "REJECT\n" + reject_message,;
+ prompt = "[R]eject, Skip, Quit ?";
+
+ while prompt.find(answer) == -1:
+ answer = utils.our_raw_input(prompt);
+ m = katie.re_default_answer.match(prompt);
+ if answer == "":
+ answer = m.group(1);
+ answer = answer[:1].upper();
+
+ if answer == 'R':
+ Katie.do_reject(0, reject_message);
+ os.unlink(Katie.pkg.changes_file[:-8]+".katie");
+ return 0;
+ elif answer == 'S':
+ return 0;
+ elif answer == 'Q':
+ sys.exit(0);
+
+ return 1;
+
+################################################################################
+
def determine_new (changes, files):
new = {};
if changes["suite"].has_key("stable"):
print "WARNING: overrides will be added for stable!";
+ if changes["suite"].has_key("oldstable"):
+ print "WARNING: overrides will be added for OLDstable!";
for pkg in new.keys():
if new[pkg].has_key("othercomponents"):
print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]);
################################################################################
-# Sort by 'have source', by ctime, by source name, by source version number, by filename
-
-def changes_compare_by_time (a, b):
- try:
- a_changes = utils.parse_changes(a, 0)
- except:
- return 1;
-
- try:
- b_changes = utils.parse_changes(b, 0)
- except:
- return -1;
-
- utils.cc_fix_changes (a_changes);
- utils.cc_fix_changes (b_changes);
+def indiv_sg_compare (a, b):
+ """Sort by source name, source, version, 'have source', and
+ finally by filename."""
+ # Sort by source version
+ q = apt_pkg.VersionCompare(a["version"], b["version"]);
+ if q:
+ return -q;
# Sort by 'have source'
-
- a_has_source = a_changes["architecture"].get("source")
- b_has_source = b_changes["architecture"].get("source")
+ a_has_source = a["architecture"].get("source");
+ b_has_source = b["architecture"].get("source");
if a_has_source and not b_has_source:
return -1;
elif b_has_source and not a_has_source:
return 1;
- # Sort by ctime
- a_ctime = os.stat(a)[stat.ST_CTIME];
- b_ctime = os.stat(b)[stat.ST_CTIME];
- q = cmp (a_ctime, b_ctime);
- if q:
- return q;
+ return cmp(a["filename"], b["filename"]);
- # Sort by source name
-
- a_source = a_changes.get("source");
- b_source = b_changes.get("source");
- q = cmp (a_source, b_source);
- if q:
- return q;
+############################################################
- # Sort by source version
+def sg_compare (a, b):
+ a = a[1];
+ b = b[1];
+ """Sort by have note, time of oldest upload."""
+ # Sort by have note
+ a_note_state = a["note_state"];
+ b_note_state = b["note_state"];
+ if a_note_state < b_note_state:
+ return -1;
+ elif a_note_state > b_note_state:
+ return 1;
- a_version = a_changes.get("version");
- b_version = b_changes.get("version");
- q = apt_pkg.VersionCompare(a_version, b_version);
- if q:
- return q
+ # Sort by time of oldest upload
+ return cmp(a["oldest"], b["oldest"]);
- # Fall back to sort by filename
+def sort_changes(changes_files):
+ """Sort into source groups, then sort each source group by version,
+ have source, filename. Finally, sort the source groups by have
+ note, time of oldest upload of each source upload."""
+ if len(changes_files) == 1:
+ return changes_files;
- return cmp(a, b);
+ sorted_list = [];
+ cache = {};
+ # Read in all the .changes files
+ for filename in changes_files:
+ try:
+ Katie.pkg.changes_file = filename;
+ Katie.init_vars();
+ Katie.update_vars();
+ cache[filename] = copy.copy(Katie.pkg.changes);
+ cache[filename]["filename"] = filename;
+ except:
+ sorted_list.append(filename);
+ break;
+ # Divide the .changes into per-source groups
+ per_source = {};
+ for filename in cache.keys():
+ source = cache[filename]["source"];
+ if not per_source.has_key(source):
+ per_source[source] = {};
+ per_source[source]["list"] = [];
+ per_source[source]["list"].append(cache[filename]);
+ # Determine oldest time and have note status for each source group
+ for source in per_source.keys():
+ source_list = per_source[source]["list"];
+ first = source_list[0];
+ oldest = os.stat(first["filename"])[stat.ST_MTIME];
+ have_note = 0;
+ for d in per_source[source]["list"]:
+ mtime = os.stat(d["filename"])[stat.ST_MTIME];
+ if mtime < oldest:
+ oldest = mtime;
+ have_note += (d.has_key("lisa note"));
+ per_source[source]["oldest"] = oldest;
+ if not have_note:
+ per_source[source]["note_state"] = 0; # none
+ elif have_note < len(source_list):
+ per_source[source]["note_state"] = 1; # some
+ else:
+ per_source[source]["note_state"] = 2; # all
+ per_source[source]["list"].sort(indiv_sg_compare);
+ per_source_items = per_source.items();
+ per_source_items.sort(sg_compare);
+ for i in per_source_items:
+ for j in i[1]["list"]:
+ sorted_list.append(j["filename"]);
+ return sorted_list;
################################################################################
def print_new (new, indexed, file=sys.stdout):
check_valid(new);
- ret_code = 0;
+ broken = 0;
index = 0;
for pkg in new.keys():
- index = index + 1;
+ index += 1;
section = new[pkg]["section"];
priority = new[pkg]["priority"];
if new[pkg]["section id"] == -1:
- section = section + "[!]";
- ret_code = 1;
+ section += "[!]";
+ broken = 1;
if new[pkg]["priority id"] == -1:
- priority = priority + "[!]";
- ret_code = 1;
+ priority += "[!]";
+ broken = 1;
if indexed:
line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section);
else:
line = "%-20s %-20s %-20s" % (pkg, priority, section);
- line = string.strip(line)+'\n';
+ line = line.strip()+'\n';
file.write(line);
- return ret_code;
+ note = Katie.pkg.changes.get("lisa note");
+ if note:
+ print "*"*75;
+ print note;
+ print "*"*75;
+ return broken, note;
################################################################################
################################################################################
################################################################################
-def spawn_editor (new):
+def edit_new (new):
# Write the current data to a temporary file
- temp_filename = tempfile.mktemp();
- fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
- os.close(fd);
+ temp_filename = utils.temp_filename();
temp_file = utils.open_file(temp_filename, 'w');
print_new (new, 0, temp_file);
temp_file.close();
editor = os.environ.get("EDITOR","vi")
result = os.system("%s %s" % (editor, temp_filename))
if result != 0:
- utils.fubar ("vi invocation failed for `%s'!" % (temp_filename), result)
- # Read the (edited) data back in
- file = utils.open_file(temp_filename);
- for line in file.readlines():
- line = string.strip(line[:-1]);
+ utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
+ # Read the edited data back in
+ temp_file = utils.open_file(temp_filename);
+ lines = temp_file.readlines();
+ temp_file.close();
+ os.unlink(temp_filename);
+ # Parse the new data
+ for line in lines:
+ line = line.strip();
if line == "":
continue;
- s = string.split(line);
+ s = line.split();
# Pad the list if necessary
s[len(s):3] = [None] * (3-len(s));
(pkg, priority, section) = s[:3];
utils.warn("Ignoring unknown package '%s'" % (pkg));
else:
# Strip off any invalid markers, print_new will readd them.
- if section[-3:] == "[!]":
+ if section.endswith("[!]"):
section = section[:-3];
- if priority[-3:] == "[!]":
+ if priority.endswith("[!]"):
priority = priority[:-3];
for file in new[pkg]["files"]:
Katie.pkg.files[file]["section"] = section;
Katie.pkg.files[file]["priority"] = priority;
new[pkg]["section"] = section;
new[pkg]["priority"] = priority;
- os.unlink(temp_filename);
################################################################################
type = new[index]["type"];
done = 0
while not done:
- print string.join([index, priority, section], '\t');
+ print "\t".join([index, priority, section]);
answer = "XXX";
if type != "dsc":
prompt = "[S]ection, Done ? ";
edit_priority = edit_section = 0;
- while string.find(prompt, answer) == -1:
+ while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt);
m = katie.re_default_answer.match(prompt)
if answer == "":
answer = m.group(1)
- answer = string.upper(answer[:1])
+ answer = answer[:1].upper()
if answer == 'P':
edit_priority = 1;
readline.set_completer(Priorities.complete);
got_priority = 0;
while not got_priority:
- new_priority = string.strip(utils.our_raw_input("New priority: "));
- if Priorities.priorities.count(new_priority) == 0:
+ new_priority = utils.our_raw_input("New priority: ").strip();
+ if new_priority not in Priorities.priorities:
print "E: '%s' is not a valid priority, try again." % (new_priority);
else:
got_priority = 1;
readline.set_completer(Sections.complete);
got_section = 0;
while not got_section:
- new_section = string.strip(utils.our_raw_input("New section: "));
- if Sections.sections.count(new_section) == 0:
+ new_section = utils.our_raw_input("New section: ").strip();
+ if new_section not in Sections.sections:
print "E: '%s' is not a valid section, try again." % (new_section);
else:
got_section = 1;
new_index = {};
index = 0;
for i in new.keys():
- index = index + 1;
+ index += 1;
new_index[index] = i;
prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index));
got_answer = 0
while not got_answer:
answer = utils.our_raw_input(prompt);
- answer = string.upper(answer[:1]);
+ if not utils.str_isnum(answer):
+ answer = answer[:1].upper();
if answer == "E" or answer == "D":
got_answer = 1;
elif katie.re_isanum.match (answer):
answer = int(answer);
if (answer < 1) or (answer > index):
- print "%s is not a valid index (%s). Please retry." % (index_range(index), answer);
+ print "%s is not a valid index (%s). Please retry." % (answer, index_range(index));
else:
got_answer = 1;
if answer == 'E':
- spawn_editor(new);
+ edit_new(new);
elif answer == 'D':
done = 1;
else:
################################################################################
+def edit_note(note):
+ # Write the current data to a temporary file
+ temp_filename = utils.temp_filename();
+ temp_file = utils.open_file(temp_filename, 'w');
+ temp_file.write(note);
+ temp_file.close();
+ editor = os.environ.get("EDITOR","vi")
+ answer = 'E';
+ while answer == 'E':
+ os.system("%s %s" % (editor, temp_filename))
+ temp_file = utils.open_file(temp_filename);
+ note = temp_file.read().rstrip();
+ temp_file.close();
+ print "Note:";
+ print utils.prefix_multi_line_string(note," ");
+ prompt = "[D]one, Edit, Abandon, Quit ?"
+ answer = "XXX";
+ while prompt.find(answer) == -1:
+ answer = utils.our_raw_input(prompt);
+ m = katie.re_default_answer.search(prompt);
+ if answer == "":
+ answer = m.group(1);
+ answer = answer[:1].upper();
+ os.unlink(temp_filename);
+ if answer == 'A':
+ return;
+ elif answer == 'Q':
+ sys.exit(0);
+ Katie.pkg.changes["lisa note"] = note;
+ Katie.dump_vars(Cnf["Dir::Queue::New"]);
+
+################################################################################
+
def check_pkg ():
try:
- less_fd = os.popen("less -", 'w', 0);
+ less_fd = os.popen("less -R -", 'w', 0);
stdout_fd = sys.stdout;
try:
sys.stdout = less_fd;
for file in files.keys():
if files[file]["type"] == "deb":
control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
- summary = summary + "\n";
- summary = summary + "Package: %s\n" % (control.Find("Package"));
- summary = summary + "Description: %s\n" % (control.Find("Description"));
+ summary += "\n";
+ summary += "Package: %s\n" % (control.Find("Package"));
+ summary += "Description: %s\n" % (control.Find("Description"));
Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary;
bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification");
- utils.send_mail(bxa_mail,"");
+ utils.send_mail(bxa_mail);
################################################################################
type_id = db_access.get_override_type_id(new[pkg]["type"]);
priority_id = new[pkg]["priority id"];
section_id = new[pkg]["section id"];
- projectB.query("INSERT INTO override (suite, component, type, package, priority, section) VALUES (%s, %s, %s, '%s', %s, %s)" % (suite_id, component_id, type_id, pkg, priority_id, section_id));
+ projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id));
for file in new[pkg]["files"]:
if files[file].has_key("new"):
del files[file]["new"];
################################################################################
+def prod_maintainer ():
+ # Here we prepare an editor and get them ready to prod...
+ temp_filename = utils.temp_filename();
+ editor = os.environ.get("EDITOR","vi")
+ answer = 'E';
+ while answer == 'E':
+ os.system("%s %s" % (editor, temp_filename))
+ file = utils.open_file(temp_filename);
+ prod_message = "".join(file.readlines());
+ file.close();
+ print "Prod message:";
+ print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1);
+ prompt = "[P]rod, Edit, Abandon, Quit ?"
+ answer = "XXX";
+ while prompt.find(answer) == -1:
+ answer = utils.our_raw_input(prompt);
+ m = katie.re_default_answer.search(prompt);
+ if answer == "":
+ answer = m.group(1);
+ answer = answer[:1].upper();
+ os.unlink(temp_filename);
+ if answer == 'A':
+ return;
+ elif answer == 'Q':
+ sys.exit(0);
+ # Otherwise, do the proding...
+ user_email_address = utils.whoami() + " <%s>" % (
+ Cnf["Dinstall::MyAdminAddress"]);
+
+ Subst = Katie.Subst;
+
+ Subst["__FROM_ADDRESS__"] = user_email_address;
+ Subst["__PROD_MESSAGE__"] = prod_message;
+ Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
+
+ prod_mail_message = utils.TemplateSubst(
+ Subst,Cnf["Dir::Templates"]+"/lisa.prod");
+
+ # Send the prod mail if appropriate
+ if not Cnf["Dinstall::Options::No-Mail"]:
+ utils.send_mail(prod_mail_message);
+
+ print "Sent proding message";
+
+################################################################################
+
def do_new():
print "NEW\n";
files = Katie.pkg.files;
if Options["No-Action"] or Options["Automatic"]:
answer = 'S';
- broken = print_new(new, 0);
+ (broken, note) = print_new(new, 0);
prompt = "";
- if not broken:
+
+ if not broken and not note:
prompt = "Add overrides, ";
- else:
+ if broken:
print "W: [!] marked entries must be fixed before package can be processed.";
- if answer == 'A':
- answer = 'E';
- prompt = prompt + "Edit overrides, Check, Manual reject, [S]kip, Quit ?";
+ if note:
+ print "W: note must be removed before package can be processed.";
+ prompt += "Remove note, ";
- while string.find(prompt, answer) == -1:
+ prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?";
+
+ while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt);
m = katie.re_default_answer.search(prompt);
if answer == "":
answer = m.group(1)
- answer = string.upper(answer[:1])
+ answer = answer[:1].upper()
if answer == 'A':
done = add_overrides (new);
if not aborted:
os.unlink(Katie.pkg.changes_file[:-8]+".katie");
done = 1;
+ elif answer == 'N':
+ edit_note(changes.get("lisa note", ""));
+ elif answer == 'P':
+ prod_maintainer();
+ elif answer == 'R':
+ confirm = utils.our_raw_input("Really clear note (y/N)? ").lower();
+ if confirm == "y":
+ del changes["lisa note"];
elif answer == 'S':
done = 1;
elif answer == 'Q':
-h, --help show this help and exit.
-m, --manual-reject=MSG manual reject with `msg'
-n, --no-action don't do anything
- -s, --sort=TYPE sort type ('time' or 'normal')
-V, --version display the version number and exit"""
sys.exit(exit_code)
('h',"help","Lisa::Options::Help"),
('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"),
('n',"no-action","Lisa::Options::No-Action"),
- ('s',"sort","Lisa::Options::Sort","HasArg"),
('V',"version","Lisa::Options::Version")];
for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
if not Cnf.has_key("Lisa::Options::%s" % (i)):
Cnf["Lisa::Options::%s" % (i)] = "";
- if not Cnf.has_key("Lisa::Options::Sort"):
- Cnf["Lisa::Options::Sort"] = "time";
changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
Options = Cnf.SubTree("Lisa::Options")
print "lisa %s" % (lisa_version);
sys.exit(0);
- if Options["Sort"] != "time" and Options["Sort"] != "normal":
- utils.fubar("Unrecognised sort type '%s'. (Recognised sort types are: time and normal)" % (Options["Sort"]));
-
Katie = katie.Katie(Cnf);
if not Options["No-Action"]:
else:
prompt = "Manual reject, [S]kip, Quit ?";
- while string.find(prompt, answer) == -1:
+ while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt);
m = katie.re_default_answer.search(prompt);
if answer == "":
answer = m.group(1);
- answer = string.upper(answer[:1]);
+ answer = answer[:1].upper();
if answer == 'A':
done = 1;
def do_accept():
print "ACCEPT";
if not Options["No-Action"]:
+ retry = 0;
+ while retry < 10:
+ try:
+ lock_fd = os.open(Cnf["Lisa::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL);
+ retry = 10;
+ except OSError, e:
+ if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
+ retry += 1;
+ if (retry >= 10):
+ utils.fubar("Couldn't obtain lock; assuming jennifer is already running.");
+ else:
+ print("Unable to get accepted lock (try %d of 10)" % retry);
+ time.sleep(60);
+ else:
+ raise;
(summary, short_summary) = Katie.build_summaries();
Katie.accept(summary, short_summary);
os.unlink(Katie.pkg.changes_file[:-8]+".katie");
+ os.unlink(Cnf["Lisa::AcceptedLockFile"]);
def check_status(files):
new = byhand = 0;
Katie.update_subst();
files = Katie.pkg.files;
+ if not recheck():
+ return;
+
(new, byhand) = check_status(files);
if new or byhand:
if new:
def main():
changes_files = init();
-
- # Sort the changes files
- if Options["Sort"] == "time":
- changes_files.sort(changes_compare_by_time);
- else:
- changes_files.sort(utils.changes_compare);
+ if len(changes_files) > 50:
+ sys.stderr.write("Sorting changes...\n");
+ changes_files = sort_changes(changes_files);
# Kill me now? **FIXME**
Cnf["Dinstall::Options::No-Mail"] = "";
- bcc = "X-Katie: %s" % (lisa_version);
+ bcc = "X-Katie: lisa %s" % (lisa_version);
if Cnf.has_key("Dinstall::Bcc"):
Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
else:
Katie.Subst["__BCC__"] = bcc;
for changes_file in changes_files:
- if not os.path.exists(changes_file):
- print "\nSkipping %s - file does not exist." % (changes_file);
+ changes_file = utils.validate_changes_file_arg(changes_file, 0);
+ if not changes_file:
continue;
print "\n" + changes_file;
do_pkg (changes_file);
projects / dak.git / blobdiff