#!/usr/bin/env python
+# vim:set et ts=4 sw=4:
-# Handles NEW and BYHAND packages
-# Copyright (C) 2001, 2002, 2003, 2004, 2005 James Troup <james@nocrew.org>
-# $Id: lisa,v 1.31 2005-11-15 09:50:32 ajt Exp $
+""" Handles NEW and BYHAND packages
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
################################################################################
-import copy, errno, os, readline, stat, sys, time;
-import apt_pkg, apt_inst;
-import db_access, fernanda, katie, logging, utils;
+from __future__ import with_statement
+
+import copy
+import errno
+import os
+import readline
+import stat
+import sys
+import time
+import contextlib
+import pwd
+import apt_pkg, apt_inst
+import examine_package
+
+from daklib.dbconn import *
+from daklib.queue import *
+from daklib import daklog
+from daklib import utils
+from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
+from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
+from daklib.summarystats import SummaryStats
+from daklib.config import Config
+from daklib.changesutils import *
# Globals
-lisa_version = "$Revision: 1.31 $";
+Options = None
+Logger = None
-Cnf = None;
-Options = None;
-Katie = None;
-projectB = None;
-Logger = None;
-
-Priorities = None;
-Sections = None;
-
-reject_message = "";
+Priorities = None
+Sections = None
################################################################################
################################################################################
################################################################################
-def reject (str, prefix="Rejected: "):
- global reject_message;
- if str:
- reject_message += prefix + str + "\n";
-
-def recheck():
- global reject_message;
- files = Katie.pkg.files;
- reject_message = "";
-
- for file in files.keys():
- # The .orig.tar.gz can disappear out from under us is it's a
- # duplicate of one in the archive.
- if not files.has_key(file):
- continue;
- # Check that the source still exists
- if files[file]["type"] == "deb":
- source_version = files[file]["source version"];
- source_package = files[file]["source package"];
- if not Katie.pkg.changes["architecture"].has_key("source") \
- and not Katie.source_exists(source_package, source_version, Katie.pkg.changes["distribution"].keys()):
- source_epochless_version = utils.re_no_epoch.sub('', source_version);
- dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version);
- if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
- reject("no source found for %s %s (%s)." % (source_package, source_version, file));
-
- # Version and file overwrite checks
- if files[file]["type"] == "deb":
- reject(Katie.check_binary_against_db(file));
- elif files[file]["type"] == "dsc":
- reject(Katie.check_source_against_db(file));
- (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
- reject(reject_msg);
-
- if reject_message:
- answer = "XXX";
- if Options["No-Action"] or Options["Automatic"]:
+def recheck(upload, session):
+# STU: I'm not sure, but I don't thin kthis is necessary any longer: upload.recheck(session)
+ if len(upload.rejects) > 0:
+ answer = "XXX"
+ if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
answer = 'S'
- print "REJECT\n" + reject_message,;
- prompt = "[R]eject, Skip, Quit ?";
+ print "REJECT\n%s" % '\n'.join(upload.rejects)
+ prompt = "[R]eject, Skip, Quit ?"
while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt);
- m = katie.re_default_answer.match(prompt);
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.match(prompt)
if answer == "":
- answer = m.group(1);
- answer = answer[:1].upper();
+ answer = m.group(1)
+ answer = answer[:1].upper()
if answer == 'R':
- Katie.do_reject(0, reject_message);
- os.unlink(Katie.pkg.changes_file[:-8]+".katie");
- return 0;
+ upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
+ upload.pkg.remove_known_changes(session=session)
+ session.commit()
+ return 0
elif answer == 'S':
- return 0;
+ return 0
elif answer == 'Q':
- sys.exit(0);
-
- return 1;
-
-################################################################################
-
-def determine_new (changes, files):
- new = {};
-
- # Build up a list of potentially new things
- for file in files.keys():
- f = files[file];
- # Skip byhand elements
- if f["type"] == "byhand":
- continue;
- pkg = f["package"];
- priority = f["priority"];
- section = f["section"];
- # FIXME: unhardcode
- if section == "non-US/main":
- section = "non-US";
- type = get_type(f);
- component = f["component"];
-
- if type == "dsc":
- priority = "source";
- if not new.has_key(pkg):
- new[pkg] = {};
- new[pkg]["priority"] = priority;
- new[pkg]["section"] = section;
- new[pkg]["type"] = type;
- new[pkg]["component"] = component;
- new[pkg]["files"] = [];
- else:
- old_type = new[pkg]["type"];
- if old_type != type:
- # source gets trumped by deb or udeb
- if old_type == "dsc":
- new[pkg]["priority"] = priority;
- new[pkg]["section"] = section;
- new[pkg]["type"] = type;
- new[pkg]["component"] = component;
- new[pkg]["files"].append(file);
- if f.has_key("othercomponents"):
- new[pkg]["othercomponents"] = f["othercomponents"];
-
- for suite in changes["suite"].keys():
- suite_id = db_access.get_suite_id(suite);
- for pkg in new.keys():
- component_id = db_access.get_component_id(new[pkg]["component"]);
- type_id = db_access.get_override_type_id(new[pkg]["type"]);
- q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id));
- ql = q.getresult();
- if ql:
- for file in new[pkg]["files"]:
- if files[file].has_key("new"):
- del files[file]["new"];
- del new[pkg];
-
- if changes["suite"].has_key("stable"):
- print "WARNING: overrides will be added for stable!";
- if changes["suite"].has_key("oldstable"):
- print "WARNING: overrides will be added for OLDstable!";
- for pkg in new.keys():
- if new[pkg].has_key("othercomponents"):
- print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]);
-
- return new;
-
-################################################################################
-
-def indiv_sg_compare (a, b):
- """Sort by source name, source, version, 'have source', and
- finally by filename."""
- # Sort by source version
- q = apt_pkg.VersionCompare(a["version"], b["version"]);
- if q:
- return -q;
-
- # Sort by 'have source'
- a_has_source = a["architecture"].get("source");
- b_has_source = b["architecture"].get("source");
- if a_has_source and not b_has_source:
- return -1;
- elif b_has_source and not a_has_source:
- return 1;
-
- return cmp(a["filename"], b["filename"]);
-
-############################################################
+ end()
+ sys.exit(0)
-def sg_compare (a, b):
- a = a[1];
- b = b[1];
- """Sort by have note, time of oldest upload."""
- # Sort by have note
- a_note_state = a["note_state"];
- b_note_state = b["note_state"];
- if a_note_state < b_note_state:
- return -1;
- elif a_note_state > b_note_state:
- return 1;
-
- # Sort by time of oldest upload
- return cmp(a["oldest"], b["oldest"]);
-
-def sort_changes(changes_files):
- """Sort into source groups, then sort each source group by version,
- have source, filename. Finally, sort the source groups by have
- note, time of oldest upload of each source upload."""
- if len(changes_files) == 1:
- return changes_files;
-
- sorted_list = [];
- cache = {};
- # Read in all the .changes files
- for filename in changes_files:
- try:
- Katie.pkg.changes_file = filename;
- Katie.init_vars();
- Katie.update_vars();
- cache[filename] = copy.copy(Katie.pkg.changes);
- cache[filename]["filename"] = filename;
- except:
- sorted_list.append(filename);
- break;
- # Divide the .changes into per-source groups
- per_source = {};
- for filename in cache.keys():
- source = cache[filename]["source"];
- if not per_source.has_key(source):
- per_source[source] = {};
- per_source[source]["list"] = [];
- per_source[source]["list"].append(cache[filename]);
- # Determine oldest time and have note status for each source group
- for source in per_source.keys():
- source_list = per_source[source]["list"];
- first = source_list[0];
- oldest = os.stat(first["filename"])[stat.ST_MTIME];
- have_note = 0;
- for d in per_source[source]["list"]:
- mtime = os.stat(d["filename"])[stat.ST_MTIME];
- if mtime < oldest:
- oldest = mtime;
- have_note += (d.has_key("lisa note"));
- per_source[source]["oldest"] = oldest;
- if not have_note:
- per_source[source]["note_state"] = 0; # none
- elif have_note < len(source_list):
- per_source[source]["note_state"] = 1; # some
- else:
- per_source[source]["note_state"] = 2; # all
- per_source[source]["list"].sort(indiv_sg_compare);
- per_source_items = per_source.items();
- per_source_items.sort(sg_compare);
- for i in per_source_items:
- for j in i[1]["list"]:
- sorted_list.append(j["filename"]);
- return sorted_list;
+ return 1
################################################################################
class Section_Completer:
- def __init__ (self):
- self.sections = [];
- q = projectB.query("SELECT section FROM section");
- for i in q.getresult():
- self.sections.append(i[0]);
+ def __init__ (self, session):
+ self.sections = []
+ self.matches = []
+ for s, in session.query(Section.section):
+ self.sections.append(s)
def complete(self, text, state):
if state == 0:
- self.matches = [];
- n = len(text);
+ self.matches = []
+ n = len(text)
for word in self.sections:
if word[:n] == text:
- self.matches.append(word);
+ self.matches.append(word)
try:
return self.matches[state]
except IndexError:
############################################################
class Priority_Completer:
- def __init__ (self):
- self.priorities = [];
- q = projectB.query("SELECT priority FROM priority");
- for i in q.getresult():
- self.priorities.append(i[0]);
+ def __init__ (self, session):
+ self.priorities = []
+ self.matches = []
+ for p, in session.query(Priority.priority):
+ self.priorities.append(p)
def complete(self, text, state):
if state == 0:
- self.matches = [];
- n = len(text);
+ self.matches = []
+ n = len(text)
for word in self.priorities:
if word[:n] == text:
- self.matches.append(word);
+ self.matches.append(word)
try:
return self.matches[state]
except IndexError:
################################################################################
-def check_valid (new):
- for pkg in new.keys():
- section = new[pkg]["section"];
- priority = new[pkg]["priority"];
- type = new[pkg]["type"];
- new[pkg]["section id"] = db_access.get_section_id(section);
- new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]);
- # Sanity checks
- if (section == "debian-installer" and type != "udeb") or \
- (section != "debian-installer" and type == "udeb"):
- new[pkg]["section id"] = -1;
- if (priority == "source" and type != "dsc") or \
- (priority != "source" and type == "dsc"):
- new[pkg]["priority id"] = -1;
-
-################################################################################
-
-def print_new (new, indexed, file=sys.stdout):
- check_valid(new);
- broken = 0;
- index = 0;
+def print_new (new, upload, indexed, file=sys.stdout):
+ check_valid(new)
+ broken = False
+ index = 0
for pkg in new.keys():
- index += 1;
- section = new[pkg]["section"];
- priority = new[pkg]["priority"];
+ index += 1
+ section = new[pkg]["section"]
+ priority = new[pkg]["priority"]
if new[pkg]["section id"] == -1:
- section += "[!]";
- broken = 1;
+ section += "[!]"
+ broken = True
if new[pkg]["priority id"] == -1:
- priority += "[!]";
- broken = 1;
+ priority += "[!]"
+ broken = True
if indexed:
- line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section);
+ line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
else:
- line = "%-20s %-20s %-20s" % (pkg, priority, section);
- line = line.strip()+'\n';
- file.write(line);
- note = Katie.pkg.changes.get("lisa note");
- if note:
- print "*"*75;
- print note;
- print "*"*75;
- return broken, note;
-
-################################################################################
-
-def get_type (f):
- # Determine the type
- if f.has_key("dbtype"):
- type = f["dbtype"];
- elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
- type = "dsc";
- else:
- utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type));
-
- # Validate the override type
- type_id = db_access.get_override_type_id(type);
- if type_id == -1:
- utils.fubar("invalid type (%s) for new. Say wha?" % (type));
-
- return type;
+ line = "%-20s %-20s %-20s" % (pkg, priority, section)
+ line = line.strip()+'\n'
+ file.write(line)
+ notes = get_new_comments(upload.pkg.changes.get("source"))
+ for note in notes:
+ print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
+ % (note.author, note.version, note.notedate, note.comment)
+ print "-" * 72
+ return broken, len(notes) > 0
################################################################################
def index_range (index):
if index == 1:
- return "1";
+ return "1"
else:
- return "1-%s" % (index);
+ return "1-%s" % (index)
################################################################################
################################################################################
-def edit_new (new):
+def edit_new (new, upload):
# Write the current data to a temporary file
- temp_filename = utils.temp_filename();
- temp_file = utils.open_file(temp_filename, 'w');
- print_new (new, 0, temp_file);
- temp_file.close();
+ (fd, temp_filename) = utils.temp_filename()
+ temp_file = os.fdopen(fd, 'w')
+ print_new (new, upload, indexed=0, file=temp_file)
+ temp_file.close()
# Spawn an editor on that file
editor = os.environ.get("EDITOR","vi")
result = os.system("%s %s" % (editor, temp_filename))
if result != 0:
utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
# Read the edited data back in
- temp_file = utils.open_file(temp_filename);
- lines = temp_file.readlines();
- temp_file.close();
- os.unlink(temp_filename);
+ temp_file = utils.open_file(temp_filename)
+ lines = temp_file.readlines()
+ temp_file.close()
+ os.unlink(temp_filename)
# Parse the new data
for line in lines:
- line = line.strip();
+ line = line.strip()
if line == "":
- continue;
- s = line.split();
+ continue
+ s = line.split()
# Pad the list if necessary
- s[len(s):3] = [None] * (3-len(s));
- (pkg, priority, section) = s[:3];
+ s[len(s):3] = [None] * (3-len(s))
+ (pkg, priority, section) = s[:3]
if not new.has_key(pkg):
- utils.warn("Ignoring unknown package '%s'" % (pkg));
+ utils.warn("Ignoring unknown package '%s'" % (pkg))
else:
# Strip off any invalid markers, print_new will readd them.
if section.endswith("[!]"):
- section = section[:-3];
+ section = section[:-3]
if priority.endswith("[!]"):
- priority = priority[:-3];
- for file in new[pkg]["files"]:
- Katie.pkg.files[file]["section"] = section;
- Katie.pkg.files[file]["priority"] = priority;
- new[pkg]["section"] = section;
- new[pkg]["priority"] = priority;
+ priority = priority[:-3]
+ for f in new[pkg]["files"]:
+ upload.pkg.files[f]["section"] = section
+ upload.pkg.files[f]["priority"] = priority
+ new[pkg]["section"] = section
+ new[pkg]["priority"] = priority
################################################################################
-def edit_index (new, index):
+def edit_index (new, upload, index):
priority = new[index]["priority"]
section = new[index]["section"]
- type = new[index]["type"];
+ ftype = new[index]["type"]
done = 0
while not done:
- print "\t".join([index, priority, section]);
+ print "\t".join([index, priority, section])
- answer = "XXX";
- if type != "dsc":
- prompt = "[B]oth, Priority, Section, Done ? ";
+ answer = "XXX"
+ if ftype != "dsc":
+ prompt = "[B]oth, Priority, Section, Done ? "
else:
- prompt = "[S]ection, Done ? ";
- edit_priority = edit_section = 0;
+ prompt = "[S]ection, Done ? "
+ edit_priority = edit_section = 0
while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt);
- m = katie.re_default_answer.match(prompt)
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.match(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
if answer == 'P':
- edit_priority = 1;
+ edit_priority = 1
elif answer == 'S':
- edit_section = 1;
+ edit_section = 1
elif answer == 'B':
- edit_priority = edit_section = 1;
+ edit_priority = edit_section = 1
elif answer == 'D':
- done = 1;
+ done = 1
# Edit the priority
if edit_priority:
- readline.set_completer(Priorities.complete);
- got_priority = 0;
+ readline.set_completer(Priorities.complete)
+ got_priority = 0
while not got_priority:
- new_priority = utils.our_raw_input("New priority: ").strip();
+ new_priority = utils.our_raw_input("New priority: ").strip()
if new_priority not in Priorities.priorities:
- print "E: '%s' is not a valid priority, try again." % (new_priority);
+ print "E: '%s' is not a valid priority, try again." % (new_priority)
else:
- got_priority = 1;
- priority = new_priority;
+ got_priority = 1
+ priority = new_priority
# Edit the section
if edit_section:
- readline.set_completer(Sections.complete);
- got_section = 0;
+ readline.set_completer(Sections.complete)
+ got_section = 0
while not got_section:
- new_section = utils.our_raw_input("New section: ").strip();
+ new_section = utils.our_raw_input("New section: ").strip()
if new_section not in Sections.sections:
- print "E: '%s' is not a valid section, try again." % (new_section);
+ print "E: '%s' is not a valid section, try again." % (new_section)
else:
- got_section = 1;
- section = new_section;
+ got_section = 1
+ section = new_section
# Reset the readline completer
- readline.set_completer(None);
+ readline.set_completer(None)
- for file in new[index]["files"]:
- Katie.pkg.files[file]["section"] = section;
- Katie.pkg.files[file]["priority"] = priority;
- new[index]["priority"] = priority;
- new[index]["section"] = section;
- return new;
+ for f in new[index]["files"]:
+ upload.pkg.files[f]["section"] = section
+ upload.pkg.files[f]["priority"] = priority
+ new[index]["priority"] = priority
+ new[index]["section"] = section
+ return new
################################################################################
-def edit_overrides (new):
- print;
+def edit_overrides (new, upload, session):
+ print
done = 0
while not done:
- print_new (new, 1);
- new_index = {};
- index = 0;
+ print_new (new, upload, indexed=1)
+ new_index = {}
+ index = 0
for i in new.keys():
- index += 1;
- new_index[index] = i;
+ index += 1
+ new_index[index] = i
- prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index));
+ prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
got_answer = 0
while not got_answer:
- answer = utils.our_raw_input(prompt);
- if not utils.str_isnum(answer):
- answer = answer[:1].upper();
+ answer = utils.our_raw_input(prompt)
+ if not answer.isdigit():
+ answer = answer[:1].upper()
if answer == "E" or answer == "D":
- got_answer = 1;
- elif katie.re_isanum.match (answer):
- answer = int(answer);
+ got_answer = 1
+ elif re_isanum.match (answer):
+ answer = int(answer)
if (answer < 1) or (answer > index):
- print "%s is not a valid index (%s). Please retry." % (answer, index_range(index));
+ print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
else:
- got_answer = 1;
+ got_answer = 1
if answer == 'E':
- edit_new(new);
+ edit_new(new, upload)
elif answer == 'D':
- done = 1;
+ done = 1
else:
- edit_index (new, new_index[answer]);
+ edit_index (new, upload, new_index[answer])
- return new;
+ return new
-################################################################################
-
-def edit_note(note):
- # Write the current data to a temporary file
- temp_filename = utils.temp_filename();
- temp_file = utils.open_file(temp_filename, 'w');
- temp_file.write(note);
- temp_file.close();
- editor = os.environ.get("EDITOR","vi")
- answer = 'E';
- while answer == 'E':
- os.system("%s %s" % (editor, temp_filename))
- temp_file = utils.open_file(temp_filename);
- note = temp_file.read().rstrip();
- temp_file.close();
- print "Note:";
- print utils.prefix_multi_line_string(note," ");
- prompt = "[D]one, Edit, Abandon, Quit ?"
- answer = "XXX";
- while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt);
- m = katie.re_default_answer.search(prompt);
- if answer == "":
- answer = m.group(1);
- answer = answer[:1].upper();
- os.unlink(temp_filename);
- if answer == 'A':
- return;
- elif answer == 'Q':
- sys.exit(0);
- Katie.pkg.changes["lisa note"] = note;
- Katie.dump_vars(Cnf["Dir::Queue::New"]);
################################################################################
-def check_pkg ():
+def check_pkg (upload):
try:
- less_fd = os.popen("less -R -", 'w', 0);
- stdout_fd = sys.stdout;
+ less_fd = os.popen("less -R -", 'w', 0)
+ stdout_fd = sys.stdout
try:
- sys.stdout = less_fd;
- fernanda.display_changes(Katie.pkg.changes_file);
- files = Katie.pkg.files;
- for file in files.keys():
- if files[file].has_key("new"):
- type = files[file]["type"];
- if type == "deb":
- fernanda.check_deb(file);
- elif type == "dsc":
- fernanda.check_dsc(file);
+ sys.stdout = less_fd
+ changes = utils.parse_changes (upload.pkg.changes_file)
+ print examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
+ files = upload.pkg.files
+ for f in files.keys():
+ if files[f].has_key("new"):
+ ftype = files[f]["type"]
+ if ftype == "deb":
+ print examine_package.check_deb(changes['distribution'], f)
+ elif ftype == "dsc":
+ print examine_package.check_dsc(changes['distribution'], f)
finally:
- sys.stdout = stdout_fd;
+ print examine_package.output_package_relations()
+ sys.stdout = stdout_fd
except IOError, e:
- if errno.errorcode[e.errno] == 'EPIPE':
- utils.warn("[fernanda] Caught EPIPE; skipping.");
- pass;
+ if e.errno == errno.EPIPE:
+ utils.warn("[examine_package] Caught EPIPE; skipping.")
+ pass
else:
- raise;
+ raise
except KeyboardInterrupt:
- utils.warn("[fernanda] Caught C-c; skipping.");
- pass;
+ utils.warn("[examine_package] Caught C-c; skipping.")
+ pass
################################################################################
## FIXME: horribly Debian specific
-def do_bxa_notification():
- files = Katie.pkg.files;
- summary = "";
- for file in files.keys():
- if files[file]["type"] == "deb":
- control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
- summary += "\n";
- summary += "Package: %s\n" % (control.Find("Package"));
- summary += "Description: %s\n" % (control.Find("Description"));
- Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary;
- bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification");
- utils.send_mail(bxa_mail);
+def do_bxa_notification(upload):
+ files = upload.pkg.files
+ summary = ""
+ for f in files.keys():
+ if files[f]["type"] == "deb":
+ control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
+ summary += "\n"
+ summary += "Package: %s\n" % (control.Find("Package"))
+ summary += "Description: %s\n" % (control.Find("Description"))
+ upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
+ bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
+ utils.send_mail(bxa_mail)
################################################################################
-def add_overrides (new):
- changes = Katie.pkg.changes;
- files = Katie.pkg.files;
+def add_overrides (new, upload, session):
+ changes = upload.pkg.changes
+ files = upload.pkg.files
+ srcpkg = changes.get("source")
- projectB.query("BEGIN WORK");
for suite in changes["suite"].keys():
- suite_id = db_access.get_suite_id(suite);
+ suite_id = get_suite(suite).suite_id
for pkg in new.keys():
- component_id = db_access.get_component_id(new[pkg]["component"]);
- type_id = db_access.get_override_type_id(new[pkg]["type"]);
- priority_id = new[pkg]["priority id"];
- section_id = new[pkg]["section id"];
- projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id));
- for file in new[pkg]["files"]:
- if files[file].has_key("new"):
- del files[file]["new"];
- del new[pkg];
-
- projectB.query("COMMIT WORK");
-
- if Cnf.FindB("Dinstall::BXANotify"):
- do_bxa_notification();
+ component_id = get_component(new[pkg]["component"]).component_id
+ type_id = get_override_type(new[pkg]["type"]).overridetype_id
+ priority_id = new[pkg]["priority id"]
+ section_id = new[pkg]["section id"]
+ Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
+ session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
+ { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
+ for f in new[pkg]["files"]:
+ if files[f].has_key("new"):
+ del files[f]["new"]
+ del new[pkg]
+
+ session.commit()
+
+ if Config().FindB("Dinstall::BXANotify"):
+ do_bxa_notification(upload)
################################################################################
-def prod_maintainer ():
- # Here we prepare an editor and get them ready to prod...
- temp_filename = utils.temp_filename();
- editor = os.environ.get("EDITOR","vi")
- answer = 'E';
- while answer == 'E':
- os.system("%s %s" % (editor, temp_filename))
- file = utils.open_file(temp_filename);
- prod_message = "".join(file.readlines());
- file.close();
- print "Prod message:";
- print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1);
- prompt = "[P]rod, Edit, Abandon, Quit ?"
- answer = "XXX";
- while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt);
- m = katie.re_default_answer.search(prompt);
- if answer == "":
- answer = m.group(1);
- answer = answer[:1].upper();
- os.unlink(temp_filename);
- if answer == 'A':
- return;
- elif answer == 'Q':
- sys.exit(0);
- # Otherwise, do the proding...
- user_email_address = utils.whoami() + " <%s>" % (
- Cnf["Dinstall::MyAdminAddress"]);
-
- Subst = Katie.Subst;
-
- Subst["__FROM_ADDRESS__"] = user_email_address;
- Subst["__PROD_MESSAGE__"] = prod_message;
- Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
-
- prod_mail_message = utils.TemplateSubst(
- Subst,Cnf["Dir::Templates"]+"/lisa.prod");
-
- # Send the prod mail if appropriate
- if not Cnf["Dinstall::Options::No-Mail"]:
- utils.send_mail(prod_mail_message);
+def do_new(upload, session):
+ print "NEW\n"
+ files = upload.pkg.files
+ upload.check_files(not Options["No-Action"])
+ changes = upload.pkg.changes
+ cnf = Config()
- print "Sent proding message";
-
-################################################################################
-
-def do_new():
- print "NEW\n";
- files = Katie.pkg.files;
- changes = Katie.pkg.changes;
+ # Check for a valid distribution
+ upload.check_distributions()
# Make a copy of distribution we can happily trample on
- changes["suite"] = copy.copy(changes["distribution"]);
-
- # Fix up the list of target suites
- for suite in changes["suite"].keys():
- override = Cnf.Find("Suite::%s::OverrideSuite" % (suite));
- if override:
- del changes["suite"][suite];
- changes["suite"][override] = 1;
- # Validate suites
- for suite in changes["suite"].keys():
- suite_id = db_access.get_suite_id(suite);
- if suite_id == -1:
- utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite));
+ changes["suite"] = copy.copy(changes["distribution"])
# The main NEW processing loop
- done = 0;
+ done = 0
while not done:
# Find out what's new
- new = determine_new(changes, files);
+ new, byhand = determine_new(upload.pkg.changes_file, changes, files, session=session)
if not new:
- break;
+ break
- answer = "XXX";
+ answer = "XXX"
if Options["No-Action"] or Options["Automatic"]:
- answer = 'S';
+ answer = 'S'
- (broken, note) = print_new(new, 0);
- prompt = "";
+ (broken, note) = print_new(new, upload, indexed=0)
+ prompt = ""
if not broken and not note:
- prompt = "Add overrides, ";
+ prompt = "Add overrides, "
if broken:
- print "W: [!] marked entries must be fixed before package can be processed.";
+ print "W: [!] marked entries must be fixed before package can be processed."
if note:
- print "W: note must be removed before package can be processed.";
- prompt += "Remove note, ";
+ print "W: note must be removed before package can be processed."
+ prompt += "RemOve all notes, Remove note, "
- prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?";
+ prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt);
- m = katie.re_default_answer.search(prompt);
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
- if answer == 'A':
- done = add_overrides (new);
+ if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
+ utils.warn("Trainees can't do that")
+ continue
+
+ if answer == 'A' and not Options["Trainee"]:
+ try:
+ check_daily_lock()
+ done = add_overrides (new, upload, session)
+ new_accept(upload, Options["No-Action"], session)
+ Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'C':
- check_pkg();
- elif answer == 'E':
- new = edit_overrides (new);
- elif answer == 'M':
- aborted = Katie.do_reject(1, Options["Manual-Reject"]);
+ check_pkg(upload)
+ elif answer == 'E' and not Options["Trainee"]:
+ new = edit_overrides (new, upload, session)
+ elif answer == 'M' and not Options["Trainee"]:
+ aborted = upload.do_reject(manual=1,
+ reject_message=Options["Manual-Reject"],
+ notes=get_new_comments(changes.get("source", ""), session=session))
if not aborted:
- os.unlink(Katie.pkg.changes_file[:-8]+".katie");
- done = 1;
+ upload.pkg.remove_known_changes(session=session)
+ session.commit()
+ Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
+ done = 1
elif answer == 'N':
- edit_note(changes.get("lisa note", ""));
- elif answer == 'P':
- prod_maintainer();
- elif answer == 'R':
- confirm = utils.our_raw_input("Really clear note (y/N)? ").lower();
+ edit_note(get_new_comments(changes.get("source", ""), session=session),
+ upload, session)
+ elif answer == 'P' and not Options["Trainee"]:
+ prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
+ upload)
+ Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
+ elif answer == 'R' and not Options["Trainee"]:
+ confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
+ if confirm == "y":
+ for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
+ session.delete(c)
+ session.commit()
+ elif answer == 'O' and not Options["Trainee"]:
+ confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
if confirm == "y":
- del changes["lisa note"];
+ for c in get_new_comments(changes.get("source", ""), session=session):
+ session.delete(c)
+ session.commit()
+
elif answer == 'S':
- done = 1;
+ done = 1
elif answer == 'Q':
+ end()
sys.exit(0)
################################################################################
################################################################################
def usage (exit_code=0):
- print """Usage: lisa [OPTION]... [CHANGES]...
+ print """Usage: dak process-new [OPTION]... [CHANGES]...
-a, --automatic automatic run
-h, --help show this help and exit.
-m, --manual-reject=MSG manual reject with `msg'
-n, --no-action don't do anything
+ -t, --trainee FTP Trainee mode
-V, --version display the version number and exit"""
sys.exit(exit_code)
################################################################################
-def init():
- global Cnf, Options, Logger, Katie, projectB, Sections, Priorities;
-
- Cnf = utils.get_conf();
-
- Arguments = [('a',"automatic","Lisa::Options::Automatic"),
- ('h',"help","Lisa::Options::Help"),
- ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"),
- ('n',"no-action","Lisa::Options::No-Action"),
- ('V',"version","Lisa::Options::Version")];
-
- for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
- if not Cnf.has_key("Lisa::Options::%s" % (i)):
- Cnf["Lisa::Options::%s" % (i)] = "";
-
- changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
- Options = Cnf.SubTree("Lisa::Options")
-
- if Options["Help"]:
- usage();
-
- if Options["Version"]:
- print "lisa %s" % (lisa_version);
- sys.exit(0);
-
- Katie = katie.Katie(Cnf);
-
- if not Options["No-Action"]:
- Logger = Katie.Logger = logging.Logger(Cnf, "lisa");
-
- projectB = Katie.projectB;
-
- Sections = Section_Completer();
- Priorities = Priority_Completer();
- readline.parse_and_bind("tab: complete");
-
- return changes_files;
-
-################################################################################
-
-def do_byhand():
- done = 0;
+def do_byhand(upload, session):
+ done = 0
while not done:
- files = Katie.pkg.files;
- will_install = 1;
- byhand = [];
-
- for file in files.keys():
- if files[file]["type"] == "byhand":
- if os.path.exists(file):
- print "W: %s still present; please process byhand components and try again." % (file);
- will_install = 0;
+ files = upload.pkg.files
+ will_install = True
+ byhand = []
+
+ for f in files.keys():
+ if files[f]["section"] == "byhand":
+ if os.path.exists(f):
+ print "W: %s still present; please process byhand components and try again." % (f)
+ will_install = False
else:
- byhand.append(file);
+ byhand.append(f)
- answer = "XXXX";
+ answer = "XXXX"
if Options["No-Action"]:
- answer = "S";
+ answer = "S"
if will_install:
if Options["Automatic"] and not Options["No-Action"]:
- answer = 'A';
- prompt = "[A]ccept, Manual reject, Skip, Quit ?";
+ answer = 'A'
+ prompt = "[A]ccept, Manual reject, Skip, Quit ?"
else:
- prompt = "Manual reject, [S]kip, Quit ?";
+ prompt = "Manual reject, [S]kip, Quit ?"
while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt);
- m = katie.re_default_answer.search(prompt);
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
- answer = m.group(1);
- answer = answer[:1].upper();
+ answer = m.group(1)
+ answer = answer[:1].upper()
if answer == 'A':
- done = 1;
- for file in byhand:
- del files[file];
+ dbchg = get_dbchange(upload.pkg.changes_file, session)
+ if dbchg is None:
+ print "Warning: cannot find changes file in database; can't process BYHAND"
+ else:
+ try:
+ check_daily_lock()
+ done = 1
+ for b in byhand:
+ # Find the file entry in the database
+ found = False
+ for f in dbchg.files:
+ if f.filename == b:
+ found = True
+ f.processed = True
+ break
+
+ if not found:
+ print "Warning: Couldn't find BYHAND item %s in the database to mark it processed" % b
+
+ session.commit()
+ Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'M':
- Katie.do_reject(1, Options["Manual-Reject"]);
- os.unlink(Katie.pkg.changes_file[:-8]+".katie");
- done = 1;
+ aborted = upload.do_reject(manual=1,
+ reject_message=Options["Manual-Reject"],
+ notes=get_new_comments(changes.get("source", ""), session=session))
+ if not aborted:
+ upload.pkg.remove_known_changes(session=session)
+ session.commit()
+ Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
+ done = 1
elif answer == 'S':
- done = 1;
+ done = 1
elif answer == 'Q':
- sys.exit(0);
+ end()
+ sys.exit(0)
################################################################################
-def do_accept():
- print "ACCEPT";
- if not Options["No-Action"]:
- retry = 0;
- while retry < 10:
- try:
- lock_fd = os.open(Cnf["Lisa::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL);
- retry = 10;
- except OSError, e:
- if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
- retry += 1;
- if (retry >= 10):
- utils.fubar("Couldn't obtain lock; assuming jennifer is already running.");
- else:
- print("Unable to get accepted lock (try %d of 10)" % retry);
- time.sleep(60);
- else:
- raise;
- (summary, short_summary) = Katie.build_summaries();
- Katie.accept(summary, short_summary);
- os.unlink(Katie.pkg.changes_file[:-8]+".katie");
- os.unlink(Cnf["Lisa::AcceptedLockFile"]);
-
-def check_status(files):
- new = byhand = 0;
- for file in files.keys():
- if files[file]["type"] == "byhand":
- byhand = 1;
- elif files[file].has_key("new"):
- new = 1;
- return (new, byhand);
-
-def do_pkg(changes_file):
- Katie.pkg.changes_file = changes_file;
- Katie.init_vars();
- Katie.update_vars();
- Katie.update_subst();
- files = Katie.pkg.files;
-
- if not recheck():
- return;
-
- (new, byhand) = check_status(files);
- if new or byhand:
- if new:
- do_new();
- if byhand:
- do_byhand();
- (new, byhand) = check_status(files);
-
- if not new and not byhand:
- do_accept();
+def check_daily_lock():
+ """
+ Raises CantGetLockError if the dinstall daily.lock exists.
+ """
+
+ cnf = Config()
+ try:
+ os.open(cnf["Process-New::DinstallLockFile"],
+ os.O_RDONLY | os.O_CREAT | os.O_EXCL)
+ except OSError, e:
+ if e.errno == errno.EEXIST or e.errno == errno.EACCES:
+ raise CantGetLockError
+
+ os.unlink(cnf["Process-New::DinstallLockFile"])
+
+
+@contextlib.contextmanager
+def lock_package(package):
+ """
+ Lock C{package} so that noone else jumps in processing it.
+
+ @type package: string
+ @param package: source package name to lock
+ """
+
+ path = os.path.join(Config()["Process-New::LockDir"], package)
+ try:
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
+ except OSError, e:
+ if e.errno == errno.EEXIST or e.errno == errno.EACCES:
+ user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
+ raise AlreadyLockedError, user
+
+ try:
+ yield fd
+ finally:
+ os.unlink(path)
+
+class clean_holding(object):
+ def __init__(self,pkg):
+ self.pkg = pkg
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, type, value, traceback):
+ h = Holding()
+
+ for f in self.pkg.files.keys():
+ if os.path.exists(os.path.join(h.holding_dir, f)):
+ os.unlink(os.path.join(h.holding_dir, f))
+
+
+def do_pkg(changes_full_path, session):
+ changes_dir = os.path.dirname(changes_full_path)
+ changes_file = os.path.basename(changes_full_path)
+
+ u = Upload()
+ u.pkg.changes_file = changes_file
+ (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
+ u.load_changes(changes_file)
+ u.pkg.directory = changes_dir
+ u.update_subst()
+ u.logger = Logger
+ origchanges = os.path.abspath(u.pkg.changes_file)
+
+ cnf = Config()
+ bcc = "X-DAK: dak process-new"
+ if cnf.has_key("Dinstall::Bcc"):
+ u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+ else:
+ u.Subst["__BCC__"] = bcc
+
+ files = u.pkg.files
+ for deb_filename, f in files.items():
+ if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
+ u.binary_file_checks(deb_filename, session)
+ u.check_binary_against_db(deb_filename, session)
+ else:
+ u.source_file_checks(deb_filename, session)
+ u.check_source_against_db(deb_filename, session)
+
+ u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
+
+ try:
+ with lock_package(u.pkg.changes["source"]):
+ with clean_holding(u.pkg):
+ if not recheck(u, session):
+ return
+
+ new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, session=session)
+ if byhand:
+ do_byhand(u, session)
+ elif new:
+ do_new(u, session)
+ else:
+ try:
+ check_daily_lock()
+ new_accept(u, Options["No-Action"], session)
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
+
+ except AlreadyLockedError, e:
+ print "Seems to be locked by %s already, skipping..." % (e)
################################################################################
def end():
- accept_count = Katie.accept_count;
- accept_bytes = Katie.accept_bytes;
+ accept_count = SummaryStats().accept_count
+ accept_bytes = SummaryStats().accept_bytes
if accept_count:
sets = "set"
if accept_count > 1:
sets = "sets"
- sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))));
- Logger.log(["total",accept_count,accept_bytes]);
+ sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
+ Logger.log(["total",accept_count,accept_bytes])
- if not Options["No-Action"]:
- Logger.close();
+ if not Options["No-Action"] and not Options["Trainee"]:
+ Logger.close()
################################################################################
def main():
- changes_files = init();
- if len(changes_files) > 50:
- sys.stderr.write("Sorting changes...\n");
- changes_files = sort_changes(changes_files);
-
- # Kill me now? **FIXME**
- Cnf["Dinstall::Options::No-Mail"] = "";
- bcc = "X-Katie: lisa %s" % (lisa_version);
- if Cnf.has_key("Dinstall::Bcc"):
- Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
+ global Options, Logger, Sections, Priorities
+
+ cnf = Config()
+ session = DBConn().session()
+
+ Arguments = [('a',"automatic","Process-New::Options::Automatic"),
+ ('h',"help","Process-New::Options::Help"),
+ ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
+ ('t',"trainee","Process-New::Options::Trainee"),
+ ('n',"no-action","Process-New::Options::No-Action")]
+
+ for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
+ if not cnf.has_key("Process-New::Options::%s" % (i)):
+ cnf["Process-New::Options::%s" % (i)] = ""
+
+ changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
+ if len(changes_files) == 0:
+ new_queue = get_policy_queue('new', session );
+ changes_paths = [ os.path.join(new_queue.path, j) for j in utils.get_changes_files(new_queue.path) ]
else:
- Katie.Subst["__BCC__"] = bcc;
+ changes_paths = [ os.path.abspath(j) for j in changes_files ]
+
+ Options = cnf.SubTree("Process-New::Options")
+
+ if Options["Help"]:
+ usage()
+
+ if not Options["No-Action"]:
+ try:
+ Logger = daklog.Logger(cnf, "process-new")
+ except CantOpenError, e:
+ Options["Trainee"] = "True"
- for changes_file in changes_files:
- changes_file = utils.validate_changes_file_arg(changes_file, 0);
+ Sections = Section_Completer(session)
+ Priorities = Priority_Completer(session)
+ readline.parse_and_bind("tab: complete")
+
+ if len(changes_paths) > 1:
+ sys.stderr.write("Sorting changes...\n")
+ changes_files = sort_changes(changes_paths, session)
+
+ for changes_file in changes_paths:
+ changes_file = utils.validate_changes_file_arg(changes_file, 0)
if not changes_file:
- continue;
- print "\n" + changes_file;
- do_pkg (changes_file);
+ continue
+ print "\n" + os.path.basename(changes_file)
+
+ do_pkg (changes_file, session)
- end();
+ end()
################################################################################
projects / dak.git / blobdiff