#!/usr/bin/env python
+# vim:set et ts=4 sw=4:
-# Handles NEW and BYHAND packages
-# Copyright (C) 2001, 2002, 2003, 2004, 2005 James Troup <james@nocrew.org>
-# $Id: lisa,v 1.31 2005-11-15 09:50:32 ajt Exp $
+""" Handles NEW and BYHAND packages
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
################################################################################
-import copy, errno, os, readline, stat, sys, time
+from __future__ import with_statement
+
+import copy
+import errno
+import os
+import readline
+import stat
+import sys
+import time
+import contextlib
+import pwd
import apt_pkg, apt_inst
-import db_access, fernanda, katie, logging, utils
+import examine_package
-# Globals
-lisa_version = "$Revision: 1.31 $"
+from daklib.dbconn import *
+from daklib.queue import *
+from daklib import daklog
+from daklib import utils
+from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
+from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
+from daklib.summarystats import SummaryStats
+from daklib.config import Config
-Cnf = None
+# Globals
Options = None
-Katie = None
-projectB = None
Logger = None
Priorities = None
Sections = None
-reject_message = ""
-
################################################################################
################################################################################
################################################################################
-def reject (str, prefix="Rejected: "):
- global reject_message
- if str:
- reject_message += prefix + str + "\n"
-
-def recheck():
- global reject_message
- files = Katie.pkg.files
- reject_message = ""
-
- for file in files.keys():
- # The .orig.tar.gz can disappear out from under us is it's a
- # duplicate of one in the archive.
- if not files.has_key(file):
- continue
- # Check that the source still exists
- if files[file]["type"] == "deb":
- source_version = files[file]["source version"]
- source_package = files[file]["source package"]
- if not Katie.pkg.changes["architecture"].has_key("source") \
- and not Katie.source_exists(source_package, source_version, Katie.pkg.changes["distribution"].keys()):
- source_epochless_version = utils.re_no_epoch.sub('', source_version)
- dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
- if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
- reject("no source found for %s %s (%s)." % (source_package, source_version, file))
-
- # Version and file overwrite checks
- if files[file]["type"] == "deb":
- reject(Katie.check_binary_against_db(file))
- elif files[file]["type"] == "dsc":
- reject(Katie.check_source_against_db(file))
- (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file)
- reject(reject_msg)
-
- if reject_message:
+def recheck(upload, session):
+# STU: I'm not sure, but I don't thin kthis is necessary any longer: upload.recheck(session)
+ if len(upload.rejects) > 0:
answer = "XXX"
- if Options["No-Action"] or Options["Automatic"]:
+ if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
answer = 'S'
- print "REJECT\n" + reject_message,
+ print "REJECT\n%s" % '\n'.join(upload.rejects)
prompt = "[R]eject, Skip, Quit ?"
while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt)
- m = katie.re_default_answer.match(prompt)
+ m = re_default_answer.match(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
if answer == 'R':
- Katie.do_reject(0, reject_message)
- os.unlink(Katie.pkg.changes_file[:-8]+".katie")
+ upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
return 0
elif answer == 'S':
return 0
elif answer == 'Q':
+ end()
sys.exit(0)
return 1
################################################################################
-def determine_new (changes, files):
- new = {}
-
- # Build up a list of potentially new things
- for file in files.keys():
- f = files[file]
- # Skip byhand elements
- if f["type"] == "byhand":
- continue
- pkg = f["package"]
- priority = f["priority"]
- section = f["section"]
- # FIXME: unhardcode
- if section == "non-US/main":
- section = "non-US"
- type = get_type(f)
- component = f["component"]
-
- if type == "dsc":
- priority = "source"
- if not new.has_key(pkg):
- new[pkg] = {}
- new[pkg]["priority"] = priority
- new[pkg]["section"] = section
- new[pkg]["type"] = type
- new[pkg]["component"] = component
- new[pkg]["files"] = []
- else:
- old_type = new[pkg]["type"]
- if old_type != type:
- # source gets trumped by deb or udeb
- if old_type == "dsc":
- new[pkg]["priority"] = priority
- new[pkg]["section"] = section
- new[pkg]["type"] = type
- new[pkg]["component"] = component
- new[pkg]["files"].append(file)
- if f.has_key("othercomponents"):
- new[pkg]["othercomponents"] = f["othercomponents"]
-
- for suite in changes["suite"].keys():
- suite_id = db_access.get_suite_id(suite)
- for pkg in new.keys():
- component_id = db_access.get_component_id(new[pkg]["component"])
- type_id = db_access.get_override_type_id(new[pkg]["type"])
- q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
- ql = q.getresult()
- if ql:
- for file in new[pkg]["files"]:
- if files[file].has_key("new"):
- del files[file]["new"]
- del new[pkg]
-
- if changes["suite"].has_key("stable"):
- print "WARNING: overrides will be added for stable!"
- if changes["suite"].has_key("oldstable"):
- print "WARNING: overrides will be added for OLDstable!"
- for pkg in new.keys():
- if new[pkg].has_key("othercomponents"):
- print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
-
- return new
-
-################################################################################
-
def indiv_sg_compare (a, b):
"""Sort by source name, source, version, 'have source', and
finally by filename."""
def sg_compare (a, b):
a = a[1]
b = b[1]
- """Sort by have note, time of oldest upload."""
+ """Sort by have note, source already in database and time of oldest upload."""
# Sort by have note
a_note_state = a["note_state"]
b_note_state = b["note_state"]
return -1
elif a_note_state > b_note_state:
return 1
+ # Sort by source already in database (descending)
+ source_in_database = cmp(a["source_in_database"], b["source_in_database"])
+ if source_in_database:
+ return -source_in_database
# Sort by time of oldest upload
return cmp(a["oldest"], b["oldest"])
-def sort_changes(changes_files):
+def sort_changes(changes_files, session):
"""Sort into source groups, then sort each source group by version,
have source, filename. Finally, sort the source groups by have
note, time of oldest upload of each source upload."""
cache = {}
# Read in all the .changes files
for filename in changes_files:
+ u = Upload()
try:
- Katie.pkg.changes_file = filename
- Katie.init_vars()
- Katie.update_vars()
- cache[filename] = copy.copy(Katie.pkg.changes)
+ u.pkg.changes_file = filename
+ u.load_changes(filename)
+ u.update_subst()
+ cache[filename] = copy.copy(u.pkg.changes)
cache[filename]["filename"] = filename
except:
sorted_list.append(filename)
per_source[source]["list"].append(cache[filename])
# Determine oldest time and have note status for each source group
for source in per_source.keys():
+ q = session.query(DBSource).filter_by(source = source).all()
+ per_source[source]["source_in_database"] = len(q)>0
source_list = per_source[source]["list"]
first = source_list[0]
oldest = os.stat(first["filename"])[stat.ST_MTIME]
mtime = os.stat(d["filename"])[stat.ST_MTIME]
if mtime < oldest:
oldest = mtime
- have_note += (d.has_key("lisa note"))
+ have_note += has_new_comment(d["source"], d["version"], session)
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0; # none
################################################################################
class Section_Completer:
- def __init__ (self):
+ def __init__ (self, session):
self.sections = []
- q = projectB.query("SELECT section FROM section")
- for i in q.getresult():
- self.sections.append(i[0])
+ self.matches = []
+ for s, in session.query(Section.section):
+ self.sections.append(s)
def complete(self, text, state):
if state == 0:
############################################################
class Priority_Completer:
- def __init__ (self):
+ def __init__ (self, session):
self.priorities = []
- q = projectB.query("SELECT priority FROM priority")
- for i in q.getresult():
- self.priorities.append(i[0])
+ self.matches = []
+ for p, in session.query(Priority.priority):
+ self.priorities.append(p)
def complete(self, text, state):
if state == 0:
################################################################################
-def check_valid (new):
- for pkg in new.keys():
- section = new[pkg]["section"]
- priority = new[pkg]["priority"]
- type = new[pkg]["type"]
- new[pkg]["section id"] = db_access.get_section_id(section)
- new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"])
- # Sanity checks
- if (section == "debian-installer" and type != "udeb") or \
- (section != "debian-installer" and type == "udeb"):
- new[pkg]["section id"] = -1
- if (priority == "source" and type != "dsc") or \
- (priority != "source" and type == "dsc"):
- new[pkg]["priority id"] = -1
-
-################################################################################
-
-def print_new (new, indexed, file=sys.stdout):
+def print_new (new, upload, indexed, file=sys.stdout):
check_valid(new)
- broken = 0
+ broken = False
index = 0
for pkg in new.keys():
index += 1
priority = new[pkg]["priority"]
if new[pkg]["section id"] == -1:
section += "[!]"
- broken = 1
+ broken = True
if new[pkg]["priority id"] == -1:
priority += "[!]"
- broken = 1
+ broken = True
if indexed:
line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
else:
line = "%-20s %-20s %-20s" % (pkg, priority, section)
line = line.strip()+'\n'
file.write(line)
- note = Katie.pkg.changes.get("lisa note")
- if note:
- print "*"*75
- print note
- print "*"*75
- return broken, note
-
-################################################################################
-
-def get_type (f):
- # Determine the type
- if f.has_key("dbtype"):
- type = f["dbtype"]
- elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
- type = "dsc"
- else:
- utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type))
-
- # Validate the override type
- type_id = db_access.get_override_type_id(type)
- if type_id == -1:
- utils.fubar("invalid type (%s) for new. Say wha?" % (type))
-
- return type
+ notes = get_new_comments(upload.pkg.changes.get("source"))
+ for note in notes:
+ print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
+ % (note.author, note.version, note.notedate, note.comment)
+ print "-" * 72
+ return broken, len(notes) > 0
################################################################################
################################################################################
################################################################################
-def edit_new (new):
+def edit_new (new, upload):
# Write the current data to a temporary file
- temp_filename = utils.temp_filename()
- temp_file = utils.open_file(temp_filename, 'w')
- print_new (new, 0, temp_file)
+ (fd, temp_filename) = utils.temp_filename()
+ temp_file = os.fdopen(fd, 'w')
+ print_new (new, upload, indexed=0, file=temp_file)
temp_file.close()
# Spawn an editor on that file
editor = os.environ.get("EDITOR","vi")
section = section[:-3]
if priority.endswith("[!]"):
priority = priority[:-3]
- for file in new[pkg]["files"]:
- Katie.pkg.files[file]["section"] = section
- Katie.pkg.files[file]["priority"] = priority
+ for f in new[pkg]["files"]:
+ upload.pkg.files[f]["section"] = section
+ upload.pkg.files[f]["priority"] = priority
new[pkg]["section"] = section
new[pkg]["priority"] = priority
################################################################################
-def edit_index (new, index):
+def edit_index (new, upload, index):
priority = new[index]["priority"]
section = new[index]["section"]
- type = new[index]["type"]
+ ftype = new[index]["type"]
done = 0
while not done:
print "\t".join([index, priority, section])
answer = "XXX"
- if type != "dsc":
+ if ftype != "dsc":
prompt = "[B]oth, Priority, Section, Done ? "
else:
prompt = "[S]ection, Done ? "
while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt)
- m = katie.re_default_answer.match(prompt)
+ m = re_default_answer.match(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
# Reset the readline completer
readline.set_completer(None)
- for file in new[index]["files"]:
- Katie.pkg.files[file]["section"] = section
- Katie.pkg.files[file]["priority"] = priority
+ for f in new[index]["files"]:
+ upload.pkg.files[f]["section"] = section
+ upload.pkg.files[f]["priority"] = priority
new[index]["priority"] = priority
new[index]["section"] = section
return new
################################################################################
-def edit_overrides (new):
+def edit_overrides (new, upload, session):
print
done = 0
while not done:
- print_new (new, 1)
+ print_new (new, upload, indexed=1)
new_index = {}
index = 0
for i in new.keys():
got_answer = 0
while not got_answer:
answer = utils.our_raw_input(prompt)
- if not utils.str_isnum(answer):
+ if not answer.isdigit():
answer = answer[:1].upper()
if answer == "E" or answer == "D":
got_answer = 1
- elif katie.re_isanum.match (answer):
+ elif re_isanum.match (answer):
answer = int(answer)
if (answer < 1) or (answer > index):
print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
got_answer = 1
if answer == 'E':
- edit_new(new)
+ edit_new(new, upload)
elif answer == 'D':
done = 1
else:
- edit_index (new, new_index[answer])
+ edit_index (new, upload, new_index[answer])
return new
################################################################################
-def edit_note(note):
+def edit_note(note, upload, session):
# Write the current data to a temporary file
- temp_filename = utils.temp_filename()
- temp_file = utils.open_file(temp_filename, 'w')
- temp_file.write(note)
- temp_file.close()
+ (fd, temp_filename) = utils.temp_filename()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
os.system("%s %s" % (editor, temp_filename))
temp_file = utils.open_file(temp_filename)
- note = temp_file.read().rstrip()
+ newnote = temp_file.read().rstrip()
temp_file.close()
- print "Note:"
- print utils.prefix_multi_line_string(note," ")
+ print "New Note:"
+ print utils.prefix_multi_line_string(newnote," ")
prompt = "[D]one, Edit, Abandon, Quit ?"
answer = "XXX"
while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt)
- m = katie.re_default_answer.search(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
if answer == 'A':
return
elif answer == 'Q':
+ end()
sys.exit(0)
- Katie.pkg.changes["lisa note"] = note
- Katie.dump_vars(Cnf["Dir::Queue::New"])
+
+ comment = NewComment()
+ comment.package = upload.pkg.changes["source"]
+ comment.version = upload.pkg.changes["version"]
+ comment.comment = newnote
+ comment.author = utils.whoami()
+ comment.trainee = bool(Options["Trainee"])
+ session.add(comment)
+ session.commit()
################################################################################
-def check_pkg ():
+def check_pkg (upload):
try:
less_fd = os.popen("less -R -", 'w', 0)
stdout_fd = sys.stdout
try:
sys.stdout = less_fd
- fernanda.display_changes(Katie.pkg.changes_file)
- files = Katie.pkg.files
- for file in files.keys():
- if files[file].has_key("new"):
- type = files[file]["type"]
- if type == "deb":
- fernanda.check_deb(file)
- elif type == "dsc":
- fernanda.check_dsc(file)
+ changes = utils.parse_changes (upload.pkg.changes_file)
+ examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
+ files = upload.pkg.files
+ for f in files.keys():
+ if files[f].has_key("new"):
+ ftype = files[f]["type"]
+ if ftype == "deb":
+ examine_package.check_deb(changes['distribution'], f)
+ elif ftype == "dsc":
+ examine_package.check_dsc(changes['distribution'], f)
finally:
+ examine_package.output_package_relations()
sys.stdout = stdout_fd
except IOError, e:
- if errno.errorcode[e.errno] == 'EPIPE':
- utils.warn("[fernanda] Caught EPIPE; skipping.")
+ if e.errno == errno.EPIPE:
+ utils.warn("[examine_package] Caught EPIPE; skipping.")
pass
else:
raise
except KeyboardInterrupt:
- utils.warn("[fernanda] Caught C-c; skipping.")
+ utils.warn("[examine_package] Caught C-c; skipping.")
pass
################################################################################
## FIXME: horribly Debian specific
-def do_bxa_notification():
- files = Katie.pkg.files
+def do_bxa_notification(upload):
+ files = upload.pkg.files
summary = ""
- for file in files.keys():
- if files[file]["type"] == "deb":
- control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))
+ for f in files.keys():
+ if files[f]["type"] == "deb":
+ control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
summary += "\n"
summary += "Package: %s\n" % (control.Find("Package"))
summary += "Description: %s\n" % (control.Find("Description"))
- Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary
- bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification")
+ upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
+ bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
utils.send_mail(bxa_mail)
################################################################################
-def add_overrides (new):
- changes = Katie.pkg.changes
- files = Katie.pkg.files
+def add_overrides (new, upload, session):
+ changes = upload.pkg.changes
+ files = upload.pkg.files
+ srcpkg = changes.get("source")
- projectB.query("BEGIN WORK")
for suite in changes["suite"].keys():
- suite_id = db_access.get_suite_id(suite)
+ suite_id = get_suite(suite).suite_id
for pkg in new.keys():
- component_id = db_access.get_component_id(new[pkg]["component"])
- type_id = db_access.get_override_type_id(new[pkg]["type"])
+ component_id = get_component(new[pkg]["component"]).component_id
+ type_id = get_override_type(new[pkg]["type"]).overridetype_id
priority_id = new[pkg]["priority id"]
section_id = new[pkg]["section id"]
- projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
- for file in new[pkg]["files"]:
- if files[file].has_key("new"):
- del files[file]["new"]
+ Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
+ session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
+ { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
+ for f in new[pkg]["files"]:
+ if files[f].has_key("new"):
+ del files[f]["new"]
del new[pkg]
- projectB.query("COMMIT WORK")
+ session.commit()
- if Cnf.FindB("Dinstall::BXANotify"):
- do_bxa_notification()
+ if Config().FindB("Dinstall::BXANotify"):
+ do_bxa_notification(upload)
################################################################################
-def prod_maintainer ():
+def prod_maintainer (note, upload):
+ cnf = Config()
# Here we prepare an editor and get them ready to prod...
- temp_filename = utils.temp_filename()
+ (fd, temp_filename) = utils.temp_filename()
+ temp_file = os.fdopen(fd, 'w')
+ if len(note) > 0:
+ for line in note:
+ temp_file.write(line)
+ temp_file.close()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
os.system("%s %s" % (editor, temp_filename))
- file = utils.open_file(temp_filename)
- prod_message = "".join(file.readlines())
- file.close()
+ temp_fh = utils.open_file(temp_filename)
+ prod_message = "".join(temp_fh.readlines())
+ temp_fh.close()
print "Prod message:"
print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
prompt = "[P]rod, Edit, Abandon, Quit ?"
answer = "XXX"
while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt)
- m = katie.re_default_answer.search(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
- os.unlink(temp_filename)
- if answer == 'A':
- return
- elif answer == 'Q':
- sys.exit(0)
+ os.unlink(temp_filename)
+ if answer == 'A':
+ return
+ elif answer == 'Q':
+ end()
+ sys.exit(0)
# Otherwise, do the proding...
user_email_address = utils.whoami() + " <%s>" % (
- Cnf["Dinstall::MyAdminAddress"])
+ cnf["Dinstall::MyAdminAddress"])
- Subst = Katie.Subst
+ Subst = upload.Subst
Subst["__FROM_ADDRESS__"] = user_email_address
Subst["__PROD_MESSAGE__"] = prod_message
- Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
+ Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
prod_mail_message = utils.TemplateSubst(
- Subst,Cnf["Dir::Templates"]+"/lisa.prod")
+ Subst,cnf["Dir::Templates"]+"/process-new.prod")
# Send the prod mail if appropriate
- if not Cnf["Dinstall::Options::No-Mail"]:
+ if not cnf["Dinstall::Options::No-Mail"]:
utils.send_mail(prod_mail_message)
print "Sent proding message"
################################################################################
-def do_new():
+def do_new(upload, session):
print "NEW\n"
- files = Katie.pkg.files
- changes = Katie.pkg.changes
+ files = upload.pkg.files
+ upload.check_files(not Options["No-Action"])
+ changes = upload.pkg.changes
+ cnf = Config()
+
+ # Check for a valid distribution
+ upload.check_distributions()
# Make a copy of distribution we can happily trample on
changes["suite"] = copy.copy(changes["distribution"])
# Fix up the list of target suites
for suite in changes["suite"].keys():
- override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
+ override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
if override:
+ (olderr, newerr) = (get_suite(suite, session) == None,
+ get_suite(override, session) == None)
+ if olderr or newerr:
+ (oinv, newinv) = ("", "")
+ if olderr: oinv = "invalid "
+ if newerr: ninv = "invalid "
+ print "warning: overriding %ssuite %s to %ssuite %s" % (
+ oinv, suite, ninv, override)
del changes["suite"][suite]
changes["suite"][override] = 1
- # Validate suites
- for suite in changes["suite"].keys():
- suite_id = db_access.get_suite_id(suite)
- if suite_id == -1:
- utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
# The main NEW processing loop
done = 0
if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
- (broken, note) = print_new(new, 0)
+ (broken, note) = print_new(new, upload, indexed=0)
prompt = ""
if not broken and not note:
print "W: [!] marked entries must be fixed before package can be processed."
if note:
print "W: note must be removed before package can be processed."
- prompt += "Remove note, "
+ prompt += "RemOve all notes, Remove note, "
prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt)
- m = katie.re_default_answer.search(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
- if answer == 'A':
- done = add_overrides (new)
+ if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
+ utils.warn("Trainees can't do that")
+ continue
+
+ if answer == 'A' and not Options["Trainee"]:
+ try:
+ check_daily_lock()
+ done = add_overrides (new, upload, session)
+ do_accept(upload, session)
+ Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'C':
- check_pkg()
- elif answer == 'E':
- new = edit_overrides (new)
- elif answer == 'M':
- aborted = Katie.do_reject(1, Options["Manual-Reject"])
+ check_pkg(upload)
+ elif answer == 'E' and not Options["Trainee"]:
+ new = edit_overrides (new, upload, session)
+ elif answer == 'M' and not Options["Trainee"]:
+ aborted = upload.do_reject(manual=1,
+ reject_message=Options["Manual-Reject"],
+ note=get_new_comments(changes.get("source", ""), session=session))
if not aborted:
- os.unlink(Katie.pkg.changes_file[:-8]+".katie")
+ upload.pkg.remove_known_changes()
+ Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
done = 1
elif answer == 'N':
- edit_note(changes.get("lisa note", ""))
- elif answer == 'P':
- prod_maintainer()
- elif answer == 'R':
+ edit_note(get_new_comments(changes.get("source", ""), session=session),
+ upload, session)
+ elif answer == 'P' and not Options["Trainee"]:
+ prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
+ upload)
+ Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
+ elif answer == 'R' and not Options["Trainee"]:
confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
if confirm == "y":
- del changes["lisa note"]
+ for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
+ session.delete(c)
+ session.commit()
+ elif answer == 'O' and not Options["Trainee"]:
+ confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
+ if confirm == "y":
+ for c in get_new_comments(changes.get("source", ""), session=session):
+ session.delete(c)
+ session.commit()
+
elif answer == 'S':
done = 1
elif answer == 'Q':
+ end()
sys.exit(0)
################################################################################
################################################################################
def usage (exit_code=0):
- print """Usage: lisa [OPTION]... [CHANGES]...
+ print """Usage: dak process-new [OPTION]... [CHANGES]...
-a, --automatic automatic run
-h, --help show this help and exit.
-m, --manual-reject=MSG manual reject with `msg'
-n, --no-action don't do anything
+ -t, --trainee FTP Trainee mode
-V, --version display the version number and exit"""
sys.exit(exit_code)
################################################################################
-def init():
- global Cnf, Options, Logger, Katie, projectB, Sections, Priorities
-
- Cnf = utils.get_conf()
-
- Arguments = [('a',"automatic","Lisa::Options::Automatic"),
- ('h',"help","Lisa::Options::Help"),
- ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"),
- ('n',"no-action","Lisa::Options::No-Action"),
- ('V',"version","Lisa::Options::Version")]
-
- for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
- if not Cnf.has_key("Lisa::Options::%s" % (i)):
- Cnf["Lisa::Options::%s" % (i)] = ""
-
- changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Lisa::Options")
-
- if Options["Help"]:
- usage()
-
- if Options["Version"]:
- print "lisa %s" % (lisa_version)
- sys.exit(0)
-
- Katie = katie.Katie(Cnf)
-
- if not Options["No-Action"]:
- Logger = Katie.Logger = logging.Logger(Cnf, "lisa")
-
- projectB = Katie.projectB
-
- Sections = Section_Completer()
- Priorities = Priority_Completer()
- readline.parse_and_bind("tab: complete")
-
- return changes_files
-
-################################################################################
-
-def do_byhand():
+def do_byhand(upload, session):
done = 0
while not done:
- files = Katie.pkg.files
+ files = upload.pkg.files
will_install = 1
byhand = []
- for file in files.keys():
- if files[file]["type"] == "byhand":
- if os.path.exists(file):
- print "W: %s still present; please process byhand components and try again." % (file)
+ for f in files.keys():
+ if files[f]["type"] == "byhand":
+ if os.path.exists(f):
+ print "W: %s still present; please process byhand components and try again." % (f)
will_install = 0
else:
- byhand.append(file)
+ byhand.append(f)
answer = "XXXX"
if Options["No-Action"]:
while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt)
- m = katie.re_default_answer.search(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
if answer == 'A':
- done = 1
- for file in byhand:
- del files[file]
+ try:
+ check_daily_lock()
+ done = 1
+ for f in byhand:
+ del files[f]
+ Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'M':
- Katie.do_reject(1, Options["Manual-Reject"])
- os.unlink(Katie.pkg.changes_file[:-8]+".katie")
+ Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
+ upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
done = 1
elif answer == 'S':
done = 1
elif answer == 'Q':
+ end()
sys.exit(0)
################################################################################
-def do_accept():
+def check_daily_lock():
+ """
+ Raises CantGetLockError if the dinstall daily.lock exists.
+ """
+
+ cnf = Config()
+ try:
+ os.open(cnf["Process-New::DinstallLockFile"],
+ os.O_RDONLY | os.O_CREAT | os.O_EXCL)
+ except OSError, e:
+ if e.errno == errno.EEXIST or e.errno == errno.EACCES:
+ raise CantGetLockError
+
+ os.unlink(cnf["Process-New::DinstallLockFile"])
+
+
+@contextlib.contextmanager
+def lock_package(package):
+ """
+ Lock C{package} so that noone else jumps in processing it.
+
+ @type package: string
+ @param package: source package name to lock
+ """
+
+ path = os.path.join(Config()["Process-New::LockDir"], package)
+ try:
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
+ except OSError, e:
+ if e.errno == errno.EEXIST or e.errno == errno.EACCES:
+ user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
+ raise AlreadyLockedError, user
+
+ try:
+ yield fd
+ finally:
+ os.unlink(path)
+
+class clean_holding(object):
+ def __init__(self,pkg):
+ self.pkg = pkg
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, type, value, traceback):
+ h = Holding()
+
+ for f in self.pkg.files.keys():
+ if os.path.exists(os.path.join(h.holding_dir, f)):
+ os.unlink(os.path.join(h.holding_dir, f))
+
+
+
+def changes_to_newstage(upload, session):
+ """move a changes file to newstage"""
+ new = get_policy_queue('new', session );
+ newstage = get_policy_queue('newstage', session );
+
+ chg = session.query(DBChange).filter_by(changesname=os.path.basename(upload.pkg.changes_file)).one()
+ chg.approved_for = newstage.policy_queue_id
+
+ for f in chg.files:
+ # update the changes_pending_files row
+ f.queue = newstage
+ utils.move(os.path.join(new.path, f.filename), newstage.path, perms=int(newstage.perms, 8))
+
+ utils.move(os.path.join(new.path, upload.pkg.changes_file), newstage.path, perms=int(newstage.perms, 8))
+ chg.in_queue = newstage
+ session.commit()
+
+def _accept(upload, session):
+ if Options["No-Action"]:
+ return
+ (summary, short_summary) = upload.build_summaries()
+ # upload.accept(summary, short_summary, targetqueue)
+
+ changes_to_newstage(upload, session)
+
+def do_accept(upload, session):
print "ACCEPT"
+ cnf = Config()
if not Options["No-Action"]:
- retry = 0
- while retry < 10:
- try:
- lock_fd = os.open(Cnf["Lisa::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
- retry = 10
- except OSError, e:
- if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
- retry += 1
- if (retry >= 10):
- utils.fubar("Couldn't obtain lock; assuming jennifer is already running.")
- else:
- print("Unable to get accepted lock (try %d of 10)" % retry)
- time.sleep(60)
- else:
- raise
- (summary, short_summary) = Katie.build_summaries()
- Katie.accept(summary, short_summary)
- os.unlink(Katie.pkg.changes_file[:-8]+".katie")
- os.unlink(Cnf["Lisa::AcceptedLockFile"])
-
-def check_status(files):
- new = byhand = 0
- for file in files.keys():
- if files[file]["type"] == "byhand":
- byhand = 1
- elif files[file].has_key("new"):
- new = 1
- return (new, byhand)
-
-def do_pkg(changes_file):
- Katie.pkg.changes_file = changes_file
- Katie.init_vars()
- Katie.update_vars()
- Katie.update_subst()
- files = Katie.pkg.files
-
- if not recheck():
- return
+ (summary, short_summary) = upload.build_summaries()
+
+ if cnf.FindB("Dinstall::SecurityQueueHandling"):
+ upload.dump_vars(cnf["Dir::Queue::Embargoed"])
+ upload.move_to_queue(get_policy_queue('embargoed'))
+ upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
+ # Check for override disparities
+ upload.Subst["__SUMMARY__"] = summary
+ else:
+ # Just a normal upload, accept it...
+ _accept(upload, session)
+
+def do_pkg(changes_file, session):
+ new_queue = get_policy_queue('new', session );
+ u = Upload()
+ u.pkg.changes_file = changes_file
+ (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
+ u.load_changes(changes_file)
+ u.pkg.directory = new_queue.path
+ u.update_subst()
+ u.logger = Logger
+ origchanges = os.path.abspath(u.pkg.changes_file)
+
+ cnf = Config()
+ bcc = "X-DAK: dak process-new"
+ if cnf.has_key("Dinstall::Bcc"):
+ u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+ else:
+ u.Subst["__BCC__"] = bcc
- (new, byhand) = check_status(files)
- if new or byhand:
- if new:
- do_new()
- if byhand:
- do_byhand()
- (new, byhand) = check_status(files)
+ files = u.pkg.files
+ for deb_filename, f in files.items():
+ if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
+ u.binary_file_checks(deb_filename, session)
+ u.check_binary_against_db(deb_filename, session)
+ else:
+ u.source_file_checks(deb_filename, session)
+ u.check_source_against_db(deb_filename, session)
- if not new and not byhand:
- do_accept()
+ u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
+
+ try:
+ with lock_package(u.pkg.changes["source"]):
+ with clean_holding(u.pkg):
+ if not recheck(u, session):
+ return
+
+ # FIXME: This does need byhand checks added!
+ new = determine_new(u.pkg.changes, files)
+ if new:
+ do_new(u, session)
+ else:
+ try:
+ check_daily_lock()
+ do_accept(u, session)
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
+# (new, byhand) = check_status(files)
+# if new or byhand:
+# if new:
+# do_new(u, session)
+# if byhand:
+# do_byhand(u, session)
+# (new, byhand) = check_status(files)
+
+# if not new and not byhand:
+# try:
+# check_daily_lock()
+# do_accept(u)
+# except CantGetLockError:
+# print "Hello? Operator! Give me the number for 911!"
+# print "Dinstall in the locked area, cant process packages, come back later"
+ except AlreadyLockedError, e:
+ print "Seems to be locked by %s already, skipping..." % (e)
################################################################################
def end():
- accept_count = Katie.accept_count
- accept_bytes = Katie.accept_bytes
+ accept_count = SummaryStats().accept_count
+ accept_bytes = SummaryStats().accept_bytes
if accept_count:
sets = "set"
sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
Logger.log(["total",accept_count,accept_bytes])
- if not Options["No-Action"]:
+ if not Options["No-Action"] and not Options["Trainee"]:
Logger.close()
################################################################################
def main():
- changes_files = init()
- if len(changes_files) > 50:
+ global Options, Logger, Sections, Priorities
+
+ cnf = Config()
+ session = DBConn().session()
+
+ Arguments = [('a',"automatic","Process-New::Options::Automatic"),
+ ('h',"help","Process-New::Options::Help"),
+ ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
+ ('t',"trainee","Process-New::Options::Trainee"),
+ ('n',"no-action","Process-New::Options::No-Action")]
+
+ for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
+ if not cnf.has_key("Process-New::Options::%s" % (i)):
+ cnf["Process-New::Options::%s" % (i)] = ""
+
+ changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
+ if len(changes_files) == 0:
+ new_queue = get_policy_queue('new', session );
+ changes_files = utils.get_changes_files(new_queue.path)
+
+ Options = cnf.SubTree("Process-New::Options")
+
+ if Options["Help"]:
+ usage()
+
+ if not Options["No-Action"]:
+ try:
+ Logger = daklog.Logger(cnf, "process-new")
+ except CantOpenError, e:
+ Options["Trainee"] = "True"
+
+ Sections = Section_Completer(session)
+ Priorities = Priority_Completer(session)
+ readline.parse_and_bind("tab: complete")
+
+ if len(changes_files) > 1:
sys.stderr.write("Sorting changes...\n")
- changes_files = sort_changes(changes_files)
+ changes_files = sort_changes(changes_files, session)
# Kill me now? **FIXME**
- Cnf["Dinstall::Options::No-Mail"] = ""
- bcc = "X-Katie: lisa %s" % (lisa_version)
- if Cnf.has_key("Dinstall::Bcc"):
- Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
- else:
- Katie.Subst["__BCC__"] = bcc
+ cnf["Dinstall::Options::No-Mail"] = ""
for changes_file in changes_files:
changes_file = utils.validate_changes_file_arg(changes_file, 0)
if not changes_file:
continue
print "\n" + changes_file
- do_pkg (changes_file)
+
+ do_pkg (changes_file, session)
end()