#!/usr/bin/env python
+# vim:set et ts=4 sw=4:
-# Handles NEW and BYHAND packages
-# Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+""" Handles NEW and BYHAND packages
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
################################################################################
-import copy, errno, os, readline, stat, sys, time
+from __future__ import with_statement
+
+import copy
+import errno
+import os
+import readline
+import stat
+import sys
+import time
+import contextlib
+import pwd
import apt_pkg, apt_inst
import examine_package
-import daklib.database
-import daklib.logging
-import daklib.queue
-import daklib.utils
+
+from daklib.dbconn import *
+from daklib.queue import *
+from daklib import daklog
+from daklib import utils
+from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
+from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
+from daklib.summarystats import SummaryStats
+from daklib.config import Config
# Globals
-Cnf = None
Options = None
-Upload = None
-projectB = None
Logger = None
Priorities = None
Sections = None
-reject_message = ""
-
################################################################################
################################################################################
################################################################################
-def reject (str, prefix="Rejected: "):
- global reject_message
- if str:
- reject_message += prefix + str + "\n"
-
-def recheck():
- global reject_message
- files = Upload.pkg.files
- reject_message = ""
-
- for file in files.keys():
- # The .orig.tar.gz can disappear out from under us is it's a
- # duplicate of one in the archive.
- if not files.has_key(file):
- continue
- # Check that the source still exists
- if files[file]["type"] == "deb":
- source_version = files[file]["source version"]
- source_package = files[file]["source package"]
- if not Upload.pkg.changes["architecture"].has_key("source") \
- and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
- source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version)
- dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
- if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
- reject("no source found for %s %s (%s)." % (source_package, source_version, file))
-
- # Version and file overwrite checks
- if files[file]["type"] == "deb":
- reject(Upload.check_binary_against_db(file))
- elif files[file]["type"] == "dsc":
- reject(Upload.check_source_against_db(file))
- (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
- reject(reject_msg)
-
- if reject_message:
+def recheck(upload, session):
+ upload.recheck()
+ if len(upload.rejects) > 0:
answer = "XXX"
- if Options["No-Action"] or Options["Automatic"]:
+ if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
answer = 'S'
- print "REJECT\n" + reject_message,
+ print "REJECT\n" + upload.rejects.join("\n"),
prompt = "[R]eject, Skip, Quit ?"
while prompt.find(answer) == -1:
- answer = daklib.utils.our_raw_input(prompt)
- m = daklib.queue.re_default_answer.match(prompt)
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.match(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
if answer == 'R':
- Upload.do_reject(0, reject_message)
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ upload.do_reject(manual=0, reject_message=upload.rejects.join("\n"))
+ os.unlink(upload.pkg.changes_file[:-8]+".dak")
return 0
elif answer == 'S':
return 0
elif answer == 'Q':
+ end()
sys.exit(0)
return 1
################################################################################
-def determine_new (changes, files):
- new = {}
-
- # Build up a list of potentially new things
- for file in files.keys():
- f = files[file]
- # Skip byhand elements
- if f["type"] == "byhand":
- continue
- pkg = f["package"]
- priority = f["priority"]
- section = f["section"]
- # FIXME: unhardcode
- if section == "non-US/main":
- section = "non-US"
- type = get_type(f)
- component = f["component"]
-
- if type == "dsc":
- priority = "source"
- if not new.has_key(pkg):
- new[pkg] = {}
- new[pkg]["priority"] = priority
- new[pkg]["section"] = section
- new[pkg]["type"] = type
- new[pkg]["component"] = component
- new[pkg]["files"] = []
- else:
- old_type = new[pkg]["type"]
- if old_type != type:
- # source gets trumped by deb or udeb
- if old_type == "dsc":
- new[pkg]["priority"] = priority
- new[pkg]["section"] = section
- new[pkg]["type"] = type
- new[pkg]["component"] = component
- new[pkg]["files"].append(file)
- if f.has_key("othercomponents"):
- new[pkg]["othercomponents"] = f["othercomponents"]
-
- for suite in changes["suite"].keys():
- suite_id = daklib.database.get_suite_id(suite)
- for pkg in new.keys():
- component_id = daklib.database.get_component_id(new[pkg]["component"])
- type_id = daklib.database.get_override_type_id(new[pkg]["type"])
- q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
- ql = q.getresult()
- if ql:
- for file in new[pkg]["files"]:
- if files[file].has_key("new"):
- del files[file]["new"]
- del new[pkg]
-
- if changes["suite"].has_key("stable"):
- print "WARNING: overrides will be added for stable!"
- if changes["suite"].has_key("oldstable"):
- print "WARNING: overrides will be added for OLDstable!"
- for pkg in new.keys():
- if new[pkg].has_key("othercomponents"):
- print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
-
- return new
-
-################################################################################
-
def indiv_sg_compare (a, b):
"""Sort by source name, source, version, 'have source', and
finally by filename."""
def sg_compare (a, b):
a = a[1]
b = b[1]
- """Sort by have note, time of oldest upload."""
+ """Sort by have note, source already in database and time of oldest upload."""
# Sort by have note
a_note_state = a["note_state"]
b_note_state = b["note_state"]
return -1
elif a_note_state > b_note_state:
return 1
+ # Sort by source already in database (descending)
+ source_in_database = cmp(a["source_in_database"], b["source_in_database"])
+ if source_in_database:
+ return -source_in_database
# Sort by time of oldest upload
return cmp(a["oldest"], b["oldest"])
-def sort_changes(changes_files):
+def sort_changes(changes_files, session):
"""Sort into source groups, then sort each source group by version,
have source, filename. Finally, sort the source groups by have
note, time of oldest upload of each source upload."""
cache = {}
# Read in all the .changes files
for filename in changes_files:
+ u = Upload()
try:
- Upload.pkg.changes_file = filename
- Upload.init_vars()
- Upload.update_vars()
- cache[filename] = copy.copy(Upload.pkg.changes)
+ u.pkg.load_dot_dak(filename)
+ u.update_subst()
+ cache[filename] = copy.copy(u.pkg.changes)
cache[filename]["filename"] = filename
except:
sorted_list.append(filename)
per_source[source]["list"].append(cache[filename])
# Determine oldest time and have note status for each source group
for source in per_source.keys():
+ q = session.query(DBSource).filter_by(source = source).all()
+ per_source[source]["source_in_database"] = len(q)>0
source_list = per_source[source]["list"]
first = source_list[0]
oldest = os.stat(first["filename"])[stat.ST_MTIME]
mtime = os.stat(d["filename"])[stat.ST_MTIME]
if mtime < oldest:
oldest = mtime
- have_note += (d.has_key("process-new note"))
+ have_note += has_new_comment(d["source"], d["version"], session)
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0; # none
################################################################################
class Section_Completer:
- def __init__ (self):
+ def __init__ (self, session):
self.sections = []
- q = projectB.query("SELECT section FROM section")
- for i in q.getresult():
- self.sections.append(i[0])
+ self.matches = []
+ for s, in session.query(Section.section):
+ self.sections.append(s)
def complete(self, text, state):
if state == 0:
############################################################
class Priority_Completer:
- def __init__ (self):
+ def __init__ (self, session):
self.priorities = []
- q = projectB.query("SELECT priority FROM priority")
- for i in q.getresult():
- self.priorities.append(i[0])
+ self.matches = []
+ for p, in session.query(Priority.priority):
+ self.priorities.append(p)
def complete(self, text, state):
if state == 0:
################################################################################
-def check_valid (new):
- for pkg in new.keys():
- section = new[pkg]["section"]
- priority = new[pkg]["priority"]
- type = new[pkg]["type"]
- new[pkg]["section id"] = daklib.database.get_section_id(section)
- new[pkg]["priority id"] = daklib.database.get_priority_id(new[pkg]["priority"])
- # Sanity checks
- di = section.find("debian-installer") != -1
- if (di and type != "udeb") or (not di and type == "udeb"):
- new[pkg]["section id"] = -1
- if (priority == "source" and type != "dsc") or \
- (priority != "source" and type == "dsc"):
- new[pkg]["priority id"] = -1
-
-################################################################################
-
-def print_new (new, indexed, file=sys.stdout):
+def print_new (new, upload, indexed, file=sys.stdout):
check_valid(new)
- broken = 0
+ broken = False
index = 0
for pkg in new.keys():
index += 1
priority = new[pkg]["priority"]
if new[pkg]["section id"] == -1:
section += "[!]"
- broken = 1
+ broken = True
if new[pkg]["priority id"] == -1:
priority += "[!]"
- broken = 1
+ broken = True
if indexed:
line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
else:
line = "%-20s %-20s %-20s" % (pkg, priority, section)
line = line.strip()+'\n'
file.write(line)
- note = Upload.pkg.changes.get("process-new note")
- if note:
- print "*"*75
- print note
- print "*"*75
- return broken, note
-
-################################################################################
-
-def get_type (f):
- # Determine the type
- if f.has_key("dbtype"):
- type = f["dbtype"]
- elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
- type = "dsc"
- else:
- daklib.utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type))
-
- # Validate the override type
- type_id = daklib.database.get_override_type_id(type)
- if type_id == -1:
- daklib.utils.fubar("invalid type (%s) for new. Say wha?" % (type))
-
- return type
+ notes = get_new_comments(upload.pkg.changes.get("source"))
+ for note in notes:
+ print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
+ % (note.author, note.version, note.notedate, note.comment)
+ print "-" * 72
+ return broken, len(notes) > 0
################################################################################
################################################################################
################################################################################
-def edit_new (new):
+def edit_new (new, upload):
# Write the current data to a temporary file
- temp_filename = daklib.utils.temp_filename()
- temp_file = daklib.utils.open_file(temp_filename, 'w')
- print_new (new, 0, temp_file)
+ (fd, temp_filename) = utils.temp_filename()
+ temp_file = os.fdopen(fd, 'w')
+ print_new (new, upload, indexed=0, file=temp_file)
temp_file.close()
# Spawn an editor on that file
editor = os.environ.get("EDITOR","vi")
result = os.system("%s %s" % (editor, temp_filename))
if result != 0:
- daklib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
+ utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
# Read the edited data back in
- temp_file = daklib.utils.open_file(temp_filename)
+ temp_file = utils.open_file(temp_filename)
lines = temp_file.readlines()
temp_file.close()
os.unlink(temp_filename)
s[len(s):3] = [None] * (3-len(s))
(pkg, priority, section) = s[:3]
if not new.has_key(pkg):
- daklib.utils.warn("Ignoring unknown package '%s'" % (pkg))
+ utils.warn("Ignoring unknown package '%s'" % (pkg))
else:
# Strip off any invalid markers, print_new will readd them.
if section.endswith("[!]"):
section = section[:-3]
if priority.endswith("[!]"):
priority = priority[:-3]
- for file in new[pkg]["files"]:
- Upload.pkg.files[file]["section"] = section
- Upload.pkg.files[file]["priority"] = priority
+ for f in new[pkg]["files"]:
+ upload.pkg.files[f]["section"] = section
+ upload.pkg.files[f]["priority"] = priority
new[pkg]["section"] = section
new[pkg]["priority"] = priority
################################################################################
-def edit_index (new, index):
+def edit_index (new, upload, index):
priority = new[index]["priority"]
section = new[index]["section"]
- type = new[index]["type"]
+ ftype = new[index]["type"]
done = 0
while not done:
print "\t".join([index, priority, section])
answer = "XXX"
- if type != "dsc":
+ if ftype != "dsc":
prompt = "[B]oth, Priority, Section, Done ? "
else:
prompt = "[S]ection, Done ? "
edit_priority = edit_section = 0
while prompt.find(answer) == -1:
- answer = daklib.utils.our_raw_input(prompt)
- m = daklib.queue.re_default_answer.match(prompt)
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.match(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
readline.set_completer(Priorities.complete)
got_priority = 0
while not got_priority:
- new_priority = daklib.utils.our_raw_input("New priority: ").strip()
+ new_priority = utils.our_raw_input("New priority: ").strip()
if new_priority not in Priorities.priorities:
print "E: '%s' is not a valid priority, try again." % (new_priority)
else:
readline.set_completer(Sections.complete)
got_section = 0
while not got_section:
- new_section = daklib.utils.our_raw_input("New section: ").strip()
+ new_section = utils.our_raw_input("New section: ").strip()
if new_section not in Sections.sections:
print "E: '%s' is not a valid section, try again." % (new_section)
else:
# Reset the readline completer
readline.set_completer(None)
- for file in new[index]["files"]:
- Upload.pkg.files[file]["section"] = section
- Upload.pkg.files[file]["priority"] = priority
+ for f in new[index]["files"]:
+ upload.pkg.files[f]["section"] = section
+ upload.pkg.files[f]["priority"] = priority
new[index]["priority"] = priority
new[index]["section"] = section
return new
################################################################################
-def edit_overrides (new):
+def edit_overrides (new, upload, session):
print
done = 0
while not done:
- print_new (new, 1)
+ print_new (new, upload, indexed=1)
new_index = {}
index = 0
for i in new.keys():
got_answer = 0
while not got_answer:
- answer = daklib.utils.our_raw_input(prompt)
+ answer = utils.our_raw_input(prompt)
if not answer.isdigit():
answer = answer[:1].upper()
if answer == "E" or answer == "D":
got_answer = 1
- elif daklib.queue.re_isanum.match (answer):
+ elif re_isanum.match (answer):
answer = int(answer)
if (answer < 1) or (answer > index):
print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
got_answer = 1
if answer == 'E':
- edit_new(new)
+ edit_new(new, upload)
elif answer == 'D':
done = 1
else:
- edit_index (new, new_index[answer])
+ edit_index (new, upload, new_index[answer])
return new
################################################################################
-def edit_note(note):
+def edit_note(note, upload, session):
# Write the current data to a temporary file
- temp_filename = daklib.utils.temp_filename()
- temp_file = daklib.utils.open_file(temp_filename, 'w')
- temp_file.write(note)
- temp_file.close()
+ (fd, temp_filename) = utils.temp_filename()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
os.system("%s %s" % (editor, temp_filename))
- temp_file = daklib.utils.open_file(temp_filename)
- note = temp_file.read().rstrip()
+ temp_file = utils.open_file(temp_filename)
+ newnote = temp_file.read().rstrip()
temp_file.close()
- print "Note:"
- print daklib.utils.prefix_multi_line_string(note," ")
+ print "New Note:"
+ print utils.prefix_multi_line_string(newnote," ")
prompt = "[D]one, Edit, Abandon, Quit ?"
answer = "XXX"
while prompt.find(answer) == -1:
- answer = daklib.utils.our_raw_input(prompt)
- m = daklib.queue.re_default_answer.search(prompt)
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
if answer == 'A':
return
elif answer == 'Q':
+ end()
sys.exit(0)
- Upload.pkg.changes["process-new note"] = note
- Upload.dump_vars(Cnf["Dir::Queue::New"])
+
+ comment = NewComment()
+ comment.package = upload.pkg.changes["source"]
+ comment.version = upload.pkg.changes["version"]
+ comment.comment = newnote
+ comment.author = utils.whoami()
+ comment.trainee = bool(Options["Trainee"])
+ session.add(comment)
+ session.commit()
################################################################################
-def check_pkg ():
+def check_pkg (upload):
try:
less_fd = os.popen("less -R -", 'w', 0)
stdout_fd = sys.stdout
try:
sys.stdout = less_fd
- examine_package.display_changes(Upload.pkg.changes_file)
- files = Upload.pkg.files
- for file in files.keys():
- if files[file].has_key("new"):
- type = files[file]["type"]
- if type == "deb":
- examine_package.check_deb(file)
- elif type == "dsc":
- examine_package.check_dsc(file)
+ changes = utils.parse_changes (upload.pkg.changes_file)
+ examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
+ files = upload.pkg.files
+ for f in files.keys():
+ if files[f].has_key("new"):
+ ftype = files[f]["type"]
+ if ftype == "deb":
+ examine_package.check_deb(changes['distribution'], f)
+ elif ftype == "dsc":
+ examine_package.check_dsc(changes['distribution'], f)
finally:
+ examine_package.output_package_relations()
sys.stdout = stdout_fd
except IOError, e:
- if errno.errorcode[e.errno] == 'EPIPE':
- daklib.utils.warn("[examine_package] Caught EPIPE; skipping.")
+ if e.errno == errno.EPIPE:
+ utils.warn("[examine_package] Caught EPIPE; skipping.")
pass
else:
raise
except KeyboardInterrupt:
- daklib.utils.warn("[examine_package] Caught C-c; skipping.")
+ utils.warn("[examine_package] Caught C-c; skipping.")
pass
################################################################################
## FIXME: horribly Debian specific
-def do_bxa_notification():
- files = Upload.pkg.files
+def do_bxa_notification(upload):
+ files = upload.pkg.files
summary = ""
- for file in files.keys():
- if files[file]["type"] == "deb":
- control = apt_pkg.ParseSection(apt_inst.debExtractControl(daklib.utils.open_file(file)))
+ for f in files.keys():
+ if files[f]["type"] == "deb":
+ control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
summary += "\n"
summary += "Package: %s\n" % (control.Find("Package"))
summary += "Description: %s\n" % (control.Find("Description"))
- Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
- bxa_mail = daklib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
- daklib.utils.send_mail(bxa_mail)
+ upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
+ bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
+ utils.send_mail(bxa_mail)
################################################################################
-def add_overrides (new):
- changes = Upload.pkg.changes
- files = Upload.pkg.files
+def add_overrides (new, upload, session):
+ changes = upload.pkg.changes
+ files = upload.pkg.files
+ srcpkg = changes.get("source")
- projectB.query("BEGIN WORK")
for suite in changes["suite"].keys():
- suite_id = daklib.database.get_suite_id(suite)
+ suite_id = get_suite(suite).suite_id
for pkg in new.keys():
- component_id = daklib.database.get_component_id(new[pkg]["component"])
- type_id = daklib.database.get_override_type_id(new[pkg]["type"])
+ component_id = get_component(new[pkg]["component"]).component_id
+ type_id = get_override_type(new[pkg]["type"]).overridetype_id
priority_id = new[pkg]["priority id"]
section_id = new[pkg]["section id"]
- projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
- for file in new[pkg]["files"]:
- if files[file].has_key("new"):
- del files[file]["new"]
+ Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
+ session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
+ { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
+ for f in new[pkg]["files"]:
+ if files[f].has_key("new"):
+ del files[f]["new"]
del new[pkg]
- projectB.query("COMMIT WORK")
+ session.commit()
- if Cnf.FindB("Dinstall::BXANotify"):
- do_bxa_notification()
+ if Config().FindB("Dinstall::BXANotify"):
+ do_bxa_notification(upload)
################################################################################
-def prod_maintainer ():
+def prod_maintainer (note, upload):
+ cnf = Config()
# Here we prepare an editor and get them ready to prod...
- temp_filename = daklib.utils.temp_filename()
+ (fd, temp_filename) = utils.temp_filename()
+ temp_file = os.fdopen(fd, 'w')
+ if len(note) > 0:
+ for line in note:
+ temp_file.write(line)
+ temp_file.close()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
os.system("%s %s" % (editor, temp_filename))
- file = daklib.utils.open_file(temp_filename)
- prod_message = "".join(file.readlines())
- file.close()
+ temp_fh = utils.open_file(temp_filename)
+ prod_message = "".join(temp_fh.readlines())
+ temp_fh.close()
print "Prod message:"
- print daklib.utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
+ print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
prompt = "[P]rod, Edit, Abandon, Quit ?"
answer = "XXX"
while prompt.find(answer) == -1:
- answer = daklib.utils.our_raw_input(prompt)
- m = daklib.queue.re_default_answer.search(prompt)
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
- os.unlink(temp_filename)
- if answer == 'A':
- return
- elif answer == 'Q':
- sys.exit(0)
+ os.unlink(temp_filename)
+ if answer == 'A':
+ return
+ elif answer == 'Q':
+ end()
+ sys.exit(0)
# Otherwise, do the proding...
- user_email_address = daklib.utils.whoami() + " <%s>" % (
- Cnf["Dinstall::MyAdminAddress"])
+ user_email_address = utils.whoami() + " <%s>" % (
+ cnf["Dinstall::MyAdminAddress"])
- Subst = Upload.Subst
+ Subst = upload.Subst
Subst["__FROM_ADDRESS__"] = user_email_address
Subst["__PROD_MESSAGE__"] = prod_message
- Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
+ Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
- prod_mail_message = daklib.utils.TemplateSubst(
- Subst,Cnf["Dir::Templates"]+"/process-new.prod")
+ prod_mail_message = utils.TemplateSubst(
+ Subst,cnf["Dir::Templates"]+"/process-new.prod")
# Send the prod mail if appropriate
- if not Cnf["Dinstall::Options::No-Mail"]:
- daklib.utils.send_mail(prod_mail_message)
+ if not cnf["Dinstall::Options::No-Mail"]:
+ utils.send_mail(prod_mail_message)
print "Sent proding message"
################################################################################
-def do_new():
+def do_new(upload, session):
print "NEW\n"
- files = Upload.pkg.files
- changes = Upload.pkg.changes
+ files = upload.pkg.files
+ changes = upload.pkg.changes
+ cnf = Config()
# Make a copy of distribution we can happily trample on
changes["suite"] = copy.copy(changes["distribution"])
# Fix up the list of target suites
for suite in changes["suite"].keys():
- override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
+ override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
if override:
- (olderr, newerr) = (daklib.database.get_suite_id(suite) == -1,
- daklib.database.get_suite_id(override) == -1)
- if olderr or newerr:
- (oinv, newinv) = ("", "")
- if olderr: oinv = "invalid "
- if newerr: ninv = "invalid "
- print "warning: overriding %ssuite %s to %ssuite %s" % (
- oinv, suite, ninv, override)
+ (olderr, newerr) = (get_suite(suite, session) == None,
+ get_suite(override, session) == None)
+ if olderr or newerr:
+ (oinv, newinv) = ("", "")
+ if olderr: oinv = "invalid "
+ if newerr: ninv = "invalid "
+ print "warning: overriding %ssuite %s to %ssuite %s" % (
+ oinv, suite, ninv, override)
del changes["suite"][suite]
changes["suite"][override] = 1
# Validate suites
for suite in changes["suite"].keys():
- suite_id = daklib.database.get_suite_id(suite)
- if suite_id == -1:
- daklib.utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
+ if get_suite(suite, session) is None:
+ utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
# The main NEW processing loop
done = 0
if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
- (broken, note) = print_new(new, 0)
+ (broken, note) = print_new(new, upload, indexed=0)
prompt = ""
if not broken and not note:
print "W: [!] marked entries must be fixed before package can be processed."
if note:
print "W: note must be removed before package can be processed."
- prompt += "Remove note, "
+ prompt += "RemOve all notes, Remove note, "
prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
while prompt.find(answer) == -1:
- answer = daklib.utils.our_raw_input(prompt)
- m = daklib.queue.re_default_answer.search(prompt)
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
- if answer == 'A':
- done = add_overrides (new)
+ if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
+ utils.warn("Trainees can't do that")
+ continue
+
+ if answer == 'A' and not Options["Trainee"]:
+ try:
+ check_daily_lock()
+ done = add_overrides (new, upload, session)
+ Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (upload.pkg.changes_file)])
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'C':
- check_pkg()
- elif answer == 'E':
- new = edit_overrides (new)
- elif answer == 'M':
- aborted = Upload.do_reject(1, Options["Manual-Reject"])
+ check_pkg(upload)
+ elif answer == 'E' and not Options["Trainee"]:
+ new = edit_overrides (new, upload, session)
+ elif answer == 'M' and not Options["Trainee"]:
+ aborted = upload.do_reject(manual=1,
+ reject_message=Options["Manual-Reject"],
+ note=get_new_comments(changes.get("source", ""), session=session))
if not aborted:
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ Logger.log([utils.getusername(), "NEW REJECT: %s" % (upload.pkg.changes_file)])
+ os.unlink(upload.pkg.changes_file[:-8]+".dak")
done = 1
elif answer == 'N':
- edit_note(changes.get("process-new note", ""))
- elif answer == 'P':
- prod_maintainer()
- elif answer == 'R':
- confirm = daklib.utils.our_raw_input("Really clear note (y/N)? ").lower()
+ edit_note(get_new_comments(changes.get("source", ""), session=session),
+ upload, session)
+ elif answer == 'P' and not Options["Trainee"]:
+ prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
+ upload)
+ Logger.log([utils.getusername(), "NEW PROD: %s" % (upload.pkg.changes_file)])
+ elif answer == 'R' and not Options["Trainee"]:
+ confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
+ if confirm == "y":
+ for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
+ session.delete(c)
+ session.commit()
+ elif answer == 'O' and not Options["Trainee"]:
+ confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
if confirm == "y":
- del changes["process-new note"]
+ for c in get_new_comments(changes.get("source", ""), session=session):
+ session.delete(c)
+ session.commit()
+
elif answer == 'S':
done = 1
elif answer == 'Q':
+ end()
sys.exit(0)
################################################################################
-h, --help show this help and exit.
-m, --manual-reject=MSG manual reject with `msg'
-n, --no-action don't do anything
+ -t, --trainee FTP Trainee mode
-V, --version display the version number and exit"""
sys.exit(exit_code)
################################################################################
-def init():
- global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
-
- Cnf = daklib.utils.get_conf()
-
- Arguments = [('a',"automatic","Process-New::Options::Automatic"),
- ('h',"help","Process-New::Options::Help"),
- ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
- ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
- ('n',"no-action","Process-New::Options::No-Action")]
-
- for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
- if not Cnf.has_key("Process-New::Options::%s" % (i)):
- Cnf["Process-New::Options::%s" % (i)] = ""
-
- changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Process-New::Options")
-
- if Options["Help"]:
- usage()
-
- Upload = daklib.queue.Upload(Cnf)
-
- if not Options["No-Action"]:
- Logger = Upload.Logger = daklib.logging.Logger(Cnf, "process-new")
-
- projectB = Upload.projectB
-
- Sections = Section_Completer()
- Priorities = Priority_Completer()
- readline.parse_and_bind("tab: complete")
-
- return changes_files
-
-################################################################################
-
-def do_byhand():
+def do_byhand(upload, session):
done = 0
while not done:
- files = Upload.pkg.files
+ files = upload.pkg.files
will_install = 1
byhand = []
- for file in files.keys():
- if files[file]["type"] == "byhand":
- if os.path.exists(file):
- print "W: %s still present; please process byhand components and try again." % (file)
+ for f in files.keys():
+ if files[f]["type"] == "byhand":
+ if os.path.exists(f):
+ print "W: %s still present; please process byhand components and try again." % (f)
will_install = 0
else:
- byhand.append(file)
+ byhand.append(f)
answer = "XXXX"
if Options["No-Action"]:
prompt = "Manual reject, [S]kip, Quit ?"
while prompt.find(answer) == -1:
- answer = daklib.utils.our_raw_input(prompt)
- m = daklib.queue.re_default_answer.search(prompt)
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
if answer == 'A':
- done = 1
- for file in byhand:
- del files[file]
+ try:
+ check_daily_lock()
+ done = 1
+ for f in byhand:
+ del files[f]
+ Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'M':
- Upload.do_reject(1, Options["Manual-Reject"])
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (upload.pkg.changes_file)])
+ upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
+ os.unlink(upload.pkg.changes_file[:-8]+".dak")
done = 1
elif answer == 'S':
done = 1
elif answer == 'Q':
+ end()
sys.exit(0)
################################################################################
-def do_accept():
+def check_daily_lock():
+ """
+ Raises CantGetLockError if the dinstall daily.lock exists.
+ """
+
+ cnf = Config()
+ try:
+ os.open(cnf["Process-New::DinstallLockFile"],
+ os.O_RDONLY | os.O_CREAT | os.O_EXCL)
+ except OSError, e:
+ if e.errno == errno.EEXIST or e.errno == errno.EACCES:
+ raise CantGetLockError
+
+ os.unlink(cnf["Process-New::DinstallLockFile"])
+
+
+@contextlib.contextmanager
+def lock_package(package):
+ """
+ Lock C{package} so that noone else jumps in processing it.
+
+ @type package: string
+ @param package: source package name to lock
+ """
+
+ path = os.path.join(Config()["Process-New::LockDir"], package)
+ try:
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
+ except OSError, e:
+ if e.errno == errno.EEXIST or e.errno == errno.EACCES:
+ user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
+ raise AlreadyLockedError, user
+
+ try:
+ yield fd
+ finally:
+ os.unlink(path)
+
+def _accept(upload):
+ if Options["No-Action"]:
+ return
+ (summary, short_summary) = upload.build_summaries()
+ upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"])
+ os.unlink(upload.pkg.changes_file[:-8]+".dak")
+
+def do_accept(upload):
print "ACCEPT"
+ cnf = Config()
if not Options["No-Action"]:
- retry = 0
- while retry < 10:
- try:
- lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
- retry = 10
- except OSError, e:
- if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
- retry += 1
- if (retry >= 10):
- daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
- else:
- print("Unable to get accepted lock (try %d of 10)" % retry)
- time.sleep(60)
- else:
- raise
- (summary, short_summary) = Upload.build_summaries()
- Upload.accept(summary, short_summary)
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
- os.unlink(Cnf["Process-New::AcceptedLockFile"])
-
-def check_status(files):
- new = byhand = 0
- for file in files.keys():
- if files[file]["type"] == "byhand":
- byhand = 1
- elif files[file].has_key("new"):
- new = 1
- return (new, byhand)
-
-def do_pkg(changes_file):
- Upload.pkg.changes_file = changes_file
- Upload.init_vars()
- Upload.update_vars()
- Upload.update_subst()
- files = Upload.pkg.files
-
- if not recheck():
- return
+ (summary, short_summary) = upload.build_summaries()
+
+ if cnf.FindB("Dinstall::SecurityQueueHandling"):
+ upload.dump_vars(cnf["Dir::Queue::Embargoed"])
+ upload.move_to_dir(cnf["Dir::Queue::Embargoed"])
+ upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
+ # Check for override disparities
+ upload.Subst["__SUMMARY__"] = summary
+ else:
+ # Just a normal upload, accept it...
+ _accept(upload)
- (new, byhand) = check_status(files)
- if new or byhand:
- if new:
- do_new()
- if byhand:
- do_byhand()
- (new, byhand) = check_status(files)
+def do_pkg(changes_file, session):
+ u = Upload()
+ u.pkg.load_dot_dak(changes_file)
+ u.update_subst()
- if not new and not byhand:
- do_accept()
+ cnf = Config()
+ bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
+ if cnf.has_key("Dinstall::Bcc"):
+ u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+ else:
+ u.Subst["__BCC__"] = bcc
+
+ files = u.pkg.files
+
+ try:
+ with lock_package(u.pkg.changes["source"]):
+ if not recheck(u, session):
+ return
+
+ (new, byhand) = check_status(files)
+ if new or byhand:
+ if new:
+ do_new(u, session)
+ if byhand:
+ do_byhand(u, session)
+ (new, byhand) = check_status(files)
+
+ if not new and not byhand:
+ try:
+ check_daily_lock()
+ do_accept(u)
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
+ except AlreadyLockedError, e:
+ print "Seems to be locked by %s already, skipping..." % (e)
################################################################################
def end():
- accept_count = Upload.accept_count
- accept_bytes = Upload.accept_bytes
+ accept_count = SummaryStats().accept_count
+ accept_bytes = SummaryStats().accept_bytes
if accept_count:
sets = "set"
if accept_count > 1:
sets = "sets"
- sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, daklib.utils.size_type(int(accept_bytes))))
- Logger.log(["total",accept_count,accept_bytes])
+ sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
+ Logger.log([utils.getusername(), "total",accept_count,accept_bytes])
- if not Options["No-Action"]:
+ if not Options["No-Action"] and not Options["Trainee"]:
Logger.close()
################################################################################
-def do_comments(dir, opref, npref, line, fn):
- for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
- lines = open("%s/%s" % (dir, comm)).readlines()
- if len(lines) == 0 or lines[0] != line + "\n": continue
- changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
- and x.endswith(".changes") ]
- changes_files = sort_changes(changes_files)
- for f in changes_files:
- f = daklib.utils.validate_changes_file_arg(f, 0)
- if not f: continue
- print "\n" + f
- fn(f, "".join(lines[1:]))
-
- if opref != npref and not Options["No-Action"]:
- newcomm = npref + comm[len(opref):]
- os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
-
-################################################################################
+def main():
+ global Options, Logger, Sections, Priorities
-def comment_accept(changes_file, comments):
- Upload.pkg.changes_file = changes_file
- Upload.init_vars()
- Upload.update_vars()
- Upload.update_subst()
- files = Upload.pkg.files
+ cnf = Config()
+ session = DBConn().session()
- if not recheck():
- return # dak wants to REJECT, crap
+ Arguments = [('a',"automatic","Process-New::Options::Automatic"),
+ ('h',"help","Process-New::Options::Help"),
+ ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
+ ('t',"trainee","Process-New::Options::Trainee"),
+ ('n',"no-action","Process-New::Options::No-Action")]
- (new, byhand) = check_status(files)
- if not new and not byhand:
- do_accept()
+ for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
+ if not cnf.has_key("Process-New::Options::%s" % (i)):
+ cnf["Process-New::Options::%s" % (i)] = ""
-################################################################################
+ changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
+ if len(changes_files) == 0:
+ changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
-def comment_reject(changes_file, comments):
- Upload.pkg.changes_file = changes_file
- Upload.init_vars()
- Upload.update_vars()
- Upload.update_subst()
- files = Upload.pkg.files
+ Options = cnf.SubTree("Process-New::Options")
- if not recheck():
- pass # dak has its own reasons to reject as well, which is fine
+ if Options["Help"]:
+ usage()
- reject(comments)
- print "REJECT\n" + reject_message,
if not Options["No-Action"]:
- Upload.do_reject(0, reject_message)
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ try:
+ Logger = daklog.Logger(cnf, "process-new")
+ except CantOpenError, e:
+ Options["Trainee"] = "True"
-################################################################################
+ Sections = Section_Completer(session)
+ Priorities = Priority_Completer(session)
+ readline.parse_and_bind("tab: complete")
-def main():
- changes_files = init()
- if len(changes_files) > 50:
+ if len(changes_files) > 1:
sys.stderr.write("Sorting changes...\n")
- changes_files = sort_changes(changes_files)
+ changes_files = sort_changes(changes_files, session)
# Kill me now? **FIXME**
- Cnf["Dinstall::Options::No-Mail"] = ""
- bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
- if Cnf.has_key("Dinstall::Bcc"):
- Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
- else:
- Upload.Subst["__BCC__"] = bcc
-
- commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
- if commentsdir:
- if changes_files != []:
- sys.stderr.write("Can't specify any changes files if working with comments-dir")
- sys.exit(1)
- do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
- do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
- else:
- for changes_file in changes_files:
- changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
- if not changes_file:
- continue
- print "\n" + changes_file
- do_pkg (changes_file)
+ cnf["Dinstall::Options::No-Mail"] = ""
+
+ for changes_file in changes_files:
+ changes_file = utils.validate_changes_file_arg(changes_file, 0)
+ if not changes_file:
+ continue
+ print "\n" + changes_file
+
+ do_pkg (changes_file, session)
end()
projects / dak.git / blobdiff