## pu: create files for BTS
## pu: create entry in queue_build
## pu: check overrides
-import errno
+
+# Integrity checks
+## GPG
+## Parsing changes (check for duplicates)
+## Parse dsc
+## file list checks
+
+# New check layout (TODO: Implement)
+## Permission checks
+### suite mappings
+### ACLs
+### version checks (suite)
+### override checks
+
+## Source checks
+### copy orig
+### unpack
+### BTS changelog
+### src contents
+### lintian
+### urgency log
+
+## Binary checks
+### timestamps
+### control checks
+### src relation check
+### contents
+
+## Database insertion (? copy from stuff)
+### BYHAND / NEW / Policy queues
+### Pool
+
+## Queue builds
+
+import datetime
+from errno import EACCES, EAGAIN
import fcntl
import os
import sys
-#from datetime import datetime
+import traceback
import apt_pkg
+import time
+from sqlalchemy.orm.exc import NoResultFound
from daklib import daklog
-#from daklib.queue import *
-from daklib import utils
from daklib.dbconn import *
-#from daklib.dak_exceptions import *
-#from daklib.regexes import re_default_answer, re_issource, re_fdnic
from daklib.urgencylog import UrgencyLog
from daklib.summarystats import SummaryStats
from daklib.config import Config
+import daklib.utils as utils
+from daklib.textutils import fix_maintainer
+from daklib.regexes import *
+
+import daklib.archive
+import daklib.upload
###############################################################################
###############################################################################
-def init():
- global Options
+def usage (exit_code=0):
+ print """Usage: dak process-upload [OPTION]... [CHANGES]...
+ -a, --automatic automatic run
+ -d, --directory <DIR> process uploads in <DIR>
+ -h, --help show this help and exit.
+ -n, --no-action don't do anything
+ -p, --no-lock don't check lockfile !! for cron.daily only !!
+ -s, --no-mail don't send any mail
+ -V, --version display the version number and exit"""
+ sys.exit(exit_code)
+
+###############################################################################
- # Initialize config and connection to db
+def try_or_reject(function):
+ """Try to call function or reject the upload if that fails
+ """
+ def wrapper(directory, upload, *args, **kwargs):
+ try:
+ return function(directory, upload, *args, **kwargs)
+ except Exception as e:
+ try:
+ reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
+ upload.rollback()
+ return real_reject(directory, upload, reason=reason)
+ except Exception as e:
+ reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
+ upload.rollback()
+ return real_reject(directory, upload, reason=reason, notify=False)
+
+ return wrapper
+
+def subst_for_upload(upload):
cnf = Config()
- DBConn()
- Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
- ('h',"help","Dinstall::Options::Help"),
- ('n',"no-action","Dinstall::Options::No-Action"),
- ('p',"no-lock", "Dinstall::Options::No-Lock"),
- ('s',"no-mail", "Dinstall::Options::No-Mail"),
- ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
+ changes = upload.changes
+ control = upload.changes.changes
- for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
- "version", "directory"]:
- if not cnf.has_key("Dinstall::Options::%s" % (i)):
- cnf["Dinstall::Options::%s" % (i)] = ""
+ if upload.final_suites is None or len(upload.final_suites) == 0:
+ suite_name = '(unknown)'
+ else:
+ suite_names = []
+ for suite in upload.final_suites:
+ if suite.policy_queue:
+ suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name))
+ else:
+ suite_names.append(suite.suite_name)
+ suite_name = ','.join(suite_names)
- changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = cnf.SubTree("Dinstall::Options")
+ maintainer_field = control.get('Changed-By', control.get('Maintainer', cnf['Dinstall::MyEmailAddress']))
+ maintainer = fix_maintainer(maintainer_field)
+ addresses = utils.mail_addresses_for_upload(control.get('Maintainer', cnf['Dinstall::MyEmailAddress']), maintainer_field, changes.primary_fingerprint)
- if Options["Help"]:
- usage()
+ bcc = 'X-DAK: dak process-upload'
+ if 'Dinstall::Bcc' in cnf:
+ bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
- # If we have a directory flag, use it to find our files
- if cnf["Dinstall::Options::Directory"] != "":
- # Note that we clobber the list of files we were given in this case
- # so warn if the user has done both
- if len(changes_files) > 0:
- utils.warn("Directory provided so ignoring files given on command line")
+ subst = {
+ '__DISTRO__': cnf['Dinstall::MyDistribution'],
+ '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'],
- changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
+ '__CHANGES_FILENAME__': upload.changes.filename,
+
+ '__SOURCE__': control.get('Source', '(unknown)'),
+ '__ARCHITECTURE__': control.get('Architecture', '(unknown)'),
+ '__VERSION__': control.get('Version', '(unknown)'),
+
+ '__SUITE__': suite_name,
+
+ '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
+ '__MAINTAINER_FROM__': maintainer[1],
+ '__MAINTAINER_TO__': ", ".join(addresses),
+ '__MAINTAINER__': maintainer_field,
+ '__BCC__': bcc,
+
+ '__BUG_SERVER__': cnf.get('Dinstall::BugServer'),
+
+ # TODO: don't use private member
+ '__FILE_CONTENTS__': upload.changes._signed_file.contents,
+
+ # __REJECT_MESSAGE__
+ }
+
+ override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
+ if override_maintainer:
+ subst['__MAINTAINER_TO__'] = subst['__MAINTAINER_FROM__'] = override_maintainer
+
+ return subst
+
+@try_or_reject
+def accept(directory, upload):
+ cnf = Config()
+
+ Logger.log(['ACCEPT', upload.changes.filename])
+
+ upload.install()
+
+ accepted_to_real_suite = False
+ for suite in upload.final_suites:
+ accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
+
+ control = upload.changes.changes
+ if 'source' in upload.changes.architectures and not Options['No-Action']:
+ urgency = control.get('Urgency')
+ if urgency not in cnf.value_list('Urgency::Valid'):
+ urgency = cnf['Urgency::Default']
+ UrgencyLog().log(control['Source'], control['Version'], urgency)
+
+ # send mail to maintainer
+ subst = subst_for_upload(upload)
+ message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted'))
+ utils.send_mail(message)
+
+ # send mail to announce lists and tracking server
+ subst = subst_for_upload(upload)
+ announce = set()
+ for suite in upload.final_suites:
+ if suite.policy_queue is None:
+ continue
+ announce.update(suite.announce or [])
+ announce_address = ", ".join(announce)
+ tracking = cnf.get('Dinstall::TrackingServer')
+ if tracking and 'source' in upload.changes.architectures:
+ announce_address = '{0}\nBcc: {1}@{2}'.format(announce_address, control['Source'], tracking)
+ message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce'))
+ utils.send_mail(message)
+
+ # Only close bugs for uploads that were not redirected to a policy queue.
+ # process-policy will close bugs for those once they are accepted.
+ subst = subst_for_upload(upload)
+ if accepted_to_real_suite and cnf.find_b('Dinstall::CloseBugs') and upload.changes.source is not None:
+ for bugnum in upload.changes.closed_bugs:
+ subst['__BUG_NUMBER__'] = str(bugnum)
+
+ message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close'))
+ utils.send_mail(message)
+
+ del subst['__BUG_NUMBER__']
+
+ # Move .changes to done, but only for uploads that were accepted to a
+ # real suite. process-policy will handle this for uploads to queues.
+ if accepted_to_real_suite:
+ src = os.path.join(upload.directory, upload.changes.filename)
+
+ now = datetime.datetime.now()
+ donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
+ dst = os.path.join(donedir, upload.changes.filename)
+ dst = utils.find_next_free(dst)
+
+ upload.transaction.fs.copy(src, dst, mode=0o644)
+
+ SummaryStats().accept_count += 1
+ SummaryStats().accept_bytes += upload.changes.bytes
+
+@try_or_reject
+def accept_to_new(directory, upload):
+ cnf = Config()
+
+ Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
+
+ upload.install_to_new()
+ # TODO: tag bugs pending, send announcement
- return changes_files
+ subst = subst_for_upload(upload)
+ message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new'))
+ utils.send_mail(message)
+
+ SummaryStats().accept_count += 1
+ SummaryStats().accept_bytes += upload.changes.bytes
+
+@try_or_reject
+def reject(directory, upload, reason=None, notify=True):
+ real_reject(directory, upload, reason, notify)
+
+def real_reject(directory, upload, reason=None, notify=True):
+ # XXX: rejection itself should go to daklib.archive.ArchiveUpload
+ cnf = Config()
+
+ Logger.log(['REJECT', upload.changes.filename])
+
+ fs = upload.transaction.fs
+ rejectdir = cnf['Dir::Reject']
+
+ files = [ f.filename for f in upload.changes.files.itervalues() ]
+ files.append(upload.changes.filename)
+
+ for fn in files:
+ src = os.path.join(upload.directory, fn)
+ dst = utils.find_next_free(os.path.join(rejectdir, fn))
+ fs.copy(src, dst)
+
+ if upload.reject_reasons is not None:
+ if reason is None:
+ reason = ''
+ reason = reason + '\n' + '\n'.join(upload.reject_reasons)
+
+ if reason is None:
+ reason = '(Unknown reason. Please check logs.)'
+
+ dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
+ fh = fs.create(dst)
+ fh.write(reason)
+ fh.close()
+
+ # TODO: fix
+ if notify:
+ subst = subst_for_upload(upload)
+ subst['__REJECTOR_ADDRESS__'] = cnf['Dinstall::MyEmailAddress']
+ subst['__MANUAL_REJECT_MESSAGE__'] = ''
+ subst['__REJECT_MESSAGE__'] = reason
+ subst['__CC__'] = 'X-DAK-Rejection: automatic (moo)'
+
+ message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected'))
+ utils.send_mail(message)
+
+ SummaryStats().reject_count += 1
###############################################################################
-def usage (exit_code=0):
- print """Usage: dak process-upload [OPTION]... [CHANGES]...
- -a, --automatic automatic run
- -h, --help show this help and exit.
- -n, --no-action don't do anything
- -p, --no-lock don't check lockfile !! for cron.daily only !!
- -s, --no-mail don't send any mail
- -V, --version display the version number and exit"""
- sys.exit(exit_code)
+def action(directory, upload):
+ changes = upload.changes
+ processed = True
+
+ global Logger
+
+ cnf = Config()
+
+ okay = upload.check()
+
+ summary = changes.changes.get('Changes', '')
+
+ package_info = []
+ if okay:
+ if changes.source is not None:
+ package_info.append("source:{0}".format(changes.source.dsc['Source']))
+ for binary in changes.binaries:
+ package_info.append("binary:{0}".format(binary.control['Package']))
+
+ (prompt, answer) = ("", "XXX")
+ if Options["No-Action"] or Options["Automatic"]:
+ answer = 'S'
+
+ queuekey = ''
+
+ print summary
+ print
+ print "\n".join(package_info)
+ print
+
+ if len(upload.reject_reasons) > 0:
+ print "Reason:"
+ print "\n".join(upload.reject_reasons)
+ print
+
+ path = os.path.join(directory, changes.filename)
+ created = os.stat(path).st_mtime
+ now = time.time()
+ too_new = (now - created < int(cnf['Dinstall::SkipTime']))
+
+ if too_new:
+ print "SKIP (too new)"
+ prompt = "[S]kip, Quit ?"
+ else:
+ prompt = "[R]eject, Skip, Quit ?"
+ if Options["Automatic"]:
+ answer = 'R'
+ elif upload.new:
+ prompt = "[N]ew, Skip, Quit ?"
+ if Options['Automatic']:
+ answer = 'N'
+ else:
+ prompt = "[A]ccept, Skip, Quit ?"
+ if Options['Automatic']:
+ answer = 'A'
+
+ while prompt.find(answer) == -1:
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.match(prompt)
+ if answer == "":
+ answer = m.group(1)
+ answer = answer[:1].upper()
+
+ if answer == 'R':
+ reject(directory, upload)
+ elif answer == 'A':
+ # upload.try_autobyhand must not be run with No-Action.
+ if Options['No-Action']:
+ accept(directory, upload)
+ elif upload.try_autobyhand():
+ accept(directory, upload)
+ else:
+ print "W: redirecting to BYHAND as automatic processing failed."
+ accept_to_new(directory, upload)
+ elif answer == 'N':
+ accept_to_new(directory, upload)
+ elif answer == 'Q':
+ sys.exit(0)
+ elif answer == 'S':
+ processed = False
+
+ #raise Exception("FAIL")
+ if not Options['No-Action']:
+ upload.commit()
+
+ return processed
###############################################################################
-def main():
+def unlink_if_exists(path):
+ try:
+ os.unlink(path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+def process_it(directory, changes, keyrings, session):
global Logger
+ print "\n{0}\n".format(changes.filename)
+ Logger.log(["Processing changes file", changes.filename])
+
+ cnf = Config()
+
+ # Some defaults in case we can't fully process the .changes file
+ #u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
+ #u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
+
+ # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
+ bcc = "X-DAK: dak process-upload"
+ #if cnf.has_key("Dinstall::Bcc"):
+ # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+ #else:
+ # u.Subst["__BCC__"] = bcc
+
+ with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
+ processed = action(directory, upload)
+ if processed and not Options['No-Action']:
+ unlink_if_exists(os.path.join(directory, changes.filename))
+ for fn in changes.files:
+ unlink_if_exists(os.path.join(directory, fn))
+
+###############################################################################
+
+def process_changes(changes_filenames):
+ session = DBConn().session()
+ keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
+ keyring_files = [ k.keyring_name for k in keyrings ]
+
+ changes = []
+ for fn in changes_filenames:
+ try:
+ directory, filename = os.path.split(fn)
+ c = daklib.upload.Changes(directory, filename, keyring_files)
+ changes.append([directory, c])
+ except Exception as e:
+ Logger.log([filename, "Error while loading changes: {0}".format(e)])
+
+ changes.sort(key=lambda x: x[1])
+
+ for directory, c in changes:
+ process_it(directory, c, keyring_files, session)
+
+ session.rollback()
+
+###############################################################################
+
+def main():
+ global Options, Logger
+
cnf = Config()
summarystats = SummaryStats()
- changes_files = init()
- log_urgency = False
- stable_queue = None
+
+ Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
+ ('h',"help","Dinstall::Options::Help"),
+ ('n',"no-action","Dinstall::Options::No-Action"),
+ ('p',"no-lock", "Dinstall::Options::No-Lock"),
+ ('s',"no-mail", "Dinstall::Options::No-Mail"),
+ ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
+
+ for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
+ "version", "directory"]:
+ if not cnf.has_key("Dinstall::Options::%s" % (i)):
+ cnf["Dinstall::Options::%s" % (i)] = ""
+
+ changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.subtree("Dinstall::Options")
+
+ if Options["Help"]:
+ usage()
# -n/--dry-run invalidates some other options which would involve things happening
if Options["No-Action"]:
Options["Automatic"] = ""
- # Check that we aren't going to clash with the daily cron job
-
# Check that we aren't going to clash with the daily cron job
if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
utils.fubar("Archive maintenance in progress. Try again later.")
# Obtain lock if not in no-action mode and initialize the log
if not Options["No-Action"]:
- lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
+ lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
try:
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- except IOError, e:
+ except IOError as e:
if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
else:
raise
- if cnf.get("Dir::UrgencyLog"):
- # Initialise UrgencyLog()
- log_urgency = True
- UrgencyLog()
- Logger = daklog.Logger(cnf, "process-upload", Options["No-Action"])
+ # Initialise UrgencyLog() - it will deal with the case where we don't
+ # want to log urgencies
+ urgencylog = UrgencyLog()
- # Sort the .changes files so that we process sourceful ones first
- changes_files.sort(utils.changes_compare)
+ Logger = daklog.Logger("process-upload", Options["No-Action"])
- # Process the changes files
- for changes_file in changes_files:
- print "\n" + changes_file
- session = DBConn().session()
- session.close()
+ # If we have a directory flag, use it to find our files
+ if cnf["Dinstall::Options::Directory"] != "":
+ # Note that we clobber the list of files we were given in this case
+ # so warn if the user has done both
+ if len(changes_files) > 0:
+ utils.warn("Directory provided so ignoring files given on command line")
+
+ changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
+ Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
+ elif not len(changes_files) > 0:
+ utils.fubar("No changes files given and no directory specified")
+ else:
+ Logger.log(["Using changes files from command-line", len(changes_files)])
+
+ process_changes(changes_files)
if summarystats.accept_count:
sets = "set"
if summarystats.accept_count > 1:
sets = "sets"
- sys.stderr.write("Installed %d package %s, %s.\n" % (summarystats.accept_count, sets,
- utils.size_type(int(summarystats.accept_bytes))))
+ print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
+ utils.size_type(int(summarystats.accept_bytes)))
Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
+ if summarystats.reject_count:
+ sets = "set"
+ if summarystats.reject_count > 1:
+ sets = "sets"
+ print "Rejected %d package %s." % (summarystats.reject_count, sets)
+ Logger.log(["rejected", summarystats.reject_count])
+
if not Options["No-Action"]:
- if log_urgency:
- UrgencyLog().close()
+ urgencylog.close()
+
Logger.close()
###############################################################################