X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_upload.py;h=93d30f85d672944842f4a8e54111e2bd76868fe6;hb=c575bba437ec1de5ceb5feb6dc9c5f87d843958e;hp=24d297c33865c5e444c345178e811a3af01425db;hpb=94055d7573bf7542145cdeb5294e5d8cad611668;p=dak.git diff --git a/dak/process_upload.py b/dak/process_upload.py index 24d297c3..93d30f85 100755 --- a/dak/process_upload.py +++ b/dak/process_upload.py @@ -125,22 +125,62 @@ Checks Debian packages from Incoming ## pu: create files for BTS ## pu: create entry in queue_build ## pu: check overrides + +# Integrity checks +## GPG +## Parsing changes (check for duplicates) +## Parse dsc +## file list checks + +# New check layout (TODO: Implement) +## Permission checks +### suite mappings +### ACLs +### version checks (suite) +### override checks + +## Source checks +### copy orig +### unpack +### BTS changelog +### src contents +### lintian +### urgency log + +## Binary checks +### timestamps +### control checks +### src relation check +### contents + +## Database insertion (? copy from stuff) +### BYHAND / NEW / Policy queues +### Pool + +## Queue builds + +import datetime import errno +from errno import EACCES, EAGAIN import fcntl import os import sys -#from datetime import datetime +import traceback import apt_pkg +import time +from sqlalchemy.orm.exc import NoResultFound from daklib import daklog -#from daklib.queue import * -from daklib import utils from daklib.dbconn import * -#from daklib.dak_exceptions import * -#from daklib.regexes import re_default_answer, re_issource, re_fdnic from daklib.urgencylog import UrgencyLog from daklib.summarystats import SummaryStats from daklib.config import Config +import daklib.utils as utils +from daklib.textutils import fix_maintainer +from daklib.regexes import * + +import daklib.archive +import daklib.upload ############################################################################### @@ -149,112 +189,464 @@ Logger = None ############################################################################### -def init(): - global Options +def usage (exit_code=0): + print """Usage: dak process-upload [OPTION]... [CHANGES]... + -a, --automatic automatic run + -d, --directory process uploads in + -h, --help show this help and exit. + -n, --no-action don't do anything + -p, --no-lock don't check lockfile !! for cron.daily only !! + -s, --no-mail don't send any mail + -V, --version display the version number and exit""" + sys.exit(exit_code) + +############################################################################### - # Initialize config and connection to db +def try_or_reject(function): + """Try to call function or reject the upload if that fails + """ + def wrapper(directory, upload, *args, **kwargs): + try: + return function(directory, upload, *args, **kwargs) + except Exception as e: + try: + reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc()) + upload.rollback() + return real_reject(directory, upload, reason=reason) + except Exception as e: + reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason) + upload.rollback() + return real_reject(directory, upload, reason=reason, notify=False) + + return wrapper + +def subst_for_upload(upload): cnf = Config() - DBConn() - Arguments = [('a',"automatic","Dinstall::Options::Automatic"), - ('h',"help","Dinstall::Options::Help"), - ('n',"no-action","Dinstall::Options::No-Action"), - ('p',"no-lock", "Dinstall::Options::No-Lock"), - ('s',"no-mail", "Dinstall::Options::No-Mail"), - ('d',"directory", "Dinstall::Options::Directory", "HasArg")] + changes = upload.changes + control = upload.changes.changes - for i in ["automatic", "help", "no-action", "no-lock", "no-mail", - "version", "directory"]: - if not cnf.has_key("Dinstall::Options::%s" % (i)): - cnf["Dinstall::Options::%s" % (i)] = "" + if upload.final_suites is None or len(upload.final_suites) == 0: + suite_name = '(unknown)' + else: + suite_names = [] + for suite in upload.final_suites: + if suite.policy_queue: + suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name)) + else: + suite_names.append(suite.suite_name) + suite_name = ','.join(suite_names) - changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) - Options = cnf.SubTree("Dinstall::Options") + maintainer_field = control.get('Maintainer', cnf['Dinstall::MyEmailAddress']) + changed_by_field = control.get('Changed-By', maintainer_field) + maintainer = fix_maintainer(changed_by_field) + if upload.changes.source is not None: + addresses = utils.mail_addresses_for_upload(maintainer_field, changed_by_field, changes.primary_fingerprint) + else: + addresses = utils.mail_addresses_for_upload(maintainer_field, maintainer_field, changes.primary_fingerprint) - if Options["Help"]: - usage() + bcc = 'X-DAK: dak process-upload' + if 'Dinstall::Bcc' in cnf: + bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc']) - # If we have a directory flag, use it to find our files - if cnf["Dinstall::Options::Directory"] != "": - # Note that we clobber the list of files we were given in this case - # so warn if the user has done both - if len(changes_files) > 0: - utils.warn("Directory provided so ignoring files given on command line") + subst = { + '__DISTRO__': cnf['Dinstall::MyDistribution'], + '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'], - changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"]) + '__CHANGES_FILENAME__': upload.changes.filename, + + '__SOURCE__': control.get('Source', '(unknown)'), + '__ARCHITECTURE__': control.get('Architecture', '(unknown)'), + '__VERSION__': control.get('Version', '(unknown)'), + + '__SUITE__': suite_name, + + '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'], + '__MAINTAINER_FROM__': maintainer[1], + '__MAINTAINER_TO__': ", ".join(addresses), + '__MAINTAINER__': changed_by_field, + '__BCC__': bcc, + + '__BUG_SERVER__': cnf.get('Dinstall::BugServer'), + + '__FILE_CONTENTS__': open(upload.changes.path, 'r').read(), + } + + override_maintainer = cnf.get('Dinstall::OverrideMaintainer') + if override_maintainer: + subst['__MAINTAINER_TO__'] = subst['__MAINTAINER_FROM__'] = override_maintainer + + return subst + +@try_or_reject +def accept(directory, upload): + cnf = Config() + + Logger.log(['ACCEPT', upload.changes.filename]) + + upload.install() + + accepted_to_real_suite = False + for suite in upload.final_suites: + accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None + + sourceful_upload = 'source' in upload.changes.architectures + + control = upload.changes.changes + if sourceful_upload and not Options['No-Action']: + urgency = control.get('Urgency') + if urgency not in cnf.value_list('Urgency::Valid'): + urgency = cnf['Urgency::Default'] + UrgencyLog().log(control['Source'], control['Version'], urgency) + + # send mail to maintainer + subst = subst_for_upload(upload) + message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted')) + utils.send_mail(message) + + # send mail to announce lists and tracking server + if accepted_to_real_suite and sourceful_upload: + subst = subst_for_upload(upload) + announce = set() + for suite in upload.final_suites: + if suite.policy_queue is not None: + continue + announce.update(suite.announce or []) + announce_address = ", ".join(announce) + + tracking = cnf.get('Dinstall::TrackingServer') + if tracking and 'source' in upload.changes.architectures: + announce_address = '{0}\nBcc: {1}@{2}'.format(announce_address, control['Source'], tracking) + + subst['__ANNOUNCE_LIST_ADDRESS__'] = announce_address + + message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce')) + utils.send_mail(message) + + # Only close bugs for uploads that were not redirected to a policy queue. + # process-policy will close bugs for those once they are accepted. + subst = subst_for_upload(upload) + if accepted_to_real_suite and cnf.find_b('Dinstall::CloseBugs') and sourceful_upload: + for bugnum in upload.changes.closed_bugs: + subst['__BUG_NUMBER__'] = str(bugnum) + + message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close')) + utils.send_mail(message) + + del subst['__BUG_NUMBER__'] + + # Move .changes to done, but only for uploads that were accepted to a + # real suite. process-policy will handle this for uploads to queues. + if accepted_to_real_suite: + src = os.path.join(upload.directory, upload.changes.filename) + + now = datetime.datetime.now() + donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) + dst = os.path.join(donedir, upload.changes.filename) + dst = utils.find_next_free(dst) + + upload.transaction.fs.copy(src, dst, mode=0o644) + + SummaryStats().accept_count += 1 + SummaryStats().accept_bytes += upload.changes.bytes + +@try_or_reject +def accept_to_new(directory, upload): + cnf = Config() + + Logger.log(['ACCEPT-TO-NEW', upload.changes.filename]) + + upload.install_to_new() + # TODO: tag bugs pending, send announcement + + subst = subst_for_upload(upload) + message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new')) + utils.send_mail(message) + + SummaryStats().accept_count += 1 + SummaryStats().accept_bytes += upload.changes.bytes + +@try_or_reject +def reject(directory, upload, reason=None, notify=True): + real_reject(directory, upload, reason, notify) + +def real_reject(directory, upload, reason=None, notify=True): + # XXX: rejection itself should go to daklib.archive.ArchiveUpload + cnf = Config() + + Logger.log(['REJECT', upload.changes.filename]) + + fs = upload.transaction.fs + rejectdir = cnf['Dir::Reject'] + + files = [ f.filename for f in upload.changes.files.itervalues() ] + files.append(upload.changes.filename) - return changes_files + for fn in files: + src = os.path.join(upload.directory, fn) + dst = utils.find_next_free(os.path.join(rejectdir, fn)) + if not os.path.exists(src): + continue + fs.copy(src, dst) + + if upload.reject_reasons is not None: + if reason is None: + reason = '' + reason = reason + '\n' + '\n'.join(upload.reject_reasons) + + if reason is None: + reason = '(Unknown reason. Please check logs.)' + + dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename))) + fh = fs.create(dst) + fh.write(reason) + fh.close() + + # TODO: fix + if notify: + subst = subst_for_upload(upload) + subst['__REJECTOR_ADDRESS__'] = cnf['Dinstall::MyEmailAddress'] + subst['__MANUAL_REJECT_MESSAGE__'] = '' + subst['__REJECT_MESSAGE__'] = reason + subst['__CC__'] = 'X-DAK-Rejection: automatic (moo)' + + message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected')) + utils.send_mail(message) + + SummaryStats().reject_count += 1 ############################################################################### -def usage (exit_code=0): - print """Usage: dak process-upload [OPTION]... [CHANGES]... - -a, --automatic automatic run - -h, --help show this help and exit. - -n, --no-action don't do anything - -p, --no-lock don't check lockfile !! for cron.daily only !! - -s, --no-mail don't send any mail - -V, --version display the version number and exit""" - sys.exit(exit_code) +def action(directory, upload): + changes = upload.changes + processed = True + + global Logger + + cnf = Config() + + okay = upload.check() + + summary = changes.changes.get('Changes', '') + + package_info = [] + if okay: + if changes.source is not None: + package_info.append("source:{0}".format(changes.source.dsc['Source'])) + for binary in changes.binaries: + package_info.append("binary:{0}".format(binary.control['Package'])) + + (prompt, answer) = ("", "XXX") + if Options["No-Action"] or Options["Automatic"]: + answer = 'S' + + queuekey = '' + + print summary + print + print "\n".join(package_info) + print + + if len(upload.reject_reasons) > 0: + print "Reason:" + print "\n".join(upload.reject_reasons) + print + + path = os.path.join(directory, changes.filename) + created = os.stat(path).st_mtime + now = time.time() + too_new = (now - created < int(cnf['Dinstall::SkipTime'])) + + if too_new: + print "SKIP (too new)" + prompt = "[S]kip, Quit ?" + else: + prompt = "[R]eject, Skip, Quit ?" + if Options["Automatic"]: + answer = 'R' + elif upload.new: + prompt = "[N]ew, Skip, Quit ?" + if Options['Automatic']: + answer = 'N' + else: + prompt = "[A]ccept, Skip, Quit ?" + if Options['Automatic']: + answer = 'A' + + while prompt.find(answer) == -1: + answer = utils.our_raw_input(prompt) + m = re_default_answer.match(prompt) + if answer == "": + answer = m.group(1) + answer = answer[:1].upper() + + if answer == 'R': + reject(directory, upload) + elif answer == 'A': + # upload.try_autobyhand must not be run with No-Action. + if Options['No-Action']: + accept(directory, upload) + elif upload.try_autobyhand(): + accept(directory, upload) + else: + print "W: redirecting to BYHAND as automatic processing failed." + accept_to_new(directory, upload) + elif answer == 'N': + accept_to_new(directory, upload) + elif answer == 'Q': + sys.exit(0) + elif answer == 'S': + processed = False + + #raise Exception("FAIL") + if not Options['No-Action']: + upload.commit() + + return processed ############################################################################### -def main(): +def unlink_if_exists(path): + try: + os.unlink(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + +def process_it(directory, changes, keyrings, session): global Logger + print "\n{0}\n".format(changes.filename) + Logger.log(["Processing changes file", changes.filename]) + + cnf = Config() + + # Some defaults in case we can't fully process the .changes file + #u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"] + #u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"] + + # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header + bcc = "X-DAK: dak process-upload" + #if cnf.has_key("Dinstall::Bcc"): + # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) + #else: + # u.Subst["__BCC__"] = bcc + + with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload: + processed = action(directory, upload) + if processed and not Options['No-Action']: + unlink_if_exists(os.path.join(directory, changes.filename)) + for fn in changes.files: + unlink_if_exists(os.path.join(directory, fn)) + +############################################################################### + +def process_changes(changes_filenames): + session = DBConn().session() + keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority) + keyring_files = [ k.keyring_name for k in keyrings ] + + changes = [] + for fn in changes_filenames: + try: + directory, filename = os.path.split(fn) + c = daklib.upload.Changes(directory, filename, keyring_files) + changes.append([directory, c]) + except Exception as e: + Logger.log([filename, "Error while loading changes: {0}".format(e)]) + + changes.sort(key=lambda x: x[1]) + + for directory, c in changes: + process_it(directory, c, keyring_files, session) + + session.rollback() + +############################################################################### + +def main(): + global Options, Logger + cnf = Config() summarystats = SummaryStats() - changes_files = init() - log_urgency = False - stable_queue = None + + Arguments = [('a',"automatic","Dinstall::Options::Automatic"), + ('h',"help","Dinstall::Options::Help"), + ('n',"no-action","Dinstall::Options::No-Action"), + ('p',"no-lock", "Dinstall::Options::No-Lock"), + ('s',"no-mail", "Dinstall::Options::No-Mail"), + ('d',"directory", "Dinstall::Options::Directory", "HasArg")] + + for i in ["automatic", "help", "no-action", "no-lock", "no-mail", + "version", "directory"]: + if not cnf.has_key("Dinstall::Options::%s" % (i)): + cnf["Dinstall::Options::%s" % (i)] = "" + + changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) + Options = cnf.subtree("Dinstall::Options") + + if Options["Help"]: + usage() # -n/--dry-run invalidates some other options which would involve things happening if Options["No-Action"]: Options["Automatic"] = "" - # Check that we aren't going to clash with the daily cron job - # Check that we aren't going to clash with the daily cron job if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later.") # Obtain lock if not in no-action mode and initialize the log if not Options["No-Action"]: - lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT) + lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT) try: fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError, e: + except IOError as e: if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN': utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.") else: raise - if cnf.get("Dir::UrgencyLog"): - # Initialise UrgencyLog() - log_urgency = True - UrgencyLog() - Logger = daklog.Logger(cnf, "process-upload", Options["No-Action"]) + # Initialise UrgencyLog() - it will deal with the case where we don't + # want to log urgencies + urgencylog = UrgencyLog() - # Sort the .changes files so that we process sourceful ones first - changes_files.sort(utils.changes_compare) + Logger = daklog.Logger("process-upload", Options["No-Action"]) - # Process the changes files - for changes_file in changes_files: - print "\n" + changes_file - session = DBConn().session() - session.close() + # If we have a directory flag, use it to find our files + if cnf["Dinstall::Options::Directory"] != "": + # Note that we clobber the list of files we were given in this case + # so warn if the user has done both + if len(changes_files) > 0: + utils.warn("Directory provided so ignoring files given on command line") + + changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"]) + Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)]) + elif not len(changes_files) > 0: + utils.fubar("No changes files given and no directory specified") + else: + Logger.log(["Using changes files from command-line", len(changes_files)]) + + process_changes(changes_files) if summarystats.accept_count: sets = "set" if summarystats.accept_count > 1: sets = "sets" - sys.stderr.write("Installed %d package %s, %s.\n" % (summarystats.accept_count, sets, - utils.size_type(int(summarystats.accept_bytes)))) + print "Installed %d package %s, %s." % (summarystats.accept_count, sets, + utils.size_type(int(summarystats.accept_bytes))) Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes]) + if summarystats.reject_count: + sets = "set" + if summarystats.reject_count > 1: + sets = "sets" + print "Rejected %d package %s." % (summarystats.reject_count, sets) + Logger.log(["rejected", summarystats.reject_count]) + if not Options["No-Action"]: - if log_urgency: - UrgencyLog().close() + urgencylog.close() + Logger.close() ###############################################################################