X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=blobdiff_plain;f=dak%2Fprocess_upload.py;h=d5425e058ccfa2de6c87dc43e8ec24cce83c5915;hp=74229e2f482b46d5f58ee351886766c6699bd4ca;hb=026bdb9580e6d18bf504d2f44d46890df5d89f1a;hpb=cec8f2c1492f4ba27babfcc16bf2ecc5fbf9d9ec diff --git a/dak/process_upload.py b/dak/process_upload.py index 74229e2f..d5425e05 100755 --- a/dak/process_upload.py +++ b/dak/process_upload.py @@ -176,10 +176,11 @@ from daklib.urgencylog import UrgencyLog from daklib.summarystats import SummaryStats from daklib.config import Config import daklib.utils as utils -from daklib.textutils import fix_maintainer from daklib.regexes import * +import daklib.announce import daklib.archive +import daklib.checks import daklib.upload ############################################################################### @@ -206,82 +207,60 @@ def try_or_reject(function): """Try to call function or reject the upload if that fails """ def wrapper(directory, upload, *args, **kwargs): + reason = 'No exception caught. This should not happen.' + try: return function(directory, upload, *args, **kwargs) + except (daklib.archive.ArchiveException, daklib.checks.Reject) as e: + reason = unicode(e) except Exception as e: - try: - reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc()) - upload.rollback() - return real_reject(directory, upload, reason=reason) - except Exception as e: - reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason) - upload.rollback() - return real_reject(directory, upload, reason=reason, notify=False) + reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc()) - return wrapper + try: + upload.rollback() + return real_reject(directory, upload, reason=reason) + except Exception as e: + reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason) + upload.rollback() + return real_reject(directory, upload, reason=reason, notify=False) -def subst_for_upload(upload): - cnf = Config() + raise Exception('Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}'.format(reason)) + return wrapper + +def get_processed_upload(upload): changes = upload.changes control = upload.changes.changes - if upload.final_suites is None or len(upload.final_suites) == 0: - suite_name = '(unknown)' - else: - suite_names = [] - for suite in upload.final_suites: - if suite.policy_queue: - suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name)) - else: - suite_names.append(suite.suite_name) - suite_name = ','.join(suite_names) - - maintainer_field = control.get('Changed-By', control.get('Maintainer', cnf['Dinstall::MyEmailAddress'])) - maintainer = fix_maintainer(maintainer_field) - addresses = utils.mail_addresses_for_upload(control.get('Maintainer', cnf['Dinstall::MyEmailAddress']), maintainer_field, changes.primary_fingerprint) - - bcc = 'X-DAK: dak process-upload' - if 'Dinstall::Bcc' in cnf: - bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc']) - - subst = { - '__DISTRO__': cnf['Dinstall::MyDistribution'], - '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'], - - '__CHANGES_FILENAME__': upload.changes.filename, - - '__SOURCE__': control.get('Source', '(unknown)'), - '__ARCHITECTURE__': control.get('Architecture', '(unknown)'), - '__VERSION__': control.get('Version', '(unknown)'), + pu = daklib.announce.ProcessedUpload() - '__SUITE__': suite_name, + pu.maintainer = control.get('Maintainer') + pu.changed_by = control.get('Changed-By') + pu.fingerprint = changes.primary_fingerprint - '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'], - '__MAINTAINER_FROM__': maintainer[1], - '__MAINTAINER_TO__': ", ".join(addresses), - '__MAINTAINER__': maintainer_field, - '__BCC__': bcc, + pu.suites = upload.final_suites or [] + pu.from_policy_suites = [] - '__BUG_SERVER__': cnf.get('Dinstall::BugServer'), + pu.changes = open(upload.changes.path, 'r').read() + pu.changes_filename = upload.changes.filename + pu.sourceful = upload.changes.sourceful + pu.source = control.get('Source') + pu.version = control.get('Version') + pu.architecture = control.get('Architecture') + pu.bugs = changes.closed_bugs - # TODO: don't use private member - '__FILE_CONTENTS__': upload.changes._signed_file.contents, + pu.program = "process-upload" - # __REJECT_MESSAGE__ - } + pu.warnings = upload.warnings - override_maintainer = cnf.get('Dinstall::OverrideMaintainer') - if override_maintainer: - subst['__MAINTAINER_TO__'] = subst['__MAINTAINER_FROM__'] = override_maintainer - - return subst + return pu @try_or_reject def accept(directory, upload): cnf = Config() Logger.log(['ACCEPT', upload.changes.filename]) + print "ACCEPT" upload.install() @@ -289,49 +268,17 @@ def accept(directory, upload): for suite in upload.final_suites: accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None + sourceful_upload = 'source' in upload.changes.architectures + control = upload.changes.changes - if 'source' in upload.changes.architectures and not Options['No-Action']: + if sourceful_upload and not Options['No-Action']: urgency = control.get('Urgency') if urgency not in cnf.value_list('Urgency::Valid'): urgency = cnf['Urgency::Default'] UrgencyLog().log(control['Source'], control['Version'], urgency) - # send mail to maintainer - subst = subst_for_upload(upload) - message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted')) - utils.send_mail(message) - - # send mail to announce lists and tracking server - subst = subst_for_upload(upload) - announce = set() - for suite in upload.final_suites: - if suite.policy_queue is not None: - continue - announce.update(suite.announce or []) - announce_address = ", ".join(announce) - - tracking = cnf.get('Dinstall::TrackingServer') - if tracking and 'source' in upload.changes.architectures: - announce_address = '{0}\nBcc: {1}@{2}'.format(announce_address, control['Source'], tracking) - - subst['__ANNOUNCE_LIST_ADDRESS__'] = announce_address - - message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce')) - utils.send_mail(message) - - del subst['__ANNOUNCE_LIST_ADDRESS__'] - - # Only close bugs for uploads that were not redirected to a policy queue. - # process-policy will close bugs for those once they are accepted. - subst = subst_for_upload(upload) - if accepted_to_real_suite and cnf.find_b('Dinstall::CloseBugs') and upload.changes.source is not None: - for bugnum in upload.changes.closed_bugs: - subst['__BUG_NUMBER__'] = str(bugnum) - - message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close')) - utils.send_mail(message) - - del subst['__BUG_NUMBER__'] + pu = get_processed_upload(upload) + daklib.announce.announce_accept(pu) # Move .changes to done, but only for uploads that were accepted to a # real suite. process-policy will handle this for uploads to queues. @@ -350,16 +297,15 @@ def accept(directory, upload): @try_or_reject def accept_to_new(directory, upload): - cnf = Config() Logger.log(['ACCEPT-TO-NEW', upload.changes.filename]) + print "ACCEPT-TO-NEW" upload.install_to_new() - # TODO: tag bugs pending, send announcement + # TODO: tag bugs pending - subst = subst_for_upload(upload) - message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new')) - utils.send_mail(message) + pu = get_processed_upload(upload) + daklib.announce.announce_new(pu) SummaryStats().accept_count += 1 SummaryStats().accept_bytes += upload.changes.bytes @@ -373,6 +319,7 @@ def real_reject(directory, upload, reason=None, notify=True): cnf = Config() Logger.log(['REJECT', upload.changes.filename]) + print "REJECT" fs = upload.transaction.fs rejectdir = cnf['Dir::Reject'] @@ -400,16 +347,9 @@ def real_reject(directory, upload, reason=None, notify=True): fh.write(reason) fh.close() - # TODO: fix if notify: - subst = subst_for_upload(upload) - subst['__REJECTOR_ADDRESS__'] = cnf['Dinstall::MyEmailAddress'] - subst['__MANUAL_REJECT_MESSAGE__'] = '' - subst['__REJECT_MESSAGE__'] = reason - subst['__CC__'] = 'X-DAK-Rejection: automatic (moo)' - - message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected')) - utils.send_mail(message) + pu = get_processed_upload(upload) + daklib.announce.announce_reject(pu, reason) SummaryStats().reject_count += 1 @@ -438,12 +378,13 @@ def action(directory, upload): if Options["No-Action"] or Options["Automatic"]: answer = 'S' - queuekey = '' - print summary print print "\n".join(package_info) print + if len(upload.warnings) > 0: + print "\n".join(upload.warnings) + print if len(upload.reject_reasons) > 0: print "Reason:" @@ -496,7 +437,6 @@ def action(directory, upload): elif answer == 'S': processed = False - #raise Exception("FAIL") if not Options['No-Action']: upload.commit() @@ -511,28 +451,22 @@ def unlink_if_exists(path): if e.errno != errno.ENOENT: raise -def process_it(directory, changes, keyrings, session): +def process_it(directory, changes, keyrings): global Logger print "\n{0}\n".format(changes.filename) Logger.log(["Processing changes file", changes.filename]) - cnf = Config() - - # Some defaults in case we can't fully process the .changes file - #u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"] - #u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"] - - # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header - bcc = "X-DAK: dak process-upload" - #if cnf.has_key("Dinstall::Bcc"): - # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) - #else: - # u.Subst["__BCC__"] = bcc - with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload: processed = action(directory, upload) if processed and not Options['No-Action']: + session = DBConn().session() + history = SignatureHistory.from_signed_file(upload.changes) + if history.query(session) is None: + session.add(history) + session.commit() + session.close() + unlink_if_exists(os.path.join(directory, changes.filename)) for fn in changes.files: unlink_if_exists(os.path.join(directory, fn)) @@ -543,6 +477,7 @@ def process_changes(changes_filenames): session = DBConn().session() keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority) keyring_files = [ k.keyring_name for k in keyrings ] + session.close() changes = [] for fn in changes_filenames: @@ -556,9 +491,7 @@ def process_changes(changes_filenames): changes.sort(key=lambda x: x[1]) for directory, c in changes: - process_it(directory, c, keyring_files, session) - - session.rollback() + process_it(directory, c, keyring_files) ###############################################################################