X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=blobdiff_plain;f=dak%2Fprocess_policy.py;h=ab37aa2216b3282fcfeb4657d36944b89b32b3fc;hp=695751c457de1e2bf46684fb6381fb866ca22b48;hb=026bdb9580e6d18bf504d2f44d46890df5d89f1a;hpb=09a1a20566dcf84ca229b4339bd8f8080eb59afd diff --git a/dak/process_policy.py b/dak/process_policy.py index 695751c4..ab37aa22 100755 --- a/dak/process_policy.py +++ b/dak/process_policy.py @@ -37,15 +37,19 @@ import re import sys import traceback import apt_pkg +from sqlalchemy.orm.exc import NoResultFound from daklib.dbconn import * from daklib import daklog from daklib import utils from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError from daklib.config import Config -from daklib.archive import ArchiveTransaction +from daklib.archive import ArchiveTransaction, source_component_from_package_list from daklib.urgencylog import UrgencyLog -from daklib.textutils import fix_maintainer +from daklib.packagelist import PackageList + +import daklib.announce +import daklib.utils # Globals Options = None @@ -55,6 +59,7 @@ Logger = None def do_comments(dir, srcqueue, opref, npref, line, fn, transaction): session = transaction.session + actions = [] for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]: lines = open(os.path.join(dir, comm)).readlines() if len(lines) == 0 or lines[0] != line + "\n": continue @@ -67,16 +72,26 @@ def do_comments(dir, srcqueue, opref, npref, line, fn, transaction): else: changes_prefix = changes_prefix + '.changes' + # We need to escape "_" as we use it with the LIKE operator (via the + # SQLA startwith) later. + changes_prefix = changes_prefix.replace("_", r"\_") + uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=srcqueue) \ .join(PolicyQueueUpload.changes).filter(DBChange.changesname.startswith(changes_prefix)) \ .order_by(PolicyQueueUpload.source_id) - for u in uploads: - print "Processing changes file: %s" % u.changes.changesname - fn(u, srcqueue, "".join(lines[1:]), transaction) + reason = "".join(lines[1:]) + actions.extend((u, reason) for u in uploads) if opref != npref: newcomm = npref + comm[len(opref):] - transaction.fs.move(os.path.join(dir, comm), os.path.join(dir, newcomm)) + newcomm = utils.find_next_free(os.path.join(dir, newcomm)) + transaction.fs.move(os.path.join(dir, comm), newcomm) + + actions.sort() + + for u, reason in actions: + print("Processing changes file: {0}".format(u.changes.changesname)) + fn(u, srcqueue, reason, transaction) ################################################################################ @@ -95,6 +110,8 @@ def try_or_reject(function): real_comment_reject(upload, srcqueue, comments, transaction, notify=False) if not Options['No-Action']: transaction.commit() + else: + transaction.rollback() return wrapper ################################################################################ @@ -119,25 +136,84 @@ def comment_accept(upload, srcqueue, comments, transaction): overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one() def binary_component_func(db_binary): - override = session.query(Override).filter_by(suite=overridesuite, package=db_binary.package) \ - .join(OverrideType).filter(OverrideType.overridetype == db_binary.binarytype) \ - .join(Component).one() - return override.component + section = db_binary.proxy['Section'] + component_name = 'main' + if section.find('/') != -1: + component_name = section.split('/', 1)[0] + return get_mapped_component(component_name, session=session) + + def is_debug_binary(db_binary): + return daklib.utils.is_in_debug_section(db_binary.proxy) + + def has_debug_binaries(upload): + return any((is_debug_binary(x) for x in upload.binaries)) def source_component_func(db_source): - override = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ + package_list = PackageList(db_source.proxy) + component = source_component_from_package_list(package_list, upload.target_suite) + if component is not None: + return get_mapped_component(component.component_name, session=session) + + # Fallback for packages without Package-List field + query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \ - .join(Component).one() - return override.component + .join(Component) + return query.one().component all_target_suites = [upload.target_suite] all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues]) for suite in all_target_suites: + debug_suite = suite.debug_suite + if upload.source is not None: - transaction.copy_source(upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted) + # If we have Source in this upload, let's include it into + # upload suite. + transaction.copy_source( + upload.source, + suite, + source_component_func(upload.source), + allow_tainted=allow_tainted, + ) + + if debug_suite is not None and has_debug_binaries(upload): + # If we're handing a debug package, we also need to include the + # source in the debug suite as well. + transaction.copy_source( + upload.source, + debug_suite, + source_component_func(upload.source), + allow_tainted=allow_tainted, + ) + for db_binary in upload.binaries: - transaction.copy_binary(db_binary, suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive]) + # Now, let's work out where to copy this guy to -- if it's + # a debug binary, and the suite has a debug suite, let's go + # ahead and target the debug suite rather then the stock + # suite. + copy_to_suite = suite + if debug_suite is not None and is_debug_binary(db_binary): + copy_to_suite = debug_suite + + # build queues and debug suites may miss the source package + # if this is a binary-only upload. + if copy_to_suite != upload.target_suite: + transaction.copy_source( + db_binary.source, + copy_to_suite, + source_component_func(db_binary.source), + allow_tainted=allow_tainted, + ) + + transaction.copy_binary( + db_binary, + copy_to_suite, + binary_component_func(db_binary), + allow_tainted=allow_tainted, + extra_archives=[upload.target_suite.archive], + ) + + suite.update_last_changed() # Copy .changes if needed if upload.target_suite.copychanges: @@ -145,6 +221,29 @@ def comment_accept(upload, srcqueue, comments, transaction): dst = os.path.join(upload.target_suite.path, upload.changes.changesname) fs.copy(src, dst, mode=upload.target_suite.archive.mode) + # Copy upload to Process-Policy::CopyDir + # Used on security.d.o to sync accepted packages to ftp-master, but this + # should eventually be replaced by something else. + copydir = cnf.get('Process-Policy::CopyDir') or None + if copydir is not None: + mode = upload.target_suite.archive.mode + if upload.source is not None: + for f in [ df.poolfile for df in upload.source.srcfiles ]: + dst = os.path.join(copydir, f.basename) + if not os.path.exists(dst): + fs.copy(f.fullpath, dst, mode=mode) + + for db_binary in upload.binaries: + f = db_binary.poolfile + dst = os.path.join(copydir, f.basename) + if not os.path.exists(dst): + fs.copy(f.fullpath, dst, mode=mode) + + src = os.path.join(upload.policy_queue.path, upload.changes.changesname) + dst = os.path.join(copydir, upload.changes.changesname) + if not os.path.exists(dst): + fs.copy(src, dst, mode=mode) + if upload.source is not None and not Options['No-Action']: urgency = upload.changes.urgency if urgency not in cnf.value_list('Urgency::Valid'): @@ -155,24 +254,8 @@ def comment_accept(upload, srcqueue, comments, transaction): if not Options['No-Action']: Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname]) - # Send announcement - subst = subst_for_upload(upload) - announce = ", ".join(upload.target_suite.announce or []) - tracking = cnf.get('Dinstall::TrackingServer') - if tracking and upload.source is not None: - announce = '{0}\nBcc: {1}@{2}'.format(announce, upload.changes.source, tracking) - subst['__ANNOUNCE_LIST_ADDRESS__'] = announce - message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce')) - utils.send_mail(message) - - # TODO: code duplication. Similar code is in process-upload. - if cnf.find_b('Dinstall::CloseBugs') and upload.changes.closes is not None and upload.source is not None: - for bugnum in upload.changes.closes: - subst['__BUG_NUMBER__'] = bugnum - message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close')) - utils.send_mail(message) - - del subst['__BUG_NUMBER__'] + pu = get_processed_upload(upload) + daklib.announce.announce_accept(pu) # TODO: code duplication. Similar code is in process-upload. # Move .changes to done @@ -189,9 +272,9 @@ def comment_accept(upload, srcqueue, comments, transaction): @try_or_reject def comment_reject(*args): - real_comment_reject(*args) + real_comment_reject(*args, manual=True) -def real_comment_reject(upload, srcqueue, comments, transaction, notify=True): +def real_comment_reject(upload, srcqueue, comments, transaction, notify=True, manual=False): cnf = Config() fs = transaction.fs @@ -230,25 +313,26 @@ def real_comment_reject(upload, srcqueue, comments, transaction, notify=True): ### Send mail notification if notify: - subst = subst_for_upload(upload) - subst['__MANUAL_REJECT_MESSAGE__'] = '' - subst['__REJECT_MESSAGE__'] = comments + rejected_by = None + reason = comments # Try to use From: from comment file if there is one. # This is not very elegant... match = re.match(r"\AFrom: ([^\n]+)\n\n", comments) if match: - subst['__REJECTOR_ADDRESS__'] = match.group(1) - subst['__REJECT_MESSAGE__'] = '\n'.join(comments.splitlines()[2:]) + rejected_by = match.group(1) + reason = '\n'.join(comments.splitlines()[2:]) - message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected')) - utils.send_mail(message) + pu = get_processed_upload(upload) + daklib.announce.announce_reject(pu, reason, rejected_by) print " REJECT" if not Options["No-Action"]: Logger.log(["Policy Queue REJECT", srcqueue.queue_name, upload.changes.changesname]) + changes = upload.changes remove_upload(upload, transaction) + session.delete(changes) ################################################################################ @@ -268,48 +352,32 @@ def remove_upload(upload, transaction): fs.unlink(os.path.join(queuedir, upload.changes.changesname)) session.delete(upload) - session.delete(changes) session.flush() ################################################################################ -def subst_for_upload(upload): - # TODO: similar code in process-upload - cnf = Config() +def get_processed_upload(upload): + pu = daklib.announce.ProcessedUpload() - maintainer_field = upload.changes.changedby or upload.changes.maintainer - addresses = utils.mail_addresses_for_upload(upload.changes.maintainer, maintainer_field, upload.changes.fingerprint) + pu.maintainer = upload.changes.maintainer + pu.changed_by = upload.changes.changedby + pu.fingerprint = upload.changes.fingerprint + + pu.suites = [ upload.target_suite ] + pu.from_policy_suites = [ upload.target_suite ] changes_path = os.path.join(upload.policy_queue.path, upload.changes.changesname) - changes_contents = open(changes_path, 'r').read() - - bcc = 'X-DAK: dak process-policy' - if 'Dinstall::Bcc' in cnf: - bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc']) - - subst = { - '__DISTRO__': cnf['Dinstall::MyDistribution'], - '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'], - - '__CHANGES_FILENAME__': upload.changes.changesname, - '__SOURCE__': upload.changes.source, - '__VERSION__': upload.changes.version, - '__ARCHITECTURE__': upload.changes.architecture, - '__MAINTAINER__': maintainer_field, - '__MAINTAINER_FROM__': fix_maintainer(maintainer_field)[1], - '__MAINTAINER_TO__': ", ".join(addresses), - '__CC__': 'X-DAK-Rejection: manual or automatic', - '__REJECTOR_ADDRESS__': cnf['Dinstall::MyEmailAddress'], - '__BCC__': bcc, - '__BUG_SERVER__': cnf.get('Dinstall::BugServer'), - '__FILE_CONTENTS__': changes_contents, - } - - override_maintainer = cnf.get('Dinstall::OverrideMaintainer') - if override_maintainer: - subst['__MAINTAINER_TO__'] = override_maintainer - - return subst + pu.changes = open(changes_path, 'r').read() + pu.changes_filename = upload.changes.changesname + pu.sourceful = upload.source is not None + pu.source = upload.changes.source + pu.version = upload.changes.version + pu.architecture = upload.changes.architecture + pu.bugs = upload.changes.closes + + pu.program = "process-policy" + + return pu ################################################################################ @@ -412,9 +480,9 @@ def main(): # The comments stuff relies on being in the right directory os.chdir(pq.path) + do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction) do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, transaction) do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, transaction) - do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction) remove_unreferenced_binaries(pq, transaction) remove_unreferenced_sources(pq, transaction)