X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_policy.py;h=0f2f22aab8a67d93e0d18b8610c1c5e1275f8648;hb=5a61a05250b1bf5d54661103e8999eb7c7fc9207;hp=61c4ef395702112020d1b67f6f1520ce10954ca4;hpb=1e945c24f59d3e92ef18f456b60aa10210a3113c;p=dak.git diff --git a/dak/process_policy.py b/dak/process_policy.py index 61c4ef39..0f2f22aa 100755 --- a/dak/process_policy.py +++ b/dak/process_policy.py @@ -37,14 +37,16 @@ import re import sys import traceback import apt_pkg +from sqlalchemy.orm.exc import NoResultFound from daklib.dbconn import * from daklib import daklog from daklib import utils from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError from daklib.config import Config -from daklib.archive import ArchiveTransaction +from daklib.archive import ArchiveTransaction, source_component_from_package_list from daklib.urgencylog import UrgencyLog +from daklib.packagelist import PackageList import daklib.announce @@ -56,6 +58,7 @@ Logger = None def do_comments(dir, srcqueue, opref, npref, line, fn, transaction): session = transaction.session + actions = [] for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]: lines = open(os.path.join(dir, comm)).readlines() if len(lines) == 0 or lines[0] != line + "\n": continue @@ -68,16 +71,26 @@ def do_comments(dir, srcqueue, opref, npref, line, fn, transaction): else: changes_prefix = changes_prefix + '.changes' + # We need to escape "_" as we use it with the LIKE operator (via the + # SQLA startwith) later. + changes_prefix = changes_prefix.replace("_", r"\_") + uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=srcqueue) \ .join(PolicyQueueUpload.changes).filter(DBChange.changesname.startswith(changes_prefix)) \ .order_by(PolicyQueueUpload.source_id) - for u in uploads: - print "Processing changes file: %s" % u.changes.changesname - fn(u, srcqueue, "".join(lines[1:]), transaction) + reason = "".join(lines[1:]) + actions.extend((u, reason) for u in uploads) if opref != npref: newcomm = npref + comm[len(opref):] - transaction.fs.move(os.path.join(dir, comm), os.path.join(dir, newcomm)) + newcomm = utils.find_next_free(os.path.join(dir, newcomm)) + transaction.fs.move(os.path.join(dir, comm), newcomm) + + actions.sort() + + for u, reason in actions: + print("Processing changes file: {0}".format(u.changes.changesname)) + fn(u, srcqueue, reason, transaction) ################################################################################ @@ -96,6 +109,8 @@ def try_or_reject(function): real_comment_reject(upload, srcqueue, comments, transaction, notify=False) if not Options['No-Action']: transaction.commit() + else: + transaction.rollback() return wrapper ################################################################################ @@ -120,16 +135,23 @@ def comment_accept(upload, srcqueue, comments, transaction): overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one() def binary_component_func(db_binary): - override = session.query(Override).filter_by(suite=overridesuite, package=db_binary.package) \ - .join(OverrideType).filter(OverrideType.overridetype == db_binary.binarytype) \ - .join(Component).one() - return override.component + section = db_binary.proxy['Section'] + component_name = 'main' + if section.find('/') != -1: + component_name = section.split('/', 1)[0] + return get_mapped_component(component_name, session=session) def source_component_func(db_source): - override = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ + package_list = PackageList(db_source.proxy) + component = source_component_from_package_list(package_list, upload.target_suite) + if component is not None: + return get_mapped_component(component.component_name, session=session) + + # Fallback for packages without Package-List field + query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \ - .join(Component).one() - return override.component + .join(Component) + return query.one().component all_target_suites = [upload.target_suite] all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues]) @@ -138,6 +160,9 @@ def comment_accept(upload, srcqueue, comments, transaction): if upload.source is not None: transaction.copy_source(upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted) for db_binary in upload.binaries: + # build queues may miss the source package if this is a binary-only upload + if suite != upload.target_suite: + transaction.copy_source(db_binary.source, suite, source_component_func(db_binary.source), allow_tainted=allow_tainted) transaction.copy_binary(db_binary, suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive]) # Copy .changes if needed @@ -146,6 +171,29 @@ def comment_accept(upload, srcqueue, comments, transaction): dst = os.path.join(upload.target_suite.path, upload.changes.changesname) fs.copy(src, dst, mode=upload.target_suite.archive.mode) + # Copy upload to Process-Policy::CopyDir + # Used on security.d.o to sync accepted packages to ftp-master, but this + # should eventually be replaced by something else. + copydir = cnf.get('Process-Policy::CopyDir') or None + if copydir is not None: + mode = upload.target_suite.archive.mode + if upload.source is not None: + for f in [ df.poolfile for df in upload.source.srcfiles ]: + dst = os.path.join(copydir, f.basename) + if not os.path.exists(dst): + fs.copy(f.fullpath, dst, mode=mode) + + for db_binary in upload.binaries: + f = db_binary.poolfile + dst = os.path.join(copydir, f.basename) + if not os.path.exists(dst): + fs.copy(f.fullpath, dst, mode=mode) + + src = os.path.join(upload.policy_queue.path, upload.changes.changesname) + dst = os.path.join(copydir, upload.changes.changesname) + if not os.path.exists(dst): + fs.copy(src, dst, mode=mode) + if upload.source is not None and not Options['No-Action']: urgency = upload.changes.urgency if urgency not in cnf.value_list('Urgency::Valid'): @@ -157,7 +205,7 @@ def comment_accept(upload, srcqueue, comments, transaction): Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname]) pu = get_processed_upload(upload) - daklib.announce.announce_accept(upload) + daklib.announce.announce_accept(pu) # TODO: code duplication. Similar code is in process-upload. # Move .changes to done @@ -232,7 +280,9 @@ def real_comment_reject(upload, srcqueue, comments, transaction, notify=True, ma if not Options["No-Action"]: Logger.log(["Policy Queue REJECT", srcqueue.queue_name, upload.changes.changesname]) + changes = upload.changes remove_upload(upload, transaction) + session.delete(changes) ################################################################################ @@ -252,7 +302,6 @@ def remove_upload(upload, transaction): fs.unlink(os.path.join(queuedir, upload.changes.changesname)) session.delete(upload) - session.delete(changes) session.flush() ################################################################################ @@ -264,7 +313,7 @@ def get_processed_upload(upload): pu.changed_by = upload.changes.changedby pu.fingerprint = upload.changes.fingerprint - pu.suites = [] + pu.suites = [ upload.target_suite ] pu.from_policy_suites = [ upload.target_suite ] changes_path = os.path.join(upload.policy_queue.path, upload.changes.changesname) @@ -381,9 +430,9 @@ def main(): # The comments stuff relies on being in the right directory os.chdir(pq.path) + do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction) do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, transaction) do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, transaction) - do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction) remove_unreferenced_binaries(pq, transaction) remove_unreferenced_sources(pq, transaction)