X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_upload.py;h=bf74baa65be064e356abaa8459b39a6688c85ad9;hb=db2d829cce8f0dd8aadb6af5e6258ab4fced871d;hp=cf1594a3098ba0d164e10b10b7cad49290fbeb23;hpb=3c3cf6b0861825abdc771e9b4c676ac52db56744;p=dak.git diff --git a/dak/process_upload.py b/dak/process_upload.py index cf1594a3..bf74baa6 100755 --- a/dak/process_upload.py +++ b/dak/process_upload.py @@ -125,25 +125,58 @@ Checks Debian packages from Incoming ## pu: create files for BTS ## pu: create entry in queue_build ## pu: check overrides -import errno + +# Integrity checks +## GPG +## Parsing changes (check for duplicates) +## Parse dsc +## file list checks + +# New check layout (TODO: Implement) +## Permission checks +### suite mappings +### ACLs +### version checks (suite) +### override checks + +## Source checks +### copy orig +### unpack +### BTS changelog +### src contents +### lintian +### urgency log + +## Binary checks +### timestamps +### control checks +### src relation check +### contents + +## Database insertion (? copy from stuff) +### BYHAND / NEW / Policy queues +### Pool + +## Queue builds + +from errno import EACCES, EAGAIN import fcntl import os import sys -#from datetime import datetime import traceback import apt_pkg +from sqlalchemy.orm.exc import NoResultFound from daklib import daklog from daklib.queue import * from daklib.queue_install import * from daklib import utils from daklib.dbconn import * -#from daklib.dak_exceptions import * -#from daklib.regexes import re_default_answer, re_issource, re_fdnic from daklib.urgencylog import UrgencyLog from daklib.summarystats import SummaryStats from daklib.holding import Holding from daklib.config import Config +from daklib.regexes import re_match_expired ############################################################################### @@ -164,13 +197,24 @@ def usage (exit_code=0): ############################################################################### -def action(u): +def byebye(): + if not Options["No-Action"]: + # Clean out the queue files + session = DBConn().session() + session.execute("DELETE FROM changes_pending_files WHERE id NOT IN (SELECT file_id FROM changes_pending_files_map )") + session.commit() + + + +def action(u, session): + global Logger + cnf = Config() holding = Holding() # changes["distribution"] may not exist in corner cases # (e.g. unreadable changes files) - if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], DictType): + if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], dict): u.pkg.changes["distribution"] = {} (summary, short_summary) = u.build_summaries() @@ -183,6 +227,11 @@ def action(u): pi = u.package_info() + try: + chg = session.query(DBChange).filter_by(changesname=os.path.basename(u.pkg.changes_file)).one() + except NoResultFound, e: + chg = None + if len(u.rejects) > 0: if u.upload_too_new(): print "SKIP (too new)\n" + pi, @@ -193,6 +242,8 @@ def action(u): if Options["Automatic"]: answer = 'R' else: + # Are we headed for NEW / BYHAND / AUTOBYHAND? + # Note that policy queues are no longer handled here qu = determine_target(u) if qu: print "%s for %s\n%s%s" % ( qu.upper(), ", ".join(u.pkg.changes["distribution"].keys()), pi, summary) @@ -205,10 +256,32 @@ def action(u): if Options["Automatic"]: answer = queuekey else: - print "ACCEPT\n" + pi + summary, - prompt = "[A]ccept, Skip, Quit ?" - if Options["Automatic"]: - answer = 'A' + # Does suite have a policy_queue configured + divert = False + for s in u.pkg.changes["distribution"].keys(): + suite = get_suite(s, session) + if suite.policy_queue: + if not chg or chg.approved_for_id != suite.policy_queue.policy_queue_id: + # This routine will check whether the upload is a binary + # upload when the source is already in the target suite. If + # so, we skip the policy queue, otherwise we go there. + divert = package_to_suite(u, suite.suite_name, session=session) + if divert: + print "%s for %s\n%s%s" % ( suite.policy_queue.queue_name.upper(), + ", ".join(u.pkg.changes["distribution"].keys()), + pi, summary) + queuekey = "P" + prompt = "[P]olicy, Skip, Quit ?" + policyqueue = suite.policy_queue + if Options["Automatic"]: + answer = 'P' + break + + if not divert: + print "ACCEPT\n" + pi + summary, + prompt = "[A]ccept, Skip, Quit ?" + if Options["Automatic"]: + answer = 'A' while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt) @@ -217,21 +290,32 @@ def action(u): answer = m.group(1) answer = answer[:1].upper() - session = DBConn().session() - if answer == 'R': os.chdir(u.pkg.directory) u.do_reject(0, pi) elif answer == 'A': - u.pkg.add_known_changes(holding.holding_dir, session) + if not chg: + chg = u.pkg.add_known_changes(holding.holding_dir, session=session, logger=Logger) + session.commit() u.accept(summary, short_summary, session) u.check_override() + chg.clean_from_queue() + session.commit() + u.remove() + elif answer == 'P': + if not chg: + chg = u.pkg.add_known_changes(holding.holding_dir, session=session, logger=Logger) + package_to_queue(u, summary, short_summary, policyqueue, chg, session) + session.commit() u.remove() elif answer == queuekey: - u.pkg.add_known_changes(holding.holding_dir, session) - QueueInfo[qu]["process"](u, summary, short_summary, session) + if not chg: + chg = u.pkg.add_known_changes(holding.holding_dir, session=session, logger=Logger) + QueueInfo[qu]["process"](u, summary, short_summary, chg, session) + session.commit() u.remove() elif answer == 'Q': + byebye() sys.exit(0) session.commit() @@ -243,7 +327,7 @@ def cleanup(): if not Options["No-Action"]: h.clean() -def process_it(changes_file): +def process_it(changes_file, session): global Logger Logger.log(["Processing changes file", changes_file]) @@ -252,6 +336,9 @@ def process_it(changes_file): holding = Holding() + # TODO: Actually implement using pending* tables so that we don't lose track + # of what is where + u = Upload() u.pkg.changes_file = changes_file u.pkg.directory = os.getcwd() @@ -277,7 +364,7 @@ def process_it(changes_file): # If this is the Real Thing(tm), copy things into a private # holding directory first to avoid replacable file races. if not Options["No-Action"]: - os.chdir(cnf["Dir::Queue::Holding"]) + holding.chdir_to_holding() # Absolutize the filename to avoid the requirement of being in the # same directory as the .changes file. @@ -294,6 +381,12 @@ def process_it(changes_file): if u.pkg.changes["fingerprint"]: valid_changes_p = u.load_changes(changespath) else: + for reason in rejects: + if re_match_expired.match(reason): + # Hrm, key expired. Lets see if we can still parse the .changes before + # we reject. Then we would be able to mail the maintainer, instead of + # just silently dropping the upload. + u.load_changes(changespath) valid_changes_p = False u.rejects.extend(rejects) @@ -303,13 +396,14 @@ def process_it(changes_file): valid_dsc_p = u.check_dsc(not Options["No-Action"]) if valid_dsc_p and not Options["No-Action"]: u.check_source() - u.check_lintian() u.check_hashes() + if valid_dsc_p and not Options["No-Action"] and not len(u.rejects): + u.check_lintian() u.check_urgency() u.check_timestamps() u.check_signed_by_key() - action(u) + action(u, session) except (SystemExit, KeyboardInterrupt): cleanup() @@ -330,7 +424,6 @@ def main(): cnf = Config() summarystats = SummaryStats() - log_urgency = False DBConn() @@ -362,7 +455,7 @@ def main(): # Obtain lock if not in no-action mode and initialize the log if not Options["No-Action"]: - lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT) + lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT) try: fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError, e: @@ -370,12 +463,12 @@ def main(): utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.") else: raise - if cnf.get("Dir::UrgencyLog"): - # Initialise UrgencyLog() - log_urgency = True - UrgencyLog() - Logger = daklog.Logger(cnf, "process-upload", Options["No-Action"]) + # Initialise UrgencyLog() - it will deal with the case where we don't + # want to log urgencies + urgencylog = UrgencyLog() + + Logger = daklog.Logger("process-upload", Options["No-Action"]) # If we have a directory flag, use it to find our files if cnf["Dinstall::Options::Directory"] != "": @@ -398,20 +491,29 @@ def main(): for changes_file in changes_files: print "\n" + changes_file session = DBConn().session() - process_it(changes_file) + process_it(changes_file, session) session.close() if summarystats.accept_count: sets = "set" if summarystats.accept_count > 1: sets = "sets" - sys.stderr.write("Installed %d package %s, %s.\n" % (summarystats.accept_count, sets, - utils.size_type(int(summarystats.accept_bytes)))) + print "Installed %d package %s, %s." % (summarystats.accept_count, sets, + utils.size_type(int(summarystats.accept_bytes))) Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes]) + if summarystats.reject_count: + sets = "set" + if summarystats.reject_count > 1: + sets = "sets" + print "Rejected %d package %s." % (summarystats.reject_count, sets) + Logger.log(["rejected", summarystats.reject_count]) + + byebye() + if not Options["No-Action"]: - if log_urgency: - UrgencyLog().close() + urgencylog.close() + Logger.close() ###############################################################################