X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_new.py;h=6bdd301f23512345c0b485f3c2241302ed3a0a1c;hb=a06a67a40443a97dccce9a3abdba68fa5150f1cd;hp=1efacb3f5376f016d0859f38a7eee8555caabed9;hpb=7a09f0723e869eb8a7d0a36365e5562f2437f2d1;p=dak.git diff --git a/dak/process_new.py b/dak/process_new.py index 1efacb3f..6bdd301f 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -60,7 +60,7 @@ from daklib.dbconn import * from daklib.queue import * from daklib import daklog from daklib import utils -from daklib.regexes import re_no_epoch, re_default_answer, re_isanum +from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError from daklib.summarystats import SummaryStats from daklib.config import Config @@ -77,43 +77,13 @@ Sections = None ################################################################################ def recheck(upload, session): - files = upload.pkg.files - - cnf = Config() - for f in files.keys(): - # The .orig.tar.gz can disappear out from under us is it's a - # duplicate of one in the archive. - if not files.has_key(f): - continue - # Check that the source still exists - if files[f]["type"] == "deb": - source_version = files[f]["source version"] - source_package = files[f]["source package"] - if not upload.pkg.changes["architecture"].has_key("source") \ - and not upload.source_exists(source_package, source_version, upload.pkg.changes["distribution"].keys()): - source_epochless_version = re_no_epoch.sub('', source_version) - dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) - found = 0 - for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]: - if cnf.has_key("Dir::Queue::%s" % (q)): - if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename): - found = 1 - if not found: - upload.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f)) - - # Version and file overwrite checks - if files[f]["type"] == "deb": - upload.check_binary_against_db(f, session) - elif files[f]["type"] == "dsc": - upload.check_source_against_db(f, session) - upload.check_dsc_against_db(f, session) - +# STU: I'm not sure, but I don't thin kthis is necessary any longer: upload.recheck(session) if len(upload.rejects) > 0: answer = "XXX" if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]: answer = 'S' - print "REJECT\n" + upload.rejects.join("\n"), + print "REJECT\n%s" % '\n'.join(upload.rejects) prompt = "[R]eject, Skip, Quit ?" while prompt.find(answer) == -1: @@ -124,8 +94,7 @@ def recheck(upload, session): answer = answer[:1].upper() if answer == 'R': - upload.do_reject(manual=0, reject_message=upload.rejects.join("\n")) - os.unlink(upload.pkg.changes_file[:-8]+".dak") + upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects)) return 0 elif answer == 'S': return 0 @@ -189,7 +158,8 @@ def sort_changes(changes_files, session): for filename in changes_files: u = Upload() try: - u.pkg.load_dot_dak(filename) + u.pkg.changes_file = filename + u.load_changes(filename) u.update_subst() cache[filename] = copy.copy(u.pkg.changes) cache[filename]["filename"] = filename @@ -629,9 +599,13 @@ def prod_maintainer (note, upload): def do_new(upload, session): print "NEW\n" files = upload.pkg.files + upload.check_files(not Options["No-Action"]) changes = upload.pkg.changes cnf = Config() + # Check for a valid distribution + upload.check_distributions() + # Make a copy of distribution we can happily trample on changes["suite"] = copy.copy(changes["distribution"]) @@ -649,10 +623,6 @@ def do_new(upload, session): oinv, suite, ninv, override) del changes["suite"][suite] changes["suite"][override] = 1 - # Validate suites - for suite in changes["suite"].keys(): - if get_suite(suite, session) is None: - utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite)) # The main NEW processing loop done = 0 @@ -695,7 +665,8 @@ def do_new(upload, session): try: check_daily_lock() done = add_overrides (new, upload, session) - Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (upload.pkg.changes_file)]) + do_accept(upload, session) + Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)]) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" print "Dinstall in the locked area, cant process packages, come back later" @@ -708,8 +679,8 @@ def do_new(upload, session): reject_message=Options["Manual-Reject"], note=get_new_comments(changes.get("source", ""), session=session)) if not aborted: - Logger.log([utils.getusername(), "NEW REJECT: %s" % (upload.pkg.changes_file)]) - os.unlink(upload.pkg.changes_file[:-8]+".dak") + upload.pkg.remove_known_changes() + Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)]) done = 1 elif answer == 'N': edit_note(get_new_comments(changes.get("source", ""), session=session), @@ -717,7 +688,7 @@ def do_new(upload, session): elif answer == 'P' and not Options["Trainee"]: prod_maintainer(get_new_comments(changes.get("source", ""), session=session), upload) - Logger.log([utils.getusername(), "NEW PROD: %s" % (upload.pkg.changes_file)]) + Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)]) elif answer == 'R' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() if confirm == "y": @@ -745,7 +716,6 @@ def usage (exit_code=0): print """Usage: dak process-new [OPTION]... [CHANGES]... -a, --automatic automatic run -h, --help show this help and exit. - -C, --comments-dir=DIR use DIR as comments-dir, for [o-]p-u-new -m, --manual-reject=MSG manual reject with `msg' -n, --no-action don't do anything -t, --trainee FTP Trainee mode @@ -792,14 +762,13 @@ def do_byhand(upload, session): done = 1 for f in byhand: del files[f] - Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) + Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'M': - Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (upload.pkg.changes_file)]) + Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)]) upload.do_reject(manual=1, reject_message=Options["Manual-Reject"]) - os.unlink(upload.pkg.changes_file[:-8]+".dak") done = 1 elif answer == 'S': done = 1 @@ -847,154 +816,124 @@ def lock_package(package): finally: os.unlink(path) -# def move_to_dir (upload, dest, perms=0660, changesperms=0664): -# utils.move (upload.pkg.changes_file, dest, perms=changesperms) -# file_keys = upload.pkg.files.keys() -# for f in file_keys: -# utils.move (f, dest, perms=perms) - -# def is_source_in_queue_dir(qdir): -# entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"]) -# and x.endswith(".changes") ] -# for entry in entries: -# # read the .dak -# u = queue.Upload(Cnf) -# u.pkg.changes_file = os.path.join(qdir, entry) -# u.update_vars() -# if not u.pkg.changes["architecture"].has_key("source"): -# # another binary upload, ignore -# continue -# if Upload.pkg.changes["version"] != u.pkg.changes["version"]: -# # another version, ignore -# continue -# # found it! -# return True -# return False - -# def move_to_holding(suite, queue_dir): -# print "Moving to %s holding area." % (suite.upper(),) -# if Options["No-Action"]: -# return -# Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file]) -# Upload.dump_vars(queue_dir) -# move_to_dir(queue_dir, perms=0664) -# os.unlink(Upload.pkg.changes_file[:-8]+".dak") - -def _accept(upload): +class clean_holding(object): + def __init__(self,pkg): + self.pkg = pkg + + def __enter__(self): + pass + + def __exit__(self, type, value, traceback): + h = Holding() + + for f in self.pkg.files.keys(): + if os.path.exists(os.path.join(h.holding_dir, f)): + os.unlink(os.path.join(h.holding_dir, f)) + + + +def changes_to_newstage(upload, session): + """move a changes file to newstage""" + new = get_policy_queue('new', session ); + newstage = get_policy_queue('newstage', session ); + + chg = session.query(DBChange).filter_by(changesname=os.path.basename(upload.pkg.changes_file)).one() + chg.approved_for = newstage.policy_queue_id + + for f in chg.files: + # update the changes_pending_files row + f.queue = newstage + utils.move(os.path.join(new.path, f.filename), newstage.path, perms=int(newstage.perms, 8)) + + utils.move(os.path.join(new.path, upload.pkg.changes_file), newstage.path, perms=int(newstage.perms, 8)) + chg.in_queue = newstage + session.commit() + +def _accept(upload, session): if Options["No-Action"]: return (summary, short_summary) = upload.build_summaries() - upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"]) - os.unlink(upload.pkg.changes_file[:-8]+".dak") - -# def do_accept_stableupdate(upload,suite, q): -# cnf = Config() -# queue_dir = cnf["Dir::Queue::%s" % (q,)] -# if not upload.pkg.changes["architecture"].has_key("source"): -# # It is not a sourceful upload. So its source may be either in p-u -# # holding, in new, in accepted or already installed. -# if is_source_in_queue_dir(queue_dir): -# # It's in p-u holding, so move it there. -# print "Binary-only upload, source in %s." % (q,) -# move_to_holding(suite, queue_dir) -# elif Upload.source_exists(Upload.pkg.changes["source"], -# Upload.pkg.changes["version"]): -# # dak tells us that there is source available. At time of -# # writing this means that it is installed, so put it into -# # accepted. -# print "Binary-only upload, source installed." -# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) -# _accept() -# elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]): -# # The source is in accepted, the binary cleared NEW: accept it. -# print "Binary-only upload, source in accepted." -# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) -# _accept() -# elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]): -# # It's in NEW. We expect the source to land in p-u holding -# # pretty soon. -# print "Binary-only upload, source in new." -# move_to_holding(suite, queue_dir) -# elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]): -# # It's in newstage. Accept into the holding area -# print "Binary-only upload, source in newstage." -# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) -# _accept() -# else: -# # No case applicable. Bail out. Return will cause the upload -# # to be skipped. -# print "ERROR" -# print "Stable update failed. Source not found." -# return -# else: -# # We are handling a sourceful upload. Move to accepted if currently -# # in p-u holding and to p-u holding otherwise. -# if is_source_in_queue_dir(queue_dir): -# print "Sourceful upload in %s, accepting." % (q,) -# _accept() -# else: -# move_to_holding(suite, queue_dir) - -def do_accept(upload): + # upload.accept(summary, short_summary, targetqueue) + + changes_to_newstage(upload, session) + +def do_accept(upload, session): print "ACCEPT" cnf = Config() if not Options["No-Action"]: (summary, short_summary) = upload.build_summaries() -# if cnf.FindB("Dinstall::SecurityQueueHandling"): -# upload.dump_vars(cnf["Dir::Queue::Embargoed"]) -# move_to_dir(cnf["Dir::Queue::Embargoed"]) -# upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"]) -# # Check for override disparities -# upload.Subst["__SUMMARY__"] = summary -# else: - # Stable updates need to be copied to proposed-updates holding - # area instead of accepted. Sourceful uploads need to go - # to it directly, binaries only if the source has not yet been - # accepted into p-u. - for suite, q in [("proposed-updates", "ProposedUpdates"), - ("oldstable-proposed-updates", "OldProposedUpdates")]: - if not upload.pkg.changes["distribution"].has_key(suite): - continue - utils.fubar("stable accept not supported yet") -# return do_accept_stableupdate(suite, q) - # Just a normal upload, accept it... - _accept(upload) - -def check_status(files): - new = byhand = 0 - for f in files.keys(): - if files[f]["type"] == "byhand": - byhand = 1 - elif files[f].has_key("new"): - new = 1 - return (new, byhand) + + if cnf.FindB("Dinstall::SecurityQueueHandling"): + upload.dump_vars(cnf["Dir::Queue::Embargoed"]) + upload.move_to_queue(get_policy_queue('embargoed')) + upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"]) + # Check for override disparities + upload.Subst["__SUMMARY__"] = summary + else: + # Just a normal upload, accept it... + _accept(upload, session) def do_pkg(changes_file, session): + new_queue = get_policy_queue('new', session ); u = Upload() - u.pkg.load_dot_dak(changes_file) + u.pkg.changes_file = changes_file + (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file) + u.load_changes(changes_file) + u.pkg.directory = new_queue.path u.update_subst() + u.logger = Logger + origchanges = os.path.abspath(u.pkg.changes_file) + + cnf = Config() + bcc = "X-DAK: dak process-new" + if cnf.has_key("Dinstall::Bcc"): + u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) + else: + u.Subst["__BCC__"] = bcc + files = u.pkg.files + for deb_filename, f in files.items(): + if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"): + u.binary_file_checks(deb_filename, session) + u.check_binary_against_db(deb_filename, session) + else: + u.source_file_checks(deb_filename, session) + u.check_source_against_db(deb_filename, session) + + u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"]) try: with lock_package(u.pkg.changes["source"]): - if not recheck(u, session): - return + with clean_holding(u.pkg): + if not recheck(u, session): + return - (new, byhand) = check_status(files) - if new or byhand: + # FIXME: This does need byhand checks added! + new = determine_new(u.pkg.changes, files) if new: do_new(u, session) - if byhand: - do_byhand(u, session) - (new, byhand) = check_status(files) - - if not new and not byhand: - try: - check_daily_lock() - do_accept(u) - except CantGetLockError: - print "Hello? Operator! Give me the number for 911!" - print "Dinstall in the locked area, cant process packages, come back later" + else: + try: + check_daily_lock() + do_accept(u, session) + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" +# (new, byhand) = check_status(files) +# if new or byhand: +# if new: +# do_new(u, session) +# if byhand: +# do_byhand(u, session) +# (new, byhand) = check_status(files) + +# if not new and not byhand: +# try: +# check_daily_lock() +# do_accept(u) +# except CantGetLockError: +# print "Hello? Operator! Give me the number for 911!" +# print "Dinstall in the locked area, cant process packages, come back later" except AlreadyLockedError, e: print "Seems to be locked by %s already, skipping..." % (e) @@ -1009,85 +948,33 @@ def end(): if accept_count > 1: sets = "sets" sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes)))) - Logger.log([utils.getusername(), "total",accept_count,accept_bytes]) + Logger.log(["total",accept_count,accept_bytes]) if not Options["No-Action"] and not Options["Trainee"]: Logger.close() ################################################################################ -# def do_comments(dir, opref, npref, line, fn): -# for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]: -# lines = open("%s/%s" % (dir, comm)).readlines() -# if len(lines) == 0 or lines[0] != line + "\n": continue -# changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_") -# and x.endswith(".changes") ] -# changes_files = sort_changes(changes_files) -# for f in changes_files: -# f = utils.validate_changes_file_arg(f, 0) -# if not f: continue -# print "\n" + f -# fn(f, "".join(lines[1:])) - -# if opref != npref and not Options["No-Action"]: -# newcomm = npref + comm[len(opref):] -# os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm)) - -# ################################################################################ - -# def comment_accept(changes_file, comments): -# Upload.pkg.changes_file = changes_file -# Upload.init_vars() -# Upload.update_vars() -# Upload.update_subst() -# files = Upload.pkg.files - -# if not recheck(): -# return # dak wants to REJECT, crap - -# (new, byhand) = check_status(files) -# if not new and not byhand: -# do_accept() - -# ################################################################################ - -# def comment_reject(changes_file, comments): -# Upload.pkg.changes_file = changes_file -# Upload.init_vars() -# Upload.update_vars() -# Upload.update_subst() - -# if not recheck(): -# pass # dak has its own reasons to reject as well, which is fine - -# reject(comments) -# print "REJECT\n" + reject_message, -# if not Options["No-Action"]: -# Upload.do_reject(0, reject_message) -# os.unlink(Upload.pkg.changes_file[:-8]+".dak") - -################################################################################ - def main(): global Options, Logger, Sections, Priorities cnf = Config() session = DBConn().session() - + Arguments = [('a',"automatic","Process-New::Options::Automatic"), ('h',"help","Process-New::Options::Help"), - ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"), ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), ('t',"trainee","Process-New::Options::Trainee"), ('n',"no-action","Process-New::Options::No-Action")] - for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]: + for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]: if not cnf.has_key("Process-New::Options::%s" % (i)): cnf["Process-New::Options::%s" % (i)] = "" changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv) - if len(changes_files) == 0 and not cnf.get("Process-New::Options::Comments-Dir",""): - changes_files = utils.get_changes_files(cnf["Dir::Queue::New"]) + if len(changes_files) == 0: + new_queue = get_policy_queue('new', session ); + changes_files = utils.get_changes_files(new_queue.path) Options = cnf.SubTree("Process-New::Options") @@ -1110,28 +997,14 @@ def main(): # Kill me now? **FIXME** cnf["Dinstall::Options::No-Mail"] = "" -# bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $" -# if cnf.has_key("Dinstall::Bcc"): -# Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) -# else: -# Upload.Subst["__BCC__"] = bcc - -# commentsdir = cnf.get("Process-New::Options::Comments-Dir","") -# if commentsdir: -# if changes_files != []: -# sys.stderr.write("Can't specify any changes files if working with comments-dir") -# sys.exit(1) -# do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept) -# do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject) -# else: - if True: - for changes_file in changes_files: - changes_file = utils.validate_changes_file_arg(changes_file, 0) - if not changes_file: - continue - print "\n" + changes_file - - do_pkg (changes_file, session) + + for changes_file in changes_files: + changes_file = utils.validate_changes_file_arg(changes_file, 0) + if not changes_file: + continue + print "\n" + changes_file + + do_pkg (changes_file, session) end()