From: Mark Hymers Date: Sat, 18 Sep 2010 11:04:05 +0000 (+0000) Subject: Merge commit 'ftpmaster/master' X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=95109ebb89d25a20ccb89fd10a857585fb23f8a6;hp=58f3289c7a3287df7fd6bcf2a35c6319b37e1e42;p=dak.git Merge commit 'ftpmaster/master' --- diff --git a/dak/process_new.py b/dak/process_new.py index f11c7c02..dd990f1f 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -428,7 +428,7 @@ def do_new(upload, session): done = 0 while not done: # Find out what's new - new = determine_new(changes, files) + new, byhand = determine_new(upload.pkg.changes_file, changes, files, session=session) if not new: break @@ -529,14 +529,14 @@ def do_byhand(upload, session): done = 0 while not done: files = upload.pkg.files - will_install = 1 + will_install = True byhand = [] for f in files.keys(): - if files[f]["type"] == "byhand": + if files[f]["section"] == "byhand": if os.path.exists(f): print "W: %s still present; please process byhand components and try again." % (f) - will_install = 0 + will_install = False else: byhand.append(f) @@ -558,21 +558,39 @@ def do_byhand(upload, session): answer = answer[:1].upper() if answer == 'A': - try: - check_daily_lock() - done = 1 - for f in byhand: - del files[f] - Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) - except CantGetLockError: - print "Hello? Operator! Give me the number for 911!" - print "Dinstall in the locked area, cant process packages, come back later" + dbchg = get_dbchange(upload.pkg.changes_file, session) + if dbchg is None: + print "Warning: cannot find changes file in database; can't process BYHAND" + else: + try: + check_daily_lock() + done = 1 + for b in byhand: + # Find the file entry in the database + found = False + for f in dbchg.files: + if f.filename == b: + found = True + f.processed = True + break + + if not found: + print "Warning: Couldn't find BYHAND item %s in the database to mark it processed" % b + + session.commit() + Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'M': - Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)]) - upload.do_reject(manual=1, reject_message=Options["Manual-Reject"]) - upload.pkg.remove_known_changes(session=session) - session.commit() - done = 1 + aborted = upload.do_reject(manual=1, + reject_message=Options["Manual-Reject"], + notes=get_new_comments(changes.get("source", ""), session=session)) + if not aborted: + upload.pkg.remove_known_changes(session=session) + session.commit() + Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)]) + done = 1 elif answer == 'S': done = 1 elif answer == 'Q': @@ -634,13 +652,15 @@ class clean_holding(object): os.unlink(os.path.join(h.holding_dir, f)) -def do_pkg(changes_file, session): - new_queue = get_policy_queue('new', session ); +def do_pkg(changes_full_path, session): + changes_dir = os.path.dirname(changes_full_path) + changes_file = os.path.basename(changes_full_path) + u = Upload() u.pkg.changes_file = changes_file (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file) u.load_changes(changes_file) - u.pkg.directory = new_queue.path + u.pkg.directory = changes_dir u.update_subst() u.logger = Logger origchanges = os.path.abspath(u.pkg.changes_file) @@ -669,12 +689,9 @@ def do_pkg(changes_file, session): if not recheck(u, session): return - new, byhand = determine_new(u.pkg.changes, files) + new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, session=session) if byhand: - # TODO: Fix this and make sure it doesn't complain when we've - # got already processed byhand components - print "Warning: This has byhand components and probably shouldn't be in NEW." - print "Contact an ftpmaster as this needs to be dealt with by them" + do_byhand(u, session) elif new: do_new(u, session) else: @@ -725,7 +742,9 @@ def main(): changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv) if len(changes_files) == 0: new_queue = get_policy_queue('new', session ); - changes_files = utils.get_changes_files(new_queue.path) + changes_paths = [ os.path.join(new_queue.path, j) for j in utils.get_changes_files(new_queue.path) ] + else: + changes_paths = [ os.path.abspath(j) for j in changes_files ] Options = cnf.SubTree("Process-New::Options") @@ -742,15 +761,15 @@ def main(): Priorities = Priority_Completer(session) readline.parse_and_bind("tab: complete") - if len(changes_files) > 1: + if len(changes_paths) > 1: sys.stderr.write("Sorting changes...\n") - changes_files = sort_changes(changes_files, session) + changes_files = sort_changes(changes_paths, session) - for changes_file in changes_files: + for changes_file in changes_paths: changes_file = utils.validate_changes_file_arg(changes_file, 0) if not changes_file: continue - print "\n" + changes_file + print "\n" + os.path.basename(changes_file) do_pkg (changes_file, session) diff --git a/dak/show_new.py b/dak/show_new.py index 6e6cb863..6d357134 100755 --- a/dak/show_new.py +++ b/dak/show_new.py @@ -172,7 +172,7 @@ def do_pkg(changes_file): u.check_source_against_db(deb_filename, session) u.pkg.changes["suite"] = u.pkg.changes["distribution"] - new, byhand = determine_new(u.pkg.changes, files, 0, session) + new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, 0, session) htmlname = changes["source"] + "_" + changes["version"] + ".html" sources.add(htmlname) diff --git a/daklib/changesutils.py b/daklib/changesutils.py index 0aca121a..35b3d57b 100644 --- a/daklib/changesutils.py +++ b/daklib/changesutils.py @@ -159,7 +159,9 @@ def changes_to_queue(upload, srcqueue, destqueue, session): for f in chg.files: # update the changes_pending_files row f.queue = destqueue - utils.move(os.path.join(srcqueue.path, f.filename), destqueue.path, perms=int(destqueue.perms, 8)) + # Only worry about unprocessed files + if not f.processed: + utils.move(os.path.join(srcqueue.path, f.filename), destqueue.path, perms=int(destqueue.perms, 8)) utils.move(os.path.join(srcqueue.path, upload.pkg.changes_file), destqueue.path, perms=int(destqueue.perms, 8)) chg.in_queue = destqueue @@ -188,9 +190,14 @@ def new_accept(upload, dry_run, session): else: # Just a normal upload, accept it... (summary, short_summary) = upload.build_summaries() - srcqueue = get_policy_queue('new', session) destqueue = get_policy_queue('newstage', session) + srcqueue = get_policy_queue_from_path(upload.pkg.directory, session) + + if not srcqueue: + # Assume NEW and hope for the best + srcqueue = get_policy_queue('new', session) + changes_to_queue(upload, srcqueue, destqueue, session) __all__.append('new_accept') diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 65e14104..fab0870e 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -1917,6 +1917,31 @@ def get_policy_queue(queuename, session=None): __all__.append('get_policy_queue') +@session_wrapper +def get_policy_queue_from_path(pathname, session=None): + """ + Returns PolicyQueue object for given C{path name} + + @type queuename: string + @param queuename: The path + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: PolicyQueue + @return: PolicyQueue object for the given queue + """ + + q = session.query(PolicyQueue).filter_by(path=pathname) + + try: + return q.one() + except NoResultFound: + return None + +__all__.append('get_policy_queue_from_path') + ################################################################################ class Priority(object): diff --git a/daklib/queue.py b/daklib/queue.py index e04b80ac..7d7faa4b 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -92,10 +92,13 @@ def get_type(f, session): # Determine what parts in a .changes are NEW -def determine_new(changes, files, warn=1, session = None): +def determine_new(filename, changes, files, warn=1, session = None): """ Determine what parts in a C{changes} file are NEW. + @type filename: str + @param filename: changes filename + @type changes: Upload.Pkg.changes dict @param changes: Changes dictionary @@ -109,15 +112,22 @@ def determine_new(changes, files, warn=1, session = None): @return: dictionary of NEW components. """ + # TODO: This should all use the database instead of parsing the changes + # file again new = {} byhand = {} + dbchg = get_dbchange(filename, session) + if dbchg is None: + print "Warning: cannot find changes file in database; won't check byhand" + # Build up a list of potentially new things for name, f in files.items(): # Keep a record of byhand elements if f["section"] == "byhand": byhand[name] = 1 continue + pkg = f["package"] priority = f["priority"] section = f["section"] @@ -165,6 +175,23 @@ def determine_new(changes, files, warn=1, session = None): del changes["suite"][suite] changes["suite"][override] = 1 + # Check for unprocessed byhand files + if dbchg is not None: + for b in byhand.keys(): + # Find the file entry in the database + found = False + for f in dbchg.files: + if f.filename == b: + found = True + # If it's processed, we can ignore it + if f.processed: + del byhand[b] + break + + if not found: + print "Warning: Couldn't find BYHAND item %s in the database; assuming unprocessed" + + # Check for new stuff for suite in changes["suite"].keys(): for pkg in new.keys(): ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session) @@ -1104,11 +1131,24 @@ class Upload(object): session = DBConn().session() self.check_source_against_db(dsc_filename, session) self.check_dsc_against_db(dsc_filename, session) - session.close() + + dbchg = get_dbchange(self.pkg.changes_file, session) # Finally, check if we're missing any files for f in self.later_check_files: - self.rejects.append("Could not find file %s references in changes" % f) + print 'XXX: %s' % f + # Check if we've already processed this file if we have a dbchg object + ok = False + if dbchg: + for pf in dbchg.files: + if pf.filename == f and pf.processed: + self.notes.append('%s was already processed so we can go ahead' % f) + ok = True + del self.pkg.files[f] + if not ok: + self.rejects.append("Could not find file %s references in changes" % f) + + session.close() return True