From d0940f70aea01a1a7ab1715757f8aae535ec0dde Mon Sep 17 00:00:00 2001 From: Mark Hymers Date: Sat, 18 Sep 2010 11:03:45 +0000 Subject: [PATCH] Attempt to fix BYHAND handling Signed-off-by: Mark Hymers --- dak/process_new.py | 61 ++++++++++++++++++++++++++---------------- dak/show_new.py | 2 +- daklib/changesutils.py | 11 ++++++-- daklib/dbconn.py | 25 +++++++++++++++++ daklib/queue.py | 46 ++++++++++++++++++++++++++++--- 5 files changed, 116 insertions(+), 29 deletions(-) diff --git a/dak/process_new.py b/dak/process_new.py index bc3ef28a..dd990f1f 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -428,7 +428,7 @@ def do_new(upload, session): done = 0 while not done: # Find out what's new - new = determine_new(changes, files) + new, byhand = determine_new(upload.pkg.changes_file, changes, files, session=session) if not new: break @@ -529,14 +529,14 @@ def do_byhand(upload, session): done = 0 while not done: files = upload.pkg.files - will_install = 1 + will_install = True byhand = [] for f in files.keys(): - if files[f]["type"] == "byhand": + if files[f]["section"] == "byhand": if os.path.exists(f): print "W: %s still present; please process byhand components and try again." % (f) - will_install = 0 + will_install = False else: byhand.append(f) @@ -558,21 +558,39 @@ def do_byhand(upload, session): answer = answer[:1].upper() if answer == 'A': - try: - check_daily_lock() - done = 1 - for f in byhand: - del files[f] - Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) - except CantGetLockError: - print "Hello? Operator! Give me the number for 911!" - print "Dinstall in the locked area, cant process packages, come back later" + dbchg = get_dbchange(upload.pkg.changes_file, session) + if dbchg is None: + print "Warning: cannot find changes file in database; can't process BYHAND" + else: + try: + check_daily_lock() + done = 1 + for b in byhand: + # Find the file entry in the database + found = False + for f in dbchg.files: + if f.filename == b: + found = True + f.processed = True + break + + if not found: + print "Warning: Couldn't find BYHAND item %s in the database to mark it processed" % b + + session.commit() + Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'M': - Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)]) - upload.do_reject(manual=1, reject_message=Options["Manual-Reject"]) - upload.pkg.remove_known_changes(session=session) - session.commit() - done = 1 + aborted = upload.do_reject(manual=1, + reject_message=Options["Manual-Reject"], + notes=get_new_comments(changes.get("source", ""), session=session)) + if not aborted: + upload.pkg.remove_known_changes(session=session) + session.commit() + Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)]) + done = 1 elif answer == 'S': done = 1 elif answer == 'Q': @@ -671,12 +689,9 @@ def do_pkg(changes_full_path, session): if not recheck(u, session): return - new, byhand = determine_new(u.pkg.changes, files) + new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, session=session) if byhand: - # TODO: Fix this and make sure it doesn't complain when we've - # got already processed byhand components - print "Warning: This has byhand components and probably shouldn't be in NEW." - print "Contact an ftpmaster as this needs to be dealt with by them" + do_byhand(u, session) elif new: do_new(u, session) else: diff --git a/dak/show_new.py b/dak/show_new.py index 6e6cb863..6d357134 100755 --- a/dak/show_new.py +++ b/dak/show_new.py @@ -172,7 +172,7 @@ def do_pkg(changes_file): u.check_source_against_db(deb_filename, session) u.pkg.changes["suite"] = u.pkg.changes["distribution"] - new, byhand = determine_new(u.pkg.changes, files, 0, session) + new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, 0, session) htmlname = changes["source"] + "_" + changes["version"] + ".html" sources.add(htmlname) diff --git a/daklib/changesutils.py b/daklib/changesutils.py index 0aca121a..35b3d57b 100644 --- a/daklib/changesutils.py +++ b/daklib/changesutils.py @@ -159,7 +159,9 @@ def changes_to_queue(upload, srcqueue, destqueue, session): for f in chg.files: # update the changes_pending_files row f.queue = destqueue - utils.move(os.path.join(srcqueue.path, f.filename), destqueue.path, perms=int(destqueue.perms, 8)) + # Only worry about unprocessed files + if not f.processed: + utils.move(os.path.join(srcqueue.path, f.filename), destqueue.path, perms=int(destqueue.perms, 8)) utils.move(os.path.join(srcqueue.path, upload.pkg.changes_file), destqueue.path, perms=int(destqueue.perms, 8)) chg.in_queue = destqueue @@ -188,9 +190,14 @@ def new_accept(upload, dry_run, session): else: # Just a normal upload, accept it... (summary, short_summary) = upload.build_summaries() - srcqueue = get_policy_queue('new', session) destqueue = get_policy_queue('newstage', session) + srcqueue = get_policy_queue_from_path(upload.pkg.directory, session) + + if not srcqueue: + # Assume NEW and hope for the best + srcqueue = get_policy_queue('new', session) + changes_to_queue(upload, srcqueue, destqueue, session) __all__.append('new_accept') diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 65e14104..fab0870e 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -1917,6 +1917,31 @@ def get_policy_queue(queuename, session=None): __all__.append('get_policy_queue') +@session_wrapper +def get_policy_queue_from_path(pathname, session=None): + """ + Returns PolicyQueue object for given C{path name} + + @type queuename: string + @param queuename: The path + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: PolicyQueue + @return: PolicyQueue object for the given queue + """ + + q = session.query(PolicyQueue).filter_by(path=pathname) + + try: + return q.one() + except NoResultFound: + return None + +__all__.append('get_policy_queue_from_path') + ################################################################################ class Priority(object): diff --git a/daklib/queue.py b/daklib/queue.py index e04b80ac..7d7faa4b 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -92,10 +92,13 @@ def get_type(f, session): # Determine what parts in a .changes are NEW -def determine_new(changes, files, warn=1, session = None): +def determine_new(filename, changes, files, warn=1, session = None): """ Determine what parts in a C{changes} file are NEW. + @type filename: str + @param filename: changes filename + @type changes: Upload.Pkg.changes dict @param changes: Changes dictionary @@ -109,15 +112,22 @@ def determine_new(changes, files, warn=1, session = None): @return: dictionary of NEW components. """ + # TODO: This should all use the database instead of parsing the changes + # file again new = {} byhand = {} + dbchg = get_dbchange(filename, session) + if dbchg is None: + print "Warning: cannot find changes file in database; won't check byhand" + # Build up a list of potentially new things for name, f in files.items(): # Keep a record of byhand elements if f["section"] == "byhand": byhand[name] = 1 continue + pkg = f["package"] priority = f["priority"] section = f["section"] @@ -165,6 +175,23 @@ def determine_new(changes, files, warn=1, session = None): del changes["suite"][suite] changes["suite"][override] = 1 + # Check for unprocessed byhand files + if dbchg is not None: + for b in byhand.keys(): + # Find the file entry in the database + found = False + for f in dbchg.files: + if f.filename == b: + found = True + # If it's processed, we can ignore it + if f.processed: + del byhand[b] + break + + if not found: + print "Warning: Couldn't find BYHAND item %s in the database; assuming unprocessed" + + # Check for new stuff for suite in changes["suite"].keys(): for pkg in new.keys(): ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session) @@ -1104,11 +1131,24 @@ class Upload(object): session = DBConn().session() self.check_source_against_db(dsc_filename, session) self.check_dsc_against_db(dsc_filename, session) - session.close() + + dbchg = get_dbchange(self.pkg.changes_file, session) # Finally, check if we're missing any files for f in self.later_check_files: - self.rejects.append("Could not find file %s references in changes" % f) + print 'XXX: %s' % f + # Check if we've already processed this file if we have a dbchg object + ok = False + if dbchg: + for pf in dbchg.files: + if pf.filename == f and pf.processed: + self.notes.append('%s was already processed so we can go ahead' % f) + ok = True + del self.pkg.files[f] + if not ok: + self.rejects.append("Could not find file %s references in changes" % f) + + session.close() return True -- 2.39.2