X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=blobdiff_plain;f=dak%2Fprocess_new.py;h=55397c7dc79a4e2df577397d9262a6addb047df1;hp=7a64277f23205d9bf8a2c4f611f5931c27f39f3b;hb=fdbef587c29814f97c192de5a8b7e9f09cc45fa4;hpb=61d1e2b14d61c045dcc07cd177dec49af0c41c45 diff --git a/dak/process_new.py b/dak/process_new.py index 7a64277f..55397c7d 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -54,6 +54,8 @@ import pwd import apt_pkg, apt_inst import examine_package import subprocess +import daklib.daksubprocess +from sqlalchemy import or_ from daklib.dbconn import * from daklib.queue import * @@ -118,6 +120,30 @@ class Priority_Completer: ################################################################################ +def takenover_binaries(upload, missing, session): + rows = [] + binaries = set([x.package for x in upload.binaries]) + for m in missing: + if m['type'] != 'dsc': + binaries.discard(m['package']) + if binaries: + source = upload.binaries[0].source.source + suite = upload.target_suite.overridesuite or \ + upload.target_suite.suite_name + suites = [s[0] for s in session.query(Suite.suite_name).filter \ + (or_(Suite.suite_name == suite, + Suite.overridesuite == suite)).all()] + rows = session.query(DBSource.source, DBBinary.package).distinct(). \ + filter(DBBinary.package.in_(binaries)). \ + join(DBBinary.source). \ + filter(DBSource.source != source). \ + join(DBBinary.suites). \ + filter(Suite.suite_name.in_(suites)). \ + order_by(DBSource.source, DBBinary.package).all() + return rows + +################################################################################ + def print_new (upload, missing, indexed, session, file=sys.stdout): check_valid(missing, session) index = 0 @@ -129,15 +155,21 @@ def print_new (upload, missing, indexed, session, file=sys.stdout): package = m['package'] section = m['section'] priority = m['priority'] + included = "" if m['included'] else "NOT UPLOADED" if indexed: - line = "(%s): %-20s %-20s %-20s" % (index, package, priority, section) + line = "(%s): %-20s %-20s %-20s %s" % (index, package, priority, section, included) else: - line = "%-20s %-20s %-20s" % (package, priority, section) + line = "%-20s %-20s %-20s %s" % (package, priority, section, included) line = line.strip() if not m['valid']: line = line + ' [!]' print >>file, line - notes = get_new_comments(upload.changes.source) + takenover = takenover_binaries(upload, missing, session) + if takenover: + print '\n\nBINARIES TAKEN OVER\n' + for t in takenover: + print '%s: %s' % (t[0], t[1]) + notes = get_new_comments(upload.policy_queue, upload.changes.source) for note in notes: print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \ % (note.author, note.version, note.notedate, note.comment) @@ -187,7 +219,8 @@ def edit_new (overrides, upload, session): type, pkg = pkg.split(':', 1) else: type = 'deb' - if (type, pkg) not in overrides_map: + o = overrides_map.get((type, pkg), None) + if o is None: utils.warn("Ignoring unknown package '%s'" % (pkg)) else: if section.find('/') != -1: @@ -200,6 +233,7 @@ def edit_new (overrides, upload, session): section=section, component=component, priority=priority, + included=o['included'], )) return new_overrides @@ -309,12 +343,18 @@ def edit_overrides (new, upload, session): ################################################################################ -def check_pkg (upload, upload_copy): +def check_pkg (upload, upload_copy, session): + missing = [] save_stdout = sys.stdout changes = os.path.join(upload_copy.directory, upload.changes.changesname) suite_name = upload.target_suite.suite_name + handler = PolicyQueueUploadHandler(upload, session) + missing = [(m['type'], m["package"]) for m in handler.missing_overrides(hints=missing)] + + less_cmd = ("less", "-R", "-") + less_process = daklib.daksubprocess.Popen(less_cmd, bufsize=0, stdin=subprocess.PIPE) try: - sys.stdout = os.popen("less -R -", 'w', 0) + sys.stdout = less_process.stdin print examine_package.display_changes(suite_name, changes) source = upload.source @@ -324,9 +364,14 @@ def check_pkg (upload, upload_copy): for binary in upload.binaries: binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename)) - print examine_package.check_deb(suite_name, binary_file) + examined = examine_package.check_deb(suite_name, binary_file) + # We always need to call check_deb to display package relations for every binary, + # but we print its output only if new overrides are being added. + if ("deb", binary.package) in missing: + print examined print examine_package.output_package_relations() + less_process.stdin.close() except IOError as e: if e.errno == errno.EPIPE: utils.warn("[examine_package] Caught EPIPE; skipping.") @@ -335,6 +380,7 @@ def check_pkg (upload, upload_copy): except KeyboardInterrupt: utils.warn("[examine_package] Caught C-c; skipping.") finally: + less_process.wait() sys.stdout = save_stdout ################################################################################ @@ -412,7 +458,7 @@ def run_user_inspect_command(upload, upload_copy): changes=changes, ) - subprocess.check_call(shell_command, shell=True) + daklib.daksubprocess.check_call(shell_command, shell=True) ################################################################################ @@ -450,7 +496,6 @@ def get_reject_reason(reason=''): ################################################################################ def do_new(upload, upload_copy, handler, session): - print "NEW\n" cnf = Config() run_user_inspect_command(upload, upload_copy) @@ -465,9 +510,22 @@ def do_new(upload, upload_copy, handler, session): missing = handler.missing_overrides(hints=missing) broken = not check_valid(missing, session) + changesname = os.path.basename(upload.changes.changesname) + + print + print changesname + print "-" * len(changesname) + print + print " Target: {0}".format(upload.target_suite.suite_name) + print " Changed-By: {0}".format(upload.changes.changedby) + print + #if len(byhand) == 0 and len(missing) == 0: # break + if missing: + print "NEW\n" + answer = "XXX" if Options["No-Action"] or Options["Automatic"]: answer = 'S' @@ -508,46 +566,45 @@ def do_new(upload, upload_copy, handler, session): continue if answer == 'A' and not Options["Trainee"]: - try: - check_daily_lock() - add_overrides(missing, upload.target_suite, session) - if Config().find_b("Dinstall::BXANotify"): - do_bxa_notification(missing, upload, session) - handler.accept() - done = True - Logger.log(["NEW ACCEPT", upload.changes.changesname]) - except CantGetLockError: - print "Hello? Operator! Give me the number for 911!" - print "Dinstall in the locked area, cant process packages, come back later" + add_overrides(missing, upload.target_suite, session) + if Config().find_b("Dinstall::BXANotify"): + do_bxa_notification(missing, upload, session) + handler.accept() + done = True + Logger.log(["NEW ACCEPT", upload.changes.changesname]) elif answer == 'C': - check_pkg(upload, upload_copy) + check_pkg(upload, upload_copy, session) elif answer == 'E' and not Options["Trainee"]: missing = edit_overrides (missing, upload, session) elif answer == 'M' and not Options["Trainee"]: reason = Options.get('Manual-Reject', '') + "\n" - reason = reason + "\n".join(get_new_comments(upload.changes.source, session=session)) + reason = reason + "\n\n=====\n\n".join([n.comment for n in get_new_comments(upload.policy_queue, upload.changes.source, session=session)]) reason = get_reject_reason(reason) if reason is not None: Logger.log(["NEW REJECT", upload.changes.changesname]) handler.reject(reason) done = True elif answer == 'N': - edit_note(get_new_comments(upload.changes.source, session=session), - upload, session, bool(Options["Trainee"])) + if edit_note(get_new_comments(upload.policy_queue, upload.changes.source, session=session), + upload, session, bool(Options["Trainee"])) == 0: + end() + sys.exit(0) elif answer == 'P' and not Options["Trainee"]: - prod_maintainer(get_new_comments(upload.changes.source, session=session), - upload) + if prod_maintainer(get_new_comments(upload.policy_queue, upload.changes.source, session=session), + upload) == 0: + end() + sys.exit(0) Logger.log(["NEW PROD", upload.changes.changesname]) elif answer == 'R' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() if confirm == "y": - for c in get_new_comments(upload.changes.source, upload.changes.version, session=session): + for c in get_new_comments(upload.policy_queue, upload.changes.source, upload.changes.version, session=session): session.delete(c) session.commit() elif answer == 'O' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower() if confirm == "y": - for c in get_new_comments(upload.changes.source, session=session): + for c in get_new_comments(upload.policy_queue, upload.changes.source, session=session): session.delete(c) session.commit() @@ -557,6 +614,9 @@ def do_new(upload, upload_copy, handler, session): end() sys.exit(0) + if handler.get_action(): + print "PENDING %s\n" % handler.get_action() + ################################################################################ ################################################################################ ################################################################################ @@ -569,6 +629,7 @@ def usage (exit_code=0): -h, --help show this help and exit. -m, --manual-reject=MSG manual reject with `msg' -n, --no-action don't do anything + -q, --queue=QUEUE operate on a different queue -t, --trainee FTP Trainee mode -V, --version display the version number and exit @@ -598,24 +659,6 @@ ENVIRONMENT VARIABLES ################################################################################ -def check_daily_lock(): - """ - Raises CantGetLockError if the dinstall daily.lock exists. - """ - - cnf = Config() - try: - lockfile = cnf.get("Process-New::DinstallLockFile", - os.path.join(cnf['Dir::Lock'], 'processnew.lock')) - - os.open(lockfile, - os.O_RDONLY | os.O_CREAT | os.O_EXCL) - except OSError as e: - if e.errno == errno.EEXIST or e.errno == errno.EACCES: - raise CantGetLockError - - os.unlink(lockfile) - @contextlib.contextmanager def lock_package(package): """ @@ -646,6 +689,8 @@ def do_pkg(upload, session): dsc = upload.source cnf = Config() + group = cnf.get('Dinstall::UnprivGroup') or None + #bcc = "X-DAK: dak process-new" #if cnf.has_key("Dinstall::Bcc"): # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) @@ -654,9 +699,10 @@ def do_pkg(upload, session): try: with lock_package(upload.changes.source): - with UploadCopy(upload) as upload_copy: + with UploadCopy(upload, group=group) as upload_copy: handler = PolicyQueueUploadHandler(upload, session) if handler.get_action() is not None: + print "PENDING %s\n" % handler.get_action() return do_new(upload, upload_copy, handler, session) @@ -673,7 +719,7 @@ def show_new_comments(uploads, session): WHERE package IN :sources ORDER BY package, version""" - r = session.execute(query, params=dict(sources=sources)) + r = session.execute(query, params=dict(sources=tuple(sources))) for i in r: print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3]) @@ -682,6 +728,43 @@ def show_new_comments(uploads, session): ################################################################################ +def sort_uploads(new_queue, uploads, session, nobinaries=False): + sources = {} + sorteduploads = [] + suitesrc = [s.source for s in session.query(DBSource.source). \ + filter(DBSource.suites.any(Suite.suite_name.in_(['unstable', 'experimental'])))] + comments = [p.package for p in session.query(NewComment.package). \ + filter_by(trainee=False, policy_queue=new_queue).distinct()] + for upload in uploads: + source = upload.changes.source + if not source in sources: + sources[source] = [] + sources[source].append({'upload': upload, + 'date': upload.changes.created, + 'stack': 1, + 'binary': True if source in suitesrc else False, + 'comments': True if source in comments else False}) + for src in sources: + if len(sources[src]) > 1: + changes = sources[src] + firstseen = sorted(changes, key=lambda k: (k['date']))[0]['date'] + changes.sort(key=lambda item:item['date']) + for i in range (0, len(changes)): + changes[i]['date'] = firstseen + changes[i]['stack'] = i + 1 + sorteduploads += sources[src] + if nobinaries: + sorteduploads = [u["upload"] for u in sorted(sorteduploads, + key=lambda k: (k["comments"], k["binary"], + k["date"], -k["stack"]))] + else: + sorteduploads = [u["upload"] for u in sorted(sorteduploads, + key=lambda k: (k["comments"], -k["binary"], + k["date"], -k["stack"]))] + return sorteduploads + +################################################################################ + def end(): accept_count = SummaryStats().accept_count accept_bytes = SummaryStats().accept_bytes @@ -744,14 +827,12 @@ def main(): if len(uploads) > 1: sys.stderr.write("Sorting changes...\n") - uploads.sort() + uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"]) if Options["Comments"]: show_new_comments(uploads, session) else: for upload in uploads: - print "\n" + os.path.basename(upload.changes.changesname) - do_pkg (upload, session) end()