X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_new.py;h=92dde2d3637d4c3675fbc88682fbc40d6465a923;hb=27e00376e81d1c37ff327ee0d39670b266418869;hp=894d07d274299363883c4561a383a1677d8c7590;hpb=716e3f4dbb11e94b13043835632e15f21ff1bfee;p=dak.git diff --git a/dak/process_new.py b/dak/process_new.py index 894d07d2..92dde2d3 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -64,6 +64,7 @@ from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError from daklib.summarystats import SummaryStats from daklib.config import Config +from daklib.changesutils import * # Globals Options = None @@ -108,104 +109,6 @@ def recheck(upload, session): ################################################################################ -def indiv_sg_compare (a, b): - """Sort by source name, source, version, 'have source', and - finally by filename.""" - # Sort by source version - q = apt_pkg.VersionCompare(a["version"], b["version"]) - if q: - return -q - - # Sort by 'have source' - a_has_source = a["architecture"].get("source") - b_has_source = b["architecture"].get("source") - if a_has_source and not b_has_source: - return -1 - elif b_has_source and not a_has_source: - return 1 - - return cmp(a["filename"], b["filename"]) - -############################################################ - -def sg_compare (a, b): - a = a[1] - b = b[1] - """Sort by have note, source already in database and time of oldest upload.""" - # Sort by have note - a_note_state = a["note_state"] - b_note_state = b["note_state"] - if a_note_state < b_note_state: - return -1 - elif a_note_state > b_note_state: - return 1 - # Sort by source already in database (descending) - source_in_database = cmp(a["source_in_database"], b["source_in_database"]) - if source_in_database: - return -source_in_database - - # Sort by time of oldest upload - return cmp(a["oldest"], b["oldest"]) - -def sort_changes(changes_files, session): - """Sort into source groups, then sort each source group by version, - have source, filename. Finally, sort the source groups by have - note, time of oldest upload of each source upload.""" - if len(changes_files) == 1: - return changes_files - - sorted_list = [] - cache = {} - # Read in all the .changes files - for filename in changes_files: - u = Upload() - try: - u.pkg.changes_file = filename - u.load_changes(filename) - u.update_subst() - cache[filename] = copy.copy(u.pkg.changes) - cache[filename]["filename"] = filename - except: - sorted_list.append(filename) - break - # Divide the .changes into per-source groups - per_source = {} - for filename in cache.keys(): - source = cache[filename]["source"] - if not per_source.has_key(source): - per_source[source] = {} - per_source[source]["list"] = [] - per_source[source]["list"].append(cache[filename]) - # Determine oldest time and have note status for each source group - for source in per_source.keys(): - q = session.query(DBSource).filter_by(source = source).all() - per_source[source]["source_in_database"] = len(q)>0 - source_list = per_source[source]["list"] - first = source_list[0] - oldest = os.stat(first["filename"])[stat.ST_MTIME] - have_note = 0 - for d in per_source[source]["list"]: - mtime = os.stat(d["filename"])[stat.ST_MTIME] - if mtime < oldest: - oldest = mtime - have_note += has_new_comment(d["source"], d["version"], session) - per_source[source]["oldest"] = oldest - if not have_note: - per_source[source]["note_state"] = 0; # none - elif have_note < len(source_list): - per_source[source]["note_state"] = 1; # some - else: - per_source[source]["note_state"] = 2; # all - per_source[source]["list"].sort(indiv_sg_compare) - per_source_items = per_source.items() - per_source_items.sort(sg_compare) - for i in per_source_items: - for j in i[1]["list"]: - sorted_list.append(j["filename"]) - return sorted_list - -################################################################################ - class Section_Completer: def __init__ (self, session): self.sections = [] @@ -430,74 +333,33 @@ def edit_overrides (new, upload, session): return new -################################################################################ - -def edit_note(note, upload, session): - # Write the current data to a temporary file - (fd, temp_filename) = utils.temp_filename() - editor = os.environ.get("EDITOR","vi") - answer = 'E' - while answer == 'E': - os.system("%s %s" % (editor, temp_filename)) - temp_file = utils.open_file(temp_filename) - newnote = temp_file.read().rstrip() - temp_file.close() - print "New Note:" - print utils.prefix_multi_line_string(newnote," ") - prompt = "[D]one, Edit, Abandon, Quit ?" - answer = "XXX" - while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt) - m = re_default_answer.search(prompt) - if answer == "": - answer = m.group(1) - answer = answer[:1].upper() - os.unlink(temp_filename) - if answer == 'A': - return - elif answer == 'Q': - end() - sys.exit(0) - - comment = NewComment() - comment.package = upload.pkg.changes["source"] - comment.version = upload.pkg.changes["version"] - comment.comment = newnote - comment.author = utils.whoami() - comment.trainee = bool(Options["Trainee"]) - session.add(comment) - session.commit() ################################################################################ def check_pkg (upload): + save_stdout = sys.stdout try: - less_fd = os.popen("less -R -", 'w', 0) - stdout_fd = sys.stdout - try: - sys.stdout = less_fd - changes = utils.parse_changes (upload.pkg.changes_file) - examine_package.display_changes(changes['distribution'], upload.pkg.changes_file) - files = upload.pkg.files - for f in files.keys(): - if files[f].has_key("new"): - ftype = files[f]["type"] - if ftype == "deb": - examine_package.check_deb(changes['distribution'], f) - elif ftype == "dsc": - examine_package.check_dsc(changes['distribution'], f) - finally: - examine_package.output_package_relations() - sys.stdout = stdout_fd - except IOError, e: + sys.stdout = os.popen("less -R -", 'w', 0) + changes = utils.parse_changes (upload.pkg.changes_file) + print examine_package.display_changes(changes['distribution'], upload.pkg.changes_file) + files = upload.pkg.files + for f in files.keys(): + if files[f].has_key("new"): + ftype = files[f]["type"] + if ftype == "deb": + print examine_package.check_deb(changes['distribution'], f) + elif ftype == "dsc": + print examine_package.check_dsc(changes['distribution'], f) + print examine_package.output_package_relations() + except IOError as e: if e.errno == errno.EPIPE: utils.warn("[examine_package] Caught EPIPE; skipping.") - pass else: + sys.stdout = save_stdout raise except KeyboardInterrupt: utils.warn("[examine_package] Caught C-c; skipping.") - pass + sys.stdout = save_stdout ################################################################################ @@ -530,7 +392,7 @@ def add_overrides (new, upload, session): type_id = get_override_type(new[pkg]["type"]).overridetype_id priority_id = new[pkg]["priority id"] section_id = new[pkg]["section id"] - Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]]) + Logger.log(["%s (%s) overrides" % (pkg, srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]]) session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')", { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id}) for f in new[pkg]["files"]: @@ -545,59 +407,6 @@ def add_overrides (new, upload, session): ################################################################################ -def prod_maintainer (note, upload): - cnf = Config() - # Here we prepare an editor and get them ready to prod... - (fd, temp_filename) = utils.temp_filename() - temp_file = os.fdopen(fd, 'w') - if len(note) > 0: - for line in note: - temp_file.write(line) - temp_file.close() - editor = os.environ.get("EDITOR","vi") - answer = 'E' - while answer == 'E': - os.system("%s %s" % (editor, temp_filename)) - temp_fh = utils.open_file(temp_filename) - prod_message = "".join(temp_fh.readlines()) - temp_fh.close() - print "Prod message:" - print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1) - prompt = "[P]rod, Edit, Abandon, Quit ?" - answer = "XXX" - while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt) - m = re_default_answer.search(prompt) - if answer == "": - answer = m.group(1) - answer = answer[:1].upper() - os.unlink(temp_filename) - if answer == 'A': - return - elif answer == 'Q': - end() - sys.exit(0) - # Otherwise, do the proding... - user_email_address = utils.whoami() + " <%s>" % ( - cnf["Dinstall::MyAdminAddress"]) - - Subst = upload.Subst - - Subst["__FROM_ADDRESS__"] = user_email_address - Subst["__PROD_MESSAGE__"] = prod_message - Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"] - - prod_mail_message = utils.TemplateSubst( - Subst,cnf["Dir::Templates"]+"/process-new.prod") - - # Send the prod mail if appropriate - if not cnf["Dinstall::Options::No-Mail"]: - utils.send_mail(prod_mail_message) - - print "Sent proding message" - -################################################################################ - def do_new(upload, session): print "NEW\n" files = upload.pkg.files @@ -611,11 +420,18 @@ def do_new(upload, session): # Make a copy of distribution we can happily trample on changes["suite"] = copy.copy(changes["distribution"]) + # Try to get an included dsc + dsc = None + (status, _) = upload.load_dsc() + if status: + dsc = upload.pkg.dsc + # The main NEW processing loop done = 0 + new = {} while not done: # Find out what's new - new = determine_new(changes, files) + new, byhand = determine_new(upload.pkg.changes_file, changes, files, dsc=dsc, session=session, new=new) if not new: break @@ -652,7 +468,7 @@ def do_new(upload, session): try: check_daily_lock() done = add_overrides (new, upload, session) - do_accept(upload, session) + new_accept(upload, Options["No-Action"], session) Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)]) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" @@ -672,7 +488,7 @@ def do_new(upload, session): done = 1 elif answer == 'N': edit_note(get_new_comments(changes.get("source", ""), session=session), - upload, session) + upload, session, bool(Options["Trainee"])) elif answer == 'P' and not Options["Trainee"]: prod_maintainer(get_new_comments(changes.get("source", ""), session=session), upload) @@ -703,6 +519,8 @@ def do_new(upload, session): def usage (exit_code=0): print """Usage: dak process-new [OPTION]... [CHANGES]... -a, --automatic automatic run + -b, --no-binaries do not sort binary-NEW packages first + -c, --comments show NEW comments -h, --help show this help and exit. -m, --manual-reject=MSG manual reject with `msg' -n, --no-action don't do anything @@ -716,14 +534,14 @@ def do_byhand(upload, session): done = 0 while not done: files = upload.pkg.files - will_install = 1 + will_install = True byhand = [] for f in files.keys(): - if files[f]["type"] == "byhand": + if files[f]["section"] == "byhand": if os.path.exists(f): print "W: %s still present; please process byhand components and try again." % (f) - will_install = 0 + will_install = False else: byhand.append(f) @@ -745,21 +563,39 @@ def do_byhand(upload, session): answer = answer[:1].upper() if answer == 'A': - try: - check_daily_lock() - done = 1 - for f in byhand: - del files[f] - Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) - except CantGetLockError: - print "Hello? Operator! Give me the number for 911!" - print "Dinstall in the locked area, cant process packages, come back later" + dbchg = get_dbchange(upload.pkg.changes_file, session) + if dbchg is None: + print "Warning: cannot find changes file in database; can't process BYHAND" + else: + try: + check_daily_lock() + done = 1 + for b in byhand: + # Find the file entry in the database + found = False + for f in dbchg.files: + if f.filename == b: + found = True + f.processed = True + break + + if not found: + print "Warning: Couldn't find BYHAND item %s in the database to mark it processed" % b + + session.commit() + Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'M': - Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)]) - upload.do_reject(manual=1, reject_message=Options["Manual-Reject"]) - upload.pkg.remove_known_changes(session=session) - session.commit() - done = 1 + aborted = upload.do_reject(manual=1, + reject_message=Options["Manual-Reject"], + notes=get_new_comments(changes.get("source", ""), session=session)) + if not aborted: + upload.pkg.remove_known_changes(session=session) + session.commit() + Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)]) + done = 1 elif answer == 'S': done = 1 elif answer == 'Q': @@ -775,13 +611,16 @@ def check_daily_lock(): cnf = Config() try: - os.open(cnf["Process-New::DinstallLockFile"], + lockfile = cnf.get("Process-New::DinstallLockFile", + os.path.join(cnf['Dir::Lock'], 'processnew.lock')) + + os.open(lockfile, os.O_RDONLY | os.O_CREAT | os.O_EXCL) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST or e.errno == errno.EACCES: raise CantGetLockError - os.unlink(cnf["Process-New::DinstallLockFile"]) + os.unlink(lockfile) @contextlib.contextmanager @@ -793,10 +632,13 @@ def lock_package(package): @param package: source package name to lock """ - path = os.path.join(Config()["Process-New::LockDir"], package) + cnf = Config() + + path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package) + try: fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST or e.errno == errno.EACCES: user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '') raise AlreadyLockedError, user @@ -821,59 +663,25 @@ class clean_holding(object): os.unlink(os.path.join(h.holding_dir, f)) +def do_pkg(changes_full_path, session): + changes_dir = os.path.dirname(changes_full_path) + changes_file = os.path.basename(changes_full_path) -def changes_to_newstage(upload, session): - """move a changes file to newstage""" - new = get_policy_queue('new', session ); - newstage = get_policy_queue('newstage', session ); - - chg = session.query(DBChange).filter_by(changesname=os.path.basename(upload.pkg.changes_file)).one() - chg.approved_for = newstage.policy_queue_id - - for f in chg.files: - # update the changes_pending_files row - f.queue = newstage - utils.move(os.path.join(new.path, f.filename), newstage.path, perms=int(newstage.perms, 8)) - - utils.move(os.path.join(new.path, upload.pkg.changes_file), newstage.path, perms=int(newstage.perms, 8)) - chg.in_queue = newstage - session.commit() - -def _accept(upload, session): - if Options["No-Action"]: - return - (summary, short_summary) = upload.build_summaries() - # upload.accept(summary, short_summary, targetqueue) - - changes_to_newstage(upload, session) - -def do_accept(upload, session): - print "ACCEPT" - cnf = Config() - if not Options["No-Action"]: - (summary, short_summary) = upload.build_summaries() - - if cnf.FindB("Dinstall::SecurityQueueHandling"): - upload.dump_vars(cnf["Dir::Queue::Embargoed"]) - upload.move_to_queue(get_policy_queue('embargoed')) - upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"]) - # Check for override disparities - upload.Subst["__SUMMARY__"] = summary - else: - # Just a normal upload, accept it... - _accept(upload, session) - -def do_pkg(changes_file, session): - new_queue = get_policy_queue('new', session ); u = Upload() u.pkg.changes_file = changes_file (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file) u.load_changes(changes_file) - u.pkg.directory = new_queue.path + u.pkg.directory = changes_dir u.update_subst() u.logger = Logger origchanges = os.path.abspath(u.pkg.changes_file) + # Try to get an included dsc + dsc = None + (status, _) = u.load_dsc() + if status: + dsc = u.pkg.dsc + cnf = Config() bcc = "X-DAK: dak process-new" if cnf.has_key("Dinstall::Bcc"): @@ -882,6 +690,7 @@ def do_pkg(changes_file, session): u.Subst["__BCC__"] = bcc files = u.pkg.files + u.check_distributions() for deb_filename, f in files.items(): if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"): u.binary_file_checks(deb_filename, session) @@ -898,35 +707,39 @@ def do_pkg(changes_file, session): if not recheck(u, session): return - # FIXME: This does need byhand checks added! - new = determine_new(u.pkg.changes, files) - if new: + new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, dsc=dsc, session=session) + if byhand: + do_byhand(u, session) + elif new: do_new(u, session) else: try: check_daily_lock() - do_accept(u, session) + new_accept(u, Options["No-Action"], session) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" print "Dinstall in the locked area, cant process packages, come back later" -# (new, byhand) = check_status(files) -# if new or byhand: -# if new: -# do_new(u, session) -# if byhand: -# do_byhand(u, session) -# (new, byhand) = check_status(files) - -# if not new and not byhand: -# try: -# check_daily_lock() -# do_accept(u) -# except CantGetLockError: -# print "Hello? Operator! Give me the number for 911!" -# print "Dinstall in the locked area, cant process packages, come back later" - except AlreadyLockedError, e: + + except AlreadyLockedError as e: print "Seems to be locked by %s already, skipping..." % (e) +def show_new_comments(changes_files, session): + sources = set() + query = """SELECT package, version, comment, author + FROM new_comments + WHERE package IN ('""" + + for changes in changes_files: + sources.add(os.path.basename(changes).split("_")[0]) + + query += "%s') ORDER BY package, version" % "', '".join(sources) + r = session.execute(query) + + for i in r: + print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3]) + + session.commit() + ################################################################################ def end(): @@ -952,19 +765,23 @@ def main(): session = DBConn().session() Arguments = [('a',"automatic","Process-New::Options::Automatic"), + ('b',"no-binaries","Process-New::Options::No-Binaries"), + ('c',"comments","Process-New::Options::Comments"), ('h',"help","Process-New::Options::Help"), ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), ('t',"trainee","Process-New::Options::Trainee"), ('n',"no-action","Process-New::Options::No-Action")] - for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]: + for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]: if not cnf.has_key("Process-New::Options::%s" % (i)): cnf["Process-New::Options::%s" % (i)] = "" changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv) if len(changes_files) == 0: new_queue = get_policy_queue('new', session ); - changes_files = utils.get_changes_files(new_queue.path) + changes_paths = [ os.path.join(new_queue.path, j) for j in utils.get_changes_files(new_queue.path) ] + else: + changes_paths = [ os.path.abspath(j) for j in changes_files ] Options = cnf.SubTree("Process-New::Options") @@ -973,28 +790,28 @@ def main(): if not Options["No-Action"]: try: - Logger = daklog.Logger(cnf, "process-new") - except CantOpenError, e: + Logger = daklog.Logger("process-new") + except CantOpenError as e: Options["Trainee"] = "True" Sections = Section_Completer(session) Priorities = Priority_Completer(session) readline.parse_and_bind("tab: complete") - if len(changes_files) > 1: + if len(changes_paths) > 1: sys.stderr.write("Sorting changes...\n") - changes_files = sort_changes(changes_files, session) - - # Kill me now? **FIXME** - cnf["Dinstall::Options::No-Mail"] = "" + changes_files = sort_changes(changes_paths, session, Options["No-Binaries"]) - for changes_file in changes_files: - changes_file = utils.validate_changes_file_arg(changes_file, 0) - if not changes_file: - continue - print "\n" + changes_file + if Options["Comments"]: + show_new_comments(changes_files, session) + else: + for changes_file in changes_files: + changes_file = utils.validate_changes_file_arg(changes_file, 0) + if not changes_file: + continue + print "\n" + os.path.basename(changes_file) - do_pkg (changes_file, session) + do_pkg (changes_file, session) end()