X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_new.py;h=d631b53aa7b90f67b6dd4d4537e4063efde90c82;hb=87bf163d12ec8328d87e1b2a2ca2239221a73bd6;hp=c0bc51f371bebe007c69441afc134e98678d8789;hpb=f4376ddbdb29017d84f876b06ebf0aed14c4e811;p=dak.git diff --git a/dak/process_new.py b/dak/process_new.py index c0bc51f3..d631b53a 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -1,8 +1,13 @@ #!/usr/bin/env python +# vim:set et ts=4 sw=4: -# Handles NEW and BYHAND packages -# Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 James Troup +""" Handles NEW and BYHAND packages +@contact: Debian FTP Master +@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -36,19 +41,31 @@ ################################################################################ -import copy, errno, os, readline, stat, sys, time +from __future__ import with_statement + +import copy +import errno +import os +import readline +import stat +import sys +import time +import contextlib +import pwd import apt_pkg, apt_inst import examine_package from daklib import database from daklib import logging from daklib import queue from daklib import utils +from daklib.regexes import re_no_epoch, re_default_answer, re_isanum +from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError # Globals -Cnf = None +Cnf = None #: Configuration, apt_pkg.Configuration Options = None Upload = None -projectB = None +projectB = None #: database connection, pgobject Logger = None Priorities = None @@ -81,7 +98,7 @@ def recheck(): source_package = files[f]["source package"] if not Upload.pkg.changes["architecture"].has_key("source") \ and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()): - source_epochless_version = utils.re_no_epoch.sub('', source_version) + source_epochless_version = re_no_epoch.sub('', source_version) dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) found = 0 for q in ["Accepted", "Embargoed", "Unembargoed"]: @@ -93,15 +110,15 @@ def recheck(): # Version and file overwrite checks if files[f]["type"] == "deb": - reject(Upload.check_binary_against_db(f)) + reject(Upload.check_binary_against_db(f), "") elif files[f]["type"] == "dsc": - reject(Upload.check_source_against_db(f)) + reject(Upload.check_source_against_db(f), "") (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f) reject(reject_msg, "") if reject_message.find("Rejected") != -1: answer = "XXX" - if Options["No-Action"] or Options["Automatic"]: + if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]: answer = 'S' print "REJECT\n" + reject_message, @@ -109,7 +126,7 @@ def recheck(): while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt) - m = queue.re_default_answer.match(prompt) + m = re_default_answer.match(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() @@ -151,7 +168,7 @@ def indiv_sg_compare (a, b): def sg_compare (a, b): a = a[1] b = b[1] - """Sort by have note, time of oldest upload.""" + """Sort by have note, source already in database and time of oldest upload.""" # Sort by have note a_note_state = a["note_state"] b_note_state = b["note_state"] @@ -159,6 +176,10 @@ def sg_compare (a, b): return -1 elif a_note_state > b_note_state: return 1 + # Sort by source already in database (descending) + source_in_database = cmp(a["source_in_database"], b["source_in_database"]) + if source_in_database: + return -source_in_database # Sort by time of oldest upload return cmp(a["oldest"], b["oldest"]) @@ -193,6 +214,9 @@ def sort_changes(changes_files): per_source[source]["list"].append(cache[filename]) # Determine oldest time and have note status for each source group for source in per_source.keys(): + q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source) + ql = q.getresult() + per_source[source]["source_in_database"] = len(ql)>0 source_list = per_source[source]["list"] first = source_list[0] oldest = os.stat(first["filename"])[stat.ST_MTIME] @@ -201,7 +225,7 @@ def sort_changes(changes_files): mtime = os.stat(d["filename"])[stat.ST_MTIME] if mtime < oldest: oldest = mtime - have_note += (d.has_key("process-new note")) + have_note += (database.has_new_comment(d["source"], d["version"])) per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none @@ -222,6 +246,7 @@ def sort_changes(changes_files): class Section_Completer: def __init__ (self): self.sections = [] + self.matches = [] q = projectB.query("SELECT section FROM section") for i in q.getresult(): self.sections.append(i[0]) @@ -243,6 +268,7 @@ class Section_Completer: class Priority_Completer: def __init__ (self): self.priorities = [] + self.matches = [] q = projectB.query("SELECT priority FROM priority") for i in q.getresult(): self.priorities.append(i[0]) @@ -281,11 +307,10 @@ def print_new (new, indexed, file=sys.stdout): line = "%-20s %-20s %-20s" % (pkg, priority, section) line = line.strip()+'\n' file.write(line) - note = Upload.pkg.changes.get("process-new note") - if note: - print "*"*75 - print note - print "*"*75 + note = database.get_new_comments(Upload.pkg.changes.get("source")) + if len(note) > 0: + for line in note: + print line return broken, note ################################################################################ @@ -301,8 +326,8 @@ def index_range (index): def edit_new (new): # Write the current data to a temporary file - temp_filename = utils.temp_filename() - temp_file = utils.open_file(temp_filename, 'w') + (fd, temp_filename) = utils.temp_filename() + temp_file = os.fdopen(fd, 'w') print_new (new, 0, temp_file) temp_file.close() # Spawn an editor on that file @@ -357,7 +382,7 @@ def edit_index (new, index): while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt) - m = queue.re_default_answer.match(prompt) + m = re_default_answer.match(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() @@ -427,7 +452,7 @@ def edit_overrides (new): answer = answer[:1].upper() if answer == "E" or answer == "D": got_answer = 1 - elif queue.re_isanum.match (answer): + elif re_isanum.match (answer): answer = int(answer) if (answer < 1) or (answer > index): print "%s is not a valid index (%s). Please retry." % (answer, index_range(index)) @@ -447,24 +472,21 @@ def edit_overrides (new): def edit_note(note): # Write the current data to a temporary file - temp_filename = utils.temp_filename() - temp_file = utils.open_file(temp_filename, 'w') - temp_file.write(note) - temp_file.close() + (fd, temp_filename) = utils.temp_filename() editor = os.environ.get("EDITOR","vi") answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) temp_file = utils.open_file(temp_filename) - note = temp_file.read().rstrip() + newnote = temp_file.read().rstrip() temp_file.close() - print "Note:" - print utils.prefix_multi_line_string(note," ") + print "New Note:" + print utils.prefix_multi_line_string(newnote," ") prompt = "[D]one, Edit, Abandon, Quit ?" answer = "XXX" while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt) - m = queue.re_default_answer.search(prompt) + m = re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() @@ -474,8 +496,7 @@ def edit_note(note): elif answer == 'Q': end() sys.exit(0) - Upload.pkg.changes["process-new note"] = note - Upload.dump_vars(Cnf["Dir::Queue::New"]) + database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami()) ################################################################################ @@ -485,16 +506,18 @@ def check_pkg (): stdout_fd = sys.stdout try: sys.stdout = less_fd - examine_package.display_changes(Upload.pkg.changes_file) + changes = utils.parse_changes (Upload.pkg.changes_file) + examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file) files = Upload.pkg.files for f in files.keys(): if files[f].has_key("new"): ftype = files[f]["type"] if ftype == "deb": - examine_package.check_deb(f) + examine_package.check_deb(changes['distribution'], f) elif ftype == "dsc": - examine_package.check_dsc(f) + examine_package.check_dsc(changes['distribution'], f) finally: + examine_package.output_package_relations() sys.stdout = stdout_fd except IOError, e: if e.errno == errno.EPIPE: @@ -550,32 +573,37 @@ def add_overrides (new): ################################################################################ -def prod_maintainer (): +def prod_maintainer (note): # Here we prepare an editor and get them ready to prod... - temp_filename = utils.temp_filename() + (fd, temp_filename) = utils.temp_filename() + temp_file = os.fdopen(fd, 'w') + if len(note) > 0: + for line in note: + temp_file.write(line) + temp_file.close() editor = os.environ.get("EDITOR","vi") answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) - f = utils.open_file(temp_filename) - prod_message = "".join(f.readlines()) - f.close() + temp_fh = utils.open_file(temp_filename) + prod_message = "".join(temp_fh.readlines()) + temp_fh.close() print "Prod message:" print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1) prompt = "[P]rod, Edit, Abandon, Quit ?" answer = "XXX" while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt) - m = queue.re_default_answer.search(prompt) + m = re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() - os.unlink(temp_filename) - if answer == 'A': - return - elif answer == 'Q': - end() - sys.exit(0) + os.unlink(temp_filename) + if answer == 'A': + return + elif answer == 'Q': + end() + sys.exit(0) # Otherwise, do the proding... user_email_address = utils.whoami() + " <%s>" % ( Cnf["Dinstall::MyAdminAddress"]) @@ -653,30 +681,37 @@ def do_new(): while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt) - m = queue.re_default_answer.search(prompt) + m = re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() - if answer == 'A': - done = add_overrides (new) + if answer == 'A' and not Options["Trainee"]: + try: + check_daily_lock() + done = add_overrides (new) + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'C': check_pkg() - elif answer == 'E': + elif answer == 'E' and not Options["Trainee"]: new = edit_overrides (new) - elif answer == 'M': - aborted = Upload.do_reject(1, Options["Manual-Reject"]) + elif answer == 'M' and not Options["Trainee"]: + aborted = Upload.do_reject(manual=1, + reject_message=Options["Manual-Reject"], + note=database.get_new_comments(changes.get("source", ""))) if not aborted: os.unlink(Upload.pkg.changes_file[:-8]+".dak") done = 1 elif answer == 'N': - edit_note(changes.get("process-new note", "")) - elif answer == 'P': - prod_maintainer() - elif answer == 'R': + edit_note(database.get_new_comments(changes.get("source", ""))) + elif answer == 'P' and not Options["Trainee"]: + prod_maintainer(database.get_new_comments(changes.get("source", ""))) + elif answer == 'R' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() if confirm == "y": - del changes["process-new note"] + database.delete_new_comments(changes.get("source"), changes.get("version")) elif answer == 'S': done = 1 elif answer == 'Q': @@ -694,6 +729,7 @@ def usage (exit_code=0): -C, --comments-dir=DIR use DIR as comments-dir, for [o-]p-u-new -m, --manual-reject=MSG manual reject with `msg' -n, --no-action don't do anything + -t, --trainee FTP Trainee mode -V, --version display the version number and exit""" sys.exit(exit_code) @@ -708,13 +744,17 @@ def init(): ('h',"help","Process-New::Options::Help"), ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"), ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), + ('t',"trainee","Process-New::Options::Trainee"), ('n',"no-action","Process-New::Options::No-Action")] - for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]: + for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]: if not Cnf.has_key("Process-New::Options::%s" % (i)): Cnf["Process-New::Options::%s" % (i)] = "" changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) + if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""): + changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"]) + Options = Cnf.SubTree("Process-New::Options") if Options["Help"]: @@ -723,7 +763,10 @@ def init(): Upload = queue.Upload(Cnf) if not Options["No-Action"]: - Logger = Upload.Logger = logging.Logger(Cnf, "process-new") + try: + Logger = Upload.Logger = logging.Logger(Cnf, "process-new") + except CantOpenError, e: + Options["Trainee"] = "Oh yes" projectB = Upload.projectB @@ -762,7 +805,7 @@ def do_byhand(): while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt) - m = queue.re_default_answer.search(prompt) + m = re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() @@ -783,22 +826,41 @@ def do_byhand(): ################################################################################ -def get_accept_lock(): - retry = 0 - while retry < 10: - try: - os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) - retry = 10 - except OSError, e: - if e.errno == errno.EACCES or e.errno == errno.EEXIST: - retry += 1 - if (retry >= 10): - utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.") - else: - print("Unable to get accepted lock (try %d of 10)" % retry) - time.sleep(60) - else: - raise +def check_daily_lock(): + """ + Raises CantGetLockError if the dinstall daily.lock exists. + """ + + try: + os.open(Cnf["Process-New::DinstallLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) + except OSError, e: + if e.errno == errno.EEXIST or e.errno == errno.EACCES: + raise CantGetLockError + + os.unlink(Cnf["Process-New::DinstallLockFile"]) + + +@contextlib.contextmanager +def lock_package(package): + """ + Lock C{package} so that noone else jumps in processing it. + + @type package: string + @param package: source package name to lock + """ + + path = os.path.join(Cnf["Process-New::LockDir"], package) + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) + except OSError, e: + if e.errno == errno.EEXIST or e.errno == errno.EACCES: + user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '') + raise AlreadyLockedError, user + + try: + yield fd + finally: + os.unlink(path) def move_to_dir (dest, perms=0660, changesperms=0664): utils.move (Upload.pkg.changes_file, dest, perms=changesperms) @@ -806,10 +868,83 @@ def move_to_dir (dest, perms=0660, changesperms=0664): for f in file_keys: utils.move (f, dest, perms=perms) +def is_source_in_queue_dir(qdir): + entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"]) + and x.endswith(".changes") ] + for entry in entries: + # read the .dak + u = queue.Upload(Cnf) + u.pkg.changes_file = os.path.join(qdir, entry) + u.update_vars() + if not u.pkg.changes["architecture"].has_key("source"): + # another binary upload, ignore + continue + if Upload.pkg.changes["version"] != u.pkg.changes["version"]: + # another version, ignore + continue + # found it! + return True + return False + +def move_to_holding(suite, queue_dir): + print "Moving to %s holding area." % (suite.upper(),) + if Options["No-Action"]: + return + Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file]) + Upload.dump_vars(queue_dir) + move_to_dir(queue_dir, perms=0664) + os.unlink(Upload.pkg.changes_file[:-8]+".dak") + +def _accept(): + if Options["No-Action"]: + return + (summary, short_summary) = Upload.build_summaries() + Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"]) + os.unlink(Upload.pkg.changes_file[:-8]+".dak") + +def do_accept_stableupdate(suite, q): + queue_dir = Cnf["Dir::Queue::%s" % (q,)] + if not Upload.pkg.changes["architecture"].has_key("source"): + # It is not a sourceful upload. So its source may be either in p-u + # holding, in new, in accepted or already installed. + if is_source_in_queue_dir(queue_dir): + # It's in p-u holding, so move it there. + print "Binary-only upload, source in %s." % (q,) + move_to_holding(suite, queue_dir) + elif Upload.source_exists(Upload.pkg.changes["source"], + Upload.pkg.changes["version"]): + # dak tells us that there is source available. At time of + # writing this means that it is installed, so put it into + # accepted. + print "Binary-only upload, source installed." + _accept() + elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]): + # The source is in accepted, the binary cleared NEW: accept it. + print "Binary-only upload, source in accepted." + _accept() + elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]): + # It's in NEW. We expect the source to land in p-u holding + # pretty soon. + print "Binary-only upload, source in new." + move_to_holding(suite, queue_dir) + else: + # No case applicable. Bail out. Return will cause the upload + # to be skipped. + print "ERROR" + print "Stable update failed. Source not found." + return + else: + # We are handling a sourceful upload. Move to accepted if currently + # in p-u holding and to p-u holding otherwise. + if is_source_in_queue_dir(queue_dir): + print "Sourceful upload in %s, accepting." % (q,) + _accept() + else: + move_to_holding(suite, queue_dir) + def do_accept(): print "ACCEPT" if not Options["No-Action"]: - get_accept_lock() (summary, short_summary) = Upload.build_summaries() if Cnf.FindB("Dinstall::SecurityQueueHandling"): Upload.dump_vars(Cnf["Dir::Queue::Embargoed"]) @@ -818,9 +953,17 @@ def do_accept(): # Check for override disparities Upload.Subst["__SUMMARY__"] = summary else: - Upload.accept(summary, short_summary) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") - os.unlink(Cnf["Process-New::AcceptedLockFile"]) + # Stable updates need to be copied to proposed-updates holding + # area instead of accepted. Sourceful uploads need to go + # to it directly, binaries only if the source has not yet been + # accepted into p-u. + for suite, q in [("proposed-updates", "ProposedUpdates"), + ("oldstable-proposed-updates", "OldProposedUpdates")]: + if not Upload.pkg.changes["distribution"].has_key(suite): + continue + return do_accept_stableupdate(suite, q) + # Just a normal upload, accept it... + _accept() def check_status(files): new = byhand = 0 @@ -838,19 +981,30 @@ def do_pkg(changes_file): Upload.update_subst() files = Upload.pkg.files - if not recheck(): - return - - (new, byhand) = check_status(files) - if new or byhand: - if new: - do_new() - if byhand: - do_byhand() - (new, byhand) = check_status(files) + try: + check_daily_lock() + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" + sys.exit(1) - if not new and not byhand: - do_accept() + try: + with lock_package(Upload.pkg.changes["source"]): + if not recheck(): + return + + (new, byhand) = check_status(files) + if new or byhand: + if new: + do_new() + if byhand: + do_byhand() + (new, byhand) = check_status(files) + + if not new and not byhand: + do_accept() + except AlreadyLockedError, e: + print "Seems to be locked by %s already, skipping..." % (e) ################################################################################ @@ -865,7 +1019,7 @@ def end(): sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes)))) Logger.log(["total",accept_count,accept_bytes]) - if not Options["No-Action"]: + if not Options["No-Action"] and not Options["Trainee"]: Logger.close() ################################################################################ @@ -949,6 +1103,7 @@ def main(): if not changes_file: continue print "\n" + changes_file + do_pkg (changes_file) end()