X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fnew_security_install.py;h=854a5834919e96aac6be9f64eb9ace94bec8d5e7;hb=f0bfd37e7286156598d79b53501ebe2000bb7924;hp=a35084620a9c3ac214572e66ee9cf230b5977649;hpb=907deca03b25dc8ce65c89062ff3d7ca7a358976;p=dak.git diff --git a/dak/new_security_install.py b/dak/new_security_install.py index a3508462..854a5834 100755 --- a/dak/new_security_install.py +++ b/dak/new_security_install.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Wrapper for Debian Security team +""" Wrapper for Debian Security team """ # Copyright (C) 2006 Anthony Towns # This program is free software; you can redistribute it and/or modify @@ -20,10 +20,13 @@ ################################################################################ -import daklib.queue, daklib.logging, daklib.utils, daklib.database -import apt_pkg, os, sys, pwd, time, re, commands +import apt_pkg, os, sys, pwd, time, commands -re_taint_free = re.compile(r"^['/;\-\+\.\s\w]+$"); +from daklib import queue +from daklib import daklog +from daklib import utils +from daklib.dbconn import DBConn, get_build_queue, get_suite_architectures +from daklib.regexes import re_taint_free Cnf = None Options = None @@ -37,13 +40,14 @@ srcverarches = {} def init(): global Cnf, Upload, Options, Logger - Cnf = daklib.utils.get_conf() + Cnf = utils.get_conf() Cnf["Dinstall::Options::No-Mail"] = "y" Arguments = [('h', "help", "Security-Install::Options::Help"), ('a', "automatic", "Security-Install::Options::Automatic"), ('n', "no-action", "Security-Install::Options::No-Action"), ('s', "sudo", "Security-Install::Options::Sudo"), (' ', "no-upload", "Security-Install::Options::No-Upload"), + ('u', "fg-upload", "Security-Install::Options::Foreground-Upload"), (' ', "drop-advisory", "Security-Install::Options::Drop-Advisory"), ('A', "approve", "Security-Install::Options::Approve"), ('R', "reject", "Security-Install::Options::Reject"), @@ -56,9 +60,7 @@ def init(): Options = Cnf.SubTree("Security-Install::Options") - whoami = os.getuid() - whoamifull = pwd.getpwuid(whoami) - username = whoamifull[0] + username = utils.getusername() if username != "dak": print "Non-dak user: %s" % username Options["Sudo"] = "y" @@ -68,11 +70,13 @@ def init(): sys.exit(0) if len(arguments) == 0: - daklib.utils.fubar("Process what?") + utils.fubar("Process what?") - Upload = daklib.queue.Upload(Cnf) + Upload = queue.Upload(Cnf) + if Options["No-Action"]: + Options["Sudo"] = "" if not Options["Sudo"] and not Options["No-Action"]: - Logger = Upload.Logger = daklib.logging.Logger(Cnf, "new-security-install") + Logger = Upload.Logger = daklog.Logger(Cnf, "new-security-install") return arguments @@ -94,9 +98,9 @@ def load_args(arguments): changesfiles = {} for a in arguments: if "/" in a: - daklib.utils.fubar("can only deal with files in the current directory") + utils.fubar("can only deal with files in the current directory") if not a.endswith(".changes"): - daklib.utils.fubar("not a .changes file: %s" % (a)) + utils.fubar("not a .changes file: %s" % (a)) Upload.init_vars() Upload.pkg.changes_file = a Upload.update_vars() @@ -108,7 +112,7 @@ def load_args(arguments): adv_ids = adv_ids.keys() if len(adv_ids) > 1: - daklib.utils.fubar("multiple advisories selected: %s" % (", ".join(adv_ids))) + utils.fubar("multiple advisories selected: %s" % (", ".join(adv_ids))) if adv_ids == []: advisory = None else: @@ -131,7 +135,7 @@ def load_adv_changes(): continue if c not in changes: changes.append(c) - srcver = "%s %s" % (Upload.pkg.changes["source"], + srcver = "%s %s" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]) srcverarches.setdefault(srcver, {}) for arch in Upload.pkg.changes["architecture"].keys(): @@ -148,9 +152,9 @@ def advisory_info(): svs = srcverarches.keys() svs.sort() for sv in svs: - as = srcverarches[sv].keys() - as.sort() - print " %s (%s)" % (sv, ", ".join(as)) + as_ = srcverarches[sv].keys() + as_.sort() + print " %s (%s)" % (sv, ", ".join(as_)) def prompt(opts, default): p = "" @@ -168,9 +172,9 @@ def prompt(opts, default): a = default while a not in v: - a = daklib.utils.our_raw_input(p) + default + a = utils.our_raw_input(p) + default a = a[:1].upper() - + return v[a] def add_changes(extras): @@ -189,7 +193,7 @@ def add_changes(extras): def yes_no(prompt): if Options["Automatic"]: return True while 1: - answer = daklib.utils.our_raw_input(prompt + " ").lower() + answer = utils.our_raw_input(prompt + " ").lower() if answer in "yn": return answer == "y" print "Invalid answer; please try again." @@ -197,9 +201,111 @@ def yes_no(prompt): def do_upload(): if Options["No-Upload"]: print "Not uploading as requested" - return + elif Options["Foreground-Upload"]: + actually_upload(changes) + else: + child = os.fork() + if child == 0: + actually_upload(changes) + os._exit(0) + print "Uploading in the background" + +def actually_upload(changes_files): + file_list = "" + suites = {} + component_mapping = {} + for component in Cnf.SubTree("Security-Install::ComponentMappings").List(): + component_mapping[component] = Cnf["Security-Install::ComponentMappings::%s" % (component)] + uploads = {}; # uploads[uri] = file_list + changesfiles = {}; # changesfiles[uri] = file_list + package_list = {} # package_list[source_name][version] + changes_files.sort(utils.changes_compare) + for changes_file in changes_files: + changes_file = utils.validate_changes_file_arg(changes_file) + # Reset variables + components = {} + upload_uris = {} + file_list = [] + Upload.init_vars() + # Parse the .dak file for the .changes file + Upload.pkg.changes_file = changes_file + Upload.update_vars() + files = Upload.pkg.files + changes = Upload.pkg.changes + dsc = Upload.pkg.dsc + # Build the file list for this .changes file + for file in files.keys(): + poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"], + utils.poolify(changes["source"], files[file]["component"]), + file) + file_list.append(poolname) + orig_component = files[file].get("original component", files[file]["component"]) + components[orig_component] = "" + # Determine the upload uri for this .changes file + for component in components.keys(): + upload_uri = component_mapping.get(component) + if upload_uri: + upload_uris[upload_uri] = "" + num_upload_uris = len(upload_uris.keys()) + if num_upload_uris == 0: + utils.fubar("%s: No valid upload URI found from components (%s)." + % (changes_file, ", ".join(components.keys()))) + elif num_upload_uris > 1: + utils.fubar("%s: more than one upload URI (%s) from components (%s)." + % (changes_file, ", ".join(upload_uris.keys()), + ", ".join(components.keys()))) + upload_uri = upload_uris.keys()[0] + # Update the file list for the upload uri + if not uploads.has_key(upload_uri): + uploads[upload_uri] = [] + uploads[upload_uri].extend(file_list) + # Update the changes list for the upload uri + if not changesfiles.has_key(upload_uri): + changesfiles[upload_uri] = [] + changesfiles[upload_uri].append(changes_file) + # Remember the suites and source name/version + for suite in changes["distribution"].keys(): + suites[suite] = "" + # Remember the source name and version + if changes["architecture"].has_key("source") and \ + changes["distribution"].has_key("testing"): + if not package_list.has_key(dsc["source"]): + package_list[dsc["source"]] = {} + package_list[dsc["source"]][dsc["version"]] = "" + + for uri in uploads.keys(): + uploads[uri].extend(changesfiles[uri]) + (host, path) = uri.split(":") + # file_list = " ".join(uploads[uri]) + print "Moving files to UploadQueue" + for filename in uploads[uri]: + utils.copy(filename, Cnf["Dir::Upload"]) + # .changes files have already been moved to queue/done by p-a + if not filename.endswith('.changes'): + remove_from_buildd(suites, filename) + #spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list)) + + if not Options["No-Action"]: + filename = "%s/testing-processed" % (Cnf["Dir::Log"]) + file = utils.open_file(filename, 'a') + for source in package_list.keys(): + for version in package_list[source].keys(): + file.write(" ".join([source, version])+'\n') + file.close() + +def remove_from_buildd(suites, filename): + """Check the buildd dir for each suite and remove the file if needed""" + builddbase = Cnf["Dir::QueueBuild"] + filebase = os.path.basename(filename) + for s in suites: + try: + os.unlink(os.path.join(builddbase, s, filebase)) + except OSError, e: + pass + # About no value printing this warning - it only confuses the security team, + # yet makes no difference otherwise. + #utils.warn("Problem removing %s from buildd queue %s [%s]" % (filebase, s, str(e))) - print "Would upload to ftp-master" # XXX def generate_advisory(template): global changes, advisory @@ -208,7 +314,7 @@ def generate_advisory(template): updated_pkgs = {}; # updated_pkgs[distro][arch][file] = {path,md5,size} for arg in changes: - arg = daklib.utils.validate_changes_file_arg(arg) + arg = utils.validate_changes_file_arg(arg) Upload.pkg.changes_file = arg Upload.init_vars() Upload.update_vars() @@ -229,7 +335,7 @@ def generate_advisory(template): md5 = files[file]["md5sum"] size = files[file]["size"] poolname = Cnf["Dir::PoolRoot"] + \ - daklib.utils.poolify(src, files[file]["component"]) + utils.poolify(src, files[file]["component"]) if arch == "source" and file.endswith(".dsc"): dscpoolname = poolname for suite in suites: @@ -273,7 +379,7 @@ def generate_advisory(template): Subst["__BCC__"] = "Bcc: %s" % (Cnf["Dinstall::Bcc"]) adv = "" - archive = Cnf["Archive::%s::PrimaryMirror" % (daklib.utils.where_am_i())] + archive = Cnf["Archive::%s::PrimaryMirror" % (utils.where_am_i())] for suite in updated_pkgs.keys(): ver = Cnf["Suite::%s::Version" % suite] if ver != "": ver += " " @@ -281,7 +387,7 @@ def generate_advisory(template): ver, suite) adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header)) - arches = Cnf.ValueList("Suite::%s::Architectures" % suite) + arches = [x.arch_name for x in get_suite_architectures(suite)] if "source" in arches: arches.remove("source") if "all" in arches: @@ -289,7 +395,7 @@ def generate_advisory(template): arches.sort() adv += "%s updates are available for %s.\n\n" % ( - suite.capitalize(), daklib.utils.join_with_commas_and(arches)) + suite.capitalize(), utils.join_with_commas_and(arches)) for a in ["source", "all"] + arches: if not updated_pkgs[suite].has_key(a): @@ -314,20 +420,19 @@ def generate_advisory(template): Subst["__ADVISORY_TEXT__"] = adv - adv = daklib.utils.TemplateSubst(Subst, template) + adv = utils.TemplateSubst(Subst, template) return adv - def spawn(command): if not re_taint_free.match(command): - daklib.utils.fubar("Invalid character in \"%s\"." % (command)) + utils.fubar("Invalid character in \"%s\"." % (command)) if Options["No-Action"]: print "[%s]" % (command) else: (result, output) = commands.getstatusoutput(command) if (result != 0): - daklib.utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result) + utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result) ##################### ! ! ! N O T E ! ! ! ##################### @@ -340,9 +445,9 @@ def spawn(command): def sudo(arg, fn, exit): if Options["Sudo"]: if advisory == None: - daklib.utils.fubar("Must set advisory name") - os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H", - "/usr/local/bin/dak new-security-install", "-"+arg, "--", advisory) + utils.fubar("Must set advisory name") + os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H", + "/usr/local/bin/dak", "new-security-install", "-"+arg, "--", advisory) else: fn() if exit: @@ -351,7 +456,7 @@ def sudo(arg, fn, exit): def do_Approve(): sudo("A", _do_Approve, True) def _do_Approve(): # 1. dump advisory in drafts - draft = "/org/security.debian.org/advisories/drafts/%s" % (advisory) + draft = "/org/security-master.debian.org/advisories/drafts/%s" % (advisory) print "Advisory in %s" % (draft) if not Options["No-Action"]: adv_file = "./advisory.%s" % (advisory) @@ -369,12 +474,15 @@ def _do_Approve(): # 3. run dak make-suite-file-list / apt-ftparchve / dak generate-releases print "Updating file lists for apt-ftparchive..." spawn("dak make-suite-file-list") + spawn("dak generate-filelist") print "Updating Packages and Sources files..." - spawn("apt-ftparchive generate %s" % (daklib.utils.which_apt_conf_file())) + spawn("/org/security-master.debian.org/dak/config/debian-security/map.sh") + spawn("apt-ftparchive generate %s" % (utils.which_apt_conf_file())) print "Updating Release files..." spawn("dak generate-releases") print "Triggering security mirrors..." - spawn("sudo -u archvsync /home/archvsync/signal_security") + spawn("/org/security-master.debian.org/dak/config/debian-security/make-mirror.sh") + spawn("sudo -u archvsync -H /home/archvsync/signal_security") # 4. chdir to done - do upload if not Options["No-Action"]: @@ -384,11 +492,13 @@ def _do_Approve(): def do_Disembargo(): sudo("D", _do_Disembargo, True) def _do_Disembargo(): if os.getcwd() != Cnf["Dir::Queue::Embargoed"].rstrip("/"): - daklib.utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"]) + utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"]) + + session = DBConn().session() dest = Cnf["Dir::Queue::Unembargoed"] - emb_q = daklib.database.get_or_set_queue_id("embargoed") - une_q = daklib.database.get_or_set_queue_id("unembargoed") + emb_q = get_build_queue("embargoed", session) + une_q = get_build_queue("unembargoed", session) for c in changes: print "Disembargoing %s" % (c) @@ -399,7 +509,8 @@ def _do_Disembargo(): if "source" in Upload.pkg.changes["architecture"].keys(): print "Adding %s %s to disembargo table" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]) - Upload.projectB.query("INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"])) + session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", + {'package': Upload.pkg.changes["source"], 'version': Upload.pkg.changes["version"]}) files = {} for suite in Upload.pkg.changes["distribution"].keys(): @@ -412,25 +523,30 @@ def _do_Disembargo(): files[os.path.join(dest_dir, file)] = 1 files = files.keys() - Upload.projectB.query("BEGIN WORK") for f in files: - Upload.projectB.query("UPDATE queue_build SET queue = %s WHERE filename = '%s' AND queue = %s" % (une_q, f, emb_q)) - Upload.projectB.query("COMMIT WORK") + session.execute("UPDATE queue_build SET queue = :unembargoed WHERE filename = :filename AND queue = :embargoed", + {'unembargoed': une_q.queue_id, 'filename': f, 'embargoed': emb_q.queue_id}) + session.commit() for file in Upload.pkg.files.keys(): - daklib.utils.copy(file, os.path.join(dest, file)) + utils.copy(file, os.path.join(dest, file)) os.unlink(file) for c in changes: - daklib.utils.copy(c, os.path.join(dest, c)) + utils.copy(c, os.path.join(dest, c)) os.unlink(c) - k = c[:8] + ".dak" - daklib.utils.copy(k, os.path.join(dest, k)) + k = c[:-8] + ".dak" + utils.copy(k, os.path.join(dest, k)) os.unlink(k) + session.commit() + def do_Reject(): sudo("R", _do_Reject, True) def _do_Reject(): global changes + + session = DBConn().session() + for c in changes: print "Rejecting %s..." % (c) Upload.init_vars() @@ -450,19 +566,21 @@ def _do_Reject(): aborted = Upload.do_reject() if not aborted: - os.unlink(c[:-8]+".katie") + os.unlink(c[:-8]+".dak") for f in files: - Upload.projectB.query( - "DELETE FROM queue_build WHERE filename = '%s'" % (f)) + session.execute("DELETE FROM queue_build WHERE filename = :filename", + {'filename': f}) os.unlink(f) print "Updating buildd information..." - spawn("/org/security.debian.org/katie/cron.buildd-security") + spawn("/org/security-master.debian.org/dak/config/debian-security/cron.buildd") adv_file = "./advisory.%s" % (advisory) if os.path.exists(adv_file): os.unlink(adv_file) + session.commit() + def do_DropAdvisory(): for c in changes: Upload.init_vars() @@ -475,11 +593,11 @@ def do_DropAdvisory(): def do_Edit(): adv_file = "./advisory.%s" % (advisory) if not os.path.exists(adv_file): - daklib.utils.copy(Cnf["Dir::Templates"]+"/security-install.advisory", adv_file) + utils.copy(Cnf["Dir::Templates"]+"/security-install.advisory", adv_file) editor = os.environ.get("EDITOR", "vi") result = os.system("%s %s" % (editor, adv_file)) if result != 0: - daklib.utils.fubar("%s invocation failed for %s." % (editor, adv_file)) + utils.fubar("%s invocation failed for %s." % (editor, adv_file)) def do_Show(): adv_file = "./advisory.%s" % (advisory) @@ -515,10 +633,10 @@ def main(): add_changes(extras) if not advisory: - daklib.utils.fubar("Must specify an advisory id") + utils.fubar("Must specify an advisory id") if not changes: - daklib.utils.fubar("No changes specified") + utils.fubar("No changes specified") if Options["Approve"]: advisory_info() @@ -543,7 +661,7 @@ def main(): if os.getcwd() == Cnf["Dir::Queue::Embargoed"].rstrip("/"): opts.append("Disembargo") opts += ["Show advisory", "Reject", "Quit"] - + advisory_info() what = prompt(opts, default) @@ -560,7 +678,7 @@ def main(): elif what == "Reject": do_Reject() else: - daklib.utils.fubar("Impossible answer '%s', wtf?" % (what)) + utils.fubar("Impossible answer '%s', wtf?" % (what)) ################################################################################