X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fnew_security_install.py;h=854a5834919e96aac6be9f64eb9ace94bec8d5e7;hb=f0bfd37e7286156598d79b53501ebe2000bb7924;hp=99378e7a47e5efac7bf92dd0aca5d02297093220;hpb=3262e8e139dadb4d47ec4575a63f43587cbf26ef;p=dak.git diff --git a/dak/new_security_install.py b/dak/new_security_install.py index 99378e7a..854a5834 100755 --- a/dak/new_security_install.py +++ b/dak/new_security_install.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Wrapper for Debian Security team +""" Wrapper for Debian Security team """ # Copyright (C) 2006 Anthony Towns # This program is free software; you can redistribute it and/or modify @@ -20,10 +20,13 @@ ################################################################################ -import daklib.queue, daklib.logging, daklib.utils, daklib.database -import apt_pkg, os, sys, pwd, time, re, commands +import apt_pkg, os, sys, pwd, time, commands -re_taint_free = re.compile(r"^['/;\-\+\.~\s\w]+$"); +from daklib import queue +from daklib import daklog +from daklib import utils +from daklib.dbconn import DBConn, get_build_queue, get_suite_architectures +from daklib.regexes import re_taint_free Cnf = None Options = None @@ -37,7 +40,7 @@ srcverarches = {} def init(): global Cnf, Upload, Options, Logger - Cnf = daklib.utils.get_conf() + Cnf = utils.get_conf() Cnf["Dinstall::Options::No-Mail"] = "y" Arguments = [('h', "help", "Security-Install::Options::Help"), ('a', "automatic", "Security-Install::Options::Automatic"), @@ -57,9 +60,7 @@ def init(): Options = Cnf.SubTree("Security-Install::Options") - whoami = os.getuid() - whoamifull = pwd.getpwuid(whoami) - username = whoamifull[0] + username = utils.getusername() if username != "dak": print "Non-dak user: %s" % username Options["Sudo"] = "y" @@ -69,13 +70,13 @@ def init(): sys.exit(0) if len(arguments) == 0: - daklib.utils.fubar("Process what?") + utils.fubar("Process what?") - Upload = daklib.queue.Upload(Cnf) + Upload = queue.Upload(Cnf) if Options["No-Action"]: Options["Sudo"] = "" if not Options["Sudo"] and not Options["No-Action"]: - Logger = Upload.Logger = daklib.logging.Logger(Cnf, "new-security-install") + Logger = Upload.Logger = daklog.Logger(Cnf, "new-security-install") return arguments @@ -97,9 +98,9 @@ def load_args(arguments): changesfiles = {} for a in arguments: if "/" in a: - daklib.utils.fubar("can only deal with files in the current directory") + utils.fubar("can only deal with files in the current directory") if not a.endswith(".changes"): - daklib.utils.fubar("not a .changes file: %s" % (a)) + utils.fubar("not a .changes file: %s" % (a)) Upload.init_vars() Upload.pkg.changes_file = a Upload.update_vars() @@ -111,7 +112,7 @@ def load_args(arguments): adv_ids = adv_ids.keys() if len(adv_ids) > 1: - daklib.utils.fubar("multiple advisories selected: %s" % (", ".join(adv_ids))) + utils.fubar("multiple advisories selected: %s" % (", ".join(adv_ids))) if adv_ids == []: advisory = None else: @@ -134,7 +135,7 @@ def load_adv_changes(): continue if c not in changes: changes.append(c) - srcver = "%s %s" % (Upload.pkg.changes["source"], + srcver = "%s %s" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]) srcverarches.setdefault(srcver, {}) for arch in Upload.pkg.changes["architecture"].keys(): @@ -151,9 +152,9 @@ def advisory_info(): svs = srcverarches.keys() svs.sort() for sv in svs: - as = srcverarches[sv].keys() - as.sort() - print " %s (%s)" % (sv, ", ".join(as)) + as_ = srcverarches[sv].keys() + as_.sort() + print " %s (%s)" % (sv, ", ".join(as_)) def prompt(opts, default): p = "" @@ -171,9 +172,9 @@ def prompt(opts, default): a = default while a not in v: - a = daklib.utils.our_raw_input(p) + default + a = utils.our_raw_input(p) + default a = a[:1].upper() - + return v[a] def add_changes(extras): @@ -192,7 +193,7 @@ def add_changes(extras): def yes_no(prompt): if Options["Automatic"]: return True while 1: - answer = daklib.utils.our_raw_input(prompt + " ").lower() + answer = utils.our_raw_input(prompt + " ").lower() if answer in "yn": return answer == "y" print "Invalid answer; please try again." @@ -204,10 +205,10 @@ def do_upload(): actually_upload(changes) else: child = os.fork() - if child == 0: - actually_upload(changes) - os._exit(0) - print "Uploading in the background" + if child == 0: + actually_upload(changes) + os._exit(0) + print "Uploading in the background" def actually_upload(changes_files): file_list = "" @@ -218,9 +219,9 @@ def actually_upload(changes_files): uploads = {}; # uploads[uri] = file_list changesfiles = {}; # changesfiles[uri] = file_list package_list = {} # package_list[source_name][version] - changes_files.sort(daklib.utils.changes_compare) + changes_files.sort(utils.changes_compare) for changes_file in changes_files: - changes_file = daklib.utils.validate_changes_file_arg(changes_file) + changes_file = utils.validate_changes_file_arg(changes_file) # Reset variables components = {} upload_uris = {} @@ -232,14 +233,10 @@ def actually_upload(changes_files): files = Upload.pkg.files changes = Upload.pkg.changes dsc = Upload.pkg.dsc - # We have the changes, now return if its amd64, to not upload them to ftp-master - if changes["distribution"].has_key("oldstable-security") and changes["architecture"].has_key("amd64"): - print "Not uploading amd64 oldstable-security changes to ftp-master\n" - continue # Build the file list for this .changes file for file in files.keys(): poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"], - daklib.utils.poolify(changes["source"], files[file]["component"]), + utils.poolify(changes["source"], files[file]["component"]), file) file_list.append(poolname) orig_component = files[file].get("original component", files[file]["component"]) @@ -251,10 +248,10 @@ def actually_upload(changes_files): upload_uris[upload_uri] = "" num_upload_uris = len(upload_uris.keys()) if num_upload_uris == 0: - daklib.utils.fubar("%s: No valid upload URI found from components (%s)." + utils.fubar("%s: No valid upload URI found from components (%s)." % (changes_file, ", ".join(components.keys()))) elif num_upload_uris > 1: - daklib.utils.fubar("%s: more than one upload URI (%s) from components (%s)." + utils.fubar("%s: more than one upload URI (%s) from components (%s)." % (changes_file, ", ".join(upload_uris.keys()), ", ".join(components.keys()))) upload_uri = upload_uris.keys()[0] @@ -279,18 +276,37 @@ def actually_upload(changes_files): for uri in uploads.keys(): uploads[uri].extend(changesfiles[uri]) (host, path) = uri.split(":") - file_list = " ".join(uploads[uri]) - print "Uploading files to %s..." % (host) - spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list)) + # file_list = " ".join(uploads[uri]) + print "Moving files to UploadQueue" + for filename in uploads[uri]: + utils.copy(filename, Cnf["Dir::Upload"]) + # .changes files have already been moved to queue/done by p-a + if not filename.endswith('.changes'): + remove_from_buildd(suites, filename) + #spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list)) if not Options["No-Action"]: filename = "%s/testing-processed" % (Cnf["Dir::Log"]) - file = daklib.utils.open_file(filename, 'a') + file = utils.open_file(filename, 'a') for source in package_list.keys(): for version in package_list[source].keys(): file.write(" ".join([source, version])+'\n') file.close() +def remove_from_buildd(suites, filename): + """Check the buildd dir for each suite and remove the file if needed""" + builddbase = Cnf["Dir::QueueBuild"] + filebase = os.path.basename(filename) + for s in suites: + try: + os.unlink(os.path.join(builddbase, s, filebase)) + except OSError, e: + pass + # About no value printing this warning - it only confuses the security team, + # yet makes no difference otherwise. + #utils.warn("Problem removing %s from buildd queue %s [%s]" % (filebase, s, str(e))) + + def generate_advisory(template): global changes, advisory @@ -298,7 +314,7 @@ def generate_advisory(template): updated_pkgs = {}; # updated_pkgs[distro][arch][file] = {path,md5,size} for arg in changes: - arg = daklib.utils.validate_changes_file_arg(arg) + arg = utils.validate_changes_file_arg(arg) Upload.pkg.changes_file = arg Upload.init_vars() Upload.update_vars() @@ -319,7 +335,7 @@ def generate_advisory(template): md5 = files[file]["md5sum"] size = files[file]["size"] poolname = Cnf["Dir::PoolRoot"] + \ - daklib.utils.poolify(src, files[file]["component"]) + utils.poolify(src, files[file]["component"]) if arch == "source" and file.endswith(".dsc"): dscpoolname = poolname for suite in suites: @@ -363,7 +379,7 @@ def generate_advisory(template): Subst["__BCC__"] = "Bcc: %s" % (Cnf["Dinstall::Bcc"]) adv = "" - archive = Cnf["Archive::%s::PrimaryMirror" % (daklib.utils.where_am_i())] + archive = Cnf["Archive::%s::PrimaryMirror" % (utils.where_am_i())] for suite in updated_pkgs.keys(): ver = Cnf["Suite::%s::Version" % suite] if ver != "": ver += " " @@ -371,7 +387,7 @@ def generate_advisory(template): ver, suite) adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header)) - arches = Cnf.ValueList("Suite::%s::Architectures" % suite) + arches = [x.arch_name for x in get_suite_architectures(suite)] if "source" in arches: arches.remove("source") if "all" in arches: @@ -379,7 +395,7 @@ def generate_advisory(template): arches.sort() adv += "%s updates are available for %s.\n\n" % ( - suite.capitalize(), daklib.utils.join_with_commas_and(arches)) + suite.capitalize(), utils.join_with_commas_and(arches)) for a in ["source", "all"] + arches: if not updated_pkgs[suite].has_key(a): @@ -404,19 +420,19 @@ def generate_advisory(template): Subst["__ADVISORY_TEXT__"] = adv - adv = daklib.utils.TemplateSubst(Subst, template) + adv = utils.TemplateSubst(Subst, template) return adv def spawn(command): if not re_taint_free.match(command): - daklib.utils.fubar("Invalid character in \"%s\"." % (command)) + utils.fubar("Invalid character in \"%s\"." % (command)) if Options["No-Action"]: print "[%s]" % (command) else: (result, output) = commands.getstatusoutput(command) if (result != 0): - daklib.utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result) + utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result) ##################### ! ! ! N O T E ! ! ! ##################### @@ -429,8 +445,8 @@ def spawn(command): def sudo(arg, fn, exit): if Options["Sudo"]: if advisory == None: - daklib.utils.fubar("Must set advisory name") - os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H", + utils.fubar("Must set advisory name") + os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H", "/usr/local/bin/dak", "new-security-install", "-"+arg, "--", advisory) else: fn() @@ -440,7 +456,7 @@ def sudo(arg, fn, exit): def do_Approve(): sudo("A", _do_Approve, True) def _do_Approve(): # 1. dump advisory in drafts - draft = "/org/security.debian.org/advisories/drafts/%s" % (advisory) + draft = "/org/security-master.debian.org/advisories/drafts/%s" % (advisory) print "Advisory in %s" % (draft) if not Options["No-Action"]: adv_file = "./advisory.%s" % (advisory) @@ -458,12 +474,15 @@ def _do_Approve(): # 3. run dak make-suite-file-list / apt-ftparchve / dak generate-releases print "Updating file lists for apt-ftparchive..." spawn("dak make-suite-file-list") + spawn("dak generate-filelist") print "Updating Packages and Sources files..." - spawn("apt-ftparchive generate %s" % (daklib.utils.which_apt_conf_file())) + spawn("/org/security-master.debian.org/dak/config/debian-security/map.sh") + spawn("apt-ftparchive generate %s" % (utils.which_apt_conf_file())) print "Updating Release files..." spawn("dak generate-releases") print "Triggering security mirrors..." - spawn("sudo -u archvsync /home/archvsync/signal_security") + spawn("/org/security-master.debian.org/dak/config/debian-security/make-mirror.sh") + spawn("sudo -u archvsync -H /home/archvsync/signal_security") # 4. chdir to done - do upload if not Options["No-Action"]: @@ -473,11 +492,13 @@ def _do_Approve(): def do_Disembargo(): sudo("D", _do_Disembargo, True) def _do_Disembargo(): if os.getcwd() != Cnf["Dir::Queue::Embargoed"].rstrip("/"): - daklib.utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"]) + utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"]) + + session = DBConn().session() dest = Cnf["Dir::Queue::Unembargoed"] - emb_q = daklib.database.get_or_set_queue_id("embargoed") - une_q = daklib.database.get_or_set_queue_id("unembargoed") + emb_q = get_build_queue("embargoed", session) + une_q = get_build_queue("unembargoed", session) for c in changes: print "Disembargoing %s" % (c) @@ -488,7 +509,8 @@ def _do_Disembargo(): if "source" in Upload.pkg.changes["architecture"].keys(): print "Adding %s %s to disembargo table" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]) - Upload.projectB.query("INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"])) + session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", + {'package': Upload.pkg.changes["source"], 'version': Upload.pkg.changes["version"]}) files = {} for suite in Upload.pkg.changes["distribution"].keys(): @@ -501,25 +523,30 @@ def _do_Disembargo(): files[os.path.join(dest_dir, file)] = 1 files = files.keys() - Upload.projectB.query("BEGIN WORK") for f in files: - Upload.projectB.query("UPDATE queue_build SET queue = %s WHERE filename = '%s' AND queue = %s" % (une_q, f, emb_q)) - Upload.projectB.query("COMMIT WORK") + session.execute("UPDATE queue_build SET queue = :unembargoed WHERE filename = :filename AND queue = :embargoed", + {'unembargoed': une_q.queue_id, 'filename': f, 'embargoed': emb_q.queue_id}) + session.commit() for file in Upload.pkg.files.keys(): - daklib.utils.copy(file, os.path.join(dest, file)) + utils.copy(file, os.path.join(dest, file)) os.unlink(file) for c in changes: - daklib.utils.copy(c, os.path.join(dest, c)) + utils.copy(c, os.path.join(dest, c)) os.unlink(c) k = c[:-8] + ".dak" - daklib.utils.copy(k, os.path.join(dest, k)) + utils.copy(k, os.path.join(dest, k)) os.unlink(k) + session.commit() + def do_Reject(): sudo("R", _do_Reject, True) def _do_Reject(): global changes + + session = DBConn().session() + for c in changes: print "Rejecting %s..." % (c) Upload.init_vars() @@ -541,17 +568,19 @@ def _do_Reject(): if not aborted: os.unlink(c[:-8]+".dak") for f in files: - Upload.projectB.query( - "DELETE FROM queue_build WHERE filename = '%s'" % (f)) + session.execute("DELETE FROM queue_build WHERE filename = :filename", + {'filename': f}) os.unlink(f) print "Updating buildd information..." - spawn("/org/security.debian.org/dak/config/debian-security/cron.buildd") + spawn("/org/security-master.debian.org/dak/config/debian-security/cron.buildd") adv_file = "./advisory.%s" % (advisory) if os.path.exists(adv_file): os.unlink(adv_file) + session.commit() + def do_DropAdvisory(): for c in changes: Upload.init_vars() @@ -564,11 +593,11 @@ def do_DropAdvisory(): def do_Edit(): adv_file = "./advisory.%s" % (advisory) if not os.path.exists(adv_file): - daklib.utils.copy(Cnf["Dir::Templates"]+"/security-install.advisory", adv_file) + utils.copy(Cnf["Dir::Templates"]+"/security-install.advisory", adv_file) editor = os.environ.get("EDITOR", "vi") result = os.system("%s %s" % (editor, adv_file)) if result != 0: - daklib.utils.fubar("%s invocation failed for %s." % (editor, adv_file)) + utils.fubar("%s invocation failed for %s." % (editor, adv_file)) def do_Show(): adv_file = "./advisory.%s" % (advisory) @@ -604,10 +633,10 @@ def main(): add_changes(extras) if not advisory: - daklib.utils.fubar("Must specify an advisory id") + utils.fubar("Must specify an advisory id") if not changes: - daklib.utils.fubar("No changes specified") + utils.fubar("No changes specified") if Options["Approve"]: advisory_info() @@ -632,7 +661,7 @@ def main(): if os.getcwd() == Cnf["Dir::Queue::Embargoed"].rstrip("/"): opts.append("Disembargo") opts += ["Show advisory", "Reject", "Quit"] - + advisory_info() what = prompt(opts, default) @@ -649,7 +678,7 @@ def main(): elif what == "Reject": do_Reject() else: - daklib.utils.fubar("Impossible answer '%s', wtf?" % (what)) + utils.fubar("Impossible answer '%s', wtf?" % (what)) ################################################################################