X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Frm.py;h=3edeac3dc35c3ccdd4a9a736f135426b859461ba;hb=27e00376e81d1c37ff327ee0d39670b266418869;hp=4c88e61dd968b42faa7f7316583d3a667597ec6a;hpb=ebb9a75ee4d43555e313451138e9a1c516c461c0;p=dak.git diff --git a/dak/rm.py b/dak/rm.py index 4c88e61d..3edeac3d 100755 --- a/dak/rm.py +++ b/dak/rm.py @@ -2,6 +2,7 @@ """ General purpose package removal tool for ftpmaster """ # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup +# Copyright (C) 2010 Alexander Reichle-Schmehl # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -44,12 +45,14 @@ import os import sys import apt_pkg import apt_inst +from re import sub from daklib.config import Config from daklib.dbconn import * from daklib import utils from daklib.dak_exceptions import * -from daklib.regexes import re_strip_source_version, re_build_dep_arch +from daklib.regexes import re_strip_source_version, re_build_dep_arch, re_bin_only_nmu +import debianbts as bts ################################################################################ @@ -66,6 +69,7 @@ Remove PACKAGE(s) from suite(s). -c, --component=COMPONENT act on this component -C, --carbon-copy=EMAIL send a CC of removal message to EMAIL -d, --done=BUG# send removal message as closure to bug# + -D, --do-close also close all bugs associated to that package -h, --help show this help and exit -m, --reason=MSG reason for removal -n, --no-action don't do anything @@ -75,7 +79,7 @@ Remove PACKAGE(s) from suite(s). -S, --source-only remove source only ARCH, BUG#, COMPONENT and SUITE can be comma (or space) separated lists, e.g. - --architecture=m68k,i386""" + --architecture=amd64,i386""" sys.exit(exit_code) @@ -94,61 +98,72 @@ def game_over(): ################################################################################ -def reverse_depends_check(removals, suites, arches=None): +def reverse_depends_check(removals, suite, arches=None, session=None): + dbsuite = get_suite(suite, session) cnf = Config() print "Checking reverse dependencies..." - components = cnf.ValueList("Suite::%s::Components" % suites[0]) dep_problem = 0 p2c = {} all_broken = {} if arches: all_arches = set(arches) else: - all_arches = set([x.arch_string for x in get_suite_architectures(suites[0])]) + all_arches = set([x.arch_string for x in get_suite_architectures(suite)]) all_arches -= set(["source", "all"]) + metakey_d = get_or_set_metadatakey("Depends", session) + metakey_p = get_or_set_metadatakey("Provides", session) + params = { + 'suite_id': dbsuite.suite_id, + 'metakey_d_id': metakey_d.key_id, + 'metakey_p_id': metakey_p.key_id, + 'arch_all_id' : get_architecture('all', session).arch_id, + } for architecture in all_arches: deps = {} sources = {} virtual_packages = {} - for component in components: - filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (cnf["Dir::Root"], suites[0], component, architecture) - # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... - (fd, temp_filename) = utils.temp_filename() - (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) - if (result != 0): - utils.fubar("Gunzip invocation failed!\n%s\n" % (output), result) - packages = utils.open_file(temp_filename) - Packages = apt_pkg.ParseTagFile(packages) - while Packages.Step(): - package = Packages.Section.Find("Package") - source = Packages.Section.Find("Source") - if not source: - source = package - elif ' ' in source: - source = source.split(' ', 1)[0] - sources[package] = source - depends = Packages.Section.Find("Depends") - if depends: - deps[package] = depends - provides = Packages.Section.Find("Provides") - # Maintain a counter for each virtual package. If a - # Provides: exists, set the counter to 0 and count all - # provides by a package not in the list for removal. - # If the counter stays 0 at the end, we know that only - # the to-be-removed packages provided this virtual - # package. - if provides: - for virtual_pkg in provides.split(","): - virtual_pkg = virtual_pkg.strip() - if virtual_pkg == package: continue - if not virtual_packages.has_key(virtual_pkg): - virtual_packages[virtual_pkg] = 0 - if package not in removals: - virtual_packages[virtual_pkg] += 1 - p2c[package] = component - packages.close() - os.unlink(temp_filename) + params['arch_id'] = get_architecture(architecture, session).arch_id + + statement = ''' + WITH suite_binaries AS + (select b.id, b.package, b.source, b.file + from binaries b WHERE b.id in + (SELECT bin FROM bin_associations WHERE suite = :suite_id) + AND b.architecture in (:arch_id, :arch_all_id)) + SELECT b.id, b.package, s.source, c.name as component, + bmd.value as depends, bmp.value as provides + FROM suite_binaries b + LEFT OUTER JOIN binaries_metadata bmd + ON b.id = bmd.bin_id AND bmd.key_id = :metakey_d_id + LEFT OUTER JOIN binaries_metadata bmp + ON b.id = bmp.bin_id AND bmp.key_id = :metakey_p_id + JOIN source s ON b.source = s.id + JOIN files f ON b.file = f.id + JOIN location l ON f.location = l.id + JOIN component c ON l.component = c.id''' + session.rollback() + query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \ + from_statement(statement).params(params) + for binary_id, package, source, component, depends, provides in query: + sources[package] = source + p2c[package] = component + if depends is not None: + deps[package] = depends + # Maintain a counter for each virtual package. If a + # Provides: exists, set the counter to 0 and count all + # provides by a package not in the list for removal. + # If the counter stays 0 at the end, we know that only + # the to-be-removed packages provided this virtual + # package. + if provides is not None: + for virtual_pkg in provides.split(","): + virtual_pkg = virtual_pkg.strip() + if virtual_pkg == package: continue + if not virtual_packages.has_key(virtual_pkg): + virtual_packages[virtual_pkg] = 0 + if package not in removals: + virtual_packages[virtual_pkg] += 1 # If a virtual package is only provided by the to-be-removed # packages, treat the virtual package as to-be-removed too. @@ -162,7 +177,7 @@ def reverse_depends_check(removals, suites, arches=None): parsed_dep = [] try: parsed_dep += apt_pkg.ParseDepends(deps[package]) - except ValueError, e: + except ValueError as e: print "Error for package %s: %s" % (package, e) for dep in parsed_dep: # Check for partial breakage. If a package has a ORed @@ -196,41 +211,44 @@ def reverse_depends_check(removals, suites, arches=None): # Check source dependencies (Build-Depends and Build-Depends-Indep) all_broken.clear() - for component in components: - filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suites[0], component) - # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... - (fd, temp_filename) = utils.temp_filename() - result, output = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) - if result != 0: - sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) - sys.exit(result) - sources = utils.open_file(temp_filename, "r") - Sources = apt_pkg.ParseTagFile(sources) - while Sources.Step(): - source = Sources.Section.Find("Package") - if source in removals: continue - parsed_dep = [] - for build_dep_type in ["Build-Depends", "Build-Depends-Indep"]: - build_dep = Sources.Section.get(build_dep_type) - if build_dep: - # Remove [arch] information since we want to see breakage on all arches - build_dep = re_build_dep_arch.sub("", build_dep) - try: - parsed_dep += apt_pkg.ParseDepends(build_dep) - except ValueError, e: - print "Error for source %s: %s" % (source, e) - for dep in parsed_dep: - unsat = 0 - for dep_package, _, _ in dep: - if dep_package in removals: - unsat += 1 - if unsat == len(dep): - if component != "main": - source = "%s/%s" % (source, component) - all_broken.setdefault(source, set()).add(utils.pp_deps(dep)) - dep_problem = 1 - sources.close() - os.unlink(temp_filename) + metakey_bd = get_or_set_metadatakey("Build-Depends", session) + metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session) + params = { + 'suite_id': dbsuite.suite_id, + 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id), + } + statement = ''' + SELECT s.id, s.source, string_agg(sm.value, ', ') as build_dep + FROM source s + JOIN source_metadata sm ON s.id = sm.src_id + WHERE s.id in + (SELECT source FROM src_associations + WHERE suite = :suite_id) + AND sm.key_id in :metakey_ids + GROUP BY s.id, s.source''' + query = session.query('id', 'source', 'build_dep').from_statement(statement). \ + params(params) + for source_id, source, build_dep in query: + if source in removals: continue + parsed_dep = [] + if build_dep is not None: + # Remove [arch] information since we want to see breakage on all arches + build_dep = re_build_dep_arch.sub("", build_dep) + try: + parsed_dep += apt_pkg.ParseDepends(build_dep) + except ValueError as e: + print "Error for source %s: %s" % (source, e) + for dep in parsed_dep: + unsat = 0 + for dep_package, _, _ in dep: + if dep_package in removals: + unsat += 1 + if unsat == len(dep): + component = DBSource.get(source_id, session).get_component_name() + if component != "main": + source = "%s/%s" % (source, component) + all_broken.setdefault(source, set()).add(utils.pp_deps(dep)) + dep_problem = 1 if all_broken: print "# Broken Build-Depends:" @@ -262,6 +280,7 @@ def main (): ('c',"component", "Rm::Options::Component", "HasArg"), ('C',"carbon-copy", "Rm::Options::Carbon-Copy", "HasArg"), # Bugs to Cc ('d',"done","Rm::Options::Done", "HasArg"), # Bugs fixed + ('D',"do-close","Rm::Options::Do-Close"), ('R',"rdep-check", "Rm::Options::Rdep-Check"), ('m',"reason", "Rm::Options::Reason", "HasArg"), # Hysterical raisins; -m is old-dinstall option for rejection reason ('n',"no-action","Rm::Options::No-Action"), @@ -272,7 +291,7 @@ def main (): for i in [ "architecture", "binary-only", "carbon-copy", "component", "done", "help", "no-action", "partial", "rdep-check", "reason", - "source-only" ]: + "source-only", "Do-Close" ]: if not cnf.has_key("Rm::Options::%s" % (i)): cnf["Rm::Options::%s" % (i)] = "" if not cnf.has_key("Rm::Options::Suite"): @@ -298,6 +317,14 @@ def main (): if Options["Architecture"] and not Options["Partial"]: utils.warn("-a/--architecture implies -p/--partial.") Options["Partial"] = "true" + if Options["Do-Close"] and not Options["Done"]: + utils.fubar("No.") + if Options["Do-Close"] and Options["Binary-Only"]: + utils.fubar("No.") + if Options["Do-Close"] and Options["Source-Only"]: + utils.fubar("No.") + if Options["Do-Close"] and Options["Suite"] != 'unstable': + utils.fubar("No.") # Force the admin to tell someone if we're not doing a 'dak # cruft-report' inspired removal (or closing a bug, which counts @@ -316,10 +343,14 @@ def main (): carbon_copy = [] for copy_to in utils.split_args(Options.get("Carbon-Copy")): if copy_to.isdigit(): - carbon_copy.append(copy_to + "@" + cnf["Dinstall::BugServer"]) + if cnf.has_key("Dinstall::BugServer"): + carbon_copy.append(copy_to + "@" + cnf["Dinstall::BugServer"]) + else: + utils.fubar("Asked to send mail to #%s in BTS but Dinstall::BugServer is not configured" % copy_to) elif copy_to == 'package': for package in arguments: - carbon_copy.append(package + "@" + cnf["Dinstall::PackagesServer"]) + if cnf.has_key("Dinstall::PackagesServer"): + carbon_copy.append(package + "@" + cnf["Dinstall::PackagesServer"]) if cnf.has_key("Dinstall::TrackingServer"): carbon_copy.append(package + "@" + cnf["Dinstall::TrackingServer"]) elif '@' in copy_to: @@ -345,8 +376,8 @@ def main (): s = get_suite(suite, session=session) if s is not None: suite_ids_list.append(s.suite_id) - if suite == "stable": - print "**WARNING** About to remove from the stable suite!" + if suite in ("oldstable", "stable"): + print "**WARNING** About to remove from the (old)stable suite!" print "This should only be done just prior to a (point) release and not at" print "any other time." game_over() @@ -363,6 +394,10 @@ def main (): # Additional component processing over_con_components = con_components.replace("c.id", "component") + # Don't do dependency checks on multiple suites + if Options["Rdep-Check"] and len(suites) > 1: + utils.fubar("Reverse dependency check on multiple suites is not implemented.") + print "Working...", sys.stdout.flush() to_remove = [] @@ -399,7 +434,7 @@ def main (): for i in source_packages.keys(): filename = "/".join(source_packages[i]) try: - dsc = utils.parse_changes(filename) + dsc = utils.parse_changes(filename, dsc_file=1) except CantOpenError: utils.warn("couldn't open '%s'." % (filename)) continue @@ -460,6 +495,7 @@ def main (): summary = "" removals = d.keys() removals.sort() + versions = [] for package in removals: versions = d[package].keys() versions.sort(apt_pkg.VersionCompare) @@ -474,6 +510,8 @@ def main (): print "Will also close bugs: "+Options["Done"] if carbon_copy: print "Will also send CCs to: " + ", ".join(carbon_copy) + if Options["Do-Close"]: + print "Will also close associated bug reports." print print "------------------- Reason -------------------" print Options["Reason"] @@ -482,7 +520,7 @@ def main (): if Options["Rdep-Check"]: arches = utils.split_args(Options["Architecture"]) - reverse_depends_check(removals, suites, arches) + reverse_depends_check(removals, suites[0], arches, session) # If -n/--no-action, drop out here if Options["No-Action"]: @@ -503,7 +541,34 @@ def main (): logfile.write("Closed bugs: %s\n" % (Options["Done"])) logfile.write("\n------------------- Reason -------------------\n%s\n" % (Options["Reason"])) logfile.write("----------------------------------------------\n") - logfile.flush() + + # Do the same in rfc822 format + logfile822 = utils.open_file(cnf["Rm::LogFile822"], 'a') + logfile822.write("Date: %s\n" % date) + logfile822.write("Ftpmaster: %s\n" % whoami) + logfile822.write("Suite: %s\n" % suites_list) + sources = [] + binaries = [] + for package in summary.split("\n"): + for row in package.split("\n"): + element = row.split("|") + if len(element) == 3: + if element[2].find("source") > 0: + sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2])) + element[2] = sub("source\s?,?", "", element[2]).strip(" ") + if element[2]: + binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element)) + if sources: + logfile822.write("Sources:\n") + for source in sources: + logfile822.write(" %s\n" % source) + if binaries: + logfile822.write("Binaries:\n") + for binary in binaries: + logfile822.write(" %s\n" % binary) + logfile822.write("Reason: %s\n" % Options["Reason"].replace('\n', '\n ')) + if Options["Done"]: + logfile822.write("Bug: %s\n" % Options["Done"]) dsc_type_id = get_override_type('dsc', session).overridetype_id deb_type_id = get_override_type('deb', session).overridetype_id @@ -536,43 +601,123 @@ def main (): session.commit() print "done." + # If we don't have a Bug server configured, we're done + if not cnf.has_key("Dinstall::BugServer"): + if Options["Done"] or Options["Do-Close"]: + print "Cannot send mail to BugServer as Dinstall::BugServer is not configured" + + logfile.write("=========================================================================\n") + logfile.close() + + logfile822.write("\n") + logfile822.close() + + return + + # read common subst variables for all bug closure mails + Subst_common = {} + Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] + Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"] + Subst_common["__CC__"] = "X-DAK: dak rm" + if carbon_copy: + Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy) + Subst_common["__SUITE_LIST__"] = suites_list + Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list) + Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"] + Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"] + Subst_common["__WHOAMI__"] = whoami + # Send the bug closing messages if Options["Done"]: - Subst = {} - Subst["__RM_ADDRESS__"] = cnf["Rm::MyEmailAddress"] - Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"] + Subst_close_rm = Subst_common bcc = [] if cnf.Find("Dinstall::Bcc") != "": bcc.append(cnf["Dinstall::Bcc"]) if cnf.Find("Rm::Bcc") != "": bcc.append(cnf["Rm::Bcc"]) if bcc: - Subst["__BCC__"] = "Bcc: " + ", ".join(bcc) + Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc) else: - Subst["__BCC__"] = "X-Filler: 42" - Subst["__CC__"] = "X-DAK: dak rm" - if carbon_copy: - Subst["__CC__"] += "\nCc: " + ", ".join(carbon_copy) - Subst["__SUITE_LIST__"] = suites_list + Subst_close_rm["__BCC__"] = "X-Filler: 42" summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, Options["Reason"]) summarymail += "----------------------------------------------\n" - Subst["__SUMMARY__"] = summarymail - Subst["__SUBJECT__"] = "Removed package(s) from %s" % (suite_list) - Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"] - Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"] - Subst["__WHOAMI__"] = whoami + Subst_close_rm["__SUMMARY__"] = summarymail + whereami = utils.where_am_i() - Archive = cnf.SubTree("Archive::%s" % (whereami)) - Subst["__MASTER_ARCHIVE__"] = Archive["OriginServer"] - Subst["__PRIMARY_MIRROR__"] = Archive["PrimaryMirror"] + Archive = get_archive(whereami, session) + if Archive is None: + utils.warn("Cannot find archive %s. Setting blank values for origin" % whereami) + Subst_close_rm["__MASTER_ARCHIVE__"] = "" + Subst_close_rm["__PRIMARY_MIRROR__"] = "" + else: + Subst_close_rm["__MASTER_ARCHIVE__"] = Archive.origin_server + Subst_close_rm["__PRIMARY_MIRROR__"] = Archive.primary_mirror + for bug in utils.split_args(Options["Done"]): - Subst["__BUG_NUMBER__"] = bug - mail_message = utils.TemplateSubst(Subst,cnf["Dir::Templates"]+"/rm.bug-close") + Subst_close_rm["__BUG_NUMBER__"] = bug + if Options["Do-Close"]: + mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related") + else: + mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close") utils.send_mail(mail_message) + # close associated bug reports + if Options["Do-Close"]: + Subst_close_other = Subst_common + bcc = [] + wnpp = utils.parse_wnpp_bug_file() + versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) + if len(versions) == 1: + Subst_close_other["__VERSION__"] = versions[0] + else: + utils.fubar("Closing bugs with multiple package versions is not supported. Do it yourself.") + if bcc: + Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc) + else: + Subst_close_other["__BCC__"] = "X-Filler: 42" + # at this point, I just assume, that the first closed bug gives + # some useful information on why the package got removed + Subst_close_other["__BUG_NUMBER__"] = utils.split_args(Options["Done"])[0] + if len(sources) == 1: + source_pkg = source.split("_", 1)[0] + else: + utils.fubar("Closing bugs for multiple source pakcages is not supported. Do it yourself.") + Subst_close_other["__BUG_NUMBER_ALSO__"] = "" + Subst_close_other["__SOURCE__"] = source_pkg + other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open') + if other_bugs: + logfile.write("Also closing bug(s):") + logfile822.write("Also-Bugs:") + for bug in other_bugs: + Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," + logfile.write(" " + str(bug)) + logfile822.write(" " + str(bug)) + logfile.write("\n") + logfile822.write("\n") + if source_pkg in wnpp.keys(): + logfile.write("Also closing WNPP bug(s):") + logfile822.write("Also-WNPP:") + for bug in wnpp[source_pkg]: + # the wnpp-rm file we parse also contains our removal + # bugs, filtering that out + if bug != Subst_close_other["__BUG_NUMBER__"]: + Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," + logfile.write(" " + str(bug)) + logfile822.write(" " + str(bug)) + logfile.write("\n") + logfile822.write("\n") + + mail_message = utils.TemplateSubst(Subst_close_other,cnf["Dir::Templates"]+"/rm.bug-close-related") + if Subst_close_other["__BUG_NUMBER_ALSO__"]: + utils.send_mail(mail_message) + + logfile.write("=========================================================================\n") logfile.close() + logfile822.write("\n") + logfile822.close() + ####################################################################################### if __name__ == '__main__':