From: Niels Thykier Date: Sat, 6 Jun 2015 07:57:33 +0000 (+0200) Subject: Extract a "remove" method from "dak rm" X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=6ff8b512d003c1b49fe2853f0a98d6386368095b;p=dak.git Extract a "remove" method from "dak rm" Signed-off-by: Niels Thykier --- diff --git a/dak/rm.py b/dak/rm.py index 97af8c2c..7a514405 100755 --- a/dak/rm.py +++ b/dak/rm.py @@ -51,6 +51,7 @@ from daklib.config import Config from daklib.dbconn import * from daklib import utils from daklib.dak_exceptions import * +from daklib.rm import remove from daklib.regexes import re_strip_source_version, re_bin_only_nmu import debianbts as bts @@ -180,7 +181,7 @@ def main (): # Accept 3 types of arguments (space separated): # 1) a number - assumed to be a bug number, i.e. nnnnn@bugs.debian.org # 2) the keyword 'package' - cc's $package@packages.debian.org for every argument - # 3) contains a '@' - assumed to be an email address, used unmofidied + # 3) contains a '@' - assumed to be an email address, used unmodified # carbon_copy = [] for copy_to in utils.split_args(Options.get("Carbon-Copy")): @@ -235,9 +236,6 @@ def main (): if Options["Architecture"] and check_source: utils.warn("'source' in -a/--argument makes no sense and is ignored.") - # Additional component processing - over_con_components = con_components.replace("c.id", "component") - # Don't do dependency checks on multiple suites if Options["Rdep-Check"] and len(suites) > 1: utils.fubar("Reverse dependency check on multiple suites is not implemented.") @@ -315,7 +313,6 @@ def main (): summary = "" removals = d.keys() removals.sort() - versions = [] for package in removals: versions = d[package].keys() versions.sort(apt_pkg.version_compare) @@ -349,191 +346,19 @@ def main (): print "Going to remove the packages now." game_over() - whoami = utils.whoami() - date = commands.getoutput('date -R') - - # Log first; if it all falls apart I want a record that we at least tried. - logfile = utils.open_file(cnf["Rm::LogFile"], 'a') - logfile.write("=========================================================================\n") - logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)) - logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)) - if Options["Done"]: - logfile.write("Closed bugs: %s\n" % (Options["Done"])) - logfile.write("\n------------------- Reason -------------------\n%s\n" % (Options["Reason"])) - logfile.write("----------------------------------------------\n") - - # Do the same in rfc822 format - logfile822 = utils.open_file(cnf["Rm::LogFile822"], 'a') - logfile822.write("Date: %s\n" % date) - logfile822.write("Ftpmaster: %s\n" % whoami) - logfile822.write("Suite: %s\n" % suites_list) - sources = [] - binaries = [] - for package in summary.split("\n"): - for row in package.split("\n"): - element = row.split("|") - if len(element) == 3: - if element[2].find("source") > 0: - sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2])) - element[2] = sub("source\s?,?", "", element[2]).strip(" ") - if element[2]: - binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element)) - if sources: - logfile822.write("Sources:\n") - for source in sources: - logfile822.write(" %s\n" % source) - if binaries: - logfile822.write("Binaries:\n") - for binary in binaries: - logfile822.write(" %s\n" % binary) - logfile822.write("Reason: %s\n" % Options["Reason"].replace('\n', '\n ')) - if Options["Done"]: - logfile822.write("Bug: %s\n" % Options["Done"]) - - dsc_type_id = get_override_type('dsc', session).overridetype_id - deb_type_id = get_override_type('deb', session).overridetype_id - # Do the actual deletion print "Deleting...", sys.stdout.flush() - for i in to_remove: - package = i[0] - architecture = i[2] - package_id = i[3] - for suite_id in suite_ids_list: - if architecture == "source": - session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid", - {'packageid': package_id, 'suiteid': suite_id}) - #print "DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id) - else: - session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid", - {'packageid': package_id, 'suiteid': suite_id}) - #print "DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id) - # Delete from the override file - if not Options["Partial"]: - if architecture == "source": - type_id = dsc_type_id - else: - type_id = deb_type_id - # TODO: Again, fix this properly to remove the remaining non-bind argument - session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (over_con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id}) - session.commit() - print "done." - - # If we don't have a Bug server configured, we're done - if not cnf.has_key("Dinstall::BugServer"): - if Options["Done"] or Options["Do-Close"]: - print "Cannot send mail to BugServer as Dinstall::BugServer is not configured" - - logfile.write("=========================================================================\n") - logfile.close() - - logfile822.write("\n") - logfile822.close() - - return - - # read common subst variables for all bug closure mails - Subst_common = {} - Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] - Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"] - Subst_common["__CC__"] = "X-DAK: dak rm" - if carbon_copy: - Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy) - Subst_common["__SUITE_LIST__"] = suites_list - Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list) - Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"] - Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"] - Subst_common["__WHOAMI__"] = whoami - - # Send the bug closing messages - if Options["Done"]: - Subst_close_rm = Subst_common - bcc = [] - if cnf.find("Dinstall::Bcc") != "": - bcc.append(cnf["Dinstall::Bcc"]) - if cnf.find("Rm::Bcc") != "": - bcc.append(cnf["Rm::Bcc"]) - if bcc: - Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc) - else: - Subst_close_rm["__BCC__"] = "X-Filler: 42" - summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, Options["Reason"]) - summarymail += "----------------------------------------------\n" - Subst_close_rm["__SUMMARY__"] = summarymail - - for bug in utils.split_args(Options["Done"]): - Subst_close_rm["__BUG_NUMBER__"] = bug - if Options["Do-Close"]: - mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related") - else: - mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close") - utils.send_mail(mail_message, whitelists=whitelists) - - # close associated bug reports - if Options["Do-Close"]: - Subst_close_other = Subst_common - bcc = [] - wnpp = utils.parse_wnpp_bug_file() - versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) - if len(versions) == 1: - Subst_close_other["__VERSION__"] = versions[0] - else: - utils.fubar("Closing bugs with multiple package versions is not supported. Do it yourself.") - if bcc: - Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc) - else: - Subst_close_other["__BCC__"] = "X-Filler: 42" - # at this point, I just assume, that the first closed bug gives - # some useful information on why the package got removed - Subst_close_other["__BUG_NUMBER__"] = utils.split_args(Options["Done"])[0] - if len(sources) == 1: - source_pkg = source.split("_", 1)[0] - else: - utils.fubar("Closing bugs for multiple source packages is not supported. Do it yourself.") - Subst_close_other["__BUG_NUMBER_ALSO__"] = "" - Subst_close_other["__SOURCE__"] = source_pkg - merged_bugs = set() - other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded') - if other_bugs: - for bugno in other_bugs: - if bugno not in merged_bugs: - for bug in bts.get_status(bugno): - for merged in bug.mergedwith: - other_bugs.remove(merged) - merged_bugs.add(merged) - logfile.write("Also closing bug(s):") - logfile822.write("Also-Bugs:") - for bug in other_bugs: - Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," - logfile.write(" " + str(bug)) - logfile822.write(" " + str(bug)) - logfile.write("\n") - logfile822.write("\n") - if source_pkg in wnpp.keys(): - logfile.write("Also closing WNPP bug(s):") - logfile822.write("Also-WNPP:") - for bug in wnpp[source_pkg]: - # the wnpp-rm file we parse also contains our removal - # bugs, filtering that out - if bug != Subst_close_other["__BUG_NUMBER__"]: - Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," - logfile.write(" " + str(bug)) - logfile822.write(" " + str(bug)) - logfile.write("\n") - logfile822.write("\n") - - mail_message = utils.TemplateSubst(Subst_close_other,cnf["Dir::Templates"]+"/rm.bug-close-related") - if Subst_close_other["__BUG_NUMBER_ALSO__"]: - utils.send_mail(mail_message) - - - logfile.write("=========================================================================\n") - logfile.close() - - logfile822.write("\n") - logfile822.close() + try: + remove(session, Options["Reason"], suites, to_remove, + partial=Options["Partial"], components=utils.split_args(Options["Components"]), + done_bugs=Options["Done"], carbon_copy=carbon_copy, close_related_bugs=Options["Do-Close"] + ) + except ValueError as ex: + utils.fubar(ex.message) + else: + print "done." ####################################################################################### diff --git a/daklib/rm.py b/daklib/rm.py new file mode 100644 index 00000000..892872f0 --- /dev/null +++ b/daklib/rm.py @@ -0,0 +1,329 @@ +"""General purpose package removal code for ftpmaster + +@contact: Debian FTP Master +@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup +@copyright: 2010 Alexander Reichle-Schmehl +@copyright: 2015 Niels Thykier +@license: GNU General Public License version 2 or later +""" +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup +# Copyright (C) 2010 Alexander Reichle-Schmehl + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# TODO: Insert "random dak quote" here + +################################################################################ + +import commands +import apt_pkg +from re import sub + +from daklib.dbconn import * +from daklib import utils +from daklib.regexes import re_bin_only_nmu +import debianbts as bts + +################################################################################ + + +def remove(session, reason, suites, removals, + whoami=None, partial=False, components=None, done_bugs=None, date=None, + carbon_copy=None, close_related_bugs=False): + """Batch remove a number of packages + Verify that the files listed in the Files field of the .dsc are + those expected given the announced Format. + + @type session: SQLA Session + @param session: The database session in use + + @type reason: string + @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by )") + + @type suites: list + @param suites: A list of the suite names in which the removal should occur + + @type removals: list + @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following + for 4 items from the database (in order): package, version, architecture, (database) id. + For source packages, the "architecture" should be set to "source". + + @type partial: bool + @param partial: Whether the removal is "partial" (e.g. architecture specific). + + @type components: list + @param components: List of components involved in a partial removal. Can be an empty list to not restrict the + removal to any components. + + @type whoami: string + @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami() + + @type date: string + @param date: The date of the removal. Defaults to commands.getoutput("date -R") + + @type done_bugs: list + @param done_bugs: A list of bugs to be closed when doing this removal. + + @type close_related_bugs: bool + @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented + for more than one suite. + + @type carbon_copy: list + @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike + "dak rm". + + @rtype: None + @return: Nothing + """ + # Generate the summary of what's to be removed + d = {} + summary = "" + sources = [] + binaries = [] + whitelists = [] + versions = [] + suite_ids_list = [] + suites_list = utils.join_with_commas_and(suites) + cnf = utils.get_conf() + con_components = None + + ####################################################################################################### + + if not reason: + raise ValueError("Empty removal reason not permitted") + + if not removals: + raise ValueError("Nothing to remove!?") + + if not suites: + raise ValueError("Removals without a suite!?") + + if whoami is None: + whoami = utils.whoami() + + if date is None: + date = commands.getoutput("date -R") + + if partial: + + component_ids_list = [] + for componentname in components: + component = get_component(componentname, session=session) + if component is None: + raise ValueError("component '%s' not recognised." % componentname) + else: + component_ids_list.append(component.component_id) + con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list]) + + for i in removals: + package = i[0] + version = i[1] + architecture = i[2] + if package not in d: + d[package] = {} + if version not in d[package]: + d[package][version] = [] + if architecture not in d[package][version]: + d[package][version].append(architecture) + + for package in sorted(removals): + versions = sorted(d[package], cmp=apt_pkg.version_compare) + for version in versions: + d[package][version].sort(utils.arch_compare_sw) + summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version])) + + for package in summary.split("\n"): + for row in package.split("\n"): + element = row.split("|") + if len(element) == 3: + if element[2].find("source") > 0: + sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2])) + element[2] = sub("source\s?,?", "", element[2]).strip(" ") + if element[2]: + binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element)) + + dsc_type_id = get_override_type('dsc', session).overridetype_id + deb_type_id = get_override_type('deb', session).overridetype_id + + for suite in suites: + s = get_suite(suite, session=session) + if s is not None: + suite_ids_list.append(s.suite_id) + whitelists.append(s.mail_whitelist) + + ####################################################################################################### + log_filename = cnf["Rm::LogFile"] + log822_filename = cnf["Rm::LogFile822"] + with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822: + logfile.write("=========================================================================\n") + logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)) + logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)) + if done_bugs: + logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs))) + logfile.write("\n------------------- Reason -------------------\n%s\n" % reason) + logfile.write("----------------------------------------------\n") + + logfile822.write("Date: %s\n" % date) + logfile822.write("Ftpmaster: %s\n" % whoami) + logfile822.write("Suite: %s\n" % suites_list) + + if sources: + logfile822.write("Sources:\n") + for source in sources: + logfile822.write(" %s\n" % source) + + if binaries: + logfile822.write("Binaries:\n") + for binary in binaries: + logfile822.write(" %s\n" % binary) + + logfile822.write("Reason: %s\n" % reason.replace('\n', '\n ')) + if done_bugs: + logfile822.write("Bug: %s\n" % (", ".join(done_bugs))) + + for i in removals: + package = i[0] + architecture = i[2] + package_id = i[3] + for suite_id in suite_ids_list: + if architecture == "source": + session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid", + {'packageid': package_id, 'suiteid': suite_id}) + else: + session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid", + {'packageid': package_id, 'suiteid': suite_id}) + # Delete from the override file + if partial: + if architecture == "source": + type_id = dsc_type_id + else: + type_id = deb_type_id + # TODO: Fix this properly to remove the remaining non-bind argument + session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id}) + + session.commit() + # ### REMOVAL COMPLETE - send mail time ### # + + # If we don't have a Bug server configured, we're done + if "Dinstall::BugServer" not in cnf: + if done_bugs or close_related_bugs: + utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured") + + logfile.write("=========================================================================\n") + logfile822.write("\n") + return + + # read common subst variables for all bug closure mails + Subst_common = {} + Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] + Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"] + Subst_common["__CC__"] = "X-DAK: dak rm" + if carbon_copy: + Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy) + Subst_common["__SUITE_LIST__"] = suites_list + Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list) + Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"] + Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"] + Subst_common["__WHOAMI__"] = whoami + + # Send the bug closing messages + if done_bugs: + Subst_close_rm = Subst_common + bcc = [] + if cnf.find("Dinstall::Bcc") != "": + bcc.append(cnf["Dinstall::Bcc"]) + if cnf.find("Rm::Bcc") != "": + bcc.append(cnf["Rm::Bcc"]) + if bcc: + Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc) + else: + Subst_close_rm["__BCC__"] = "X-Filler: 42" + summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason) + summarymail += "----------------------------------------------\n" + Subst_close_rm["__SUMMARY__"] = summarymail + + for bug in done_bugs: + Subst_close_rm["__BUG_NUMBER__"] = bug + if close_related_bugs: + mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related") + else: + mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close") + utils.send_mail(mail_message, whitelists=whitelists) + + # close associated bug reports + if close_related_bugs: + Subst_close_other = Subst_common + bcc = [] + wnpp = utils.parse_wnpp_bug_file() + versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) + if len(versions) == 1: + Subst_close_other["__VERSION__"] = versions[0] + else: + logfile.write("=========================================================================\n") + logfile822.write("\n") + raise ValueError("Closing bugs with multiple package versions is not supported. Do it yourself.") + if bcc: + Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc) + else: + Subst_close_other["__BCC__"] = "X-Filler: 42" + # at this point, I just assume, that the first closed bug gives + # some useful information on why the package got removed + Subst_close_other["__BUG_NUMBER__"] = done_bugs[0] + if len(sources) == 1: + source_pkg = source.split("_", 1)[0] + else: + logfile.write("=========================================================================\n") + logfile822.write("\n") + raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.") + Subst_close_other["__BUG_NUMBER_ALSO__"] = "" + Subst_close_other["__SOURCE__"] = source_pkg + merged_bugs = set() + other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded') + if other_bugs: + for bugno in other_bugs: + if bugno not in merged_bugs: + for bug in bts.get_status(bugno): + for merged in bug.mergedwith: + other_bugs.remove(merged) + merged_bugs.add(merged) + logfile.write("Also closing bug(s):") + logfile822.write("Also-Bugs:") + for bug in other_bugs: + Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," + logfile.write(" " + str(bug)) + logfile822.write(" " + str(bug)) + logfile.write("\n") + logfile822.write("\n") + if source_pkg in wnpp: + logfile.write("Also closing WNPP bug(s):") + logfile822.write("Also-WNPP:") + for bug in wnpp[source_pkg]: + # the wnpp-rm file we parse also contains our removal + # bugs, filtering that out + if bug != Subst_close_other["__BUG_NUMBER__"]: + Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," + logfile.write(" " + str(bug)) + logfile822.write(" " + str(bug)) + logfile.write("\n") + logfile822.write("\n") + + mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"]+"/rm.bug-close-related") + if Subst_close_other["__BUG_NUMBER_ALSO__"]: + utils.send_mail(mail_message) + + logfile.write("=========================================================================\n") + logfile822.write("\n")