X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Frm.py;fp=daklib%2Frm.py;h=892872f07626285139118072e20e15f1d630c57b;hb=6ff8b512d003c1b49fe2853f0a98d6386368095b;hp=0000000000000000000000000000000000000000;hpb=90885e3cbeee73afe199d6c77cd8d4e58cedf5ac;p=dak.git diff --git a/daklib/rm.py b/daklib/rm.py new file mode 100644 index 00000000..892872f0 --- /dev/null +++ b/daklib/rm.py @@ -0,0 +1,329 @@ +"""General purpose package removal code for ftpmaster + +@contact: Debian FTP Master +@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup +@copyright: 2010 Alexander Reichle-Schmehl +@copyright: 2015 Niels Thykier +@license: GNU General Public License version 2 or later +""" +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup +# Copyright (C) 2010 Alexander Reichle-Schmehl + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# TODO: Insert "random dak quote" here + +################################################################################ + +import commands +import apt_pkg +from re import sub + +from daklib.dbconn import * +from daklib import utils +from daklib.regexes import re_bin_only_nmu +import debianbts as bts + +################################################################################ + + +def remove(session, reason, suites, removals, + whoami=None, partial=False, components=None, done_bugs=None, date=None, + carbon_copy=None, close_related_bugs=False): + """Batch remove a number of packages + Verify that the files listed in the Files field of the .dsc are + those expected given the announced Format. + + @type session: SQLA Session + @param session: The database session in use + + @type reason: string + @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by )") + + @type suites: list + @param suites: A list of the suite names in which the removal should occur + + @type removals: list + @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following + for 4 items from the database (in order): package, version, architecture, (database) id. + For source packages, the "architecture" should be set to "source". + + @type partial: bool + @param partial: Whether the removal is "partial" (e.g. architecture specific). + + @type components: list + @param components: List of components involved in a partial removal. Can be an empty list to not restrict the + removal to any components. + + @type whoami: string + @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami() + + @type date: string + @param date: The date of the removal. Defaults to commands.getoutput("date -R") + + @type done_bugs: list + @param done_bugs: A list of bugs to be closed when doing this removal. + + @type close_related_bugs: bool + @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented + for more than one suite. + + @type carbon_copy: list + @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike + "dak rm". + + @rtype: None + @return: Nothing + """ + # Generate the summary of what's to be removed + d = {} + summary = "" + sources = [] + binaries = [] + whitelists = [] + versions = [] + suite_ids_list = [] + suites_list = utils.join_with_commas_and(suites) + cnf = utils.get_conf() + con_components = None + + ####################################################################################################### + + if not reason: + raise ValueError("Empty removal reason not permitted") + + if not removals: + raise ValueError("Nothing to remove!?") + + if not suites: + raise ValueError("Removals without a suite!?") + + if whoami is None: + whoami = utils.whoami() + + if date is None: + date = commands.getoutput("date -R") + + if partial: + + component_ids_list = [] + for componentname in components: + component = get_component(componentname, session=session) + if component is None: + raise ValueError("component '%s' not recognised." % componentname) + else: + component_ids_list.append(component.component_id) + con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list]) + + for i in removals: + package = i[0] + version = i[1] + architecture = i[2] + if package not in d: + d[package] = {} + if version not in d[package]: + d[package][version] = [] + if architecture not in d[package][version]: + d[package][version].append(architecture) + + for package in sorted(removals): + versions = sorted(d[package], cmp=apt_pkg.version_compare) + for version in versions: + d[package][version].sort(utils.arch_compare_sw) + summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version])) + + for package in summary.split("\n"): + for row in package.split("\n"): + element = row.split("|") + if len(element) == 3: + if element[2].find("source") > 0: + sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2])) + element[2] = sub("source\s?,?", "", element[2]).strip(" ") + if element[2]: + binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element)) + + dsc_type_id = get_override_type('dsc', session).overridetype_id + deb_type_id = get_override_type('deb', session).overridetype_id + + for suite in suites: + s = get_suite(suite, session=session) + if s is not None: + suite_ids_list.append(s.suite_id) + whitelists.append(s.mail_whitelist) + + ####################################################################################################### + log_filename = cnf["Rm::LogFile"] + log822_filename = cnf["Rm::LogFile822"] + with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822: + logfile.write("=========================================================================\n") + logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)) + logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)) + if done_bugs: + logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs))) + logfile.write("\n------------------- Reason -------------------\n%s\n" % reason) + logfile.write("----------------------------------------------\n") + + logfile822.write("Date: %s\n" % date) + logfile822.write("Ftpmaster: %s\n" % whoami) + logfile822.write("Suite: %s\n" % suites_list) + + if sources: + logfile822.write("Sources:\n") + for source in sources: + logfile822.write(" %s\n" % source) + + if binaries: + logfile822.write("Binaries:\n") + for binary in binaries: + logfile822.write(" %s\n" % binary) + + logfile822.write("Reason: %s\n" % reason.replace('\n', '\n ')) + if done_bugs: + logfile822.write("Bug: %s\n" % (", ".join(done_bugs))) + + for i in removals: + package = i[0] + architecture = i[2] + package_id = i[3] + for suite_id in suite_ids_list: + if architecture == "source": + session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid", + {'packageid': package_id, 'suiteid': suite_id}) + else: + session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid", + {'packageid': package_id, 'suiteid': suite_id}) + # Delete from the override file + if partial: + if architecture == "source": + type_id = dsc_type_id + else: + type_id = deb_type_id + # TODO: Fix this properly to remove the remaining non-bind argument + session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id}) + + session.commit() + # ### REMOVAL COMPLETE - send mail time ### # + + # If we don't have a Bug server configured, we're done + if "Dinstall::BugServer" not in cnf: + if done_bugs or close_related_bugs: + utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured") + + logfile.write("=========================================================================\n") + logfile822.write("\n") + return + + # read common subst variables for all bug closure mails + Subst_common = {} + Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] + Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"] + Subst_common["__CC__"] = "X-DAK: dak rm" + if carbon_copy: + Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy) + Subst_common["__SUITE_LIST__"] = suites_list + Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list) + Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"] + Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"] + Subst_common["__WHOAMI__"] = whoami + + # Send the bug closing messages + if done_bugs: + Subst_close_rm = Subst_common + bcc = [] + if cnf.find("Dinstall::Bcc") != "": + bcc.append(cnf["Dinstall::Bcc"]) + if cnf.find("Rm::Bcc") != "": + bcc.append(cnf["Rm::Bcc"]) + if bcc: + Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc) + else: + Subst_close_rm["__BCC__"] = "X-Filler: 42" + summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason) + summarymail += "----------------------------------------------\n" + Subst_close_rm["__SUMMARY__"] = summarymail + + for bug in done_bugs: + Subst_close_rm["__BUG_NUMBER__"] = bug + if close_related_bugs: + mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related") + else: + mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close") + utils.send_mail(mail_message, whitelists=whitelists) + + # close associated bug reports + if close_related_bugs: + Subst_close_other = Subst_common + bcc = [] + wnpp = utils.parse_wnpp_bug_file() + versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) + if len(versions) == 1: + Subst_close_other["__VERSION__"] = versions[0] + else: + logfile.write("=========================================================================\n") + logfile822.write("\n") + raise ValueError("Closing bugs with multiple package versions is not supported. Do it yourself.") + if bcc: + Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc) + else: + Subst_close_other["__BCC__"] = "X-Filler: 42" + # at this point, I just assume, that the first closed bug gives + # some useful information on why the package got removed + Subst_close_other["__BUG_NUMBER__"] = done_bugs[0] + if len(sources) == 1: + source_pkg = source.split("_", 1)[0] + else: + logfile.write("=========================================================================\n") + logfile822.write("\n") + raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.") + Subst_close_other["__BUG_NUMBER_ALSO__"] = "" + Subst_close_other["__SOURCE__"] = source_pkg + merged_bugs = set() + other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded') + if other_bugs: + for bugno in other_bugs: + if bugno not in merged_bugs: + for bug in bts.get_status(bugno): + for merged in bug.mergedwith: + other_bugs.remove(merged) + merged_bugs.add(merged) + logfile.write("Also closing bug(s):") + logfile822.write("Also-Bugs:") + for bug in other_bugs: + Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," + logfile.write(" " + str(bug)) + logfile822.write(" " + str(bug)) + logfile.write("\n") + logfile822.write("\n") + if source_pkg in wnpp: + logfile.write("Also closing WNPP bug(s):") + logfile822.write("Also-WNPP:") + for bug in wnpp[source_pkg]: + # the wnpp-rm file we parse also contains our removal + # bugs, filtering that out + if bug != Subst_close_other["__BUG_NUMBER__"]: + Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," + logfile.write(" " + str(bug)) + logfile822.write(" " + str(bug)) + logfile.write("\n") + logfile822.write("\n") + + mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"]+"/rm.bug-close-related") + if Subst_close_other["__BUG_NUMBER_ALSO__"]: + utils.send_mail(mail_message) + + logfile.write("=========================================================================\n") + logfile822.write("\n")