from daklib.dbconn import *
from daklib import utils
from daklib.dak_exceptions import *
+from daklib.rm import remove
from daklib.regexes import re_strip_source_version, re_bin_only_nmu
import debianbts as bts
# Accept 3 types of arguments (space separated):
# 1) a number - assumed to be a bug number, i.e. nnnnn@bugs.debian.org
# 2) the keyword 'package' - cc's $package@packages.debian.org for every argument
- # 3) contains a '@' - assumed to be an email address, used unmofidied
+ # 3) contains a '@' - assumed to be an email address, used unmodified
#
carbon_copy = []
for copy_to in utils.split_args(Options.get("Carbon-Copy")):
if Options["Architecture"] and check_source:
utils.warn("'source' in -a/--argument makes no sense and is ignored.")
- # Additional component processing
- over_con_components = con_components.replace("c.id", "component")
-
# Don't do dependency checks on multiple suites
if Options["Rdep-Check"] and len(suites) > 1:
utils.fubar("Reverse dependency check on multiple suites is not implemented.")
summary = ""
removals = d.keys()
removals.sort()
- versions = []
for package in removals:
versions = d[package].keys()
versions.sort(apt_pkg.version_compare)
print "Going to remove the packages now."
game_over()
- whoami = utils.whoami()
- date = commands.getoutput('date -R')
-
- # Log first; if it all falls apart I want a record that we at least tried.
- logfile = utils.open_file(cnf["Rm::LogFile"], 'a')
- logfile.write("=========================================================================\n")
- logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
- logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
- if Options["Done"]:
- logfile.write("Closed bugs: %s\n" % (Options["Done"]))
- logfile.write("\n------------------- Reason -------------------\n%s\n" % (Options["Reason"]))
- logfile.write("----------------------------------------------\n")
-
- # Do the same in rfc822 format
- logfile822 = utils.open_file(cnf["Rm::LogFile822"], 'a')
- logfile822.write("Date: %s\n" % date)
- logfile822.write("Ftpmaster: %s\n" % whoami)
- logfile822.write("Suite: %s\n" % suites_list)
- sources = []
- binaries = []
- for package in summary.split("\n"):
- for row in package.split("\n"):
- element = row.split("|")
- if len(element) == 3:
- if element[2].find("source") > 0:
- sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2]))
- element[2] = sub("source\s?,?", "", element[2]).strip(" ")
- if element[2]:
- binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element))
- if sources:
- logfile822.write("Sources:\n")
- for source in sources:
- logfile822.write(" %s\n" % source)
- if binaries:
- logfile822.write("Binaries:\n")
- for binary in binaries:
- logfile822.write(" %s\n" % binary)
- logfile822.write("Reason: %s\n" % Options["Reason"].replace('\n', '\n '))
- if Options["Done"]:
- logfile822.write("Bug: %s\n" % Options["Done"])
-
- dsc_type_id = get_override_type('dsc', session).overridetype_id
- deb_type_id = get_override_type('deb', session).overridetype_id
-
# Do the actual deletion
print "Deleting...",
sys.stdout.flush()
- for i in to_remove:
- package = i[0]
- architecture = i[2]
- package_id = i[3]
- for suite_id in suite_ids_list:
- if architecture == "source":
- session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
- {'packageid': package_id, 'suiteid': suite_id})
- #print "DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id)
- else:
- session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
- {'packageid': package_id, 'suiteid': suite_id})
- #print "DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id)
- # Delete from the override file
- if not Options["Partial"]:
- if architecture == "source":
- type_id = dsc_type_id
- else:
- type_id = deb_type_id
- # TODO: Again, fix this properly to remove the remaining non-bind argument
- session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (over_con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
- session.commit()
- print "done."
-
- # If we don't have a Bug server configured, we're done
- if not cnf.has_key("Dinstall::BugServer"):
- if Options["Done"] or Options["Do-Close"]:
- print "Cannot send mail to BugServer as Dinstall::BugServer is not configured"
-
- logfile.write("=========================================================================\n")
- logfile.close()
-
- logfile822.write("\n")
- logfile822.close()
-
- return
-
- # read common subst variables for all bug closure mails
- Subst_common = {}
- Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
- Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
- Subst_common["__CC__"] = "X-DAK: dak rm"
- if carbon_copy:
- Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
- Subst_common["__SUITE_LIST__"] = suites_list
- Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list)
- Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
- Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
- Subst_common["__WHOAMI__"] = whoami
-
- # Send the bug closing messages
- if Options["Done"]:
- Subst_close_rm = Subst_common
- bcc = []
- if cnf.find("Dinstall::Bcc") != "":
- bcc.append(cnf["Dinstall::Bcc"])
- if cnf.find("Rm::Bcc") != "":
- bcc.append(cnf["Rm::Bcc"])
- if bcc:
- Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
- else:
- Subst_close_rm["__BCC__"] = "X-Filler: 42"
- summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, Options["Reason"])
- summarymail += "----------------------------------------------\n"
- Subst_close_rm["__SUMMARY__"] = summarymail
-
- for bug in utils.split_args(Options["Done"]):
- Subst_close_rm["__BUG_NUMBER__"] = bug
- if Options["Do-Close"]:
- mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
- else:
- mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
- utils.send_mail(mail_message, whitelists=whitelists)
-
- # close associated bug reports
- if Options["Do-Close"]:
- Subst_close_other = Subst_common
- bcc = []
- wnpp = utils.parse_wnpp_bug_file()
- versions = list(set([re_bin_only_nmu.sub('', v) for v in versions]))
- if len(versions) == 1:
- Subst_close_other["__VERSION__"] = versions[0]
- else:
- utils.fubar("Closing bugs with multiple package versions is not supported. Do it yourself.")
- if bcc:
- Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
- else:
- Subst_close_other["__BCC__"] = "X-Filler: 42"
- # at this point, I just assume, that the first closed bug gives
- # some useful information on why the package got removed
- Subst_close_other["__BUG_NUMBER__"] = utils.split_args(Options["Done"])[0]
- if len(sources) == 1:
- source_pkg = source.split("_", 1)[0]
- else:
- utils.fubar("Closing bugs for multiple source packages is not supported. Do it yourself.")
- Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
- Subst_close_other["__SOURCE__"] = source_pkg
- merged_bugs = set()
- other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded')
- if other_bugs:
- for bugno in other_bugs:
- if bugno not in merged_bugs:
- for bug in bts.get_status(bugno):
- for merged in bug.mergedwith:
- other_bugs.remove(merged)
- merged_bugs.add(merged)
- logfile.write("Also closing bug(s):")
- logfile822.write("Also-Bugs:")
- for bug in other_bugs:
- Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
- logfile.write(" " + str(bug))
- logfile822.write(" " + str(bug))
- logfile.write("\n")
- logfile822.write("\n")
- if source_pkg in wnpp.keys():
- logfile.write("Also closing WNPP bug(s):")
- logfile822.write("Also-WNPP:")
- for bug in wnpp[source_pkg]:
- # the wnpp-rm file we parse also contains our removal
- # bugs, filtering that out
- if bug != Subst_close_other["__BUG_NUMBER__"]:
- Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
- logfile.write(" " + str(bug))
- logfile822.write(" " + str(bug))
- logfile.write("\n")
- logfile822.write("\n")
-
- mail_message = utils.TemplateSubst(Subst_close_other,cnf["Dir::Templates"]+"/rm.bug-close-related")
- if Subst_close_other["__BUG_NUMBER_ALSO__"]:
- utils.send_mail(mail_message)
-
-
- logfile.write("=========================================================================\n")
- logfile.close()
-
- logfile822.write("\n")
- logfile822.close()
+ try:
+ remove(session, Options["Reason"], suites, to_remove,
+ partial=Options["Partial"], components=utils.split_args(Options["Components"]),
+ done_bugs=Options["Done"], carbon_copy=carbon_copy, close_related_bugs=Options["Do-Close"]
+ )
+ except ValueError as ex:
+ utils.fubar(ex.message)
+ else:
+ print "done."
#######################################################################################
--- /dev/null
+"""General purpose package removal code for ftpmaster
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
+@copyright: 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
+@copyright: 2015 Niels Thykier <niels@thykier.net>
+@license: GNU General Public License version 2 or later
+"""
+# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
+# Copyright (C) 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# TODO: Insert "random dak quote" here
+
+################################################################################
+
+import commands
+import apt_pkg
+from re import sub
+
+from daklib.dbconn import *
+from daklib import utils
+from daklib.regexes import re_bin_only_nmu
+import debianbts as bts
+
+################################################################################
+
+
+def remove(session, reason, suites, removals,
+ whoami=None, partial=False, components=None, done_bugs=None, date=None,
+ carbon_copy=None, close_related_bugs=False):
+ """Batch remove a number of packages
+ Verify that the files listed in the Files field of the .dsc are
+ those expected given the announced Format.
+
+ @type session: SQLA Session
+ @param session: The database session in use
+
+ @type reason: string
+ @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by <source>)")
+
+ @type suites: list
+ @param suites: A list of the suite names in which the removal should occur
+
+ @type removals: list
+ @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following
+ for 4 items from the database (in order): package, version, architecture, (database) id.
+ For source packages, the "architecture" should be set to "source".
+
+ @type partial: bool
+ @param partial: Whether the removal is "partial" (e.g. architecture specific).
+
+ @type components: list
+ @param components: List of components involved in a partial removal. Can be an empty list to not restrict the
+ removal to any components.
+
+ @type whoami: string
+ @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami()
+
+ @type date: string
+ @param date: The date of the removal. Defaults to commands.getoutput("date -R")
+
+ @type done_bugs: list
+ @param done_bugs: A list of bugs to be closed when doing this removal.
+
+ @type close_related_bugs: bool
+ @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented
+ for more than one suite.
+
+ @type carbon_copy: list
+ @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike
+ "dak rm".
+
+ @rtype: None
+ @return: Nothing
+ """
+ # Generate the summary of what's to be removed
+ d = {}
+ summary = ""
+ sources = []
+ binaries = []
+ whitelists = []
+ versions = []
+ suite_ids_list = []
+ suites_list = utils.join_with_commas_and(suites)
+ cnf = utils.get_conf()
+ con_components = None
+
+ #######################################################################################################
+
+ if not reason:
+ raise ValueError("Empty removal reason not permitted")
+
+ if not removals:
+ raise ValueError("Nothing to remove!?")
+
+ if not suites:
+ raise ValueError("Removals without a suite!?")
+
+ if whoami is None:
+ whoami = utils.whoami()
+
+ if date is None:
+ date = commands.getoutput("date -R")
+
+ if partial:
+
+ component_ids_list = []
+ for componentname in components:
+ component = get_component(componentname, session=session)
+ if component is None:
+ raise ValueError("component '%s' not recognised." % componentname)
+ else:
+ component_ids_list.append(component.component_id)
+ con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list])
+
+ for i in removals:
+ package = i[0]
+ version = i[1]
+ architecture = i[2]
+ if package not in d:
+ d[package] = {}
+ if version not in d[package]:
+ d[package][version] = []
+ if architecture not in d[package][version]:
+ d[package][version].append(architecture)
+
+ for package in sorted(removals):
+ versions = sorted(d[package], cmp=apt_pkg.version_compare)
+ for version in versions:
+ d[package][version].sort(utils.arch_compare_sw)
+ summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
+
+ for package in summary.split("\n"):
+ for row in package.split("\n"):
+ element = row.split("|")
+ if len(element) == 3:
+ if element[2].find("source") > 0:
+ sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2]))
+ element[2] = sub("source\s?,?", "", element[2]).strip(" ")
+ if element[2]:
+ binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element))
+
+ dsc_type_id = get_override_type('dsc', session).overridetype_id
+ deb_type_id = get_override_type('deb', session).overridetype_id
+
+ for suite in suites:
+ s = get_suite(suite, session=session)
+ if s is not None:
+ suite_ids_list.append(s.suite_id)
+ whitelists.append(s.mail_whitelist)
+
+ #######################################################################################################
+ log_filename = cnf["Rm::LogFile"]
+ log822_filename = cnf["Rm::LogFile822"]
+ with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822:
+ logfile.write("=========================================================================\n")
+ logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
+ logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
+ if done_bugs:
+ logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs)))
+ logfile.write("\n------------------- Reason -------------------\n%s\n" % reason)
+ logfile.write("----------------------------------------------\n")
+
+ logfile822.write("Date: %s\n" % date)
+ logfile822.write("Ftpmaster: %s\n" % whoami)
+ logfile822.write("Suite: %s\n" % suites_list)
+
+ if sources:
+ logfile822.write("Sources:\n")
+ for source in sources:
+ logfile822.write(" %s\n" % source)
+
+ if binaries:
+ logfile822.write("Binaries:\n")
+ for binary in binaries:
+ logfile822.write(" %s\n" % binary)
+
+ logfile822.write("Reason: %s\n" % reason.replace('\n', '\n '))
+ if done_bugs:
+ logfile822.write("Bug: %s\n" % (", ".join(done_bugs)))
+
+ for i in removals:
+ package = i[0]
+ architecture = i[2]
+ package_id = i[3]
+ for suite_id in suite_ids_list:
+ if architecture == "source":
+ session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
+ {'packageid': package_id, 'suiteid': suite_id})
+ else:
+ session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
+ {'packageid': package_id, 'suiteid': suite_id})
+ # Delete from the override file
+ if partial:
+ if architecture == "source":
+ type_id = dsc_type_id
+ else:
+ type_id = deb_type_id
+ # TODO: Fix this properly to remove the remaining non-bind argument
+ session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
+
+ session.commit()
+ # ### REMOVAL COMPLETE - send mail time ### #
+
+ # If we don't have a Bug server configured, we're done
+ if "Dinstall::BugServer" not in cnf:
+ if done_bugs or close_related_bugs:
+ utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured")
+
+ logfile.write("=========================================================================\n")
+ logfile822.write("\n")
+ return
+
+ # read common subst variables for all bug closure mails
+ Subst_common = {}
+ Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
+ Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
+ Subst_common["__CC__"] = "X-DAK: dak rm"
+ if carbon_copy:
+ Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
+ Subst_common["__SUITE_LIST__"] = suites_list
+ Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list)
+ Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
+ Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
+ Subst_common["__WHOAMI__"] = whoami
+
+ # Send the bug closing messages
+ if done_bugs:
+ Subst_close_rm = Subst_common
+ bcc = []
+ if cnf.find("Dinstall::Bcc") != "":
+ bcc.append(cnf["Dinstall::Bcc"])
+ if cnf.find("Rm::Bcc") != "":
+ bcc.append(cnf["Rm::Bcc"])
+ if bcc:
+ Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
+ else:
+ Subst_close_rm["__BCC__"] = "X-Filler: 42"
+ summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason)
+ summarymail += "----------------------------------------------\n"
+ Subst_close_rm["__SUMMARY__"] = summarymail
+
+ for bug in done_bugs:
+ Subst_close_rm["__BUG_NUMBER__"] = bug
+ if close_related_bugs:
+ mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
+ else:
+ mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
+ utils.send_mail(mail_message, whitelists=whitelists)
+
+ # close associated bug reports
+ if close_related_bugs:
+ Subst_close_other = Subst_common
+ bcc = []
+ wnpp = utils.parse_wnpp_bug_file()
+ versions = list(set([re_bin_only_nmu.sub('', v) for v in versions]))
+ if len(versions) == 1:
+ Subst_close_other["__VERSION__"] = versions[0]
+ else:
+ logfile.write("=========================================================================\n")
+ logfile822.write("\n")
+ raise ValueError("Closing bugs with multiple package versions is not supported. Do it yourself.")
+ if bcc:
+ Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
+ else:
+ Subst_close_other["__BCC__"] = "X-Filler: 42"
+ # at this point, I just assume, that the first closed bug gives
+ # some useful information on why the package got removed
+ Subst_close_other["__BUG_NUMBER__"] = done_bugs[0]
+ if len(sources) == 1:
+ source_pkg = source.split("_", 1)[0]
+ else:
+ logfile.write("=========================================================================\n")
+ logfile822.write("\n")
+ raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.")
+ Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
+ Subst_close_other["__SOURCE__"] = source_pkg
+ merged_bugs = set()
+ other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded')
+ if other_bugs:
+ for bugno in other_bugs:
+ if bugno not in merged_bugs:
+ for bug in bts.get_status(bugno):
+ for merged in bug.mergedwith:
+ other_bugs.remove(merged)
+ merged_bugs.add(merged)
+ logfile.write("Also closing bug(s):")
+ logfile822.write("Also-Bugs:")
+ for bug in other_bugs:
+ Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
+ logfile.write(" " + str(bug))
+ logfile822.write(" " + str(bug))
+ logfile.write("\n")
+ logfile822.write("\n")
+ if source_pkg in wnpp:
+ logfile.write("Also closing WNPP bug(s):")
+ logfile822.write("Also-WNPP:")
+ for bug in wnpp[source_pkg]:
+ # the wnpp-rm file we parse also contains our removal
+ # bugs, filtering that out
+ if bug != Subst_close_other["__BUG_NUMBER__"]:
+ Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
+ logfile.write(" " + str(bug))
+ logfile822.write(" " + str(bug))
+ logfile.write("\n")
+ logfile822.write("\n")
+
+ mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"]+"/rm.bug-close-related")
+ if Subst_close_other["__BUG_NUMBER_ALSO__"]:
+ utils.send_mail(mail_message)
+
+ logfile.write("=========================================================================\n")
+ logfile822.write("\n")
projects / dak.git / commitdiff