1 """General purpose package removal code for ftpmaster
3 @contact: Debian FTP Master <ftpmaster@debian.org>
4 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
5 @copyright: 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
6 @copyright: 2015 Niels Thykier <niels@thykier.net>
7 @license: GNU General Public License version 2 or later
9 # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
10 # Copyright (C) 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ################################################################################
28 # From: Andrew Morton <akpm@osdl.org>
30 # To: linux-kernel@vger.kernel.org
31 # Date: Sat, 22 May 2004 01:36:36 -0700
32 # X-Mailer: Sylpheed version 0.9.7 (GTK+ 1.2.10; i386-redhat-linux-gnu)
36 # Although this feature has been around for a while it is new code, and the
37 # usual cautions apply. If it munches all your files please tell Jens and
38 # he'll type them in again for you.
40 ################################################################################
46 from collections import defaultdict
47 from regexes import re_build_dep_arch
49 from daklib.dbconn import *
50 from daklib import utils
51 from daklib.regexes import re_bin_only_nmu
52 import debianbts as bts
54 ################################################################################
57 class ReverseDependencyChecker(object):
58 """A bulk tester for reverse dependency checks
60 This class is similar to the check_reverse_depends method from "utils". However,
61 it is primarily focused on facilitating bulk testing of reverse dependencies.
62 It caches the state of the suite and then uses that as basis for answering queries.
63 This saves a significant amount of time if multiple reverse dependency checks are
67 def __init__(self, session, suite):
68 """Creates a new ReverseDependencyChecker instance
70 This will spend a significant amount of time caching data.
72 @type session: SQLA Session
73 @param session: The database session in use
76 @param suite: The name of the suite that is used as basis for removal tests.
78 self._session = session
79 dbsuite = get_suite(suite, session)
80 suite_archs2id = dict((x.arch_string, x.arch_id) for x in get_suite_architectures(suite))
81 package_dependencies, arch_providers_of, arch_provided_by = self._load_package_information(session,
84 self._package_dependencies = package_dependencies
85 self._arch_providers_of = arch_providers_of
86 self._arch_provided_by = arch_provided_by
87 self._archs_in_suite = set(suite_archs2id)
90 def _load_package_information(session, suite_id, suite_archs2id):
91 package_dependencies = defaultdict(lambda: defaultdict(set))
92 arch_providers_of = defaultdict(lambda: defaultdict(set))
93 arch_provided_by = defaultdict(lambda: defaultdict(set))
94 source_deps = defaultdict(set)
95 metakey_d = get_or_set_metadatakey("Depends", session)
96 metakey_p = get_or_set_metadatakey("Provides", session)
99 'arch_all_id': suite_archs2id['all'],
100 'metakey_d_id': metakey_d.key_id,
101 'metakey_p_id': metakey_p.key_id,
103 all_arches = set(suite_archs2id)
104 all_arches.discard('source')
106 package_dependencies['source'] = source_deps
108 for architecture in all_arches:
109 deps = defaultdict(set)
110 providers_of = defaultdict(set)
111 provided_by = defaultdict(set)
112 arch_providers_of[architecture] = providers_of
113 arch_provided_by[architecture] = provided_by
114 package_dependencies[architecture] = deps
116 params['arch_id'] = suite_archs2id[architecture]
120 (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
121 (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
123 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
124 WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id'''
125 query = session.query('package', 'depends', 'provides'). \
126 from_statement(statement).params(params)
127 for package, depends, provides in query:
129 if depends is not None:
132 for dep in apt_pkg.parse_depends(depends):
133 parsed_dep.append(frozenset(d[0] for d in dep))
134 deps[package].update(parsed_dep)
135 except ValueError as e:
136 print "Error for package %s: %s" % (package, e)
137 # Maintain a counter for each virtual package. If a
138 # Provides: exists, set the counter to 0 and count all
139 # provides by a package not in the list for removal.
140 # If the counter stays 0 at the end, we know that only
141 # the to-be-removed packages provided this virtual
143 if provides is not None:
144 for virtual_pkg in provides.split(","):
145 virtual_pkg = virtual_pkg.strip()
146 if virtual_pkg == package:
148 provided_by[virtual_pkg].add(package)
149 providers_of[package].add(virtual_pkg)
151 # Check source dependencies (Build-Depends and Build-Depends-Indep)
152 metakey_bd = get_or_set_metadatakey("Build-Depends", session)
153 metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
155 'suite_id': suite_id,
156 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
159 SELECT s.source, string_agg(sm.value, ', ') as build_dep
161 JOIN source_metadata sm ON s.id = sm.src_id
163 (SELECT source FROM src_associations
164 WHERE suite = :suite_id)
165 AND sm.key_id in :metakey_ids
166 GROUP BY s.id, s.source'''
167 query = session.query('source', 'build_dep').from_statement(statement). \
169 for source, build_dep in query:
170 if build_dep is not None:
171 # Remove [arch] information since we want to see breakage on all arches
172 build_dep = re_build_dep_arch.sub("", build_dep)
175 for dep in apt_pkg.parse_src_depends(build_dep):
176 parsed_dep.append(frozenset(d[0] for d in dep))
177 source_deps[source].update(parsed_dep)
178 except ValueError as e:
179 print "Error for package %s: %s" % (source, e)
181 return package_dependencies, arch_providers_of, arch_provided_by
183 def check_reverse_depends(self, removal_requests):
184 """Bulk check reverse dependencies
188 "eclipse-rcp": None, # means ALL architectures (incl. source)
189 "eclipse": None, # means ALL architectures (incl. source)
190 "lintian": ["source", "all"], # Only these two "architectures".
192 obj.check_reverse_depends(removal_request)
194 @type removal_requests: dict (or a list of tuples)
195 @param removal_requests: A dictionary mapping a package name to a list of architectures. The list of
196 architectures decides from which the package will be removed - if the list is empty the package will
197 be removed on ALL architectures in the suite (including "source").
200 @return: A mapping of "removed package" (as a "(pkg, arch)"-tuple) to a set of broken
201 broken packages (also as "(pkg, arch)"-tuple). Note that the architecture values
202 in these tuples /can/ be "source" to reflect a breakage in build-dependencies.
205 archs_in_suite = self._archs_in_suite
206 removals_by_arch = defaultdict(set)
207 affected_virtual_by_arch = defaultdict(set)
208 package_dependencies = self._package_dependencies
209 arch_providers_of = self._arch_providers_of
210 arch_provided_by = self._arch_provided_by
211 arch_provides2removal = defaultdict(lambda: defaultdict(set))
212 dep_problems = defaultdict(set)
213 src_deps = package_dependencies['source']
215 arch_all_removals = set()
217 if isinstance(removal_requests, dict):
218 removal_requests = removal_requests.iteritems()
220 for pkg, arch_list in removal_requests:
222 arch_list = archs_in_suite
223 for arch in arch_list:
225 src_removals.add(pkg)
228 arch_all_removals.add(pkg)
230 removals_by_arch[arch].add(pkg)
231 if pkg in arch_providers_of[arch]:
232 affected_virtual_by_arch[arch].add(pkg)
234 if arch_all_removals:
235 for arch in archs_in_suite:
236 if arch in ('all', 'source'):
238 removals_by_arch[arch].update(arch_all_removals)
239 for pkg in arch_all_removals:
240 if pkg in arch_providers_of[arch]:
241 affected_virtual_by_arch[arch].add(pkg)
243 if not removals_by_arch:
244 # Nothing to remove => no problems
247 for arch, removed_providers in affected_virtual_by_arch.iteritems():
248 provides2removal = arch_provides2removal[arch]
249 removals = removals_by_arch[arch]
250 for virtual_pkg, virtual_providers in arch_provided_by[arch].iteritems():
251 v = virtual_providers & removed_providers
252 if len(v) == len(virtual_providers):
253 # We removed all the providers of virtual_pkg
254 removals.add(virtual_pkg)
255 # Pick one to take the blame for the removal
256 # - we sort for determinism, optimally we would prefer to blame the same package
257 # to minimise the number of blamed packages.
258 provides2removal[virtual_pkg] = sorted(v)[0]
260 for arch, removals in removals_by_arch.iteritems():
261 deps = package_dependencies[arch]
262 provides2removal = arch_provides2removal[arch]
264 # Check binary dependencies (Depends)
265 for package, dependencies in deps.iteritems():
266 if package in removals:
268 for clause in dependencies:
269 if not (clause <= removals):
270 # Something probably still satisfies this relation
272 # whoops, we seemed to have removed all packages that could possibly satisfy
273 # this relation. Lets blame something for it
274 for dep_package in clause:
275 removal = dep_package
276 if dep_package in provides2removal:
277 removal = provides2removal[dep_package]
278 dep_problems[(removal, arch)].add((package, arch))
280 for source, build_dependencies in src_deps.iteritems():
281 if source in src_removals:
283 for clause in build_dependencies:
284 if not (clause <= removals):
285 # Something probably still satisfies this relation
287 # whoops, we seemed to have removed all packages that could possibly satisfy
288 # this relation. Lets blame something for it
289 for dep_package in clause:
290 removal = dep_package
291 if dep_package in provides2removal:
292 removal = provides2removal[dep_package]
293 dep_problems[(removal, arch)].add((source, 'source'))
298 def remove(session, reason, suites, removals,
299 whoami=None, partial=False, components=None, done_bugs=None, date=None,
300 carbon_copy=None, close_related_bugs=False):
301 """Batch remove a number of packages
302 Verify that the files listed in the Files field of the .dsc are
303 those expected given the announced Format.
305 @type session: SQLA Session
306 @param session: The database session in use
309 @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by <source>)")
312 @param suites: A list of the suite names in which the removal should occur
315 @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following
316 for 4 items from the database (in order): package, version, architecture, (database) id.
317 For source packages, the "architecture" should be set to "source".
320 @param partial: Whether the removal is "partial" (e.g. architecture specific).
322 @type components: list
323 @param components: List of components involved in a partial removal. Can be an empty list to not restrict the
324 removal to any components.
327 @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami()
330 @param date: The date of the removal. Defaults to commands.getoutput("date -R")
332 @type done_bugs: list
333 @param done_bugs: A list of bugs to be closed when doing this removal.
335 @type close_related_bugs: bool
336 @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented
337 for more than one suite.
339 @type carbon_copy: list
340 @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike
346 # Generate the summary of what's to be removed
355 suites_list = utils.join_with_commas_and(suites)
356 cnf = utils.get_conf()
359 #######################################################################################################
362 raise ValueError("Empty removal reason not permitted")
365 raise ValueError("Nothing to remove!?")
368 raise ValueError("Removals without a suite!?")
371 whoami = utils.whoami()
374 date = commands.getoutput("date -R")
376 if partial and components:
378 component_ids_list = []
379 for componentname in components:
380 component = get_component(componentname, session=session)
381 if component is None:
382 raise ValueError("component '%s' not recognised." % componentname)
384 component_ids_list.append(component.component_id)
385 if component_ids_list:
386 con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list])
394 if version not in d[package]:
395 d[package][version] = []
396 if architecture not in d[package][version]:
397 d[package][version].append(architecture)
399 for package in sorted(d):
400 versions = sorted(d[package], cmp=apt_pkg.version_compare)
401 for version in versions:
402 d[package][version].sort(utils.arch_compare_sw)
403 summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
404 if apt_pkg.version_compare(version, newest_source) > 0:
405 newest_source = version
407 for package in summary.split("\n"):
408 for row in package.split("\n"):
409 element = row.split("|")
410 if len(element) == 3:
411 if element[2].find("source") > 0:
412 sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2]))
413 element[2] = sub("source\s?,?", "", element[2]).strip(" ")
415 binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element))
417 dsc_type_id = get_override_type('dsc', session).overridetype_id
418 deb_type_id = get_override_type('deb', session).overridetype_id
421 s = get_suite(suite, session=session)
423 suite_ids_list.append(s.suite_id)
424 whitelists.append(s.mail_whitelist)
426 #######################################################################################################
427 log_filename = cnf["Rm::LogFile"]
428 log822_filename = cnf["Rm::LogFile822"]
429 with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822:
430 fcntl.lockf(logfile, fcntl.LOCK_EX)
431 fcntl.lockf(logfile822, fcntl.LOCK_EX)
433 logfile.write("=========================================================================\n")
434 logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
435 logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
437 logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs)))
438 logfile.write("\n------------------- Reason -------------------\n%s\n" % reason)
439 logfile.write("----------------------------------------------\n")
441 logfile822.write("Date: %s\n" % date)
442 logfile822.write("Ftpmaster: %s\n" % whoami)
443 logfile822.write("Suite: %s\n" % suites_list)
446 logfile822.write("Sources:\n")
447 for source in sources:
448 logfile822.write(" %s\n" % source)
451 logfile822.write("Binaries:\n")
452 for binary in binaries:
453 logfile822.write(" %s\n" % binary)
455 logfile822.write("Reason: %s\n" % reason.replace('\n', '\n '))
457 logfile822.write("Bug: %s\n" % (", ".join(done_bugs)))
463 for suite_id in suite_ids_list:
464 if architecture == "source":
465 session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
466 {'packageid': package_id, 'suiteid': suite_id})
468 session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
469 {'packageid': package_id, 'suiteid': suite_id})
470 # Delete from the override file
472 if architecture == "source":
473 type_id = dsc_type_id
475 type_id = deb_type_id
476 # TODO: Fix this properly to remove the remaining non-bind argument
477 session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
480 # ### REMOVAL COMPLETE - send mail time ### #
482 # If we don't have a Bug server configured, we're done
483 if "Dinstall::BugServer" not in cnf:
484 if done_bugs or close_related_bugs:
485 utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured")
487 logfile.write("=========================================================================\n")
488 logfile822.write("\n")
491 # read common subst variables for all bug closure mails
493 Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
494 Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
495 Subst_common["__CC__"] = "X-DAK: dak rm"
497 Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
498 Subst_common["__SUITE_LIST__"] = suites_list
499 Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list)
500 Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
501 Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
502 Subst_common["__WHOAMI__"] = whoami
504 # Send the bug closing messages
506 Subst_close_rm = Subst_common
508 if cnf.find("Dinstall::Bcc") != "":
509 bcc.append(cnf["Dinstall::Bcc"])
510 if cnf.find("Rm::Bcc") != "":
511 bcc.append(cnf["Rm::Bcc"])
513 Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
515 Subst_close_rm["__BCC__"] = "X-Filler: 42"
516 summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason)
517 summarymail += "----------------------------------------------\n"
518 Subst_close_rm["__SUMMARY__"] = summarymail
520 for bug in done_bugs:
521 Subst_close_rm["__BUG_NUMBER__"] = bug
522 if close_related_bugs:
523 mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
525 mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
526 utils.send_mail(mail_message, whitelists=whitelists)
528 # close associated bug reports
529 if close_related_bugs:
530 Subst_close_other = Subst_common
532 wnpp = utils.parse_wnpp_bug_file()
533 newest_source = re_bin_only_nmu.sub('', newest_source)
534 if len(set(s.split("_", 1)[0] for s in sources)) == 1:
535 source_pkg = source.split("_", 1)[0]
537 logfile.write("=========================================================================\n")
538 logfile822.write("\n")
539 raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.")
540 if newest_source != '':
541 Subst_close_other["__VERSION__"] = newest_source
543 logfile.write("=========================================================================\n")
544 logfile822.write("\n")
545 raise ValueError("No versions can be found. Close bugs yourself.")
547 Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
549 Subst_close_other["__BCC__"] = "X-Filler: 42"
550 # at this point, I just assume, that the first closed bug gives
551 # some useful information on why the package got removed
552 Subst_close_other["__BUG_NUMBER__"] = done_bugs[0]
553 Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
554 Subst_close_other["__SOURCE__"] = source_pkg
556 other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded')
558 for bugno in other_bugs:
559 if bugno not in merged_bugs:
560 for bug in bts.get_status(bugno):
561 for merged in bug.mergedwith:
562 other_bugs.remove(merged)
563 merged_bugs.add(merged)
564 logfile.write("Also closing bug(s):")
565 logfile822.write("Also-Bugs:")
566 for bug in other_bugs:
567 Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
568 logfile.write(" " + str(bug))
569 logfile822.write(" " + str(bug))
571 logfile822.write("\n")
572 if source_pkg in wnpp:
573 logfile.write("Also closing WNPP bug(s):")
574 logfile822.write("Also-WNPP:")
575 for bug in wnpp[source_pkg]:
576 # the wnpp-rm file we parse also contains our removal
577 # bugs, filtering that out
578 if bug != Subst_close_other["__BUG_NUMBER__"]:
579 Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
580 logfile.write(" " + str(bug))
581 logfile822.write(" " + str(bug))
583 logfile822.write("\n")
585 mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"]+"/rm.bug-close-related")
586 if Subst_close_other["__BUG_NUMBER_ALSO__"]:
587 utils.send_mail(mail_message)
589 logfile.write("=========================================================================\n")
590 logfile822.write("\n")