1 """General purpose package removal code for ftpmaster
3 @contact: Debian FTP Master <ftpmaster@debian.org>
4 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
5 @copyright: 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
6 @copyright: 2015 Niels Thykier <niels@thykier.net>
7 @license: GNU General Public License version 2 or later
9 # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
10 # Copyright (C) 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ################################################################################
28 # From: Andrew Morton <akpm@osdl.org>
30 # To: linux-kernel@vger.kernel.org
31 # Date: Sat, 22 May 2004 01:36:36 -0700
32 # X-Mailer: Sylpheed version 0.9.7 (GTK+ 1.2.10; i386-redhat-linux-gnu)
36 # Although this feature has been around for a while it is new code, and the
37 # usual cautions apply. If it munches all your files please tell Jens and
38 # he'll type them in again for you.
40 ################################################################################
46 from collections import defaultdict
47 from regexes import re_build_dep_arch
49 from daklib.dbconn import *
50 from daklib import utils
51 from daklib.regexes import re_bin_only_nmu
52 import debianbts as bts
54 ################################################################################
57 class ReverseDependencyChecker(object):
58 """A bulk tester for reverse dependency checks
60 This class is similar to the check_reverse_depends method from "utils". However,
61 it is primarily focused on facilitating bulk testing of reverse dependencies.
62 It caches the state of the suite and then uses that as basis for answering queries.
63 This saves a significant amount of time if multiple reverse dependency checks are
67 def __init__(self, session, suite):
68 """Creates a new ReverseDependencyChecker instance
70 This will spend a significant amount of time caching data.
72 @type session: SQLA Session
73 @param session: The database session in use
76 @param suite: The name of the suite that is used as basis for removal tests.
78 self._session = session
79 dbsuite = get_suite(suite, session)
80 suite_archs2id = dict((x.arch_string, x.arch_id) for x in get_suite_architectures(suite))
81 package_dependencies, arch_providers_of, arch_provided_by = self._load_package_information(session,
84 self._package_dependencies = package_dependencies
85 self._arch_providers_of = arch_providers_of
86 self._arch_provided_by = arch_provided_by
87 self._archs_in_suite = set(suite_archs2id)
90 def _load_package_information(session, suite_id, suite_archs2id):
91 package_dependencies = defaultdict(lambda: defaultdict(set))
92 arch_providers_of = defaultdict(lambda: defaultdict(set))
93 arch_provided_by = defaultdict(lambda: defaultdict(set))
94 source_deps = defaultdict(set)
95 metakey_d = get_or_set_metadatakey("Depends", session)
96 metakey_p = get_or_set_metadatakey("Provides", session)
99 'arch_all_id': suite_archs2id['all'],
100 'metakey_d_id': metakey_d.key_id,
101 'metakey_p_id': metakey_p.key_id,
103 all_arches = set(suite_archs2id)
104 all_arches.discard('source')
106 package_dependencies['source'] = source_deps
108 for architecture in all_arches:
109 deps = defaultdict(set)
110 providers_of = defaultdict(set)
111 provided_by = defaultdict(set)
112 arch_providers_of[architecture] = providers_of
113 arch_provided_by[architecture] = provided_by
114 package_dependencies[architecture] = deps
116 params['arch_id'] = suite_archs2id[architecture]
120 (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
121 (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
123 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
124 WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id'''
125 query = session.query('package', 'depends', 'provides'). \
126 from_statement(statement).params(params)
127 for package, depends, provides in query:
129 if depends is not None:
132 for dep in apt_pkg.parse_depends(depends):
133 parsed_dep.append(frozenset(d[0] for d in dep))
134 deps[package].update(parsed_dep)
135 except ValueError as e:
136 print "Error for package %s: %s" % (package, e)
137 # Maintain a counter for each virtual package. If a
138 # Provides: exists, set the counter to 0 and count all
139 # provides by a package not in the list for removal.
140 # If the counter stays 0 at the end, we know that only
141 # the to-be-removed packages provided this virtual
143 if provides is not None:
144 for virtual_pkg in provides.split(","):
145 virtual_pkg = virtual_pkg.strip()
146 if virtual_pkg == package:
148 provided_by[virtual_pkg].add(package)
149 providers_of[package].add(virtual_pkg)
151 # Check source dependencies (Build-Depends and Build-Depends-Indep)
152 metakey_bd = get_or_set_metadatakey("Build-Depends", session)
153 metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
155 'suite_id': suite_id,
156 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
159 SELECT s.source, string_agg(sm.value, ', ') as build_dep
161 JOIN source_metadata sm ON s.id = sm.src_id
163 (SELECT source FROM src_associations
164 WHERE suite = :suite_id)
165 AND sm.key_id in :metakey_ids
166 GROUP BY s.id, s.source'''
167 query = session.query('source', 'build_dep').from_statement(statement). \
169 for source, build_dep in query:
170 if build_dep is not None:
171 # Remove [arch] information since we want to see breakage on all arches
172 build_dep = re_build_dep_arch.sub("", build_dep)
175 for dep in apt_pkg.parse_src_depends(build_dep):
176 parsed_dep.append(frozenset(d[0] for d in dep))
177 source_deps[source].update(parsed_dep)
178 except ValueError as e:
179 print "Error for package %s: %s" % (source, e)
181 return package_dependencies, arch_providers_of, arch_provided_by
183 def check_reverse_depends(self, removal_requests):
184 """Bulk check reverse dependencies
188 "eclipse-rcp": None, # means ALL architectures (incl. source)
189 "eclipse": None, # means ALL architectures (incl. source)
190 "lintian": ["source", "all"], # Only these two "architectures".
192 obj.check_reverse_depends(removal_request)
194 @type removal_requests: dict (or a list of tuples)
195 @param removal_requests: A dictionary mapping a package name to a list of architectures. The list of
196 architectures decides from which the package will be removed - if the list is empty the package will
197 be removed on ALL architectures in the suite (including "source").
200 @return: A mapping of "removed package" (as a "(pkg, arch)"-tuple) to a set of broken
201 broken packages (also as "(pkg, arch)"-tuple). Note that the architecture values
202 in these tuples /can/ be "source" to reflect a breakage in build-dependencies.
205 archs_in_suite = self._archs_in_suite
206 removals_by_arch = defaultdict(set)
207 affected_virtual_by_arch = defaultdict(set)
208 package_dependencies = self._package_dependencies
209 arch_providers_of = self._arch_providers_of
210 arch_provided_by = self._arch_provided_by
211 arch_provides2removal = defaultdict(lambda: defaultdict(set))
212 dep_problems = defaultdict(set)
213 src_deps = package_dependencies['source']
215 arch_all_removals = set()
217 if isinstance(removal_requests, dict):
218 removal_requests = removal_requests.iteritems()
220 for pkg, arch_list in removal_requests:
222 arch_list = archs_in_suite
223 for arch in arch_list:
225 src_removals.add(pkg)
228 arch_all_removals.add(pkg)
230 removals_by_arch[arch].add(pkg)
231 if pkg in arch_providers_of[arch]:
232 affected_virtual_by_arch[arch].add(pkg)
234 if arch_all_removals:
235 for arch in archs_in_suite:
236 if arch in ('all', 'source'):
238 removals_by_arch[arch].update(arch_all_removals)
239 for pkg in arch_all_removals:
240 if pkg in arch_providers_of[arch]:
241 affected_virtual_by_arch[arch].add(pkg)
243 if not removals_by_arch:
244 # Nothing to remove => no problems
247 for arch, removed_providers in affected_virtual_by_arch.iteritems():
248 provides2removal = arch_provides2removal[arch]
249 removals = removals_by_arch[arch]
250 for virtual_pkg, virtual_providers in arch_provided_by[arch].iteritems():
251 v = virtual_providers & removed_providers
252 if len(v) == len(virtual_providers):
253 # We removed all the providers of virtual_pkg
254 removals.add(virtual_pkg)
255 # Pick one to take the blame for the removal
256 # - we sort for determinism, optimally we would prefer to blame the same package
257 # to minimise the number of blamed packages.
258 provides2removal[virtual_pkg] = sorted(v)[0]
260 for arch, removals in removals_by_arch.iteritems():
261 deps = package_dependencies[arch]
262 provides2removal = arch_provides2removal[arch]
264 # Check binary dependencies (Depends)
265 for package, dependencies in deps.iteritems():
266 if package in removals:
268 for clause in dependencies:
269 if not (clause <= removals):
270 # Something probably still satisfies this relation
272 # whoops, we seemed to have removed all packages that could possibly satisfy
273 # this relation. Lets blame something for it
274 for dep_package in clause:
275 removal = dep_package
276 if dep_package in provides2removal:
277 removal = provides2removal[dep_package]
278 dep_problems[(removal, arch)].add((package, arch))
280 for source, build_dependencies in src_deps.iteritems():
281 if source in src_removals:
283 for clause in build_dependencies:
284 if not (clause <= removals):
285 # Something probably still satisfies this relation
287 # whoops, we seemed to have removed all packages that could possibly satisfy
288 # this relation. Lets blame something for it
289 for dep_package in clause:
290 removal = dep_package
291 if dep_package in provides2removal:
292 removal = provides2removal[dep_package]
293 dep_problems[(removal, arch)].add((source, 'source'))
298 def remove(session, reason, suites, removals,
299 whoami=None, partial=False, components=None, done_bugs=None, date=None,
300 carbon_copy=None, close_related_bugs=False):
301 """Batch remove a number of packages
302 Verify that the files listed in the Files field of the .dsc are
303 those expected given the announced Format.
305 @type session: SQLA Session
306 @param session: The database session in use
309 @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by <source>)")
312 @param suites: A list of the suite names in which the removal should occur
315 @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following
316 for 4 items from the database (in order): package, version, architecture, (database) id.
317 For source packages, the "architecture" should be set to "source".
320 @param partial: Whether the removal is "partial" (e.g. architecture specific).
322 @type components: list
323 @param components: List of components involved in a partial removal. Can be an empty list to not restrict the
324 removal to any components.
327 @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami()
330 @param date: The date of the removal. Defaults to commands.getoutput("date -R")
332 @type done_bugs: list
333 @param done_bugs: A list of bugs to be closed when doing this removal.
335 @type close_related_bugs: bool
336 @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented
337 for more than one suite.
339 @type carbon_copy: list
340 @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike
346 # Generate the summary of what's to be removed
354 suites_list = utils.join_with_commas_and(suites)
355 cnf = utils.get_conf()
358 #######################################################################################################
361 raise ValueError("Empty removal reason not permitted")
364 raise ValueError("Nothing to remove!?")
367 raise ValueError("Removals without a suite!?")
370 whoami = utils.whoami()
373 date = commands.getoutput("date -R")
375 if partial and components:
377 component_ids_list = []
378 for componentname in components:
379 component = get_component(componentname, session=session)
380 if component is None:
381 raise ValueError("component '%s' not recognised." % componentname)
383 component_ids_list.append(component.component_id)
384 if component_ids_list:
385 con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list])
393 if version not in d[package]:
394 d[package][version] = []
395 if architecture not in d[package][version]:
396 d[package][version].append(architecture)
398 for package in sorted(d):
399 versions = sorted(d[package], cmp=apt_pkg.version_compare)
400 for version in versions:
401 d[package][version].sort(utils.arch_compare_sw)
402 summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
404 for package in summary.split("\n"):
405 for row in package.split("\n"):
406 element = row.split("|")
407 if len(element) == 3:
408 if element[2].find("source") > 0:
409 sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2]))
410 element[2] = sub("source\s?,?", "", element[2]).strip(" ")
412 binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element))
414 dsc_type_id = get_override_type('dsc', session).overridetype_id
415 deb_type_id = get_override_type('deb', session).overridetype_id
418 s = get_suite(suite, session=session)
420 suite_ids_list.append(s.suite_id)
421 whitelists.append(s.mail_whitelist)
423 #######################################################################################################
424 log_filename = cnf["Rm::LogFile"]
425 log822_filename = cnf["Rm::LogFile822"]
426 with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822:
427 fcntl.lockf(logfile, fcntl.LOCK_EX)
428 fcntl.lockf(logfile822, fcntl.LOCK_EX)
430 logfile.write("=========================================================================\n")
431 logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
432 logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
434 logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs)))
435 logfile.write("\n------------------- Reason -------------------\n%s\n" % reason)
436 logfile.write("----------------------------------------------\n")
438 logfile822.write("Date: %s\n" % date)
439 logfile822.write("Ftpmaster: %s\n" % whoami)
440 logfile822.write("Suite: %s\n" % suites_list)
443 logfile822.write("Sources:\n")
444 for source in sources:
445 logfile822.write(" %s\n" % source)
448 logfile822.write("Binaries:\n")
449 for binary in binaries:
450 logfile822.write(" %s\n" % binary)
452 logfile822.write("Reason: %s\n" % reason.replace('\n', '\n '))
454 logfile822.write("Bug: %s\n" % (", ".join(done_bugs)))
460 for suite_id in suite_ids_list:
461 if architecture == "source":
462 session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
463 {'packageid': package_id, 'suiteid': suite_id})
465 session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
466 {'packageid': package_id, 'suiteid': suite_id})
467 # Delete from the override file
469 if architecture == "source":
470 type_id = dsc_type_id
472 type_id = deb_type_id
473 # TODO: Fix this properly to remove the remaining non-bind argument
474 session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
477 # ### REMOVAL COMPLETE - send mail time ### #
479 # If we don't have a Bug server configured, we're done
480 if "Dinstall::BugServer" not in cnf:
481 if done_bugs or close_related_bugs:
482 utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured")
484 logfile.write("=========================================================================\n")
485 logfile822.write("\n")
488 # read common subst variables for all bug closure mails
490 Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
491 Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
492 Subst_common["__CC__"] = "X-DAK: dak rm"
494 Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
495 Subst_common["__SUITE_LIST__"] = suites_list
496 Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list)
497 Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
498 Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
499 Subst_common["__WHOAMI__"] = whoami
501 # Send the bug closing messages
503 Subst_close_rm = Subst_common
505 if cnf.find("Dinstall::Bcc") != "":
506 bcc.append(cnf["Dinstall::Bcc"])
507 if cnf.find("Rm::Bcc") != "":
508 bcc.append(cnf["Rm::Bcc"])
510 Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
512 Subst_close_rm["__BCC__"] = "X-Filler: 42"
513 summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason)
514 summarymail += "----------------------------------------------\n"
515 Subst_close_rm["__SUMMARY__"] = summarymail
517 for bug in done_bugs:
518 Subst_close_rm["__BUG_NUMBER__"] = bug
519 if close_related_bugs:
520 mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
522 mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
523 utils.send_mail(mail_message, whitelists=whitelists)
525 # close associated bug reports
526 if close_related_bugs:
527 Subst_close_other = Subst_common
529 wnpp = utils.parse_wnpp_bug_file()
530 versions = list(set([re_bin_only_nmu.sub('', v) for v in versions]))
531 if len(versions) == 1:
532 Subst_close_other["__VERSION__"] = versions[0]
534 logfile.write("=========================================================================\n")
535 logfile822.write("\n")
536 raise ValueError("Closing bugs with multiple package versions is not supported. Do it yourself.")
538 Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
540 Subst_close_other["__BCC__"] = "X-Filler: 42"
541 # at this point, I just assume, that the first closed bug gives
542 # some useful information on why the package got removed
543 Subst_close_other["__BUG_NUMBER__"] = done_bugs[0]
544 if len(sources) == 1:
545 source_pkg = source.split("_", 1)[0]
547 logfile.write("=========================================================================\n")
548 logfile822.write("\n")
549 raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.")
550 Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
551 Subst_close_other["__SOURCE__"] = source_pkg
553 other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded')
555 for bugno in other_bugs:
556 if bugno not in merged_bugs:
557 for bug in bts.get_status(bugno):
558 for merged in bug.mergedwith:
559 other_bugs.remove(merged)
560 merged_bugs.add(merged)
561 logfile.write("Also closing bug(s):")
562 logfile822.write("Also-Bugs:")
563 for bug in other_bugs:
564 Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
565 logfile.write(" " + str(bug))
566 logfile822.write(" " + str(bug))
568 logfile822.write("\n")
569 if source_pkg in wnpp:
570 logfile.write("Also closing WNPP bug(s):")
571 logfile822.write("Also-WNPP:")
572 for bug in wnpp[source_pkg]:
573 # the wnpp-rm file we parse also contains our removal
574 # bugs, filtering that out
575 if bug != Subst_close_other["__BUG_NUMBER__"]:
576 Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
577 logfile.write(" " + str(bug))
578 logfile822.write(" " + str(bug))
580 logfile822.write("\n")
582 mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"]+"/rm.bug-close-related")
583 if Subst_close_other["__BUG_NUMBER_ALSO__"]:
584 utils.send_mail(mail_message)
586 logfile.write("=========================================================================\n")
587 logfile822.write("\n")