1 """General purpose package removal code for ftpmaster
3 @contact: Debian FTP Master <ftpmaster@debian.org>
4 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
5 @copyright: 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
6 @copyright: 2015 Niels Thykier <niels@thykier.net>
7 @license: GNU General Public License version 2 or later
9 # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
10 # Copyright (C) 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ################################################################################
28 # From: Andrew Morton <akpm@osdl.org>
30 # To: linux-kernel@vger.kernel.org
31 # Date: Sat, 22 May 2004 01:36:36 -0700
32 # X-Mailer: Sylpheed version 0.9.7 (GTK+ 1.2.10; i386-redhat-linux-gnu)
36 # Although this feature has been around for a while it is new code, and the
37 # usual cautions apply. If it munches all your files please tell Jens and
38 # he'll type them in again for you.
40 ################################################################################
45 from collections import defaultdict
46 from regexes import re_build_dep_arch
48 from daklib.dbconn import *
49 from daklib import utils
50 from daklib.regexes import re_bin_only_nmu
51 import debianbts as bts
53 ################################################################################
56 class ReverseDependencyChecker(object):
57 """A bulk tester for reverse dependency checks
59 This class is similar to the check_reverse_depends method from "utils". However,
60 it is primarily focused on facilitating bulk testing of reverse dependencies.
61 It caches the state of the suite and then uses that as basis for answering queries.
62 This saves a significant amount of time if multiple reverse dependency checks are
66 def __init__(self, session, suite):
67 """Creates a new ReverseDependencyChecker instance
69 This will spend a significant amount of time caching data.
71 @type session: SQLA Session
72 @param session: The database session in use
75 @param suite: The name of the suite that is used as basis for removal tests.
77 self._session = session
78 dbsuite = get_suite(suite, session)
79 suite_archs2id = dict((x.arch_string, x.arch_id) for x in get_suite_architectures(suite))
80 package_dependencies, arch_providers_of, arch_provided_by = self._load_package_information(session,
83 self._package_dependencies = package_dependencies
84 self._arch_providers_of = arch_providers_of
85 self._arch_provided_by = arch_provided_by
86 self._archs_in_suite = set(suite_archs2id)
89 def _load_package_information(session, suite_id, suite_archs2id):
90 package_dependencies = defaultdict(lambda: defaultdict(set))
91 arch_providers_of = defaultdict(lambda: defaultdict(set))
92 arch_provided_by = defaultdict(lambda: defaultdict(set))
93 source_deps = defaultdict(set)
94 metakey_d = get_or_set_metadatakey("Depends", session)
95 metakey_p = get_or_set_metadatakey("Provides", session)
98 'arch_all_id': suite_archs2id['all'],
99 'metakey_d_id': metakey_d.key_id,
100 'metakey_p_id': metakey_p.key_id,
102 all_arches = set(suite_archs2id)
103 all_arches.discard('source')
105 package_dependencies['source'] = source_deps
107 for architecture in all_arches:
108 deps = defaultdict(set)
109 providers_of = defaultdict(set)
110 provided_by = defaultdict(set)
111 arch_providers_of[architecture] = providers_of
112 arch_provided_by[architecture] = provided_by
113 package_dependencies[architecture] = deps
115 params['arch_id'] = suite_archs2id[architecture]
119 (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
120 (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
122 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
123 WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id'''
124 query = session.query('package', 'depends', 'provides'). \
125 from_statement(statement).params(params)
126 for package, depends, provides in query:
128 if depends is not None:
131 for dep in apt_pkg.parse_depends(depends):
132 parsed_dep.append(frozenset(d[0] for d in dep))
133 deps[package].update(parsed_dep)
134 except ValueError as e:
135 print "Error for package %s: %s" % (package, e)
136 # Maintain a counter for each virtual package. If a
137 # Provides: exists, set the counter to 0 and count all
138 # provides by a package not in the list for removal.
139 # If the counter stays 0 at the end, we know that only
140 # the to-be-removed packages provided this virtual
142 if provides is not None:
143 for virtual_pkg in provides.split(","):
144 virtual_pkg = virtual_pkg.strip()
145 if virtual_pkg == package:
147 provided_by[virtual_pkg].add(package)
148 providers_of[package].add(virtual_pkg)
150 # Check source dependencies (Build-Depends and Build-Depends-Indep)
151 metakey_bd = get_or_set_metadatakey("Build-Depends", session)
152 metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
154 'suite_id': suite_id,
155 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
158 SELECT s.source, string_agg(sm.value, ', ') as build_dep
160 JOIN source_metadata sm ON s.id = sm.src_id
162 (SELECT source FROM src_associations
163 WHERE suite = :suite_id)
164 AND sm.key_id in :metakey_ids
165 GROUP BY s.id, s.source'''
166 query = session.query('source', 'build_dep').from_statement(statement). \
168 for source, build_dep in query:
169 if build_dep is not None:
170 # Remove [arch] information since we want to see breakage on all arches
171 build_dep = re_build_dep_arch.sub("", build_dep)
174 for dep in apt_pkg.parse_src_depends(build_dep):
175 parsed_dep.append(frozenset(d[0] for d in dep))
176 source_deps[source].update(parsed_dep)
177 except ValueError as e:
178 print "Error for package %s: %s" % (source, e)
180 return package_dependencies, arch_providers_of, arch_provided_by
182 def check_reverse_depends(self, removal_requests):
183 """Bulk check reverse dependencies
187 "eclipse-rcp": None, # means ALL architectures (incl. source)
188 "eclipse": None, # means ALL architectures (incl. source)
189 "lintian": ["source", "all"], # Only these two "architectures".
191 obj.check_reverse_depends(removal_request)
193 @type removal_requests: dict (or a list of tuples)
194 @param removal_requests: A dictionary mapping a package name to a list of architectures. The list of
195 architectures decides from which the package will be removed - if the list is empty the package will
196 be removed on ALL architectures in the suite (including "source").
199 @return: A mapping of "removed package" (as a "(pkg, arch)"-tuple) to a set of broken
200 broken packages (also as "(pkg, arch)"-tuple). Note that the architecture values
201 in these tuples /can/ be "source" to reflect a breakage in build-dependencies.
204 archs_in_suite = self._archs_in_suite
205 removals_by_arch = defaultdict(set)
206 affected_virtual_by_arch = defaultdict(set)
207 package_dependencies = self._package_dependencies
208 arch_providers_of = self._arch_providers_of
209 arch_provided_by = self._arch_provided_by
210 arch_provides2removal = defaultdict(lambda: defaultdict(set))
211 dep_problems = defaultdict(set)
212 src_deps = package_dependencies['source']
214 arch_all_removals = set()
216 if isinstance(removal_requests, dict):
217 removal_requests = removal_requests.iteritems()
219 for pkg, arch_list in removal_requests:
221 arch_list = archs_in_suite
222 for arch in arch_list:
224 src_removals.add(pkg)
227 arch_all_removals.add(pkg)
229 removals_by_arch[arch].add(pkg)
230 if pkg in arch_providers_of[arch]:
231 affected_virtual_by_arch[arch].add(pkg)
233 if arch_all_removals:
234 for arch in archs_in_suite:
235 if arch in ('all', 'source'):
237 removals_by_arch[arch].update(arch_all_removals)
238 for pkg in arch_all_removals:
239 if pkg in arch_providers_of[arch]:
240 affected_virtual_by_arch[arch].add(pkg)
242 if not removals_by_arch:
243 # Nothing to remove => no problems
246 for arch, removed_providers in affected_virtual_by_arch.iteritems():
247 provides2removal = arch_provides2removal[arch]
248 removals = removals_by_arch[arch]
249 for virtual_pkg, virtual_providers in arch_provided_by[arch].iteritems():
250 v = virtual_providers & removed_providers
251 if len(v) == len(virtual_providers):
252 # We removed all the providers of virtual_pkg
253 removals.add(virtual_pkg)
254 # Pick one to take the blame for the removal
255 # - we sort for determinism, optimally we would prefer to blame the same package
256 # to minimise the number of blamed packages.
257 provides2removal[virtual_pkg] = sorted(v)[0]
259 for arch, removals in removals_by_arch.iteritems():
260 deps = package_dependencies[arch]
261 provides2removal = arch_provides2removal[arch]
263 # Check binary dependencies (Depends)
264 for package, dependencies in deps.iteritems():
265 if package in removals:
267 for clause in dependencies:
268 if not (clause <= removals):
269 # Something probably still satisfies this relation
271 # whoops, we seemed to have removed all packages that could possibly satisfy
272 # this relation. Lets blame something for it
273 for dep_package in clause:
274 removal = dep_package
275 if dep_package in provides2removal:
276 removal = provides2removal[dep_package]
277 dep_problems[(removal, arch)].add((package, arch))
279 for source, build_dependencies in src_deps.iteritems():
280 if source in src_removals:
282 for clause in build_dependencies:
283 if not (clause <= removals):
284 # Something probably still satisfies this relation
286 # whoops, we seemed to have removed all packages that could possibly satisfy
287 # this relation. Lets blame something for it
288 for dep_package in clause:
289 removal = dep_package
290 if dep_package in provides2removal:
291 removal = provides2removal[dep_package]
292 dep_problems[(removal, arch)].add((source, 'source'))
297 def remove(session, reason, suites, removals,
298 whoami=None, partial=False, components=None, done_bugs=None, date=None,
299 carbon_copy=None, close_related_bugs=False):
300 """Batch remove a number of packages
301 Verify that the files listed in the Files field of the .dsc are
302 those expected given the announced Format.
304 @type session: SQLA Session
305 @param session: The database session in use
308 @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by <source>)")
311 @param suites: A list of the suite names in which the removal should occur
314 @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following
315 for 4 items from the database (in order): package, version, architecture, (database) id.
316 For source packages, the "architecture" should be set to "source".
319 @param partial: Whether the removal is "partial" (e.g. architecture specific).
321 @type components: list
322 @param components: List of components involved in a partial removal. Can be an empty list to not restrict the
323 removal to any components.
326 @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami()
329 @param date: The date of the removal. Defaults to commands.getoutput("date -R")
331 @type done_bugs: list
332 @param done_bugs: A list of bugs to be closed when doing this removal.
334 @type close_related_bugs: bool
335 @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented
336 for more than one suite.
338 @type carbon_copy: list
339 @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike
345 # Generate the summary of what's to be removed
353 suites_list = utils.join_with_commas_and(suites)
354 cnf = utils.get_conf()
357 #######################################################################################################
360 raise ValueError("Empty removal reason not permitted")
363 raise ValueError("Nothing to remove!?")
366 raise ValueError("Removals without a suite!?")
369 whoami = utils.whoami()
372 date = commands.getoutput("date -R")
376 component_ids_list = []
377 for componentname in components:
378 component = get_component(componentname, session=session)
379 if component is None:
380 raise ValueError("component '%s' not recognised." % componentname)
382 component_ids_list.append(component.component_id)
383 if component_ids_list:
384 con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list])
392 if version not in d[package]:
393 d[package][version] = []
394 if architecture not in d[package][version]:
395 d[package][version].append(architecture)
397 for package in sorted(d):
398 versions = sorted(d[package], cmp=apt_pkg.version_compare)
399 for version in versions:
400 d[package][version].sort(utils.arch_compare_sw)
401 summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
403 for package in summary.split("\n"):
404 for row in package.split("\n"):
405 element = row.split("|")
406 if len(element) == 3:
407 if element[2].find("source") > 0:
408 sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2]))
409 element[2] = sub("source\s?,?", "", element[2]).strip(" ")
411 binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element))
413 dsc_type_id = get_override_type('dsc', session).overridetype_id
414 deb_type_id = get_override_type('deb', session).overridetype_id
417 s = get_suite(suite, session=session)
419 suite_ids_list.append(s.suite_id)
420 whitelists.append(s.mail_whitelist)
422 #######################################################################################################
423 log_filename = cnf["Rm::LogFile"]
424 log822_filename = cnf["Rm::LogFile822"]
425 with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822:
426 logfile.write("=========================================================================\n")
427 logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
428 logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
430 logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs)))
431 logfile.write("\n------------------- Reason -------------------\n%s\n" % reason)
432 logfile.write("----------------------------------------------\n")
434 logfile822.write("Date: %s\n" % date)
435 logfile822.write("Ftpmaster: %s\n" % whoami)
436 logfile822.write("Suite: %s\n" % suites_list)
439 logfile822.write("Sources:\n")
440 for source in sources:
441 logfile822.write(" %s\n" % source)
444 logfile822.write("Binaries:\n")
445 for binary in binaries:
446 logfile822.write(" %s\n" % binary)
448 logfile822.write("Reason: %s\n" % reason.replace('\n', '\n '))
450 logfile822.write("Bug: %s\n" % (", ".join(done_bugs)))
456 for suite_id in suite_ids_list:
457 if architecture == "source":
458 session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
459 {'packageid': package_id, 'suiteid': suite_id})
461 session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
462 {'packageid': package_id, 'suiteid': suite_id})
463 # Delete from the override file
465 if architecture == "source":
466 type_id = dsc_type_id
468 type_id = deb_type_id
469 # TODO: Fix this properly to remove the remaining non-bind argument
470 session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
473 # ### REMOVAL COMPLETE - send mail time ### #
475 # If we don't have a Bug server configured, we're done
476 if "Dinstall::BugServer" not in cnf:
477 if done_bugs or close_related_bugs:
478 utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured")
480 logfile.write("=========================================================================\n")
481 logfile822.write("\n")
484 # read common subst variables for all bug closure mails
486 Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
487 Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
488 Subst_common["__CC__"] = "X-DAK: dak rm"
490 Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
491 Subst_common["__SUITE_LIST__"] = suites_list
492 Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list)
493 Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
494 Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
495 Subst_common["__WHOAMI__"] = whoami
497 # Send the bug closing messages
499 Subst_close_rm = Subst_common
501 if cnf.find("Dinstall::Bcc") != "":
502 bcc.append(cnf["Dinstall::Bcc"])
503 if cnf.find("Rm::Bcc") != "":
504 bcc.append(cnf["Rm::Bcc"])
506 Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
508 Subst_close_rm["__BCC__"] = "X-Filler: 42"
509 summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason)
510 summarymail += "----------------------------------------------\n"
511 Subst_close_rm["__SUMMARY__"] = summarymail
513 for bug in done_bugs:
514 Subst_close_rm["__BUG_NUMBER__"] = bug
515 if close_related_bugs:
516 mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
518 mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
519 utils.send_mail(mail_message, whitelists=whitelists)
521 # close associated bug reports
522 if close_related_bugs:
523 Subst_close_other = Subst_common
525 wnpp = utils.parse_wnpp_bug_file()
526 versions = list(set([re_bin_only_nmu.sub('', v) for v in versions]))
527 if len(versions) == 1:
528 Subst_close_other["__VERSION__"] = versions[0]
530 logfile.write("=========================================================================\n")
531 logfile822.write("\n")
532 raise ValueError("Closing bugs with multiple package versions is not supported. Do it yourself.")
534 Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
536 Subst_close_other["__BCC__"] = "X-Filler: 42"
537 # at this point, I just assume, that the first closed bug gives
538 # some useful information on why the package got removed
539 Subst_close_other["__BUG_NUMBER__"] = done_bugs[0]
540 if len(sources) == 1:
541 source_pkg = source.split("_", 1)[0]
543 logfile.write("=========================================================================\n")
544 logfile822.write("\n")
545 raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.")
546 Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
547 Subst_close_other["__SOURCE__"] = source_pkg
549 other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded')
551 for bugno in other_bugs:
552 if bugno not in merged_bugs:
553 for bug in bts.get_status(bugno):
554 for merged in bug.mergedwith:
555 other_bugs.remove(merged)
556 merged_bugs.add(merged)
557 logfile.write("Also closing bug(s):")
558 logfile822.write("Also-Bugs:")
559 for bug in other_bugs:
560 Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
561 logfile.write(" " + str(bug))
562 logfile822.write(" " + str(bug))
564 logfile822.write("\n")
565 if source_pkg in wnpp:
566 logfile.write("Also closing WNPP bug(s):")
567 logfile822.write("Also-WNPP:")
568 for bug in wnpp[source_pkg]:
569 # the wnpp-rm file we parse also contains our removal
570 # bugs, filtering that out
571 if bug != Subst_close_other["__BUG_NUMBER__"]:
572 Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
573 logfile.write(" " + str(bug))
574 logfile822.write(" " + str(bug))
576 logfile822.write("\n")
578 mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"]+"/rm.bug-close-related")
579 if Subst_close_other["__BUG_NUMBER_ALSO__"]:
580 utils.send_mail(mail_message)
582 logfile.write("=========================================================================\n")
583 logfile822.write("\n")