4 Check for obsolete binary packages
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2000-2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Torsten Werner <twerner@debian.org>
9 @copyright: 2015 Niels Thykier <niels@thykier.net>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # | priviledged positions? What privilege? The honour of working harder
30 # | than most people for absolutely no recognition?
32 # Manoj Srivastava <srivasta@debian.org> in <87lln8aqfm.fsf@glaurung.internal.golden-gryphon.com>
34 ################################################################################
38 from itertools import chain, product
39 from collections import defaultdict
41 from daklib.config import Config
42 from daklib.dbconn import *
43 from daklib import utils
44 from daklib.cruft import *
45 from daklib.rm import remove, ReverseDependencyChecker
47 ################################################################################
50 def usage(exit_code=0):
51 print """Usage: dak auto-decruft
52 Automatic removal of common kinds of cruft
54 -h, --help show this help and exit.
55 -n, --dry-run don't do anything, just show what would have been done
56 -s, --suite=SUITE check suite SUITE.
57 --if-newer-version-in OS remove all packages in SUITE with a lower version than
58 in OS (e.g. -s experimental --if-newer-version-in
60 --if-newer-version-in-rm-msg RMMSG
61 use RMMSG in the removal message (e.g. "NVIU")
65 ################################################################################
68 def compute_sourceless_groups(suite_id, session):
69 """Find binaries without a source
72 @param suite_id: The id of the suite denoted by suite_name
74 @type session: SQLA Session
75 @param session: The database session in use
77 rows = query_without_source(suite_id, session)
78 message = '[auto-cruft] no longer built from source, no reverse dependencies'
79 arch = get_architecture('all', session=session)
80 arch_all_id_tuple = tuple([arch.arch_id])
81 arch_all_list = ["all"]
85 "name": "sourceless:%s" % package,
86 "packages": tuple([package]),
87 "architectures": arch_all_list,
88 "architecture_ids": arch_all_id_tuple,
91 package: arch_all_list,
97 def compute_nbs_groups(suite_id, suite_name, session):
98 """Find binaries no longer built
101 @param suite_id: The id of the suite denoted by suite_name
103 @type suite_name: string
104 @param suite_name: The name of the suite to remove from
106 @type session: SQLA Session
107 @param session: The database session in use
109 rows = queryNBS(suite_id, session)
110 arch2ids = dict((a.arch_string, a.arch_id) for a in get_suite_architectures(suite_name))
113 (pkg_list, arch_list, source, _) = row
114 message = '[auto-cruft] NBS (no longer built by %s, no reverse dependencies)' % source
115 removal_request = dict((pkg, arch_list) for pkg in pkg_list)
117 "name": "NBS:%s" % source,
118 "packages": tuple(sorted(pkg_list)),
119 "architectures": sorted(arch_list, cmp=utils.arch_compare_sw),
120 "architecture_ids": tuple(arch2ids[arch] for arch in arch_list),
122 "removal_request": removal_request,
127 def remove_groups(groups, suite_id, suite_name, session):
129 message = group["message"]
131 "architecture_ids": group["architecture_ids"],
132 "packages": group["packages"],
135 q = session.execute("""
136 SELECT b.package, b.version, a.arch_string, b.id
138 JOIN bin_associations ba ON b.id = ba.bin
139 JOIN architecture a ON b.architecture = a.id
140 JOIN suite su ON ba.suite = su.id
141 WHERE a.id IN :architecture_ids AND b.package IN :packages AND su.id = :suite_id
144 remove(session, message, [suite_name], list(q), partial=True, whoami="DAK's auto-decrufter")
149 for iterable in args:
150 for value in iterable:
151 if value not in seen:
156 def merge_group(groupA, groupB):
157 """Merges two removal groups into one
159 Note that some values are taken entirely from groupA (e.g. name and message)
162 @param groupA: A removal group
165 @param groupB: Another removal group
168 @returns: A merged group
170 pkg_list = sorted(dedup(groupA["packages"], groupB["packages"]))
171 arch_list = sorted(dedup(groupA["architectures"], groupB["architectures"]), cmp=utils.arch_compare_sw)
172 arch_list_id = dedup(groupA["architecture_ids"], groupB["architecture_ids"])
173 removalA = groupA["removal_request"]
174 removalB = groupB["removal_request"]
176 for pkg in dedup(removalA, removalB):
177 listA = removalA[pkg] if pkg in removalA else []
178 listB = removalB[pkg] if pkg in removalB else []
179 new_removal[pkg] = sorted(dedup(listA, listB), cmp=utils.arch_compare_sw)
182 "name": groupA["name"],
183 "packages": tuple(pkg_list),
184 "architectures": arch_list,
185 "architecture_ids": tuple(arch_list_id),
186 "message": groupA["message"],
187 "removal_request": new_removal,
193 def auto_decruft_suite(suite_name, suite_id, session, dryrun, debug):
194 """Run the auto-decrufter on a given suite
196 @type suite_name: string
197 @param suite_name: The name of the suite to remove from
200 @param suite_id: The id of the suite denoted by suite_name
202 @type session: SQLA Session
203 @param session: The database session in use
206 @param dryrun: If True, just print the actions rather than actually doing them
209 @param debug: If True, print some extra information
211 all_architectures = [a.arch_string for a in get_suite_architectures(suite_name)]
212 pkg_arch2groups = defaultdict(set)
215 full_removal_request = []
216 group_generator = chain(
217 compute_sourceless_groups(suite_id, session),
218 compute_nbs_groups(suite_id, suite_name, session)
220 for group in group_generator:
221 group_name = group["name"]
222 pkgs = group["packages"]
223 affected_archs = group["architectures"]
224 # If we remove an arch:all package, then the breakage can occur on any
225 # of the architectures.
226 if "all" in affected_archs:
227 affected_archs = all_architectures
228 for pkg_arch in product(pkgs, affected_archs):
229 pkg_arch2groups[pkg_arch].add(group_name)
230 if group_name not in groups:
231 groups[group_name] = group
232 group_order.append(group_name)
234 # This case usually happens when versions differ between architectures...
236 print "N: Merging group %s" % (group_name)
237 groups[group_name] = merge_group(groups[group_name], group)
239 for group_name in group_order:
240 removal_request = groups[group_name]["removal_request"]
241 full_removal_request.extend(removal_request.iteritems())
245 print "N: Found no candidates"
249 print "N: Considering to remove the following packages:"
250 for group_name in sorted(groups):
251 group_info = groups[group_name]
252 pkgs = group_info["packages"]
253 archs = group_info["architectures"]
254 print "N: * %s: %s [%s]" % (group_name, ", ".join(pkgs), " ".join(archs))
257 print "N: Compiling ReverseDependencyChecker (RDC) - please hold ..."
258 rdc = ReverseDependencyChecker(session, suite_name)
260 print "N: Computing initial breakage..."
262 breakage = rdc.check_reverse_depends(full_removal_request)
264 by_breakers = [(len(breakage[x]), x, breakage[x]) for x in breakage]
265 by_breakers.sort(reverse=True)
267 print "N: - Removal would break %s (package, architecture)-pairs" % (len(breakage))
268 print "N: - full breakage:"
269 for _, breaker, broken in by_breakers:
270 bname = "%s/%s" % breaker
271 broken_str = ", ".join("%s/%s" % b for b in sorted(broken))
272 print "N: * %s => %s" % (bname, broken_str)
274 averted_breakage = set()
276 for _, package_arch, breakage in by_breakers:
277 if breakage <= averted_breakage:
278 # We already avoided this break
280 guilty_groups = pkg_arch2groups[package_arch]
282 if not guilty_groups:
283 utils.fubar("Cannot figure what group provided %s" % str(package_arch))
286 # Only output it, if it truly a new group being discarded
287 # - a group can reach this part multiple times, if it breaks things on
288 # more than one architecture. This being rather common in fact.
289 already_discard = True
290 if any(group_name for group_name in guilty_groups if group_name in groups):
291 already_discard = False
293 if not already_discard:
294 avoided = sorted(breakage - averted_breakage)
295 print "N: - skipping removal of %s (breakage: %s)" % (", ".join(sorted(guilty_groups)), str(avoided))
297 averted_breakage |= breakage
298 for group_name in guilty_groups:
299 if group_name in groups:
300 del groups[group_name]
304 print "N: Nothing left to remove"
308 print "N: Now considering to remove: %s" % str(", ".join(sorted(groups.iterkeys())))
310 # Rebuild the removal request with the remaining groups and off
311 # we go to (not) break the world once more time
312 full_removal_request = []
313 for group_info in groups.itervalues():
314 full_removal_request.extend(group_info["removal_request"].iteritems())
315 breakage = rdc.check_reverse_depends(full_removal_request)
318 print "N: Removal looks good"
321 print "Would remove the equivalent of:"
322 for group_name in group_order:
323 if group_name not in groups:
325 group_info = groups[group_name]
326 pkgs = group_info["packages"]
327 archs = group_info["architectures"]
328 message = group_info["message"]
330 # Embed the -R just in case someone wants to run it manually later
331 print ' dak rm -m "{message}" -s {suite} -a {architectures} -p -R -b {packages}'.format(
332 message=message, suite=suite_name,
333 architectures=",".join(archs), packages=" ".join(pkgs),
337 print "Note: The removals may be interdependent. A non-breaking result may require the execution of all"
338 print "of the removals"
340 remove_groups(groups.itervalues(), suite_id, suite_name, session)
343 def sources2removals(source_list, suite_id, session):
344 """Compute removals items given a list of names of source packages
346 @type source_list: list
347 @param source_list: A list of names of source packages
350 @param suite_id: The id of the suite from which these sources should be removed
352 @type session: SQLA Session
353 @param session: The database session in use
356 @return: A list of items to be removed to remove all sources and their binaries from the given suite
359 params = {"suite_id": suite_id, "sources": tuple(source_list)}
360 q = session.execute("""
361 SELECT s.source, s.version, 'source', s.id
363 JOIN src_associations sa ON sa.source = s.id
364 WHERE sa.suite = :suite_id AND s.source IN :sources""", params)
366 q = session.execute("""
367 SELECT b.package, b.version, a.arch_string, b.id
369 JOIN bin_associations ba ON b.id = ba.bin
370 JOIN architecture a ON b.architecture = a.id
371 JOIN source s ON b.source = s.id
372 WHERE ba.suite = :suite_id AND s.source IN :sources""", params)
377 def decruft_newer_version_in(othersuite, suite_name, suite_id, rm_msg, session, dryrun):
378 """Compute removals items given a list of names of source packages
380 @type othersuite: str
381 @param othersuite: The name of the suite to compare with (e.g. "unstable" for "NVIU")
384 @param suite: The name of the suite from which to do removals (e.g. "experimental" for "NVIU")
387 @param suite_id: The id of the suite from which these sources should be removed
390 @param rm_msg: The removal message (or tag, e.g. "NVIU")
392 @type session: SQLA Session
393 @param session: The database session in use
396 @param dryrun: If True, just print the actions rather than actually doing them
398 nvi_list = [x[0] for x in newer_version(othersuite, suite_name, session)]
400 message = "[auto-cruft] %s" % rm_msg
402 print " dak rm -m \"%s\" -s %s %s" % (message, suite_name, " ".join(nvi_list))
404 removals = sources2removals(nvi_list, suite_id, session)
405 remove(session, message, [suite_name], removals, whoami="DAK's auto-decrufter")
407 ################################################################################
413 Arguments = [('h',"help","Auto-Decruft::Options::Help"),
414 ('n',"dry-run","Auto-Decruft::Options::Dry-Run"),
415 ('d',"debug","Auto-Decruft::Options::Debug"),
416 ('s',"suite","Auto-Decruft::Options::Suite","HasArg"),
417 # The "\0" seems to be the only way to disable short options.
418 ("\0",'if-newer-version-in',"Auto-Decruft::Options::OtherSuite", "HasArg"),
419 ("\0",'if-newer-version-in-rm-msg',"Auto-Decruft::Options::OtherSuiteRMMsg", "HasArg")]
420 for i in ["help", "Dry-Run", "Debug", "OtherSuite", "OtherSuiteRMMsg"]:
421 if not cnf.has_key("Auto-Decruft::Options::%s" % (i)):
422 cnf["Auto-Decruft::Options::%s" % (i)] = ""
424 cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
426 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
428 Options = cnf.subtree("Auto-Decruft::Options")
434 if Options["Dry-Run"]:
439 if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]:
440 utils.fubar("--if-newer-version-in requires --if-newer-version-in-rm-msg")
442 session = DBConn().session()
444 suite = get_suite(Options["Suite"].lower(), session)
446 utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
448 suite_id = suite.suite_id
449 suite_name = suite.suite_name.lower()
451 auto_decruft_suite(suite_name, suite_id, session, dryrun, debug)
453 if Options["OtherSuite"]:
454 osuite = get_suite(Options["OtherSuite"].lower(), session).suite_name
455 decruft_newer_version_in(osuite, suite_name, suite_id, Options["OtherSuiteRMMsg"], session, dryrun)
460 ################################################################################
462 if __name__ == '__main__':