X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=blobdiff_plain;f=dak%2Fauto_decruft.py;h=e08e793b11af1d0a6b5b11f9d9d11d950c3a065c;hp=78bc5a283c92e66180e318615d330e8f11764d50;hb=1339ed64e64f6ef3fd2ea93a8165ab56d55fc856;hpb=ac84e3ea94787e97cc8354ac9132dcbf9dc922da diff --git a/dak/auto_decruft.py b/dak/auto_decruft.py index 78bc5a28..e08e793b 100644 --- a/dak/auto_decruft.py +++ b/dak/auto_decruft.py @@ -48,12 +48,18 @@ from daklib.rm import remove, ReverseDependencyChecker def usage(exit_code=0): - print """Usage: dak cruft-report -Check for obsolete or duplicated packages. + print """Usage: dak auto-decruft +Automatic removal of common kinds of cruft -h, --help show this help and exit. -n, --dry-run don't do anything, just show what would have been done - -s, --suite=SUITE check suite SUITE.""" + -s, --suite=SUITE check suite SUITE. + --if-newer-version-in OS remove all packages in SUITE with a lower version than + in OS (e.g. -s experimental --if-newer-version-in + unstable) + --if-newer-version-in-rm-msg RMMSG + use RMMSG in the removal message (e.g. "NVIU") + """ sys.exit(exit_code) ################################################################################ @@ -63,14 +69,15 @@ def compute_sourceless_groups(suite_id, session): """Find binaries without a source @type suite_id: int - @param suite_id: The id of the suite donated by suite_name + @param suite_id: The id of the suite denoted by suite_name @type session: SQLA Session @param session: The database session in use """"" rows = query_without_source(suite_id, session) message = '[auto-cruft] no longer built from source, no reverse dependencies' - arch_all_id_tuple = tuple([get_architecture('all', session=session)]) + arch = get_architecture('all', session=session) + arch_all_id_tuple = tuple([arch.arch_id]) arch_all_list = ["all"] for row in rows: package = row[0] @@ -91,7 +98,7 @@ def compute_nbs_groups(suite_id, suite_name, session): """Find binaries no longer built @type suite_id: int - @param suite_id: The id of the suite donated by suite_name + @param suite_id: The id of the suite denoted by suite_name @type suite_name: string @param suite_name: The name of the suite to remove from @@ -137,6 +144,52 @@ def remove_groups(groups, suite_id, suite_name, session): remove(session, message, [suite_name], list(q), partial=True, whoami="DAK's auto-decrufter") +def dedup(*args): + seen = set() + for iterable in args: + for value in iterable: + if value not in seen: + seen.add(value) + yield value + + +def merge_group(groupA, groupB): + """Merges two removal groups into one + + Note that some values are taken entirely from groupA (e.g. name and message) + + @type groupA: dict + @param groupA: A removal group + + @type groupB: dict + @param groupB: Another removal group + + @rtype: dict + @returns: A merged group + """ + pkg_list = sorted(dedup(groupA["packages"], groupB["packages"])) + arch_list = sorted(dedup(groupA["architectures"], groupB["architectures"]), cmp=utils.arch_compare_sw) + arch_list_id = dedup(groupA["architecture_ids"], groupB["architecture_ids"]) + removalA = groupA["removal_request"] + removalB = groupB["removal_request"] + new_removal = {} + for pkg in dedup(removalA, removalB): + listA = removalA[pkg] if pkg in removalA else [] + listB = removalB[pkg] if pkg in removalB else [] + new_removal[pkg] = sorted(dedup(listA, listB), cmp=utils.arch_compare_sw) + + merged_group = { + "name": groupA["name"], + "packages": tuple(pkg_list), + "architectures": arch_list, + "architecture_ids": tuple(arch_list_id), + "message": groupA["message"], + "removal_request": new_removal, + } + + return merged_group + + def auto_decruft_suite(suite_name, suite_id, session, dryrun, debug): """Run the auto-decrufter on a given suite @@ -144,7 +197,7 @@ def auto_decruft_suite(suite_name, suite_id, session, dryrun, debug): @param suite_name: The name of the suite to remove from @type suite_id: int - @param suite_id: The id of the suite donated by suite_name + @param suite_id: The id of the suite denoted by suite_name @type session: SQLA Session @param session: The database session in use @@ -166,24 +219,32 @@ def auto_decruft_suite(suite_name, suite_id, session, dryrun, debug): ) for group in group_generator: group_name = group["name"] - pkgs = group["packages"] - affected_archs = group["architectures"] - removal_request = group["removal_request"] - # If we remove an arch:all package, then the breakage can occur on any - # of the architectures. - if "all" in affected_archs: - affected_archs = all_architectures - for pkg_arch in product(pkgs, affected_archs): - pkg_arch2groups[pkg_arch].add(group_name) - groups[group_name] = group - group_order.append(group_name) + if group_name not in groups: + pkgs = group["packages"] + affected_archs = group["architectures"] + # If we remove an arch:all package, then the breakage can occur on any + # of the architectures. + if "all" in affected_archs: + affected_archs = all_architectures + for pkg_arch in product(pkgs, affected_archs): + pkg_arch2groups[pkg_arch].add(group_name) + groups[group_name] = group + group_order.append(group_name) + else: + # This case usually happens when versions differ between architectures... + if debug: + print "N: Merging group %s" % (group_name) + groups[group_name] = merge_group(groups[group_name], group) + for group_name in group_order: + removal_request = groups[group_name]["removal_request"] full_removal_request.extend(removal_request.iteritems()) if not groups: if debug: print "N: Found no candidates" return + if debug: print "N: Considering to remove the following packages:" for group_name in sorted(groups): @@ -279,6 +340,70 @@ def auto_decruft_suite(suite_name, suite_id, session, dryrun, debug): remove_groups(groups.itervalues(), suite_id, suite_name, session) +def sources2removals(source_list, suite_id, session): + """Compute removals items given a list of names of source packages + + @type source_list: list + @param source_list: A list of names of source packages + + @type suite_id: int + @param suite_id: The id of the suite from which these sources should be removed + + @type session: SQLA Session + @param session: The database session in use + + @rtype: list + @return: A list of items to be removed to remove all sources and their binaries from the given suite + """ + to_remove = [] + params = {"suite_id": suite_id, "sources": tuple(source_list)} + q = session.execute(""" + SELECT s.source, s.version, 'source', s.id + FROM source s + JOIN src_associations sa ON sa.source = s.id + WHERE sa.suite = :suite_id AND s.source IN :sources""", params) + to_remove.extend(q) + q = session.execute(""" + SELECT b.package, b.version, a.arch_string, b.id + FROM binaries b + JOIN bin_associations ba ON b.id = ba.bin + JOIN architecture a ON b.architecture = a.id + JOIN source s ON b.source = s.id + WHERE ba.suite = :suite_id AND s.source IN :sources""", params) + to_remove.extend(q) + return to_remove + + +def decruft_newer_version_in(othersuite, suite_name, suite_id, rm_msg, session, dryrun): + """Compute removals items given a list of names of source packages + + @type othersuite: str + @param othersuite: The name of the suite to compare with (e.g. "unstable" for "NVIU") + + @type suite: str + @param suite: The name of the suite from which to do removals (e.g. "experimental" for "NVIU") + + @type suite_id: int + @param suite_id: The id of the suite from which these sources should be removed + + @type rm_msg: str + @param rm_msg: The removal message (or tag, e.g. "NVIU") + + @type session: SQLA Session + @param session: The database session in use + + @type dryrun: bool + @param dryrun: If True, just print the actions rather than actually doing them + """ + nvi_list = [x[0] for x in newer_version(othersuite, suite_name, session)] + if nvi_list: + message = "[auto-cruft] %s" % rm_msg + if dryrun: + print " dak rm -m \"%s\" -s %s %s" % (message, suite_name, " ".join(nvi_list)) + else: + removals = sources2removals(nvi_list, suite_id, session) + remove(session, message, [suite_name], removals, whoami="DAK's auto-decrufter") + ################################################################################ def main (): @@ -288,8 +413,11 @@ def main (): Arguments = [('h',"help","Auto-Decruft::Options::Help"), ('n',"dry-run","Auto-Decruft::Options::Dry-Run"), ('d',"debug","Auto-Decruft::Options::Debug"), - ('s',"suite","Auto-Decruft::Options::Suite","HasArg")] - for i in ["help", "Dry-Run", "Debug"]: + ('s',"suite","Auto-Decruft::Options::Suite","HasArg"), + # The "\0" seems to be the only way to disable short options. + ("\0",'if-newer-version-in',"Auto-Decruft::Options::OtherSuite", "HasArg"), + ("\0",'if-newer-version-in-rm-msg',"Auto-Decruft::Options::OtherSuiteRMMsg", "HasArg")] + for i in ["help", "Dry-Run", "Debug", "OtherSuite", "OtherSuiteRMMsg"]: if not cnf.has_key("Auto-Decruft::Options::%s" % (i)): cnf["Auto-Decruft::Options::%s" % (i)] = "" @@ -308,6 +436,9 @@ def main (): if Options["Debug"]: debug = True + if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]: + utils.fubar("--if-newer-version-in requires --if-newer-version-in-rm-msg") + session = DBConn().session() suite = get_suite(Options["Suite"].lower(), session) @@ -319,6 +450,13 @@ def main (): auto_decruft_suite(suite_name, suite_id, session, dryrun, debug) + if Options["OtherSuite"]: + osuite = get_suite(Options["OtherSuite"].lower(), session).suite_name + decruft_newer_version_in(osuite, suite_name, suite_id, Options["OtherSuiteRMMsg"], session, dryrun) + + if not dryrun: + session.commit() + ################################################################################ if __name__ == '__main__':