X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Frm.py;h=1e523d2e0a0dc4662bc90452fe24232c0b47f421;hb=391f5ec09a119131dc846b796ca791f4cecc69e4;hp=892872f07626285139118072e20e15f1d630c57b;hpb=6ff8b512d003c1b49fe2853f0a98d6386368095b;p=dak.git diff --git a/daklib/rm.py b/daklib/rm.py index 892872f0..1e523d2e 100644 --- a/daklib/rm.py +++ b/daklib/rm.py @@ -25,13 +25,26 @@ ################################################################################ -# TODO: Insert "random dak quote" here +# From: Andrew Morton +# Subject: 2.6.6-mm5 +# To: linux-kernel@vger.kernel.org +# Date: Sat, 22 May 2004 01:36:36 -0700 +# X-Mailer: Sylpheed version 0.9.7 (GTK+ 1.2.10; i386-redhat-linux-gnu) +# +# [...] +# +# Although this feature has been around for a while it is new code, and the +# usual cautions apply. If it munches all your files please tell Jens and +# he'll type them in again for you. ################################################################################ import commands import apt_pkg +import fcntl from re import sub +from collections import defaultdict +from regexes import re_build_dep_arch from daklib.dbconn import * from daklib import utils @@ -41,6 +54,247 @@ import debianbts as bts ################################################################################ +class ReverseDependencyChecker(object): + """A bulk tester for reverse dependency checks + + This class is similar to the check_reverse_depends method from "utils". However, + it is primarily focused on facilitating bulk testing of reverse dependencies. + It caches the state of the suite and then uses that as basis for answering queries. + This saves a significant amount of time if multiple reverse dependency checks are + required. + """ + + def __init__(self, session, suite): + """Creates a new ReverseDependencyChecker instance + + This will spend a significant amount of time caching data. + + @type session: SQLA Session + @param session: The database session in use + + @type suite: str + @param suite: The name of the suite that is used as basis for removal tests. + """ + self._session = session + dbsuite = get_suite(suite, session) + suite_archs2id = dict((x.arch_string, x.arch_id) for x in get_suite_architectures(suite)) + package_dependencies, arch_providers_of, arch_provided_by = self._load_package_information(session, + dbsuite.suite_id, + suite_archs2id) + self._package_dependencies = package_dependencies + self._arch_providers_of = arch_providers_of + self._arch_provided_by = arch_provided_by + self._archs_in_suite = set(suite_archs2id) + + @staticmethod + def _load_package_information(session, suite_id, suite_archs2id): + package_dependencies = defaultdict(lambda: defaultdict(set)) + arch_providers_of = defaultdict(lambda: defaultdict(set)) + arch_provided_by = defaultdict(lambda: defaultdict(set)) + source_deps = defaultdict(set) + metakey_d = get_or_set_metadatakey("Depends", session) + metakey_p = get_or_set_metadatakey("Provides", session) + params = { + 'suite_id': suite_id, + 'arch_all_id': suite_archs2id['all'], + 'metakey_d_id': metakey_d.key_id, + 'metakey_p_id': metakey_p.key_id, + } + all_arches = set(suite_archs2id) + all_arches.discard('source') + + package_dependencies['source'] = source_deps + + for architecture in all_arches: + deps = defaultdict(set) + providers_of = defaultdict(set) + provided_by = defaultdict(set) + arch_providers_of[architecture] = providers_of + arch_provided_by[architecture] = provided_by + package_dependencies[architecture] = deps + + params['arch_id'] = suite_archs2id[architecture] + + statement = ''' + SELECT b.package, + (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends, + (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides + FROM binaries b + JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id + WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id''' + query = session.query('package', 'depends', 'provides'). \ + from_statement(statement).params(params) + for package, depends, provides in query: + + if depends is not None: + try: + parsed_dep = [] + for dep in apt_pkg.parse_depends(depends): + parsed_dep.append(frozenset(d[0] for d in dep)) + deps[package].update(parsed_dep) + except ValueError as e: + print "Error for package %s: %s" % (package, e) + # Maintain a counter for each virtual package. If a + # Provides: exists, set the counter to 0 and count all + # provides by a package not in the list for removal. + # If the counter stays 0 at the end, we know that only + # the to-be-removed packages provided this virtual + # package. + if provides is not None: + for virtual_pkg in provides.split(","): + virtual_pkg = virtual_pkg.strip() + if virtual_pkg == package: + continue + provided_by[virtual_pkg].add(package) + providers_of[package].add(virtual_pkg) + + # Check source dependencies (Build-Depends and Build-Depends-Indep) + metakey_bd = get_or_set_metadatakey("Build-Depends", session) + metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session) + params = { + 'suite_id': suite_id, + 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id), + } + statement = ''' + SELECT s.source, string_agg(sm.value, ', ') as build_dep + FROM source s + JOIN source_metadata sm ON s.id = sm.src_id + WHERE s.id in + (SELECT source FROM src_associations + WHERE suite = :suite_id) + AND sm.key_id in :metakey_ids + GROUP BY s.id, s.source''' + query = session.query('source', 'build_dep').from_statement(statement). \ + params(params) + for source, build_dep in query: + if build_dep is not None: + # Remove [arch] information since we want to see breakage on all arches + build_dep = re_build_dep_arch.sub("", build_dep) + try: + parsed_dep = [] + for dep in apt_pkg.parse_src_depends(build_dep): + parsed_dep.append(frozenset(d[0] for d in dep)) + source_deps[source].update(parsed_dep) + except ValueError as e: + print "Error for package %s: %s" % (source, e) + + return package_dependencies, arch_providers_of, arch_provided_by + + def check_reverse_depends(self, removal_requests): + """Bulk check reverse dependencies + + Example: + removal_request = { + "eclipse-rcp": None, # means ALL architectures (incl. source) + "eclipse": None, # means ALL architectures (incl. source) + "lintian": ["source", "all"], # Only these two "architectures". + } + obj.check_reverse_depends(removal_request) + + @type removal_requests: dict (or a list of tuples) + @param removal_requests: A dictionary mapping a package name to a list of architectures. The list of + architectures decides from which the package will be removed - if the list is empty the package will + be removed on ALL architectures in the suite (including "source"). + + @rtype: dict + @return: A mapping of "removed package" (as a "(pkg, arch)"-tuple) to a set of broken + broken packages (also as "(pkg, arch)"-tuple). Note that the architecture values + in these tuples /can/ be "source" to reflect a breakage in build-dependencies. + """ + + archs_in_suite = self._archs_in_suite + removals_by_arch = defaultdict(set) + affected_virtual_by_arch = defaultdict(set) + package_dependencies = self._package_dependencies + arch_providers_of = self._arch_providers_of + arch_provided_by = self._arch_provided_by + arch_provides2removal = defaultdict(lambda: defaultdict(set)) + dep_problems = defaultdict(set) + src_deps = package_dependencies['source'] + src_removals = set() + arch_all_removals = set() + + if isinstance(removal_requests, dict): + removal_requests = removal_requests.iteritems() + + for pkg, arch_list in removal_requests: + if not arch_list: + arch_list = archs_in_suite + for arch in arch_list: + if arch == 'source': + src_removals.add(pkg) + continue + if arch == 'all': + arch_all_removals.add(pkg) + continue + removals_by_arch[arch].add(pkg) + if pkg in arch_providers_of[arch]: + affected_virtual_by_arch[arch].add(pkg) + + if arch_all_removals: + for arch in archs_in_suite: + if arch in ('all', 'source'): + continue + removals_by_arch[arch].update(arch_all_removals) + for pkg in arch_all_removals: + if pkg in arch_providers_of[arch]: + affected_virtual_by_arch[arch].add(pkg) + + if not removals_by_arch: + # Nothing to remove => no problems + return dep_problems + + for arch, removed_providers in affected_virtual_by_arch.iteritems(): + provides2removal = arch_provides2removal[arch] + removals = removals_by_arch[arch] + for virtual_pkg, virtual_providers in arch_provided_by[arch].iteritems(): + v = virtual_providers & removed_providers + if len(v) == len(virtual_providers): + # We removed all the providers of virtual_pkg + removals.add(virtual_pkg) + # Pick one to take the blame for the removal + # - we sort for determinism, optimally we would prefer to blame the same package + # to minimise the number of blamed packages. + provides2removal[virtual_pkg] = sorted(v)[0] + + for arch, removals in removals_by_arch.iteritems(): + deps = package_dependencies[arch] + provides2removal = arch_provides2removal[arch] + + # Check binary dependencies (Depends) + for package, dependencies in deps.iteritems(): + if package in removals: + continue + for clause in dependencies: + if not (clause <= removals): + # Something probably still satisfies this relation + continue + # whoops, we seemed to have removed all packages that could possibly satisfy + # this relation. Lets blame something for it + for dep_package in clause: + removal = dep_package + if dep_package in provides2removal: + removal = provides2removal[dep_package] + dep_problems[(removal, arch)].add((package, arch)) + + for source, build_dependencies in src_deps.iteritems(): + if source in src_removals: + continue + for clause in build_dependencies: + if not (clause <= removals): + # Something probably still satisfies this relation + continue + # whoops, we seemed to have removed all packages that could possibly satisfy + # this relation. Lets blame something for it + for dep_package in clause: + removal = dep_package + if dep_package in provides2removal: + removal = provides2removal[dep_package] + dep_problems[(removal, arch)].add((source, 'source')) + + return dep_problems + + def remove(session, reason, suites, removals, whoami=None, partial=False, components=None, done_bugs=None, date=None, carbon_copy=None, close_related_bugs=False): @@ -96,10 +350,11 @@ def remove(session, reason, suites, removals, binaries = [] whitelists = [] versions = [] + newest_source = '' suite_ids_list = [] suites_list = utils.join_with_commas_and(suites) cnf = utils.get_conf() - con_components = None + con_components = '' ####################################################################################################### @@ -118,7 +373,7 @@ def remove(session, reason, suites, removals, if date is None: date = commands.getoutput("date -R") - if partial: + if partial and components: component_ids_list = [] for componentname in components: @@ -127,7 +382,8 @@ def remove(session, reason, suites, removals, raise ValueError("component '%s' not recognised." % componentname) else: component_ids_list.append(component.component_id) - con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list]) + if component_ids_list: + con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list]) for i in removals: package = i[0] @@ -140,11 +396,13 @@ def remove(session, reason, suites, removals, if architecture not in d[package][version]: d[package][version].append(architecture) - for package in sorted(removals): + for package in sorted(d): versions = sorted(d[package], cmp=apt_pkg.version_compare) for version in versions: d[package][version].sort(utils.arch_compare_sw) summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version])) + if apt_pkg.version_compare(version, newest_source) > 0: + newest_source = version for package in summary.split("\n"): for row in package.split("\n"): @@ -169,6 +427,9 @@ def remove(session, reason, suites, removals, log_filename = cnf["Rm::LogFile"] log822_filename = cnf["Rm::LogFile822"] with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822: + fcntl.lockf(logfile, fcntl.LOCK_EX) + fcntl.lockf(logfile822, fcntl.LOCK_EX) + logfile.write("=========================================================================\n") logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)) logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)) @@ -207,7 +468,7 @@ def remove(session, reason, suites, removals, session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid", {'packageid': package_id, 'suiteid': suite_id}) # Delete from the override file - if partial: + if not partial: if architecture == "source": type_id = dsc_type_id else: @@ -269,13 +530,19 @@ def remove(session, reason, suites, removals, Subst_close_other = Subst_common bcc = [] wnpp = utils.parse_wnpp_bug_file() - versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) - if len(versions) == 1: - Subst_close_other["__VERSION__"] = versions[0] + newest_source = re_bin_only_nmu.sub('', newest_source) + if len(set(s.split("_", 1)[0] for s in sources)) == 1: + source_pkg = source.split("_", 1)[0] + else: + logfile.write("=========================================================================\n") + logfile822.write("\n") + raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.") + if newest_source != '': + Subst_close_other["__VERSION__"] = newest_source else: logfile.write("=========================================================================\n") logfile822.write("\n") - raise ValueError("Closing bugs with multiple package versions is not supported. Do it yourself.") + raise ValueError("No versions can be found. Close bugs yourself.") if bcc: Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc) else: @@ -283,12 +550,6 @@ def remove(session, reason, suites, removals, # at this point, I just assume, that the first closed bug gives # some useful information on why the package got removed Subst_close_other["__BUG_NUMBER__"] = done_bugs[0] - if len(sources) == 1: - source_pkg = source.split("_", 1)[0] - else: - logfile.write("=========================================================================\n") - logfile822.write("\n") - raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.") Subst_close_other["__BUG_NUMBER_ALSO__"] = "" Subst_close_other["__SOURCE__"] = source_pkg merged_bugs = set()