From: Niels Thykier Date: Thu, 11 Jun 2015 11:36:04 +0000 (+0200) Subject: Merge branch 'master' into auto-decruft-passive X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=commitdiff_plain;h=321e39b64490fb825769f7ed9f6534cebeedd923;hp=28b86b197fffbd068c2f030e987bc145829d9601 Merge branch 'master' into auto-decruft-passive Conflicts: dak/generate_filelist.py --- diff --git a/dak/auto_decruft.py b/dak/auto_decruft.py new file mode 100644 index 00000000..78bc5a28 --- /dev/null +++ b/dak/auto_decruft.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python + +""" +Check for obsolete binary packages + +@contact: Debian FTP Master +@copyright: 2000-2006 James Troup +@copyright: 2009 Torsten Werner +@copyright: 2015 Niels Thykier +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# | priviledged positions? What privilege? The honour of working harder +# | than most people for absolutely no recognition? +# +# Manoj Srivastava in <87lln8aqfm.fsf@glaurung.internal.golden-gryphon.com> + +################################################################################ + +import sys +import apt_pkg +from itertools import chain, product +from collections import defaultdict + +from daklib.config import Config +from daklib.dbconn import * +from daklib import utils +from daklib.cruft import * +from daklib.rm import remove, ReverseDependencyChecker + +################################################################################ + + +def usage(exit_code=0): + print """Usage: dak cruft-report +Check for obsolete or duplicated packages. + + -h, --help show this help and exit. + -n, --dry-run don't do anything, just show what would have been done + -s, --suite=SUITE check suite SUITE.""" + sys.exit(exit_code) + +################################################################################ + + +def compute_sourceless_groups(suite_id, session): + """Find binaries without a source + + @type suite_id: int + @param suite_id: The id of the suite donated by suite_name + + @type session: SQLA Session + @param session: The database session in use + """"" + rows = query_without_source(suite_id, session) + message = '[auto-cruft] no longer built from source, no reverse dependencies' + arch_all_id_tuple = tuple([get_architecture('all', session=session)]) + arch_all_list = ["all"] + for row in rows: + package = row[0] + group_info = { + "name": "sourceless:%s" % package, + "packages": tuple([package]), + "architectures": arch_all_list, + "architecture_ids": arch_all_id_tuple, + "message": message, + "removal_request": { + package: arch_all_list, + }, + } + yield group_info + + +def compute_nbs_groups(suite_id, suite_name, session): + """Find binaries no longer built + + @type suite_id: int + @param suite_id: The id of the suite donated by suite_name + + @type suite_name: string + @param suite_name: The name of the suite to remove from + + @type session: SQLA Session + @param session: The database session in use + """"" + rows = queryNBS(suite_id, session) + arch2ids = dict((a.arch_string, a.arch_id) for a in get_suite_architectures(suite_name)) + + for row in rows: + (pkg_list, arch_list, source, _) = row + message = '[auto-cruft] NBS (no longer built by %s, no reverse dependencies)' % source + removal_request = dict((pkg, arch_list) for pkg in pkg_list) + group_info = { + "name": "NBS:%s" % source, + "packages": tuple(sorted(pkg_list)), + "architectures": sorted(arch_list, cmp=utils.arch_compare_sw), + "architecture_ids": tuple(arch2ids[arch] for arch in arch_list), + "message": message, + "removal_request": removal_request, + } + yield group_info + + +def remove_groups(groups, suite_id, suite_name, session): + for group in groups: + message = group["message"] + params = { + "architecture_ids": group["architecture_ids"], + "packages": group["packages"], + "suite_id": suite_id + } + q = session.execute(""" + SELECT b.package, b.version, a.arch_string, b.id + FROM binaries b + JOIN bin_associations ba ON b.id = ba.bin + JOIN architecture a ON b.architecture = a.id + JOIN suite su ON ba.suite = su.id + WHERE a.id IN :architecture_ids AND b.package IN :packages AND su.id = :suite_id + """, params) + + remove(session, message, [suite_name], list(q), partial=True, whoami="DAK's auto-decrufter") + + +def auto_decruft_suite(suite_name, suite_id, session, dryrun, debug): + """Run the auto-decrufter on a given suite + + @type suite_name: string + @param suite_name: The name of the suite to remove from + + @type suite_id: int + @param suite_id: The id of the suite donated by suite_name + + @type session: SQLA Session + @param session: The database session in use + + @type dryrun: bool + @param dryrun: If True, just print the actions rather than actually doing them + + @type debug: bool + @param debug: If True, print some extra information + """ + all_architectures = [a.arch_string for a in get_suite_architectures(suite_name)] + pkg_arch2groups = defaultdict(set) + group_order = [] + groups = {} + full_removal_request = [] + group_generator = chain( + compute_sourceless_groups(suite_id, session), + compute_nbs_groups(suite_id, suite_name, session) + ) + for group in group_generator: + group_name = group["name"] + pkgs = group["packages"] + affected_archs = group["architectures"] + removal_request = group["removal_request"] + # If we remove an arch:all package, then the breakage can occur on any + # of the architectures. + if "all" in affected_archs: + affected_archs = all_architectures + for pkg_arch in product(pkgs, affected_archs): + pkg_arch2groups[pkg_arch].add(group_name) + groups[group_name] = group + group_order.append(group_name) + + full_removal_request.extend(removal_request.iteritems()) + + if not groups: + if debug: + print "N: Found no candidates" + return + if debug: + print "N: Considering to remove the following packages:" + for group_name in sorted(groups): + group_info = groups[group_name] + pkgs = group_info["packages"] + archs = group_info["architectures"] + print "N: * %s: %s [%s]" % (group_name, ", ".join(pkgs), " ".join(archs)) + + if debug: + print "N: Compiling ReverseDependencyChecker (RDC) - please hold ..." + rdc = ReverseDependencyChecker(session, suite_name) + if debug: + print "N: Computing initial breakage..." + + breakage = rdc.check_reverse_depends(full_removal_request) + while breakage: + by_breakers = [(len(breakage[x]), x, breakage[x]) for x in breakage] + by_breakers.sort(reverse=True) + if debug: + print "N: - Removal would break %s (package, architecture)-pairs" % (len(breakage)) + print "N: - full breakage:" + for _, breaker, broken in by_breakers: + bname = "%s/%s" % breaker + broken_str = ", ".join("%s/%s" % b for b in sorted(broken)) + print "N: * %s => %s" % (bname, broken_str) + + averted_breakage = set() + + for _, package_arch, breakage in by_breakers: + if breakage <= averted_breakage: + # We already avoided this break + continue + guilty_groups = pkg_arch2groups[package_arch] + + if not guilty_groups: + utils.fubar("Cannot figure what group provided %s" % str(package_arch)) + + if debug: + # Only output it, if it truly a new group being discarded + # - a group can reach this part multiple times, if it breaks things on + # more than one architecture. This being rather common in fact. + already_discard = True + if any(group_name for group_name in guilty_groups if group_name in groups): + already_discard = False + + if not already_discard: + avoided = sorted(breakage - averted_breakage) + print "N: - skipping removal of %s (breakage: %s)" % (", ".join(sorted(guilty_groups)), str(avoided)) + + averted_breakage |= breakage + for group_name in guilty_groups: + if group_name in groups: + del groups[group_name] + + if not groups: + if debug: + print "N: Nothing left to remove" + return + + if debug: + print "N: Now considering to remove: %s" % str(", ".join(sorted(groups.iterkeys()))) + + # Rebuild the removal request with the remaining groups and off + # we go to (not) break the world once more time + full_removal_request = [] + for group_info in groups.itervalues(): + full_removal_request.extend(group_info["removal_request"].iteritems()) + breakage = rdc.check_reverse_depends(full_removal_request) + + if debug: + print "N: Removal looks good" + + if dryrun: + print "Would remove the equivalent of:" + for group_name in group_order: + if group_name not in groups: + continue + group_info = groups[group_name] + pkgs = group_info["packages"] + archs = group_info["architectures"] + message = group_info["message"] + + # Embed the -R just in case someone wants to run it manually later + print ' dak rm -m "{message}" -s {suite} -a {architectures} -p -R -b {packages}'.format( + message=message, suite=suite_name, + architectures=",".join(archs), packages=" ".join(pkgs), + ) + + print + print "Note: The removals may be interdependent. A non-breaking result may require the execution of all" + print "of the removals" + else: + remove_groups(groups.itervalues(), suite_id, suite_name, session) + + +################################################################################ + +def main (): + global Options + cnf = Config() + + Arguments = [('h',"help","Auto-Decruft::Options::Help"), + ('n',"dry-run","Auto-Decruft::Options::Dry-Run"), + ('d',"debug","Auto-Decruft::Options::Debug"), + ('s',"suite","Auto-Decruft::Options::Suite","HasArg")] + for i in ["help", "Dry-Run", "Debug"]: + if not cnf.has_key("Auto-Decruft::Options::%s" % (i)): + cnf["Auto-Decruft::Options::%s" % (i)] = "" + + cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") + + apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) + + Options = cnf.subtree("Auto-Decruft::Options") + if Options["Help"]: + usage() + + debug = False + dryrun = False + if Options["Dry-Run"]: + dryrun = True + if Options["Debug"]: + debug = True + + session = DBConn().session() + + suite = get_suite(Options["Suite"].lower(), session) + if not suite: + utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) + + suite_id = suite.suite_id + suite_name = suite.suite_name.lower() + + auto_decruft_suite(suite_name, suite_id, session, dryrun, debug) + +################################################################################ + +if __name__ == '__main__': + main() diff --git a/dak/cruft_report.py b/dak/cruft_report.py index a960a686..5c6ee8d0 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -209,29 +209,9 @@ def do_newer_version(lowersuite_name, highersuite_name, code, session): ################################################################################ -def queryWithoutSource(suite_id, session): - """searches for arch: all packages from suite that do no longer - reference a source package in the same suite - - subquery unique_binaries: selects all packages with only 1 version - in suite since 'dak rm' does not allow to specify version numbers""" - - query = """ - with unique_binaries as - (select package, max(version) as version, max(source) as source - from bin_associations_binaries - where architecture = 2 and suite = :suite_id - group by package having count(*) = 1) - select ub.package, ub.version - from unique_binaries ub - left join src_associations_src sas - on ub.source = sas.src and sas.suite = :suite_id - where sas.id is null - order by ub.package""" - return session.execute(query, { 'suite_id': suite_id }) def reportWithoutSource(suite_name, suite_id, session, rdeps=False): - rows = queryWithoutSource(suite_id, session) + rows = query_without_source(suite_id, session) title = 'packages without source in suite %s' % suite_name if rows.rowcount > 0: print '%s\n%s\n' % (title, '-' * len(title)) @@ -284,90 +264,7 @@ def reportNewerAll(suite_name, session): print " dak rm -m %s -s %s -a %s -p -b %s\n" % \ (message, suite_name, oldarch, package) -def queryNBS(suite_id, session): - """This one is really complex. It searches arch != all packages that - are no longer built from current source packages in suite. - - temp table unique_binaries: will be populated with packages that - have only one version in suite because 'dak rm' does not allow - specifying version numbers - - temp table newest_binaries: will be populated with packages that are - built from current sources - - subquery uptodate_arch: returns all architectures built from current - sources - subquery unique_binaries_uptodate_arch: returns all packages in - architectures from uptodate_arch - - subquery unique_binaries_uptodate_arch_agg: same as - unique_binaries_uptodate_arch but with column architecture - aggregated to array - - subquery uptodate_packages: similar to uptodate_arch but returns all - packages built from current sources - - subquery outdated_packages: returns all packages with architectures - no longer built from current source - """ - - query = """ -create temp table unique_binaries ( - package text not null, - architecture integer not null, - source integer not null); - -insert into unique_binaries - select bab.package, bab.architecture, max(bab.source) - from bin_associations_binaries bab - where bab.suite = :suite_id and bab.architecture > 2 - group by package, architecture having count(*) = 1; - -create temp table newest_binaries ( - package text not null, - architecture integer not null, - source text not null, - version debversion not null); - -insert into newest_binaries - select ub.package, ub.architecture, nsa.source, nsa.version - from unique_binaries ub - join newest_src_association nsa - on ub.source = nsa.src and nsa.suite = :suite_id; - -with uptodate_arch as - (select architecture, source, version - from newest_binaries - group by architecture, source, version), - unique_binaries_uptodate_arch as - (select ub.package, ub.architecture, ua.source, ua.version - from unique_binaries ub - join source s - on ub.source = s.id - join uptodate_arch ua - on ub.architecture = ua.architecture and s.source = ua.source), - unique_binaries_uptodate_arch_agg as - (select ubua.package, - array(select unnest(array_agg(a.arch_string)) order by 1) as arch_list, - ubua.source, ubua.version - from unique_binaries_uptodate_arch ubua - join architecture a - on ubua.architecture = a.id - group by ubua.source, ubua.version, ubua.package), - uptodate_packages as - (select package, source, version - from newest_binaries - group by package, source, version), - outdated_packages as - (select array(select unnest(array_agg(package)) order by 1) as pkg_list, - arch_list, source, version - from unique_binaries_uptodate_arch_agg - where package not in - (select package from uptodate_packages) - group by arch_list, source, version) - select * from outdated_packages order by source""" - return session.execute(query, { 'suite_id': suite_id }) def reportNBS(suite_name, suite_id, rdeps=False): session = DBConn().session() diff --git a/dak/dak.py b/dak/dak.py index 334bde08..c6c979a7 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -113,6 +113,8 @@ def init(): "Update suite with packages from a different suite"), ("cruft-report", "Check for obsolete or duplicated packages"), + ("auto-decruft", + "Clean cruft without reverse dependencies automatically"), ("examine-package", "Show information useful for NEW processing"), ("import", diff --git a/dak/generate_filelist.py b/dak/generate_filelist.py new file mode 100755 index 00000000..b839a5cc --- /dev/null +++ b/dak/generate_filelist.py @@ -0,0 +1,217 @@ +#!/usr/bin/python + +""" +Generate file lists for apt-ftparchive. + +@contact: Debian FTP Master +@copyright: 2009 Torsten Werner +@copyright: 2011 Ansgar Burchardt +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# Ganneff> Please go and try to lock mhy now. After than try to lock NEW. +# twerner> !lock mhy +# dak> twerner: You suck, this is already locked by Ganneff +# Ganneff> now try with NEW +# twerner> !lock NEW +# dak> twerner: also locked NEW +# mhy> Ganneff: oy, stop using me for locks and highlighting me you tall muppet +# Ganneff> hehe :) + +################################################################################ + +from daklib.dbconn import * +from daklib.config import Config +from daklib import utils, daklog +from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED +import apt_pkg, os, stat, sys + +from daklib.lists import getSources, getBinaries, getArchAll + +def listPath(suite, component, architecture = None, type = None, + incremental_mode = False): + """returns full path to the list file""" + suffixMap = { 'deb': "binary-", + 'udeb': "debian-installer_binary-" } + if architecture: + suffix = suffixMap[type] + architecture.arch_string + else: + suffix = "source" + filename = "%s_%s_%s.list" % \ + (suite.suite_name, component.component_name, suffix) + pathname = os.path.join(Config()["Dir::Lists"], filename) + file = utils.open_file(pathname, "a") + timestamp = None + if incremental_mode: + timestamp = os.fstat(file.fileno())[stat.ST_MTIME] + else: + file.seek(0) + file.truncate() + return (file, timestamp) + +def writeSourceList(suite_id, component_id, incremental_mode): + session = DBConn().session() + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) + (file, timestamp) = listPath(suite, component, + incremental_mode = incremental_mode) + + message = "sources list for %s %s" % (suite.suite_name, component.component_name) + + for _, filename in getSources(suite, component, session, timestamp): + file.write(filename + '\n') + session.rollback() + file.close() + return (PROC_STATUS_SUCCESS, message) + +def writeAllList(suite_id, component_id, architecture_id, type, incremental_mode): + session = DBConn().session() + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) + architecture = Architecture.get(architecture_id, session) + (file, timestamp) = listPath(suite, component, architecture, type, + incremental_mode) + + message = "all list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type) + + for _, filename in getArchAll(suite, component, architecture, type, + session, timestamp): + file.write(filename + '\n') + session.rollback() + file.close() + return (PROC_STATUS_SUCCESS, message) + +def writeBinaryList(suite_id, component_id, architecture_id, type, incremental_mode): + session = DBConn().session() + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) + architecture = Architecture.get(architecture_id, session) + (file, timestamp) = listPath(suite, component, architecture, type, + incremental_mode) + + message = "binary list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type) + + for _, filename in getBinaries(suite, component, architecture, type, + session, timestamp): + file.write(filename + '\n') + session.rollback() + file.close() + return (PROC_STATUS_SUCCESS, message) + +def usage(): + print """Usage: dak generate_filelist [OPTIONS] +Create filename lists for apt-ftparchive. + + -s, --suite=SUITE act on this suite + -c, --component=COMPONENT act on this component + -a, --architecture=ARCH act on this architecture + -h, --help show this help and exit + -i, --incremental activate incremental mode + +ARCH, COMPONENT and SUITE can be comma (or space) separated list, e.g. + --suite=testing,unstable + +Incremental mode appends only newer files to existing lists.""" + sys.exit() + +def main(): + cnf = Config() + Logger = daklog.Logger('generate-filelist') + Arguments = [('h', "help", "Filelist::Options::Help"), + ('s', "suite", "Filelist::Options::Suite", "HasArg"), + ('c', "component", "Filelist::Options::Component", "HasArg"), + ('a', "architecture", "Filelist::Options::Architecture", "HasArg"), + ('i', "incremental", "Filelist::Options::Incremental")] + session = DBConn().session() + query_suites = session.query(Suite) + suites = [suite.suite_name for suite in query_suites] + if not cnf.has_key('Filelist::Options::Suite'): + cnf['Filelist::Options::Suite'] = ','.join(suites).encode() + query_components = session.query(Component) + components = \ + [component.component_name for component in query_components] + if not cnf.has_key('Filelist::Options::Component'): + cnf['Filelist::Options::Component'] = ','.join(components).encode() + query_architectures = session.query(Architecture) + architectures = \ + [architecture.arch_string for architecture in query_architectures] + if not cnf.has_key('Filelist::Options::Architecture'): + cnf['Filelist::Options::Architecture'] = ','.join(architectures).encode() + cnf['Filelist::Options::Help'] = '' + cnf['Filelist::Options::Incremental'] = '' + apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) + Options = cnf.subtree("Filelist::Options") + if Options['Help']: + usage() + pool = DakProcessPool() + query_suites = query_suites. \ + filter(Suite.suite_name.in_(utils.split_args(Options['Suite']))) + query_components = query_components. \ + filter(Component.component_name.in_(utils.split_args(Options['Component']))) + query_architectures = query_architectures. \ + filter(Architecture.arch_string.in_(utils.split_args(Options['Architecture']))) + + def parse_results(message): + # Split out into (code, msg) + code, msg = message + if code == PROC_STATUS_SUCCESS: + Logger.log([msg]) + elif code == PROC_STATUS_SIGNALRAISED: + Logger.log(['E: Subprocess received signal ', msg]) + else: + Logger.log(['E: ', msg]) + + for suite in query_suites: + suite_id = suite.suite_id + for component in query_components: + component_id = component.component_id + for architecture in query_architectures: + architecture_id = architecture.arch_id + if architecture not in suite.architectures: + pass + elif architecture.arch_string == 'source': + pool.apply_async(writeSourceList, + (suite_id, component_id, Options['Incremental']), callback=parse_results) + elif architecture.arch_string == 'all': + pool.apply_async(writeAllList, + (suite_id, component_id, architecture_id, 'deb', + Options['Incremental']), callback=parse_results) + pool.apply_async(writeAllList, + (suite_id, component_id, architecture_id, 'udeb', + Options['Incremental']), callback=parse_results) + else: # arch any + pool.apply_async(writeBinaryList, + (suite_id, component_id, architecture_id, 'deb', + Options['Incremental']), callback=parse_results) + pool.apply_async(writeBinaryList, + (suite_id, component_id, architecture_id, 'udeb', + Options['Incremental']), callback=parse_results) + pool.close() + pool.join() + + # this script doesn't change the database + session.close() + + Logger.close() + + sys.exit(pool.overall_status()) + +if __name__ == '__main__': + main() + diff --git a/dak/rm.py b/dak/rm.py index 97af8c2c..7a514405 100755 --- a/dak/rm.py +++ b/dak/rm.py @@ -51,6 +51,7 @@ from daklib.config import Config from daklib.dbconn import * from daklib import utils from daklib.dak_exceptions import * +from daklib.rm import remove from daklib.regexes import re_strip_source_version, re_bin_only_nmu import debianbts as bts @@ -180,7 +181,7 @@ def main (): # Accept 3 types of arguments (space separated): # 1) a number - assumed to be a bug number, i.e. nnnnn@bugs.debian.org # 2) the keyword 'package' - cc's $package@packages.debian.org for every argument - # 3) contains a '@' - assumed to be an email address, used unmofidied + # 3) contains a '@' - assumed to be an email address, used unmodified # carbon_copy = [] for copy_to in utils.split_args(Options.get("Carbon-Copy")): @@ -235,9 +236,6 @@ def main (): if Options["Architecture"] and check_source: utils.warn("'source' in -a/--argument makes no sense and is ignored.") - # Additional component processing - over_con_components = con_components.replace("c.id", "component") - # Don't do dependency checks on multiple suites if Options["Rdep-Check"] and len(suites) > 1: utils.fubar("Reverse dependency check on multiple suites is not implemented.") @@ -315,7 +313,6 @@ def main (): summary = "" removals = d.keys() removals.sort() - versions = [] for package in removals: versions = d[package].keys() versions.sort(apt_pkg.version_compare) @@ -349,191 +346,19 @@ def main (): print "Going to remove the packages now." game_over() - whoami = utils.whoami() - date = commands.getoutput('date -R') - - # Log first; if it all falls apart I want a record that we at least tried. - logfile = utils.open_file(cnf["Rm::LogFile"], 'a') - logfile.write("=========================================================================\n") - logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)) - logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)) - if Options["Done"]: - logfile.write("Closed bugs: %s\n" % (Options["Done"])) - logfile.write("\n------------------- Reason -------------------\n%s\n" % (Options["Reason"])) - logfile.write("----------------------------------------------\n") - - # Do the same in rfc822 format - logfile822 = utils.open_file(cnf["Rm::LogFile822"], 'a') - logfile822.write("Date: %s\n" % date) - logfile822.write("Ftpmaster: %s\n" % whoami) - logfile822.write("Suite: %s\n" % suites_list) - sources = [] - binaries = [] - for package in summary.split("\n"): - for row in package.split("\n"): - element = row.split("|") - if len(element) == 3: - if element[2].find("source") > 0: - sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2])) - element[2] = sub("source\s?,?", "", element[2]).strip(" ") - if element[2]: - binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element)) - if sources: - logfile822.write("Sources:\n") - for source in sources: - logfile822.write(" %s\n" % source) - if binaries: - logfile822.write("Binaries:\n") - for binary in binaries: - logfile822.write(" %s\n" % binary) - logfile822.write("Reason: %s\n" % Options["Reason"].replace('\n', '\n ')) - if Options["Done"]: - logfile822.write("Bug: %s\n" % Options["Done"]) - - dsc_type_id = get_override_type('dsc', session).overridetype_id - deb_type_id = get_override_type('deb', session).overridetype_id - # Do the actual deletion print "Deleting...", sys.stdout.flush() - for i in to_remove: - package = i[0] - architecture = i[2] - package_id = i[3] - for suite_id in suite_ids_list: - if architecture == "source": - session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid", - {'packageid': package_id, 'suiteid': suite_id}) - #print "DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id) - else: - session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid", - {'packageid': package_id, 'suiteid': suite_id}) - #print "DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id) - # Delete from the override file - if not Options["Partial"]: - if architecture == "source": - type_id = dsc_type_id - else: - type_id = deb_type_id - # TODO: Again, fix this properly to remove the remaining non-bind argument - session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (over_con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id}) - session.commit() - print "done." - - # If we don't have a Bug server configured, we're done - if not cnf.has_key("Dinstall::BugServer"): - if Options["Done"] or Options["Do-Close"]: - print "Cannot send mail to BugServer as Dinstall::BugServer is not configured" - - logfile.write("=========================================================================\n") - logfile.close() - - logfile822.write("\n") - logfile822.close() - - return - - # read common subst variables for all bug closure mails - Subst_common = {} - Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] - Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"] - Subst_common["__CC__"] = "X-DAK: dak rm" - if carbon_copy: - Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy) - Subst_common["__SUITE_LIST__"] = suites_list - Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list) - Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"] - Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"] - Subst_common["__WHOAMI__"] = whoami - - # Send the bug closing messages - if Options["Done"]: - Subst_close_rm = Subst_common - bcc = [] - if cnf.find("Dinstall::Bcc") != "": - bcc.append(cnf["Dinstall::Bcc"]) - if cnf.find("Rm::Bcc") != "": - bcc.append(cnf["Rm::Bcc"]) - if bcc: - Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc) - else: - Subst_close_rm["__BCC__"] = "X-Filler: 42" - summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, Options["Reason"]) - summarymail += "----------------------------------------------\n" - Subst_close_rm["__SUMMARY__"] = summarymail - - for bug in utils.split_args(Options["Done"]): - Subst_close_rm["__BUG_NUMBER__"] = bug - if Options["Do-Close"]: - mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related") - else: - mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close") - utils.send_mail(mail_message, whitelists=whitelists) - - # close associated bug reports - if Options["Do-Close"]: - Subst_close_other = Subst_common - bcc = [] - wnpp = utils.parse_wnpp_bug_file() - versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) - if len(versions) == 1: - Subst_close_other["__VERSION__"] = versions[0] - else: - utils.fubar("Closing bugs with multiple package versions is not supported. Do it yourself.") - if bcc: - Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc) - else: - Subst_close_other["__BCC__"] = "X-Filler: 42" - # at this point, I just assume, that the first closed bug gives - # some useful information on why the package got removed - Subst_close_other["__BUG_NUMBER__"] = utils.split_args(Options["Done"])[0] - if len(sources) == 1: - source_pkg = source.split("_", 1)[0] - else: - utils.fubar("Closing bugs for multiple source packages is not supported. Do it yourself.") - Subst_close_other["__BUG_NUMBER_ALSO__"] = "" - Subst_close_other["__SOURCE__"] = source_pkg - merged_bugs = set() - other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded') - if other_bugs: - for bugno in other_bugs: - if bugno not in merged_bugs: - for bug in bts.get_status(bugno): - for merged in bug.mergedwith: - other_bugs.remove(merged) - merged_bugs.add(merged) - logfile.write("Also closing bug(s):") - logfile822.write("Also-Bugs:") - for bug in other_bugs: - Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," - logfile.write(" " + str(bug)) - logfile822.write(" " + str(bug)) - logfile.write("\n") - logfile822.write("\n") - if source_pkg in wnpp.keys(): - logfile.write("Also closing WNPP bug(s):") - logfile822.write("Also-WNPP:") - for bug in wnpp[source_pkg]: - # the wnpp-rm file we parse also contains our removal - # bugs, filtering that out - if bug != Subst_close_other["__BUG_NUMBER__"]: - Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," - logfile.write(" " + str(bug)) - logfile822.write(" " + str(bug)) - logfile.write("\n") - logfile822.write("\n") - - mail_message = utils.TemplateSubst(Subst_close_other,cnf["Dir::Templates"]+"/rm.bug-close-related") - if Subst_close_other["__BUG_NUMBER_ALSO__"]: - utils.send_mail(mail_message) - - - logfile.write("=========================================================================\n") - logfile.close() - - logfile822.write("\n") - logfile822.close() + try: + remove(session, Options["Reason"], suites, to_remove, + partial=Options["Partial"], components=utils.split_args(Options["Components"]), + done_bugs=Options["Done"], carbon_copy=carbon_copy, close_related_bugs=Options["Do-Close"] + ) + except ValueError as ex: + utils.fubar(ex.message) + else: + print "done." ####################################################################################### diff --git a/daklib/cruft.py b/daklib/cruft.py index a685bcb5..05666ceb 100644 --- a/daklib/cruft.py +++ b/daklib/cruft.py @@ -121,3 +121,111 @@ def report_multiple_source(suite): if binary.has_multiple_sources(): print binary print + + +def query_without_source(suite_id, session): + """searches for arch: all packages from suite that do no longer + reference a source package in the same suite + + subquery unique_binaries: selects all packages with only 1 version + in suite since 'dak rm' does not allow to specify version numbers""" + + query = """ + with unique_binaries as + (select package, max(version) as version, max(source) as source + from bin_associations_binaries + where architecture = 2 and suite = :suite_id + group by package having count(*) = 1) + select ub.package, ub.version + from unique_binaries ub + left join src_associations_src sas + on ub.source = sas.src and sas.suite = :suite_id + where sas.id is null + order by ub.package""" + return session.execute(query, {'suite_id': suite_id}) + + +def queryNBS(suite_id, session): + """This one is really complex. It searches arch != all packages that + are no longer built from current source packages in suite. + + temp table unique_binaries: will be populated with packages that + have only one version in suite because 'dak rm' does not allow + specifying version numbers + + temp table newest_binaries: will be populated with packages that are + built from current sources + + subquery uptodate_arch: returns all architectures built from current + sources + + subquery unique_binaries_uptodate_arch: returns all packages in + architectures from uptodate_arch + + subquery unique_binaries_uptodate_arch_agg: same as + unique_binaries_uptodate_arch but with column architecture + aggregated to array + + subquery uptodate_packages: similar to uptodate_arch but returns all + packages built from current sources + + subquery outdated_packages: returns all packages with architectures + no longer built from current source + """ + + query = """ +create temp table unique_binaries ( + package text not null, + architecture integer not null, + source integer not null); + +insert into unique_binaries + select bab.package, bab.architecture, max(bab.source) + from bin_associations_binaries bab + where bab.suite = :suite_id and bab.architecture > 2 + group by package, architecture having count(*) = 1; + +create temp table newest_binaries ( + package text not null, + architecture integer not null, + source text not null, + version debversion not null); + +insert into newest_binaries + select ub.package, ub.architecture, nsa.source, nsa.version + from unique_binaries ub + join newest_src_association nsa + on ub.source = nsa.src and nsa.suite = :suite_id; + +with uptodate_arch as + (select architecture, source, version + from newest_binaries + group by architecture, source, version), + unique_binaries_uptodate_arch as + (select ub.package, ub.architecture, ua.source, ua.version + from unique_binaries ub + join source s + on ub.source = s.id + join uptodate_arch ua + on ub.architecture = ua.architecture and s.source = ua.source), + unique_binaries_uptodate_arch_agg as + (select ubua.package, + array(select unnest(array_agg(a.arch_string)) order by 1) as arch_list, + ubua.source, ubua.version + from unique_binaries_uptodate_arch ubua + join architecture a + on ubua.architecture = a.id + group by ubua.source, ubua.version, ubua.package), + uptodate_packages as + (select package, source, version + from newest_binaries + group by package, source, version), + outdated_packages as + (select array(select unnest(array_agg(package)) order by 1) as pkg_list, + arch_list, source, version + from unique_binaries_uptodate_arch_agg + where package not in + (select package from uptodate_packages) + group by arch_list, source, version) + select * from outdated_packages order by source""" + return session.execute(query, {'suite_id': suite_id}) diff --git a/daklib/rm.py b/daklib/rm.py new file mode 100644 index 00000000..b4793255 --- /dev/null +++ b/daklib/rm.py @@ -0,0 +1,582 @@ +"""General purpose package removal code for ftpmaster + +@contact: Debian FTP Master +@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup +@copyright: 2010 Alexander Reichle-Schmehl +@copyright: 2015 Niels Thykier +@license: GNU General Public License version 2 or later +""" +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup +# Copyright (C) 2010 Alexander Reichle-Schmehl + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# From: Andrew Morton +# Subject: 2.6.6-mm5 +# To: linux-kernel@vger.kernel.org +# Date: Sat, 22 May 2004 01:36:36 -0700 +# X-Mailer: Sylpheed version 0.9.7 (GTK+ 1.2.10; i386-redhat-linux-gnu) +# +# [...] +# +# Although this feature has been around for a while it is new code, and the +# usual cautions apply. If it munches all your files please tell Jens and +# he'll type them in again for you. + +################################################################################ + +import commands +import apt_pkg +from re import sub +from collections import defaultdict +from regexes import re_build_dep_arch + +from daklib.dbconn import * +from daklib import utils +from daklib.regexes import re_bin_only_nmu +import debianbts as bts + +################################################################################ + + +class ReverseDependencyChecker(object): + """A bulk tester for reverse dependency checks + + This class is similar to the check_reverse_depends method from "utils". However, + it is primarily focused on facilitating bulk testing of reverse dependencies. + It caches the state of the suite and then uses that as basis for answering queries. + This saves a significant amount of time if multiple reverse dependency checks are + required. + """ + + def __init__(self, session, suite): + """Creates a new ReverseDependencyChecker instance + + This will spend a significant amount of time caching data. + + @type session: SQLA Session + @param session: The database session in use + + @type suite: str + @param suite: The name of the suite that is used as basis for removal tests. + """ + self._session = session + dbsuite = get_suite(suite, session) + suite_archs2id = dict((x.arch_string, x.arch_id) for x in get_suite_architectures(suite)) + package_dependencies, arch_providors_of, arch_provided_by = self._load_package_information(session, + dbsuite.suite_id, + suite_archs2id) + self._package_dependencies = package_dependencies + self._arch_providors_of = arch_providors_of + self._arch_provided_by = arch_provided_by + self._archs_in_suite = set(suite_archs2id) + + @staticmethod + def _load_package_information(session, suite_id, suite_archs2id): + package_dependencies = defaultdict(lambda: defaultdict(set)) + arch_providors_of = defaultdict(lambda: defaultdict(set)) + arch_provided_by = defaultdict(lambda: defaultdict(set)) + source_deps = defaultdict(set) + metakey_d = get_or_set_metadatakey("Depends", session) + metakey_p = get_or_set_metadatakey("Provides", session) + params = { + 'suite_id': suite_id, + 'arch_all_id': suite_archs2id['all'], + 'metakey_d_id': metakey_d.key_id, + 'metakey_p_id': metakey_p.key_id, + } + all_arches = set(suite_archs2id) + all_arches.discard('source') + + package_dependencies['source'] = source_deps + + for architecture in all_arches: + deps = defaultdict(set) + providers_of = defaultdict(set) + provided_by = defaultdict(set) + arch_providors_of[architecture] = providers_of + arch_provided_by[architecture] = provided_by + package_dependencies[architecture] = deps + + params['arch_id'] = suite_archs2id[architecture] + + statement = ''' + SELECT b.package, + (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends, + (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides + FROM binaries b + JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id + WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id''' + query = session.query('package', 'depends', 'provides'). \ + from_statement(statement).params(params) + for package, depends, provides in query: + + if depends is not None: + try: + parsed_dep = [] + for dep in apt_pkg.parse_depends(depends): + parsed_dep.append(frozenset(d[0] for d in dep)) + deps[package].update(parsed_dep) + except ValueError as e: + print "Error for package %s: %s" % (package, e) + # Maintain a counter for each virtual package. If a + # Provides: exists, set the counter to 0 and count all + # provides by a package not in the list for removal. + # If the counter stays 0 at the end, we know that only + # the to-be-removed packages provided this virtual + # package. + if provides is not None: + for virtual_pkg in provides.split(","): + virtual_pkg = virtual_pkg.strip() + if virtual_pkg == package: + continue + provided_by[virtual_pkg].add(package) + providers_of[package].add(virtual_pkg) + + # Check source dependencies (Build-Depends and Build-Depends-Indep) + metakey_bd = get_or_set_metadatakey("Build-Depends", session) + metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session) + params = { + 'suite_id': suite_id, + 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id), + } + statement = ''' + SELECT s.source, string_agg(sm.value, ', ') as build_dep + FROM source s + JOIN source_metadata sm ON s.id = sm.src_id + WHERE s.id in + (SELECT source FROM src_associations + WHERE suite = :suite_id) + AND sm.key_id in :metakey_ids + GROUP BY s.id, s.source''' + query = session.query('source', 'build_dep').from_statement(statement). \ + params(params) + for source, build_dep in query: + if build_dep is not None: + # Remove [arch] information since we want to see breakage on all arches + build_dep = re_build_dep_arch.sub("", build_dep) + try: + parsed_dep = [] + for dep in apt_pkg.parse_src_depends(build_dep): + parsed_dep.append(frozenset(d[0] for d in dep)) + source_deps[source].update(parsed_dep) + except ValueError as e: + print "Error for package %s: %s" % (source, e) + + return package_dependencies, arch_providors_of, arch_provided_by + + def check_reverse_depends(self, removal_requests): + """Bulk check reverse dependencies + + Example: + removal_request = { + "eclipse-rcp": None, # means ALL architectures (incl. source) + "eclipse": None, # means ALL architectures (incl. source) + "lintian": ["source", "all"], # Only these two "architectures". + } + obj.check_reverse_depends(removal_request) + + @type removal_requests: dict (or a list of tuples) + @param removal_requests: A dictionary mapping a package name to a list of architectures. The list of + architectures decides from which the package will be removed - if the list is empty the package will + be removed on ALL architectures in the suite (including "source"). + + @rtype: dict + @return: A mapping of "removed package" (as a "(pkg, arch)"-tuple) to a set of broken + broken packages (also as "(pkg, arch)"-tuple). Note that the architecture values + in these tuples /can/ be "source" to reflect a breakage in build-dependencies. + """ + + archs_in_suite = self._archs_in_suite + removals_by_arch = defaultdict(set) + affected_virtual_by_arch = defaultdict(set) + package_dependencies = self._package_dependencies + arch_providors_of = self._arch_providors_of + arch_provided_by = self._arch_provided_by + arch_provides2removal = defaultdict(lambda: defaultdict(set)) + dep_problems = defaultdict(set) + src_deps = package_dependencies['source'] + src_removals = set() + arch_all_removals = set() + + if isinstance(removal_requests, dict): + removal_requests = removal_requests.iteritems() + + for pkg, arch_list in removal_requests: + if not arch_list: + arch_list = archs_in_suite + for arch in arch_list: + if arch == 'source': + src_removals.add(pkg) + continue + if arch == 'all': + arch_all_removals.add(pkg) + continue + removals_by_arch[arch].add(pkg) + if pkg in arch_providors_of[arch]: + affected_virtual_by_arch[arch].add(pkg) + + if arch_all_removals: + for arch in archs_in_suite: + if arch in ('all', 'source'): + continue + removals_by_arch[arch].update(arch_all_removals) + for pkg in arch_all_removals: + if pkg in arch_providors_of[arch]: + affected_virtual_by_arch[arch].add(pkg) + + if not removals_by_arch: + # Nothing to remove => no problems + return dep_problems + + for arch, removed_providers in affected_virtual_by_arch.iteritems(): + provides2removal = arch_provides2removal[arch] + removals = removals_by_arch[arch] + for virtual_pkg, virtual_providers in arch_provided_by[arch].iteritems(): + v = virtual_providers & removed_providers + if len(v) == len(virtual_providers): + # We removed all the providers of virtual_pkg + removals.add(virtual_pkg) + # Pick one to take the blame for the removal + # - we sort for determinism, optimally we would prefer to blame the same package + # to minimise the number of blamed packages. + provides2removal[virtual_pkg] = sorted(v)[0] + + for arch, removals in removals_by_arch.iteritems(): + deps = package_dependencies[arch] + provides2removal = arch_provides2removal[arch] + + # Check binary dependencies (Depends) + for package, dependencies in deps.iteritems(): + if package in removals: + continue + for clause in dependencies: + if not (clause <= removals): + # Something probably still satisfies this relation + continue + # whoops, we seemed to have removed all packages that could possibly satisfy + # this relation. Lets blame something for it + for dep_package in clause: + removal = dep_package + if dep_package in provides2removal: + removal = provides2removal[dep_package] + dep_problems[(removal, arch)].add((package, arch)) + + for source, build_dependencies in src_deps.iteritems(): + if source in src_removals: + continue + for clause in build_dependencies: + if not (clause <= removals): + # Something probably still satisfies this relation + continue + # whoops, we seemed to have removed all packages that could possibly satisfy + # this relation. Lets blame something for it + for dep_package in clause: + removal = dep_package + if dep_package in provides2removal: + removal = provides2removal[dep_package] + dep_problems[(removal, arch)].add((source, 'source')) + + return dep_problems + + +def remove(session, reason, suites, removals, + whoami=None, partial=False, components=None, done_bugs=None, date=None, + carbon_copy=None, close_related_bugs=False): + """Batch remove a number of packages + Verify that the files listed in the Files field of the .dsc are + those expected given the announced Format. + + @type session: SQLA Session + @param session: The database session in use + + @type reason: string + @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by )") + + @type suites: list + @param suites: A list of the suite names in which the removal should occur + + @type removals: list + @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following + for 4 items from the database (in order): package, version, architecture, (database) id. + For source packages, the "architecture" should be set to "source". + + @type partial: bool + @param partial: Whether the removal is "partial" (e.g. architecture specific). + + @type components: list + @param components: List of components involved in a partial removal. Can be an empty list to not restrict the + removal to any components. + + @type whoami: string + @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami() + + @type date: string + @param date: The date of the removal. Defaults to commands.getoutput("date -R") + + @type done_bugs: list + @param done_bugs: A list of bugs to be closed when doing this removal. + + @type close_related_bugs: bool + @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented + for more than one suite. + + @type carbon_copy: list + @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike + "dak rm". + + @rtype: None + @return: Nothing + """ + # Generate the summary of what's to be removed + d = {} + summary = "" + sources = [] + binaries = [] + whitelists = [] + versions = [] + suite_ids_list = [] + suites_list = utils.join_with_commas_and(suites) + cnf = utils.get_conf() + con_components = None + + ####################################################################################################### + + if not reason: + raise ValueError("Empty removal reason not permitted") + + if not removals: + raise ValueError("Nothing to remove!?") + + if not suites: + raise ValueError("Removals without a suite!?") + + if whoami is None: + whoami = utils.whoami() + + if date is None: + date = commands.getoutput("date -R") + + if partial: + + component_ids_list = [] + for componentname in components: + component = get_component(componentname, session=session) + if component is None: + raise ValueError("component '%s' not recognised." % componentname) + else: + component_ids_list.append(component.component_id) + con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list]) + + for i in removals: + package = i[0] + version = i[1] + architecture = i[2] + if package not in d: + d[package] = {} + if version not in d[package]: + d[package][version] = [] + if architecture not in d[package][version]: + d[package][version].append(architecture) + + for package in sorted(removals): + versions = sorted(d[package], cmp=apt_pkg.version_compare) + for version in versions: + d[package][version].sort(utils.arch_compare_sw) + summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version])) + + for package in summary.split("\n"): + for row in package.split("\n"): + element = row.split("|") + if len(element) == 3: + if element[2].find("source") > 0: + sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2])) + element[2] = sub("source\s?,?", "", element[2]).strip(" ") + if element[2]: + binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element)) + + dsc_type_id = get_override_type('dsc', session).overridetype_id + deb_type_id = get_override_type('deb', session).overridetype_id + + for suite in suites: + s = get_suite(suite, session=session) + if s is not None: + suite_ids_list.append(s.suite_id) + whitelists.append(s.mail_whitelist) + + ####################################################################################################### + log_filename = cnf["Rm::LogFile"] + log822_filename = cnf["Rm::LogFile822"] + with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822: + logfile.write("=========================================================================\n") + logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)) + logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)) + if done_bugs: + logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs))) + logfile.write("\n------------------- Reason -------------------\n%s\n" % reason) + logfile.write("----------------------------------------------\n") + + logfile822.write("Date: %s\n" % date) + logfile822.write("Ftpmaster: %s\n" % whoami) + logfile822.write("Suite: %s\n" % suites_list) + + if sources: + logfile822.write("Sources:\n") + for source in sources: + logfile822.write(" %s\n" % source) + + if binaries: + logfile822.write("Binaries:\n") + for binary in binaries: + logfile822.write(" %s\n" % binary) + + logfile822.write("Reason: %s\n" % reason.replace('\n', '\n ')) + if done_bugs: + logfile822.write("Bug: %s\n" % (", ".join(done_bugs))) + + for i in removals: + package = i[0] + architecture = i[2] + package_id = i[3] + for suite_id in suite_ids_list: + if architecture == "source": + session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid", + {'packageid': package_id, 'suiteid': suite_id}) + else: + session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid", + {'packageid': package_id, 'suiteid': suite_id}) + # Delete from the override file + if partial: + if architecture == "source": + type_id = dsc_type_id + else: + type_id = deb_type_id + # TODO: Fix this properly to remove the remaining non-bind argument + session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id}) + + session.commit() + # ### REMOVAL COMPLETE - send mail time ### # + + # If we don't have a Bug server configured, we're done + if "Dinstall::BugServer" not in cnf: + if done_bugs or close_related_bugs: + utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured") + + logfile.write("=========================================================================\n") + logfile822.write("\n") + return + + # read common subst variables for all bug closure mails + Subst_common = {} + Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] + Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"] + Subst_common["__CC__"] = "X-DAK: dak rm" + if carbon_copy: + Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy) + Subst_common["__SUITE_LIST__"] = suites_list + Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list) + Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"] + Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"] + Subst_common["__WHOAMI__"] = whoami + + # Send the bug closing messages + if done_bugs: + Subst_close_rm = Subst_common + bcc = [] + if cnf.find("Dinstall::Bcc") != "": + bcc.append(cnf["Dinstall::Bcc"]) + if cnf.find("Rm::Bcc") != "": + bcc.append(cnf["Rm::Bcc"]) + if bcc: + Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc) + else: + Subst_close_rm["__BCC__"] = "X-Filler: 42" + summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason) + summarymail += "----------------------------------------------\n" + Subst_close_rm["__SUMMARY__"] = summarymail + + for bug in done_bugs: + Subst_close_rm["__BUG_NUMBER__"] = bug + if close_related_bugs: + mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related") + else: + mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close") + utils.send_mail(mail_message, whitelists=whitelists) + + # close associated bug reports + if close_related_bugs: + Subst_close_other = Subst_common + bcc = [] + wnpp = utils.parse_wnpp_bug_file() + versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) + if len(versions) == 1: + Subst_close_other["__VERSION__"] = versions[0] + else: + logfile.write("=========================================================================\n") + logfile822.write("\n") + raise ValueError("Closing bugs with multiple package versions is not supported. Do it yourself.") + if bcc: + Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc) + else: + Subst_close_other["__BCC__"] = "X-Filler: 42" + # at this point, I just assume, that the first closed bug gives + # some useful information on why the package got removed + Subst_close_other["__BUG_NUMBER__"] = done_bugs[0] + if len(sources) == 1: + source_pkg = source.split("_", 1)[0] + else: + logfile.write("=========================================================================\n") + logfile822.write("\n") + raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.") + Subst_close_other["__BUG_NUMBER_ALSO__"] = "" + Subst_close_other["__SOURCE__"] = source_pkg + merged_bugs = set() + other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded') + if other_bugs: + for bugno in other_bugs: + if bugno not in merged_bugs: + for bug in bts.get_status(bugno): + for merged in bug.mergedwith: + other_bugs.remove(merged) + merged_bugs.add(merged) + logfile.write("Also closing bug(s):") + logfile822.write("Also-Bugs:") + for bug in other_bugs: + Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," + logfile.write(" " + str(bug)) + logfile822.write(" " + str(bug)) + logfile.write("\n") + logfile822.write("\n") + if source_pkg in wnpp: + logfile.write("Also closing WNPP bug(s):") + logfile822.write("Also-WNPP:") + for bug in wnpp[source_pkg]: + # the wnpp-rm file we parse also contains our removal + # bugs, filtering that out + if bug != Subst_close_other["__BUG_NUMBER__"]: + Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," + logfile.write(" " + str(bug)) + logfile822.write(" " + str(bug)) + logfile.write("\n") + logfile822.write("\n") + + mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"]+"/rm.bug-close-related") + if Subst_close_other["__BUG_NUMBER_ALSO__"]: + utils.send_mail(mail_message) + + logfile.write("=========================================================================\n") + logfile822.write("\n") diff --git a/daklib/utils.py b/daklib/utils.py index 64f75fbe..12798ab2 100644 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -1123,19 +1123,20 @@ def call_editor(text="", suffix=".txt"): ################################################################################ -def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False): +def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False, quiet=False): dbsuite = get_suite(suite, session) overridesuite = dbsuite if dbsuite.overridesuite is not None: overridesuite = get_suite(dbsuite.overridesuite, session) dep_problem = 0 p2c = {} - all_broken = {} + all_broken = defaultdict(lambda: defaultdict(set)) if arches: all_arches = set(arches) else: - all_arches = set([x.arch_string for x in get_suite_architectures(suite)]) + all_arches = set(x.arch_string for x in get_suite_architectures(suite)) all_arches -= set(["source", "all"]) + removal_set = set(removals) metakey_d = get_or_set_metadatakey("Depends", session) metakey_p = get_or_set_metadatakey("Provides", session) params = { @@ -1150,7 +1151,7 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals params['arch_id'] = get_architecture(architecture, session).arch_id statement = ''' - SELECT b.id, b.package, s.source, c.name as component, + SELECT b.package, s.source, c.name as component, (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends, (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides FROM binaries b @@ -1159,9 +1160,9 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals JOIN files_archive_map af ON b.file = af.file_id JOIN component c ON af.component_id = c.id WHERE b.architecture = :arch_id''' - query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \ + query = session.query('package', 'source', 'component', 'depends', 'provides'). \ from_statement(statement).params(params) - for binary_id, package, source, component, depends, provides in query: + for package, source, component, depends, provides in query: sources[package] = source p2c[package] = component if depends is not None: @@ -1183,18 +1184,16 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals # If a virtual package is only provided by the to-be-removed # packages, treat the virtual package as to-be-removed too. - for virtual_pkg in virtual_packages.keys(): - if virtual_packages[virtual_pkg] == 0: - removals.append(virtual_pkg) + removal_set.update(virtual_pkg for virtual_pkg in virtual_packages if not virtual_packages[virtual_pkg]) # Check binary dependencies (Depends) - for package in deps.keys(): + for package in deps: if package in removals: continue - parsed_dep = [] try: - parsed_dep += apt_pkg.parse_depends(deps[package]) + parsed_dep = apt_pkg.parse_depends(deps[package]) except ValueError as e: print "Error for package %s: %s" % (package, e) + parsed_dep = [] for dep in parsed_dep: # Check for partial breakage. If a package has a ORed # dependency, there is only a dependency problem if all @@ -1208,10 +1207,10 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals source = sources[package] if component != "main": source = "%s/%s" % (source, component) - all_broken.setdefault(source, {}).setdefault(package, set()).add(architecture) + all_broken[source][package].add(architecture) dep_problem = 1 - if all_broken: + if all_broken and not quiet: if cruft: print " - broken Depends:" else: @@ -1236,7 +1235,7 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals print # Check source dependencies (Build-Depends and Build-Depends-Indep) - all_broken.clear() + all_broken = defaultdict(set) metakey_bd = get_or_set_metadatakey("Build-Depends", session) metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session) params = { @@ -1244,7 +1243,7 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id), } statement = ''' - SELECT s.id, s.source, string_agg(sm.value, ', ') as build_dep + SELECT s.source, string_agg(sm.value, ', ') as build_dep FROM source s JOIN source_metadata sm ON s.id = sm.src_id WHERE s.id in @@ -1252,16 +1251,16 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals WHERE suite = :suite_id) AND sm.key_id in :metakey_ids GROUP BY s.id, s.source''' - query = session.query('id', 'source', 'build_dep').from_statement(statement). \ + query = session.query('source', 'build_dep').from_statement(statement). \ params(params) - for source_id, source, build_dep in query: + for source, build_dep in query: if source in removals: continue parsed_dep = [] if build_dep is not None: # Remove [arch] information since we want to see breakage on all arches build_dep = re_build_dep_arch.sub("", build_dep) try: - parsed_dep += apt_pkg.parse_src_depends(build_dep) + parsed_dep = apt_pkg.parse_src_depends(build_dep) except ValueError as e: print "Error for source %s: %s" % (source, e) for dep in parsed_dep: @@ -1279,10 +1278,10 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals key = source if component != "main": key = "%s/%s" % (source, component) - all_broken.setdefault(key, set()).add(pp_deps(dep)) + all_broken[key].add(pp_deps(dep)) dep_problem = 1 - if all_broken: + if all_broken and not quiet: if cruft: print " - broken Build-Depends:" else: diff --git a/docs/README.quotes b/docs/README.quotes index ff6810f7..a5987461 100644 --- a/docs/README.quotes +++ b/docs/README.quotes @@ -134,13 +134,6 @@ DON'T PUT http://172.16.100.107/ IN YOUR URLS, YOU INCOMPETENT FUCKMONKEYS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -| priviledged positions? What privilege? The honour of working harder -| than most people for absolutely no recognition? - -Manoj Srivastava in <87lln8aqfm.fsf@glaurung.internal.golden-gryphon.com> - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - you could just r00t klecker through [...] and do it yourself heh I think there's a bit in the DMUP about that @@ -241,20 +234,6 @@ Next week on "When Good Buildds Go Bad"[...] %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -From: Andrew Morton -Subject: 2.6.6-mm5 -To: linux-kernel@vger.kernel.org -Date: Sat, 22 May 2004 01:36:36 -0700 -X-Mailer: Sylpheed version 0.9.7 (GTK+ 1.2.10; i386-redhat-linux-gnu) - -[...] - - Although this feature has been around for a while it is new code, and the - usual cautions apply. If it munches all your files please tell Jens and - he'll type them in again for you. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - From: Randall Donald Subject: foo_3.4-1_i386.changes REJECTED To: John Doe