--- /dev/null
+#!/usr/bin/env python
+
+"""
+Check for obsolete binary packages
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000-2006 James Troup <james@nocrew.org>
+@copyright: 2009 Torsten Werner <twerner@debian.org>
+@copyright: 2015 Niels Thykier <niels@thykier.net>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# | priviledged positions? What privilege? The honour of working harder
+# | than most people for absolutely no recognition?
+#
+# Manoj Srivastava <srivasta@debian.org> in <87lln8aqfm.fsf@glaurung.internal.golden-gryphon.com>
+
+################################################################################
+
+import sys
+import apt_pkg
+from itertools import chain, product
+from collections import defaultdict
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import utils
+from daklib.cruft import *
+from daklib.rm import remove, ReverseDependencyChecker
+
+################################################################################
+
+
+def usage(exit_code=0):
+ print """Usage: dak cruft-report
+Check for obsolete or duplicated packages.
+
+ -h, --help show this help and exit.
+ -n, --dry-run don't do anything, just show what would have been done
+ -s, --suite=SUITE check suite SUITE."""
+ sys.exit(exit_code)
+
+################################################################################
+
+
+def compute_sourceless_groups(suite_id, session):
+ """Find binaries without a source
+
+ @type suite_id: int
+ @param suite_id: The id of the suite donated by suite_name
+
+ @type session: SQLA Session
+ @param session: The database session in use
+ """""
+ rows = query_without_source(suite_id, session)
+ message = '[auto-cruft] no longer built from source, no reverse dependencies'
+ arch_all_id_tuple = tuple([get_architecture('all', session=session)])
+ arch_all_list = ["all"]
+ for row in rows:
+ package = row[0]
+ group_info = {
+ "name": "sourceless:%s" % package,
+ "packages": tuple([package]),
+ "architectures": arch_all_list,
+ "architecture_ids": arch_all_id_tuple,
+ "message": message,
+ "removal_request": {
+ package: arch_all_list,
+ },
+ }
+ yield group_info
+
+
+def compute_nbs_groups(suite_id, suite_name, session):
+ """Find binaries no longer built
+
+ @type suite_id: int
+ @param suite_id: The id of the suite donated by suite_name
+
+ @type suite_name: string
+ @param suite_name: The name of the suite to remove from
+
+ @type session: SQLA Session
+ @param session: The database session in use
+ """""
+ rows = queryNBS(suite_id, session)
+ arch2ids = dict((a.arch_string, a.arch_id) for a in get_suite_architectures(suite_name))
+
+ for row in rows:
+ (pkg_list, arch_list, source, _) = row
+ message = '[auto-cruft] NBS (no longer built by %s, no reverse dependencies)' % source
+ removal_request = dict((pkg, arch_list) for pkg in pkg_list)
+ group_info = {
+ "name": "NBS:%s" % source,
+ "packages": tuple(sorted(pkg_list)),
+ "architectures": sorted(arch_list, cmp=utils.arch_compare_sw),
+ "architecture_ids": tuple(arch2ids[arch] for arch in arch_list),
+ "message": message,
+ "removal_request": removal_request,
+ }
+ yield group_info
+
+
+def remove_groups(groups, suite_id, suite_name, session):
+ for group in groups:
+ message = group["message"]
+ params = {
+ "architecture_ids": group["architecture_ids"],
+ "packages": group["packages"],
+ "suite_id": suite_id
+ }
+ q = session.execute("""
+ SELECT b.package, b.version, a.arch_string, b.id
+ FROM binaries b
+ JOIN bin_associations ba ON b.id = ba.bin
+ JOIN architecture a ON b.architecture = a.id
+ JOIN suite su ON ba.suite = su.id
+ WHERE a.id IN :architecture_ids AND b.package IN :packages AND su.id = :suite_id
+ """, params)
+
+ remove(session, message, [suite_name], list(q), partial=True, whoami="DAK's auto-decrufter")
+
+
+def auto_decruft_suite(suite_name, suite_id, session, dryrun, debug):
+ """Run the auto-decrufter on a given suite
+
+ @type suite_name: string
+ @param suite_name: The name of the suite to remove from
+
+ @type suite_id: int
+ @param suite_id: The id of the suite donated by suite_name
+
+ @type session: SQLA Session
+ @param session: The database session in use
+
+ @type dryrun: bool
+ @param dryrun: If True, just print the actions rather than actually doing them
+
+ @type debug: bool
+ @param debug: If True, print some extra information
+ """
+ all_architectures = [a.arch_string for a in get_suite_architectures(suite_name)]
+ pkg_arch2groups = defaultdict(set)
+ group_order = []
+ groups = {}
+ full_removal_request = []
+ group_generator = chain(
+ compute_sourceless_groups(suite_id, session),
+ compute_nbs_groups(suite_id, suite_name, session)
+ )
+ for group in group_generator:
+ group_name = group["name"]
+ pkgs = group["packages"]
+ affected_archs = group["architectures"]
+ removal_request = group["removal_request"]
+ # If we remove an arch:all package, then the breakage can occur on any
+ # of the architectures.
+ if "all" in affected_archs:
+ affected_archs = all_architectures
+ for pkg_arch in product(pkgs, affected_archs):
+ pkg_arch2groups[pkg_arch].add(group_name)
+ groups[group_name] = group
+ group_order.append(group_name)
+
+ full_removal_request.extend(removal_request.iteritems())
+
+ if not groups:
+ if debug:
+ print "N: Found no candidates"
+ return
+ if debug:
+ print "N: Considering to remove the following packages:"
+ for group_name in sorted(groups):
+ group_info = groups[group_name]
+ pkgs = group_info["packages"]
+ archs = group_info["architectures"]
+ print "N: * %s: %s [%s]" % (group_name, ", ".join(pkgs), " ".join(archs))
+
+ if debug:
+ print "N: Compiling ReverseDependencyChecker (RDC) - please hold ..."
+ rdc = ReverseDependencyChecker(session, suite_name)
+ if debug:
+ print "N: Computing initial breakage..."
+
+ breakage = rdc.check_reverse_depends(full_removal_request)
+ while breakage:
+ by_breakers = [(len(breakage[x]), x, breakage[x]) for x in breakage]
+ by_breakers.sort(reverse=True)
+ if debug:
+ print "N: - Removal would break %s (package, architecture)-pairs" % (len(breakage))
+ print "N: - full breakage:"
+ for _, breaker, broken in by_breakers:
+ bname = "%s/%s" % breaker
+ broken_str = ", ".join("%s/%s" % b for b in sorted(broken))
+ print "N: * %s => %s" % (bname, broken_str)
+
+ averted_breakage = set()
+
+ for _, package_arch, breakage in by_breakers:
+ if breakage <= averted_breakage:
+ # We already avoided this break
+ continue
+ guilty_groups = pkg_arch2groups[package_arch]
+
+ if not guilty_groups:
+ utils.fubar("Cannot figure what group provided %s" % str(package_arch))
+
+ if debug:
+ # Only output it, if it truly a new group being discarded
+ # - a group can reach this part multiple times, if it breaks things on
+ # more than one architecture. This being rather common in fact.
+ already_discard = True
+ if any(group_name for group_name in guilty_groups if group_name in groups):
+ already_discard = False
+
+ if not already_discard:
+ avoided = sorted(breakage - averted_breakage)
+ print "N: - skipping removal of %s (breakage: %s)" % (", ".join(sorted(guilty_groups)), str(avoided))
+
+ averted_breakage |= breakage
+ for group_name in guilty_groups:
+ if group_name in groups:
+ del groups[group_name]
+
+ if not groups:
+ if debug:
+ print "N: Nothing left to remove"
+ return
+
+ if debug:
+ print "N: Now considering to remove: %s" % str(", ".join(sorted(groups.iterkeys())))
+
+ # Rebuild the removal request with the remaining groups and off
+ # we go to (not) break the world once more time
+ full_removal_request = []
+ for group_info in groups.itervalues():
+ full_removal_request.extend(group_info["removal_request"].iteritems())
+ breakage = rdc.check_reverse_depends(full_removal_request)
+
+ if debug:
+ print "N: Removal looks good"
+
+ if dryrun:
+ print "Would remove the equivalent of:"
+ for group_name in group_order:
+ if group_name not in groups:
+ continue
+ group_info = groups[group_name]
+ pkgs = group_info["packages"]
+ archs = group_info["architectures"]
+ message = group_info["message"]
+
+ # Embed the -R just in case someone wants to run it manually later
+ print ' dak rm -m "{message}" -s {suite} -a {architectures} -p -R -b {packages}'.format(
+ message=message, suite=suite_name,
+ architectures=",".join(archs), packages=" ".join(pkgs),
+ )
+
+ print
+ print "Note: The removals may be interdependent. A non-breaking result may require the execution of all"
+ print "of the removals"
+ else:
+ remove_groups(groups.itervalues(), suite_id, suite_name, session)
+
+
+################################################################################
+
+def main ():
+ global Options
+ cnf = Config()
+
+ Arguments = [('h',"help","Auto-Decruft::Options::Help"),
+ ('n',"dry-run","Auto-Decruft::Options::Dry-Run"),
+ ('d',"debug","Auto-Decruft::Options::Debug"),
+ ('s',"suite","Auto-Decruft::Options::Suite","HasArg")]
+ for i in ["help", "Dry-Run", "Debug"]:
+ if not cnf.has_key("Auto-Decruft::Options::%s" % (i)):
+ cnf["Auto-Decruft::Options::%s" % (i)] = ""
+
+ cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
+
+ apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
+
+ Options = cnf.subtree("Auto-Decruft::Options")
+ if Options["Help"]:
+ usage()
+
+ debug = False
+ dryrun = False
+ if Options["Dry-Run"]:
+ dryrun = True
+ if Options["Debug"]:
+ debug = True
+
+ session = DBConn().session()
+
+ suite = get_suite(Options["Suite"].lower(), session)
+ if not suite:
+ utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
+
+ suite_id = suite.suite_id
+ suite_name = suite.suite_name.lower()
+
+ auto_decruft_suite(suite_name, suite_id, session, dryrun, debug)
+
+################################################################################
+
+if __name__ == '__main__':
+ main()
################################################################################
-def queryWithoutSource(suite_id, session):
- """searches for arch: all packages from suite that do no longer
- reference a source package in the same suite
-
- subquery unique_binaries: selects all packages with only 1 version
- in suite since 'dak rm' does not allow to specify version numbers"""
-
- query = """
- with unique_binaries as
- (select package, max(version) as version, max(source) as source
- from bin_associations_binaries
- where architecture = 2 and suite = :suite_id
- group by package having count(*) = 1)
- select ub.package, ub.version
- from unique_binaries ub
- left join src_associations_src sas
- on ub.source = sas.src and sas.suite = :suite_id
- where sas.id is null
- order by ub.package"""
- return session.execute(query, { 'suite_id': suite_id })
def reportWithoutSource(suite_name, suite_id, session, rdeps=False):
- rows = queryWithoutSource(suite_id, session)
+ rows = query_without_source(suite_id, session)
title = 'packages without source in suite %s' % suite_name
if rows.rowcount > 0:
print '%s\n%s\n' % (title, '-' * len(title))
print " dak rm -m %s -s %s -a %s -p -b %s\n" % \
(message, suite_name, oldarch, package)
-def queryNBS(suite_id, session):
- """This one is really complex. It searches arch != all packages that
- are no longer built from current source packages in suite.
-
- temp table unique_binaries: will be populated with packages that
- have only one version in suite because 'dak rm' does not allow
- specifying version numbers
-
- temp table newest_binaries: will be populated with packages that are
- built from current sources
-
- subquery uptodate_arch: returns all architectures built from current
- sources
- subquery unique_binaries_uptodate_arch: returns all packages in
- architectures from uptodate_arch
-
- subquery unique_binaries_uptodate_arch_agg: same as
- unique_binaries_uptodate_arch but with column architecture
- aggregated to array
-
- subquery uptodate_packages: similar to uptodate_arch but returns all
- packages built from current sources
-
- subquery outdated_packages: returns all packages with architectures
- no longer built from current source
- """
-
- query = """
-create temp table unique_binaries (
- package text not null,
- architecture integer not null,
- source integer not null);
-
-insert into unique_binaries
- select bab.package, bab.architecture, max(bab.source)
- from bin_associations_binaries bab
- where bab.suite = :suite_id and bab.architecture > 2
- group by package, architecture having count(*) = 1;
-
-create temp table newest_binaries (
- package text not null,
- architecture integer not null,
- source text not null,
- version debversion not null);
-
-insert into newest_binaries
- select ub.package, ub.architecture, nsa.source, nsa.version
- from unique_binaries ub
- join newest_src_association nsa
- on ub.source = nsa.src and nsa.suite = :suite_id;
-
-with uptodate_arch as
- (select architecture, source, version
- from newest_binaries
- group by architecture, source, version),
- unique_binaries_uptodate_arch as
- (select ub.package, ub.architecture, ua.source, ua.version
- from unique_binaries ub
- join source s
- on ub.source = s.id
- join uptodate_arch ua
- on ub.architecture = ua.architecture and s.source = ua.source),
- unique_binaries_uptodate_arch_agg as
- (select ubua.package,
- array(select unnest(array_agg(a.arch_string)) order by 1) as arch_list,
- ubua.source, ubua.version
- from unique_binaries_uptodate_arch ubua
- join architecture a
- on ubua.architecture = a.id
- group by ubua.source, ubua.version, ubua.package),
- uptodate_packages as
- (select package, source, version
- from newest_binaries
- group by package, source, version),
- outdated_packages as
- (select array(select unnest(array_agg(package)) order by 1) as pkg_list,
- arch_list, source, version
- from unique_binaries_uptodate_arch_agg
- where package not in
- (select package from uptodate_packages)
- group by arch_list, source, version)
- select * from outdated_packages order by source"""
- return session.execute(query, { 'suite_id': suite_id })
def reportNBS(suite_name, suite_id, rdeps=False):
session = DBConn().session()
"Update suite with packages from a different suite"),
("cruft-report",
"Check for obsolete or duplicated packages"),
+ ("auto-decruft",
+ "Clean cruft without reverse dependencies automatically"),
("examine-package",
"Show information useful for NEW processing"),
("import",
--- /dev/null
+#!/usr/bin/python
+
+"""
+Generate file lists for apt-ftparchive.
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Torsten Werner <twerner@debian.org>
+@copyright: 2011 Ansgar Burchardt <ansgar@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# Ganneff> Please go and try to lock mhy now. After than try to lock NEW.
+# twerner> !lock mhy
+# dak> twerner: You suck, this is already locked by Ganneff
+# Ganneff> now try with NEW
+# twerner> !lock NEW
+# dak> twerner: also locked NEW
+# mhy> Ganneff: oy, stop using me for locks and highlighting me you tall muppet
+# Ganneff> hehe :)
+
+################################################################################
+
+from daklib.dbconn import *
+from daklib.config import Config
+from daklib import utils, daklog
+from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
+import apt_pkg, os, stat, sys
+
+from daklib.lists import getSources, getBinaries, getArchAll
+
+def listPath(suite, component, architecture = None, type = None,
+ incremental_mode = False):
+ """returns full path to the list file"""
+ suffixMap = { 'deb': "binary-",
+ 'udeb': "debian-installer_binary-" }
+ if architecture:
+ suffix = suffixMap[type] + architecture.arch_string
+ else:
+ suffix = "source"
+ filename = "%s_%s_%s.list" % \
+ (suite.suite_name, component.component_name, suffix)
+ pathname = os.path.join(Config()["Dir::Lists"], filename)
+ file = utils.open_file(pathname, "a")
+ timestamp = None
+ if incremental_mode:
+ timestamp = os.fstat(file.fileno())[stat.ST_MTIME]
+ else:
+ file.seek(0)
+ file.truncate()
+ return (file, timestamp)
+
+def writeSourceList(suite_id, component_id, incremental_mode):
+ session = DBConn().session()
+ suite = Suite.get(suite_id, session)
+ component = Component.get(component_id, session)
+ (file, timestamp) = listPath(suite, component,
+ incremental_mode = incremental_mode)
+
+ message = "sources list for %s %s" % (suite.suite_name, component.component_name)
+
+ for _, filename in getSources(suite, component, session, timestamp):
+ file.write(filename + '\n')
+ session.rollback()
+ file.close()
+ return (PROC_STATUS_SUCCESS, message)
+
+def writeAllList(suite_id, component_id, architecture_id, type, incremental_mode):
+ session = DBConn().session()
+ suite = Suite.get(suite_id, session)
+ component = Component.get(component_id, session)
+ architecture = Architecture.get(architecture_id, session)
+ (file, timestamp) = listPath(suite, component, architecture, type,
+ incremental_mode)
+
+ message = "all list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type)
+
+ for _, filename in getArchAll(suite, component, architecture, type,
+ session, timestamp):
+ file.write(filename + '\n')
+ session.rollback()
+ file.close()
+ return (PROC_STATUS_SUCCESS, message)
+
+def writeBinaryList(suite_id, component_id, architecture_id, type, incremental_mode):
+ session = DBConn().session()
+ suite = Suite.get(suite_id, session)
+ component = Component.get(component_id, session)
+ architecture = Architecture.get(architecture_id, session)
+ (file, timestamp) = listPath(suite, component, architecture, type,
+ incremental_mode)
+
+ message = "binary list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type)
+
+ for _, filename in getBinaries(suite, component, architecture, type,
+ session, timestamp):
+ file.write(filename + '\n')
+ session.rollback()
+ file.close()
+ return (PROC_STATUS_SUCCESS, message)
+
+def usage():
+ print """Usage: dak generate_filelist [OPTIONS]
+Create filename lists for apt-ftparchive.
+
+ -s, --suite=SUITE act on this suite
+ -c, --component=COMPONENT act on this component
+ -a, --architecture=ARCH act on this architecture
+ -h, --help show this help and exit
+ -i, --incremental activate incremental mode
+
+ARCH, COMPONENT and SUITE can be comma (or space) separated list, e.g.
+ --suite=testing,unstable
+
+Incremental mode appends only newer files to existing lists."""
+ sys.exit()
+
+def main():
+ cnf = Config()
+ Logger = daklog.Logger('generate-filelist')
+ Arguments = [('h', "help", "Filelist::Options::Help"),
+ ('s', "suite", "Filelist::Options::Suite", "HasArg"),
+ ('c', "component", "Filelist::Options::Component", "HasArg"),
+ ('a', "architecture", "Filelist::Options::Architecture", "HasArg"),
+ ('i', "incremental", "Filelist::Options::Incremental")]
+ session = DBConn().session()
+ query_suites = session.query(Suite)
+ suites = [suite.suite_name for suite in query_suites]
+ if not cnf.has_key('Filelist::Options::Suite'):
+ cnf['Filelist::Options::Suite'] = ','.join(suites).encode()
+ query_components = session.query(Component)
+ components = \
+ [component.component_name for component in query_components]
+ if not cnf.has_key('Filelist::Options::Component'):
+ cnf['Filelist::Options::Component'] = ','.join(components).encode()
+ query_architectures = session.query(Architecture)
+ architectures = \
+ [architecture.arch_string for architecture in query_architectures]
+ if not cnf.has_key('Filelist::Options::Architecture'):
+ cnf['Filelist::Options::Architecture'] = ','.join(architectures).encode()
+ cnf['Filelist::Options::Help'] = ''
+ cnf['Filelist::Options::Incremental'] = ''
+ apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.subtree("Filelist::Options")
+ if Options['Help']:
+ usage()
+ pool = DakProcessPool()
+ query_suites = query_suites. \
+ filter(Suite.suite_name.in_(utils.split_args(Options['Suite'])))
+ query_components = query_components. \
+ filter(Component.component_name.in_(utils.split_args(Options['Component'])))
+ query_architectures = query_architectures. \
+ filter(Architecture.arch_string.in_(utils.split_args(Options['Architecture'])))
+
+ def parse_results(message):
+ # Split out into (code, msg)
+ code, msg = message
+ if code == PROC_STATUS_SUCCESS:
+ Logger.log([msg])
+ elif code == PROC_STATUS_SIGNALRAISED:
+ Logger.log(['E: Subprocess received signal ', msg])
+ else:
+ Logger.log(['E: ', msg])
+
+ for suite in query_suites:
+ suite_id = suite.suite_id
+ for component in query_components:
+ component_id = component.component_id
+ for architecture in query_architectures:
+ architecture_id = architecture.arch_id
+ if architecture not in suite.architectures:
+ pass
+ elif architecture.arch_string == 'source':
+ pool.apply_async(writeSourceList,
+ (suite_id, component_id, Options['Incremental']), callback=parse_results)
+ elif architecture.arch_string == 'all':
+ pool.apply_async(writeAllList,
+ (suite_id, component_id, architecture_id, 'deb',
+ Options['Incremental']), callback=parse_results)
+ pool.apply_async(writeAllList,
+ (suite_id, component_id, architecture_id, 'udeb',
+ Options['Incremental']), callback=parse_results)
+ else: # arch any
+ pool.apply_async(writeBinaryList,
+ (suite_id, component_id, architecture_id, 'deb',
+ Options['Incremental']), callback=parse_results)
+ pool.apply_async(writeBinaryList,
+ (suite_id, component_id, architecture_id, 'udeb',
+ Options['Incremental']), callback=parse_results)
+ pool.close()
+ pool.join()
+
+ # this script doesn't change the database
+ session.close()
+
+ Logger.close()
+
+ sys.exit(pool.overall_status())
+
+if __name__ == '__main__':
+ main()
+
from daklib.dbconn import *
from daklib import utils
from daklib.dak_exceptions import *
+from daklib.rm import remove
from daklib.regexes import re_strip_source_version, re_bin_only_nmu
import debianbts as bts
# Accept 3 types of arguments (space separated):
# 1) a number - assumed to be a bug number, i.e. nnnnn@bugs.debian.org
# 2) the keyword 'package' - cc's $package@packages.debian.org for every argument
- # 3) contains a '@' - assumed to be an email address, used unmofidied
+ # 3) contains a '@' - assumed to be an email address, used unmodified
#
carbon_copy = []
for copy_to in utils.split_args(Options.get("Carbon-Copy")):
if Options["Architecture"] and check_source:
utils.warn("'source' in -a/--argument makes no sense and is ignored.")
- # Additional component processing
- over_con_components = con_components.replace("c.id", "component")
-
# Don't do dependency checks on multiple suites
if Options["Rdep-Check"] and len(suites) > 1:
utils.fubar("Reverse dependency check on multiple suites is not implemented.")
summary = ""
removals = d.keys()
removals.sort()
- versions = []
for package in removals:
versions = d[package].keys()
versions.sort(apt_pkg.version_compare)
print "Going to remove the packages now."
game_over()
- whoami = utils.whoami()
- date = commands.getoutput('date -R')
-
- # Log first; if it all falls apart I want a record that we at least tried.
- logfile = utils.open_file(cnf["Rm::LogFile"], 'a')
- logfile.write("=========================================================================\n")
- logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
- logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
- if Options["Done"]:
- logfile.write("Closed bugs: %s\n" % (Options["Done"]))
- logfile.write("\n------------------- Reason -------------------\n%s\n" % (Options["Reason"]))
- logfile.write("----------------------------------------------\n")
-
- # Do the same in rfc822 format
- logfile822 = utils.open_file(cnf["Rm::LogFile822"], 'a')
- logfile822.write("Date: %s\n" % date)
- logfile822.write("Ftpmaster: %s\n" % whoami)
- logfile822.write("Suite: %s\n" % suites_list)
- sources = []
- binaries = []
- for package in summary.split("\n"):
- for row in package.split("\n"):
- element = row.split("|")
- if len(element) == 3:
- if element[2].find("source") > 0:
- sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2]))
- element[2] = sub("source\s?,?", "", element[2]).strip(" ")
- if element[2]:
- binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element))
- if sources:
- logfile822.write("Sources:\n")
- for source in sources:
- logfile822.write(" %s\n" % source)
- if binaries:
- logfile822.write("Binaries:\n")
- for binary in binaries:
- logfile822.write(" %s\n" % binary)
- logfile822.write("Reason: %s\n" % Options["Reason"].replace('\n', '\n '))
- if Options["Done"]:
- logfile822.write("Bug: %s\n" % Options["Done"])
-
- dsc_type_id = get_override_type('dsc', session).overridetype_id
- deb_type_id = get_override_type('deb', session).overridetype_id
-
# Do the actual deletion
print "Deleting...",
sys.stdout.flush()
- for i in to_remove:
- package = i[0]
- architecture = i[2]
- package_id = i[3]
- for suite_id in suite_ids_list:
- if architecture == "source":
- session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
- {'packageid': package_id, 'suiteid': suite_id})
- #print "DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id)
- else:
- session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
- {'packageid': package_id, 'suiteid': suite_id})
- #print "DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id)
- # Delete from the override file
- if not Options["Partial"]:
- if architecture == "source":
- type_id = dsc_type_id
- else:
- type_id = deb_type_id
- # TODO: Again, fix this properly to remove the remaining non-bind argument
- session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (over_con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
- session.commit()
- print "done."
-
- # If we don't have a Bug server configured, we're done
- if not cnf.has_key("Dinstall::BugServer"):
- if Options["Done"] or Options["Do-Close"]:
- print "Cannot send mail to BugServer as Dinstall::BugServer is not configured"
-
- logfile.write("=========================================================================\n")
- logfile.close()
-
- logfile822.write("\n")
- logfile822.close()
-
- return
-
- # read common subst variables for all bug closure mails
- Subst_common = {}
- Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
- Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
- Subst_common["__CC__"] = "X-DAK: dak rm"
- if carbon_copy:
- Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
- Subst_common["__SUITE_LIST__"] = suites_list
- Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list)
- Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
- Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
- Subst_common["__WHOAMI__"] = whoami
-
- # Send the bug closing messages
- if Options["Done"]:
- Subst_close_rm = Subst_common
- bcc = []
- if cnf.find("Dinstall::Bcc") != "":
- bcc.append(cnf["Dinstall::Bcc"])
- if cnf.find("Rm::Bcc") != "":
- bcc.append(cnf["Rm::Bcc"])
- if bcc:
- Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
- else:
- Subst_close_rm["__BCC__"] = "X-Filler: 42"
- summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, Options["Reason"])
- summarymail += "----------------------------------------------\n"
- Subst_close_rm["__SUMMARY__"] = summarymail
-
- for bug in utils.split_args(Options["Done"]):
- Subst_close_rm["__BUG_NUMBER__"] = bug
- if Options["Do-Close"]:
- mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
- else:
- mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
- utils.send_mail(mail_message, whitelists=whitelists)
-
- # close associated bug reports
- if Options["Do-Close"]:
- Subst_close_other = Subst_common
- bcc = []
- wnpp = utils.parse_wnpp_bug_file()
- versions = list(set([re_bin_only_nmu.sub('', v) for v in versions]))
- if len(versions) == 1:
- Subst_close_other["__VERSION__"] = versions[0]
- else:
- utils.fubar("Closing bugs with multiple package versions is not supported. Do it yourself.")
- if bcc:
- Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
- else:
- Subst_close_other["__BCC__"] = "X-Filler: 42"
- # at this point, I just assume, that the first closed bug gives
- # some useful information on why the package got removed
- Subst_close_other["__BUG_NUMBER__"] = utils.split_args(Options["Done"])[0]
- if len(sources) == 1:
- source_pkg = source.split("_", 1)[0]
- else:
- utils.fubar("Closing bugs for multiple source packages is not supported. Do it yourself.")
- Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
- Subst_close_other["__SOURCE__"] = source_pkg
- merged_bugs = set()
- other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded')
- if other_bugs:
- for bugno in other_bugs:
- if bugno not in merged_bugs:
- for bug in bts.get_status(bugno):
- for merged in bug.mergedwith:
- other_bugs.remove(merged)
- merged_bugs.add(merged)
- logfile.write("Also closing bug(s):")
- logfile822.write("Also-Bugs:")
- for bug in other_bugs:
- Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
- logfile.write(" " + str(bug))
- logfile822.write(" " + str(bug))
- logfile.write("\n")
- logfile822.write("\n")
- if source_pkg in wnpp.keys():
- logfile.write("Also closing WNPP bug(s):")
- logfile822.write("Also-WNPP:")
- for bug in wnpp[source_pkg]:
- # the wnpp-rm file we parse also contains our removal
- # bugs, filtering that out
- if bug != Subst_close_other["__BUG_NUMBER__"]:
- Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
- logfile.write(" " + str(bug))
- logfile822.write(" " + str(bug))
- logfile.write("\n")
- logfile822.write("\n")
-
- mail_message = utils.TemplateSubst(Subst_close_other,cnf["Dir::Templates"]+"/rm.bug-close-related")
- if Subst_close_other["__BUG_NUMBER_ALSO__"]:
- utils.send_mail(mail_message)
-
-
- logfile.write("=========================================================================\n")
- logfile.close()
-
- logfile822.write("\n")
- logfile822.close()
+ try:
+ remove(session, Options["Reason"], suites, to_remove,
+ partial=Options["Partial"], components=utils.split_args(Options["Components"]),
+ done_bugs=Options["Done"], carbon_copy=carbon_copy, close_related_bugs=Options["Do-Close"]
+ )
+ except ValueError as ex:
+ utils.fubar(ex.message)
+ else:
+ print "done."
#######################################################################################
if binary.has_multiple_sources():
print binary
print
+
+
+def query_without_source(suite_id, session):
+ """searches for arch: all packages from suite that do no longer
+ reference a source package in the same suite
+
+ subquery unique_binaries: selects all packages with only 1 version
+ in suite since 'dak rm' does not allow to specify version numbers"""
+
+ query = """
+ with unique_binaries as
+ (select package, max(version) as version, max(source) as source
+ from bin_associations_binaries
+ where architecture = 2 and suite = :suite_id
+ group by package having count(*) = 1)
+ select ub.package, ub.version
+ from unique_binaries ub
+ left join src_associations_src sas
+ on ub.source = sas.src and sas.suite = :suite_id
+ where sas.id is null
+ order by ub.package"""
+ return session.execute(query, {'suite_id': suite_id})
+
+
+def queryNBS(suite_id, session):
+ """This one is really complex. It searches arch != all packages that
+ are no longer built from current source packages in suite.
+
+ temp table unique_binaries: will be populated with packages that
+ have only one version in suite because 'dak rm' does not allow
+ specifying version numbers
+
+ temp table newest_binaries: will be populated with packages that are
+ built from current sources
+
+ subquery uptodate_arch: returns all architectures built from current
+ sources
+
+ subquery unique_binaries_uptodate_arch: returns all packages in
+ architectures from uptodate_arch
+
+ subquery unique_binaries_uptodate_arch_agg: same as
+ unique_binaries_uptodate_arch but with column architecture
+ aggregated to array
+
+ subquery uptodate_packages: similar to uptodate_arch but returns all
+ packages built from current sources
+
+ subquery outdated_packages: returns all packages with architectures
+ no longer built from current source
+ """
+
+ query = """
+create temp table unique_binaries (
+ package text not null,
+ architecture integer not null,
+ source integer not null);
+
+insert into unique_binaries
+ select bab.package, bab.architecture, max(bab.source)
+ from bin_associations_binaries bab
+ where bab.suite = :suite_id and bab.architecture > 2
+ group by package, architecture having count(*) = 1;
+
+create temp table newest_binaries (
+ package text not null,
+ architecture integer not null,
+ source text not null,
+ version debversion not null);
+
+insert into newest_binaries
+ select ub.package, ub.architecture, nsa.source, nsa.version
+ from unique_binaries ub
+ join newest_src_association nsa
+ on ub.source = nsa.src and nsa.suite = :suite_id;
+
+with uptodate_arch as
+ (select architecture, source, version
+ from newest_binaries
+ group by architecture, source, version),
+ unique_binaries_uptodate_arch as
+ (select ub.package, ub.architecture, ua.source, ua.version
+ from unique_binaries ub
+ join source s
+ on ub.source = s.id
+ join uptodate_arch ua
+ on ub.architecture = ua.architecture and s.source = ua.source),
+ unique_binaries_uptodate_arch_agg as
+ (select ubua.package,
+ array(select unnest(array_agg(a.arch_string)) order by 1) as arch_list,
+ ubua.source, ubua.version
+ from unique_binaries_uptodate_arch ubua
+ join architecture a
+ on ubua.architecture = a.id
+ group by ubua.source, ubua.version, ubua.package),
+ uptodate_packages as
+ (select package, source, version
+ from newest_binaries
+ group by package, source, version),
+ outdated_packages as
+ (select array(select unnest(array_agg(package)) order by 1) as pkg_list,
+ arch_list, source, version
+ from unique_binaries_uptodate_arch_agg
+ where package not in
+ (select package from uptodate_packages)
+ group by arch_list, source, version)
+ select * from outdated_packages order by source"""
+ return session.execute(query, {'suite_id': suite_id})
--- /dev/null
+"""General purpose package removal code for ftpmaster
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
+@copyright: 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
+@copyright: 2015 Niels Thykier <niels@thykier.net>
+@license: GNU General Public License version 2 or later
+"""
+# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
+# Copyright (C) 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# From: Andrew Morton <akpm@osdl.org>
+# Subject: 2.6.6-mm5
+# To: linux-kernel@vger.kernel.org
+# Date: Sat, 22 May 2004 01:36:36 -0700
+# X-Mailer: Sylpheed version 0.9.7 (GTK+ 1.2.10; i386-redhat-linux-gnu)
+#
+# [...]
+#
+# Although this feature has been around for a while it is new code, and the
+# usual cautions apply. If it munches all your files please tell Jens and
+# he'll type them in again for you.
+
+################################################################################
+
+import commands
+import apt_pkg
+from re import sub
+from collections import defaultdict
+from regexes import re_build_dep_arch
+
+from daklib.dbconn import *
+from daklib import utils
+from daklib.regexes import re_bin_only_nmu
+import debianbts as bts
+
+################################################################################
+
+
+class ReverseDependencyChecker(object):
+ """A bulk tester for reverse dependency checks
+
+ This class is similar to the check_reverse_depends method from "utils". However,
+ it is primarily focused on facilitating bulk testing of reverse dependencies.
+ It caches the state of the suite and then uses that as basis for answering queries.
+ This saves a significant amount of time if multiple reverse dependency checks are
+ required.
+ """
+
+ def __init__(self, session, suite):
+ """Creates a new ReverseDependencyChecker instance
+
+ This will spend a significant amount of time caching data.
+
+ @type session: SQLA Session
+ @param session: The database session in use
+
+ @type suite: str
+ @param suite: The name of the suite that is used as basis for removal tests.
+ """
+ self._session = session
+ dbsuite = get_suite(suite, session)
+ suite_archs2id = dict((x.arch_string, x.arch_id) for x in get_suite_architectures(suite))
+ package_dependencies, arch_providors_of, arch_provided_by = self._load_package_information(session,
+ dbsuite.suite_id,
+ suite_archs2id)
+ self._package_dependencies = package_dependencies
+ self._arch_providors_of = arch_providors_of
+ self._arch_provided_by = arch_provided_by
+ self._archs_in_suite = set(suite_archs2id)
+
+ @staticmethod
+ def _load_package_information(session, suite_id, suite_archs2id):
+ package_dependencies = defaultdict(lambda: defaultdict(set))
+ arch_providors_of = defaultdict(lambda: defaultdict(set))
+ arch_provided_by = defaultdict(lambda: defaultdict(set))
+ source_deps = defaultdict(set)
+ metakey_d = get_or_set_metadatakey("Depends", session)
+ metakey_p = get_or_set_metadatakey("Provides", session)
+ params = {
+ 'suite_id': suite_id,
+ 'arch_all_id': suite_archs2id['all'],
+ 'metakey_d_id': metakey_d.key_id,
+ 'metakey_p_id': metakey_p.key_id,
+ }
+ all_arches = set(suite_archs2id)
+ all_arches.discard('source')
+
+ package_dependencies['source'] = source_deps
+
+ for architecture in all_arches:
+ deps = defaultdict(set)
+ providers_of = defaultdict(set)
+ provided_by = defaultdict(set)
+ arch_providors_of[architecture] = providers_of
+ arch_provided_by[architecture] = provided_by
+ package_dependencies[architecture] = deps
+
+ params['arch_id'] = suite_archs2id[architecture]
+
+ statement = '''
+ SELECT b.package,
+ (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
+ (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
+ FROM binaries b
+ JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
+ WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id'''
+ query = session.query('package', 'depends', 'provides'). \
+ from_statement(statement).params(params)
+ for package, depends, provides in query:
+
+ if depends is not None:
+ try:
+ parsed_dep = []
+ for dep in apt_pkg.parse_depends(depends):
+ parsed_dep.append(frozenset(d[0] for d in dep))
+ deps[package].update(parsed_dep)
+ except ValueError as e:
+ print "Error for package %s: %s" % (package, e)
+ # Maintain a counter for each virtual package. If a
+ # Provides: exists, set the counter to 0 and count all
+ # provides by a package not in the list for removal.
+ # If the counter stays 0 at the end, we know that only
+ # the to-be-removed packages provided this virtual
+ # package.
+ if provides is not None:
+ for virtual_pkg in provides.split(","):
+ virtual_pkg = virtual_pkg.strip()
+ if virtual_pkg == package:
+ continue
+ provided_by[virtual_pkg].add(package)
+ providers_of[package].add(virtual_pkg)
+
+ # Check source dependencies (Build-Depends and Build-Depends-Indep)
+ metakey_bd = get_or_set_metadatakey("Build-Depends", session)
+ metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
+ params = {
+ 'suite_id': suite_id,
+ 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
+ }
+ statement = '''
+ SELECT s.source, string_agg(sm.value, ', ') as build_dep
+ FROM source s
+ JOIN source_metadata sm ON s.id = sm.src_id
+ WHERE s.id in
+ (SELECT source FROM src_associations
+ WHERE suite = :suite_id)
+ AND sm.key_id in :metakey_ids
+ GROUP BY s.id, s.source'''
+ query = session.query('source', 'build_dep').from_statement(statement). \
+ params(params)
+ for source, build_dep in query:
+ if build_dep is not None:
+ # Remove [arch] information since we want to see breakage on all arches
+ build_dep = re_build_dep_arch.sub("", build_dep)
+ try:
+ parsed_dep = []
+ for dep in apt_pkg.parse_src_depends(build_dep):
+ parsed_dep.append(frozenset(d[0] for d in dep))
+ source_deps[source].update(parsed_dep)
+ except ValueError as e:
+ print "Error for package %s: %s" % (source, e)
+
+ return package_dependencies, arch_providors_of, arch_provided_by
+
+ def check_reverse_depends(self, removal_requests):
+ """Bulk check reverse dependencies
+
+ Example:
+ removal_request = {
+ "eclipse-rcp": None, # means ALL architectures (incl. source)
+ "eclipse": None, # means ALL architectures (incl. source)
+ "lintian": ["source", "all"], # Only these two "architectures".
+ }
+ obj.check_reverse_depends(removal_request)
+
+ @type removal_requests: dict (or a list of tuples)
+ @param removal_requests: A dictionary mapping a package name to a list of architectures. The list of
+ architectures decides from which the package will be removed - if the list is empty the package will
+ be removed on ALL architectures in the suite (including "source").
+
+ @rtype: dict
+ @return: A mapping of "removed package" (as a "(pkg, arch)"-tuple) to a set of broken
+ broken packages (also as "(pkg, arch)"-tuple). Note that the architecture values
+ in these tuples /can/ be "source" to reflect a breakage in build-dependencies.
+ """
+
+ archs_in_suite = self._archs_in_suite
+ removals_by_arch = defaultdict(set)
+ affected_virtual_by_arch = defaultdict(set)
+ package_dependencies = self._package_dependencies
+ arch_providors_of = self._arch_providors_of
+ arch_provided_by = self._arch_provided_by
+ arch_provides2removal = defaultdict(lambda: defaultdict(set))
+ dep_problems = defaultdict(set)
+ src_deps = package_dependencies['source']
+ src_removals = set()
+ arch_all_removals = set()
+
+ if isinstance(removal_requests, dict):
+ removal_requests = removal_requests.iteritems()
+
+ for pkg, arch_list in removal_requests:
+ if not arch_list:
+ arch_list = archs_in_suite
+ for arch in arch_list:
+ if arch == 'source':
+ src_removals.add(pkg)
+ continue
+ if arch == 'all':
+ arch_all_removals.add(pkg)
+ continue
+ removals_by_arch[arch].add(pkg)
+ if pkg in arch_providors_of[arch]:
+ affected_virtual_by_arch[arch].add(pkg)
+
+ if arch_all_removals:
+ for arch in archs_in_suite:
+ if arch in ('all', 'source'):
+ continue
+ removals_by_arch[arch].update(arch_all_removals)
+ for pkg in arch_all_removals:
+ if pkg in arch_providors_of[arch]:
+ affected_virtual_by_arch[arch].add(pkg)
+
+ if not removals_by_arch:
+ # Nothing to remove => no problems
+ return dep_problems
+
+ for arch, removed_providers in affected_virtual_by_arch.iteritems():
+ provides2removal = arch_provides2removal[arch]
+ removals = removals_by_arch[arch]
+ for virtual_pkg, virtual_providers in arch_provided_by[arch].iteritems():
+ v = virtual_providers & removed_providers
+ if len(v) == len(virtual_providers):
+ # We removed all the providers of virtual_pkg
+ removals.add(virtual_pkg)
+ # Pick one to take the blame for the removal
+ # - we sort for determinism, optimally we would prefer to blame the same package
+ # to minimise the number of blamed packages.
+ provides2removal[virtual_pkg] = sorted(v)[0]
+
+ for arch, removals in removals_by_arch.iteritems():
+ deps = package_dependencies[arch]
+ provides2removal = arch_provides2removal[arch]
+
+ # Check binary dependencies (Depends)
+ for package, dependencies in deps.iteritems():
+ if package in removals:
+ continue
+ for clause in dependencies:
+ if not (clause <= removals):
+ # Something probably still satisfies this relation
+ continue
+ # whoops, we seemed to have removed all packages that could possibly satisfy
+ # this relation. Lets blame something for it
+ for dep_package in clause:
+ removal = dep_package
+ if dep_package in provides2removal:
+ removal = provides2removal[dep_package]
+ dep_problems[(removal, arch)].add((package, arch))
+
+ for source, build_dependencies in src_deps.iteritems():
+ if source in src_removals:
+ continue
+ for clause in build_dependencies:
+ if not (clause <= removals):
+ # Something probably still satisfies this relation
+ continue
+ # whoops, we seemed to have removed all packages that could possibly satisfy
+ # this relation. Lets blame something for it
+ for dep_package in clause:
+ removal = dep_package
+ if dep_package in provides2removal:
+ removal = provides2removal[dep_package]
+ dep_problems[(removal, arch)].add((source, 'source'))
+
+ return dep_problems
+
+
+def remove(session, reason, suites, removals,
+ whoami=None, partial=False, components=None, done_bugs=None, date=None,
+ carbon_copy=None, close_related_bugs=False):
+ """Batch remove a number of packages
+ Verify that the files listed in the Files field of the .dsc are
+ those expected given the announced Format.
+
+ @type session: SQLA Session
+ @param session: The database session in use
+
+ @type reason: string
+ @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by <source>)")
+
+ @type suites: list
+ @param suites: A list of the suite names in which the removal should occur
+
+ @type removals: list
+ @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following
+ for 4 items from the database (in order): package, version, architecture, (database) id.
+ For source packages, the "architecture" should be set to "source".
+
+ @type partial: bool
+ @param partial: Whether the removal is "partial" (e.g. architecture specific).
+
+ @type components: list
+ @param components: List of components involved in a partial removal. Can be an empty list to not restrict the
+ removal to any components.
+
+ @type whoami: string
+ @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami()
+
+ @type date: string
+ @param date: The date of the removal. Defaults to commands.getoutput("date -R")
+
+ @type done_bugs: list
+ @param done_bugs: A list of bugs to be closed when doing this removal.
+
+ @type close_related_bugs: bool
+ @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented
+ for more than one suite.
+
+ @type carbon_copy: list
+ @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike
+ "dak rm".
+
+ @rtype: None
+ @return: Nothing
+ """
+ # Generate the summary of what's to be removed
+ d = {}
+ summary = ""
+ sources = []
+ binaries = []
+ whitelists = []
+ versions = []
+ suite_ids_list = []
+ suites_list = utils.join_with_commas_and(suites)
+ cnf = utils.get_conf()
+ con_components = None
+
+ #######################################################################################################
+
+ if not reason:
+ raise ValueError("Empty removal reason not permitted")
+
+ if not removals:
+ raise ValueError("Nothing to remove!?")
+
+ if not suites:
+ raise ValueError("Removals without a suite!?")
+
+ if whoami is None:
+ whoami = utils.whoami()
+
+ if date is None:
+ date = commands.getoutput("date -R")
+
+ if partial:
+
+ component_ids_list = []
+ for componentname in components:
+ component = get_component(componentname, session=session)
+ if component is None:
+ raise ValueError("component '%s' not recognised." % componentname)
+ else:
+ component_ids_list.append(component.component_id)
+ con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list])
+
+ for i in removals:
+ package = i[0]
+ version = i[1]
+ architecture = i[2]
+ if package not in d:
+ d[package] = {}
+ if version not in d[package]:
+ d[package][version] = []
+ if architecture not in d[package][version]:
+ d[package][version].append(architecture)
+
+ for package in sorted(removals):
+ versions = sorted(d[package], cmp=apt_pkg.version_compare)
+ for version in versions:
+ d[package][version].sort(utils.arch_compare_sw)
+ summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
+
+ for package in summary.split("\n"):
+ for row in package.split("\n"):
+ element = row.split("|")
+ if len(element) == 3:
+ if element[2].find("source") > 0:
+ sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2]))
+ element[2] = sub("source\s?,?", "", element[2]).strip(" ")
+ if element[2]:
+ binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element))
+
+ dsc_type_id = get_override_type('dsc', session).overridetype_id
+ deb_type_id = get_override_type('deb', session).overridetype_id
+
+ for suite in suites:
+ s = get_suite(suite, session=session)
+ if s is not None:
+ suite_ids_list.append(s.suite_id)
+ whitelists.append(s.mail_whitelist)
+
+ #######################################################################################################
+ log_filename = cnf["Rm::LogFile"]
+ log822_filename = cnf["Rm::LogFile822"]
+ with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822:
+ logfile.write("=========================================================================\n")
+ logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
+ logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
+ if done_bugs:
+ logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs)))
+ logfile.write("\n------------------- Reason -------------------\n%s\n" % reason)
+ logfile.write("----------------------------------------------\n")
+
+ logfile822.write("Date: %s\n" % date)
+ logfile822.write("Ftpmaster: %s\n" % whoami)
+ logfile822.write("Suite: %s\n" % suites_list)
+
+ if sources:
+ logfile822.write("Sources:\n")
+ for source in sources:
+ logfile822.write(" %s\n" % source)
+
+ if binaries:
+ logfile822.write("Binaries:\n")
+ for binary in binaries:
+ logfile822.write(" %s\n" % binary)
+
+ logfile822.write("Reason: %s\n" % reason.replace('\n', '\n '))
+ if done_bugs:
+ logfile822.write("Bug: %s\n" % (", ".join(done_bugs)))
+
+ for i in removals:
+ package = i[0]
+ architecture = i[2]
+ package_id = i[3]
+ for suite_id in suite_ids_list:
+ if architecture == "source":
+ session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
+ {'packageid': package_id, 'suiteid': suite_id})
+ else:
+ session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
+ {'packageid': package_id, 'suiteid': suite_id})
+ # Delete from the override file
+ if partial:
+ if architecture == "source":
+ type_id = dsc_type_id
+ else:
+ type_id = deb_type_id
+ # TODO: Fix this properly to remove the remaining non-bind argument
+ session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
+
+ session.commit()
+ # ### REMOVAL COMPLETE - send mail time ### #
+
+ # If we don't have a Bug server configured, we're done
+ if "Dinstall::BugServer" not in cnf:
+ if done_bugs or close_related_bugs:
+ utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured")
+
+ logfile.write("=========================================================================\n")
+ logfile822.write("\n")
+ return
+
+ # read common subst variables for all bug closure mails
+ Subst_common = {}
+ Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
+ Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
+ Subst_common["__CC__"] = "X-DAK: dak rm"
+ if carbon_copy:
+ Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
+ Subst_common["__SUITE_LIST__"] = suites_list
+ Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list)
+ Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
+ Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
+ Subst_common["__WHOAMI__"] = whoami
+
+ # Send the bug closing messages
+ if done_bugs:
+ Subst_close_rm = Subst_common
+ bcc = []
+ if cnf.find("Dinstall::Bcc") != "":
+ bcc.append(cnf["Dinstall::Bcc"])
+ if cnf.find("Rm::Bcc") != "":
+ bcc.append(cnf["Rm::Bcc"])
+ if bcc:
+ Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
+ else:
+ Subst_close_rm["__BCC__"] = "X-Filler: 42"
+ summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason)
+ summarymail += "----------------------------------------------\n"
+ Subst_close_rm["__SUMMARY__"] = summarymail
+
+ for bug in done_bugs:
+ Subst_close_rm["__BUG_NUMBER__"] = bug
+ if close_related_bugs:
+ mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
+ else:
+ mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
+ utils.send_mail(mail_message, whitelists=whitelists)
+
+ # close associated bug reports
+ if close_related_bugs:
+ Subst_close_other = Subst_common
+ bcc = []
+ wnpp = utils.parse_wnpp_bug_file()
+ versions = list(set([re_bin_only_nmu.sub('', v) for v in versions]))
+ if len(versions) == 1:
+ Subst_close_other["__VERSION__"] = versions[0]
+ else:
+ logfile.write("=========================================================================\n")
+ logfile822.write("\n")
+ raise ValueError("Closing bugs with multiple package versions is not supported. Do it yourself.")
+ if bcc:
+ Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
+ else:
+ Subst_close_other["__BCC__"] = "X-Filler: 42"
+ # at this point, I just assume, that the first closed bug gives
+ # some useful information on why the package got removed
+ Subst_close_other["__BUG_NUMBER__"] = done_bugs[0]
+ if len(sources) == 1:
+ source_pkg = source.split("_", 1)[0]
+ else:
+ logfile.write("=========================================================================\n")
+ logfile822.write("\n")
+ raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.")
+ Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
+ Subst_close_other["__SOURCE__"] = source_pkg
+ merged_bugs = set()
+ other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded')
+ if other_bugs:
+ for bugno in other_bugs:
+ if bugno not in merged_bugs:
+ for bug in bts.get_status(bugno):
+ for merged in bug.mergedwith:
+ other_bugs.remove(merged)
+ merged_bugs.add(merged)
+ logfile.write("Also closing bug(s):")
+ logfile822.write("Also-Bugs:")
+ for bug in other_bugs:
+ Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
+ logfile.write(" " + str(bug))
+ logfile822.write(" " + str(bug))
+ logfile.write("\n")
+ logfile822.write("\n")
+ if source_pkg in wnpp:
+ logfile.write("Also closing WNPP bug(s):")
+ logfile822.write("Also-WNPP:")
+ for bug in wnpp[source_pkg]:
+ # the wnpp-rm file we parse also contains our removal
+ # bugs, filtering that out
+ if bug != Subst_close_other["__BUG_NUMBER__"]:
+ Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
+ logfile.write(" " + str(bug))
+ logfile822.write(" " + str(bug))
+ logfile.write("\n")
+ logfile822.write("\n")
+
+ mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"]+"/rm.bug-close-related")
+ if Subst_close_other["__BUG_NUMBER_ALSO__"]:
+ utils.send_mail(mail_message)
+
+ logfile.write("=========================================================================\n")
+ logfile822.write("\n")
################################################################################
-def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False):
+def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False, quiet=False):
dbsuite = get_suite(suite, session)
overridesuite = dbsuite
if dbsuite.overridesuite is not None:
overridesuite = get_suite(dbsuite.overridesuite, session)
dep_problem = 0
p2c = {}
- all_broken = {}
+ all_broken = defaultdict(lambda: defaultdict(set))
if arches:
all_arches = set(arches)
else:
- all_arches = set([x.arch_string for x in get_suite_architectures(suite)])
+ all_arches = set(x.arch_string for x in get_suite_architectures(suite))
all_arches -= set(["source", "all"])
+ removal_set = set(removals)
metakey_d = get_or_set_metadatakey("Depends", session)
metakey_p = get_or_set_metadatakey("Provides", session)
params = {
params['arch_id'] = get_architecture(architecture, session).arch_id
statement = '''
- SELECT b.id, b.package, s.source, c.name as component,
+ SELECT b.package, s.source, c.name as component,
(SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
(SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
FROM binaries b
JOIN files_archive_map af ON b.file = af.file_id
JOIN component c ON af.component_id = c.id
WHERE b.architecture = :arch_id'''
- query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \
+ query = session.query('package', 'source', 'component', 'depends', 'provides'). \
from_statement(statement).params(params)
- for binary_id, package, source, component, depends, provides in query:
+ for package, source, component, depends, provides in query:
sources[package] = source
p2c[package] = component
if depends is not None:
# If a virtual package is only provided by the to-be-removed
# packages, treat the virtual package as to-be-removed too.
- for virtual_pkg in virtual_packages.keys():
- if virtual_packages[virtual_pkg] == 0:
- removals.append(virtual_pkg)
+ removal_set.update(virtual_pkg for virtual_pkg in virtual_packages if not virtual_packages[virtual_pkg])
# Check binary dependencies (Depends)
- for package in deps.keys():
+ for package in deps:
if package in removals: continue
- parsed_dep = []
try:
- parsed_dep += apt_pkg.parse_depends(deps[package])
+ parsed_dep = apt_pkg.parse_depends(deps[package])
except ValueError as e:
print "Error for package %s: %s" % (package, e)
+ parsed_dep = []
for dep in parsed_dep:
# Check for partial breakage. If a package has a ORed
# dependency, there is only a dependency problem if all
source = sources[package]
if component != "main":
source = "%s/%s" % (source, component)
- all_broken.setdefault(source, {}).setdefault(package, set()).add(architecture)
+ all_broken[source][package].add(architecture)
dep_problem = 1
- if all_broken:
+ if all_broken and not quiet:
if cruft:
print " - broken Depends:"
else:
print
# Check source dependencies (Build-Depends and Build-Depends-Indep)
- all_broken.clear()
+ all_broken = defaultdict(set)
metakey_bd = get_or_set_metadatakey("Build-Depends", session)
metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
params = {
'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
}
statement = '''
- SELECT s.id, s.source, string_agg(sm.value, ', ') as build_dep
+ SELECT s.source, string_agg(sm.value, ', ') as build_dep
FROM source s
JOIN source_metadata sm ON s.id = sm.src_id
WHERE s.id in
WHERE suite = :suite_id)
AND sm.key_id in :metakey_ids
GROUP BY s.id, s.source'''
- query = session.query('id', 'source', 'build_dep').from_statement(statement). \
+ query = session.query('source', 'build_dep').from_statement(statement). \
params(params)
- for source_id, source, build_dep in query:
+ for source, build_dep in query:
if source in removals: continue
parsed_dep = []
if build_dep is not None:
# Remove [arch] information since we want to see breakage on all arches
build_dep = re_build_dep_arch.sub("", build_dep)
try:
- parsed_dep += apt_pkg.parse_src_depends(build_dep)
+ parsed_dep = apt_pkg.parse_src_depends(build_dep)
except ValueError as e:
print "Error for source %s: %s" % (source, e)
for dep in parsed_dep:
key = source
if component != "main":
key = "%s/%s" % (source, component)
- all_broken.setdefault(key, set()).add(pp_deps(dep))
+ all_broken[key].add(pp_deps(dep))
dep_problem = 1
- if all_broken:
+ if all_broken and not quiet:
if cruft:
print " - broken Build-Depends:"
else:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-| priviledged positions? What privilege? The honour of working harder
-| than most people for absolutely no recognition?
-
-Manoj Srivastava <srivasta@debian.org> in <87lln8aqfm.fsf@glaurung.internal.golden-gryphon.com>
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
<elmo_h> you could just r00t klecker through [...] and do it yourself
<mdz> heh
<mdz> I think there's a bit in the DMUP about that
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-From: Andrew Morton <akpm@osdl.org>
-Subject: 2.6.6-mm5
-To: linux-kernel@vger.kernel.org
-Date: Sat, 22 May 2004 01:36:36 -0700
-X-Mailer: Sylpheed version 0.9.7 (GTK+ 1.2.10; i386-redhat-linux-gnu)
-
-[...]
-
- Although this feature has been around for a while it is new code, and the
- usual cautions apply. If it munches all your files please tell Jens and
- he'll type them in again for you.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
From: Randall Donald <ftpmaster@debian.org>
Subject: foo_3.4-1_i386.changes REJECTED
To: John Doe <jdoe@debian.org>
projects / dak.git / commitdiff