From: Ansgar Burchardt Date: Wed, 24 Aug 2011 12:34:13 +0000 (+0200) Subject: Merge remote-tracking branch 'origin/master' into auditpackages X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=4c6d9d52eddb26028e5328e30c2d312bc45f3906;hp=ba39a263d7248c981fcc6e7283c71e3138efe024;p=dak.git Merge remote-tracking branch 'origin/master' into auditpackages Conflicts: dak/update_db.py --- diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index c4a25a61..ed6f48bc 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -243,22 +243,10 @@ function mkmaintainers() { function copyoverrides() { log 'Copying override files into public view ...' - for ofile in $copyoverrides ; do - cd $overridedir - chmod g+w override.$ofile - - cd $indices - - newofile=override.$ofile.gz - rm -f .newover-$ofile.gz - pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`" - if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then - log " installing new $newofile $pc" - mv -f .newover-$ofile.gz $newofile - chmod g+w $newofile - else - rm -f .newover-$ofile.gz - fi + for ofile in ${overridedir}/override.{lenny,squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do + bname=${ofile##*/} + gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz + chmod g+w ${indices}/${bname}.gz done } diff --git a/config/debian/vars b/config/debian/vars index 19dee850..cea5344f 100644 --- a/config/debian/vars +++ b/config/debian/vars @@ -30,8 +30,6 @@ exportpublic=$public/rsync/export/ ftpgroup=debadmin -copyoverrides="wheezy.contrib wheezy.contrib.src wheezy.main wheezy.main.src wheezy.non-free wheezy.non-free.src wheezy.extra.main wheezy.extra.non-free wheezy.extra.contrib wheezy.main.debian-installer sid.contrib sid.contrib.src sid.main sid.main.debian-installer sid.main.src sid.non-free sid.non-free.src sid.extra.contrib sid.extra.main sid.extra.non-free lenny.contrib lenny.contrib.src lenny.main lenny.main.src lenny.non-free lenny.non-free.src lenny.extra.main lenny.extra.contrib lenny.extra.non-free squeeze.contrib squeeze.contrib.src squeeze.main squeeze.main.src squeeze.non-free squeeze.non-free.src squeeze.extra.main squeeze.extra.contrib squeeze.extra.non-free" - TMPDIR=${base}/tmp PATH=$masterdir:$PATH diff --git a/dak/admin.py b/dak/admin.py index c6128b9b..fe369b51 100755 --- a/dak/admin.py +++ b/dak/admin.py @@ -100,7 +100,7 @@ Perform administrative work on the dak database. v-c list show version checks for all suites v-c list-suite SUITE show version checks for suite SUITE v-c add SUITE CHECK REFERENCE add a version check for suite SUITE - v-c rm SUITE CHECK REFERENCE rmove a version check + v-c rm SUITE CHECK REFERENCE remove a version check where CHECK is one of Enhances, MustBeNewerThan, MustBeOlderThan REFERENCE is another suite name diff --git a/dak/control_suite.py b/dak/control_suite.py index ed4f6b1a..6dd79b8f 100755 --- a/dak/control_suite.py +++ b/dak/control_suite.py @@ -193,6 +193,17 @@ def version_checks(package, architecture, target_suite, new_version, session, fo ####################################################################################### +def cmp_package_version(a, b): + """ + comparison function for tuples of the form (package-name, version ...) + """ + cmp_package = cmp(a[0], b[0]) + if cmp_package != 0: + return cmp_package + return apt_pkg.VersionCompare(a[1], b[1]) + +####################################################################################### + def set_suite(file, suite, session, britney=False, force=False): suite_id = suite.suite_id lines = file.readlines() @@ -227,19 +238,8 @@ def set_suite(file, suite, session, britney=False, force=False): key = " ".join(split_line) desired[key] = "" - # Check to see which packages need removed and remove them - for key in current.keys(): - if not desired.has_key(key): - (package, version, architecture) = key.split() - pkid = current[key] - if architecture == "source": - session.execute("""DELETE FROM src_associations WHERE id = :pkid""", {'pkid': pkid}) - else: - session.execute("""DELETE FROM bin_associations WHERE id = :pkid""", {'pkid': pkid}) - Logger.log(["removed", key, pkid]) - # Check to see which packages need added and add them - for key in desired.keys(): + for key in sorted(desired.keys(), cmp=cmp_package_version): if not current.has_key(key): (package, version, architecture) = key.split() version_checks(package, architecture, suite.suite_name, version, session, force) @@ -254,6 +254,17 @@ def set_suite(file, suite, session, britney=False, force=False): VALUES (:suiteid, :pkid)""", {'suiteid': suite_id, 'pkid': pkid}) Logger.log(["added", key, pkid]) + # Check to see which packages need removed and remove them + for key in current.keys(): + if not desired.has_key(key): + (package, version, architecture) = key.split() + pkid = current[key] + if architecture == "source": + session.execute("""DELETE FROM src_associations WHERE id = :pkid""", {'pkid': pkid}) + else: + session.execute("""DELETE FROM bin_associations WHERE id = :pkid""", {'pkid': pkid}) + Logger.log(["removed", key, pkid]) + session.commit() if britney: @@ -268,17 +279,19 @@ def process_file(file, suite, action, session, britney=False, force=False): suite_id = suite.suite_id - lines = file.readlines() + request = [] # Our session is already in a transaction - for line in lines: + for line in file: split_line = line.strip().split() if len(split_line) != 3: utils.warn("'%s' does not break into 'package version architecture'." % (line[:-1])) continue + request.append(split_line) - (package, version, architecture) = split_line + request.sort(cmp=cmp_package_version) + for package, version, architecture in request: pkid = get_id(package, version, architecture, session) if not pkid: continue diff --git a/dak/dakdb/update66.py b/dak/dakdb/update66.py new file mode 100755 index 00000000..af61c04a --- /dev/null +++ b/dak/dakdb/update66.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Add 2 partial indexes to speed up dak rm. + +@contact: Debian FTP Master +@copyright: 2011 Torsten Werner +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError +from daklib.config import Config + +################################################################################ +def do_update(self): + """ + Add 2 partial indexes to speed up dak rm. + """ + print __doc__ + try: + cnf = Config() + + c = self.db.cursor() + + # partial index for Depends + c.execute("SELECT key_id FROM metadata_keys WHERE key = 'Depends'") + key = c.fetchone()[0] + c.execute("""CREATE INDEX binaries_metadata_depends + ON binaries_metadata (bin_id) WHERE key_id = %d""" % key) + + # partial index for Provides + c.execute("SELECT key_id FROM metadata_keys WHERE key = 'Provides'") + key = c.fetchone()[0] + c.execute("""CREATE INDEX binaries_metadata_provides + ON binaries_metadata (bin_id) WHERE key_id = %d""" % key) + + c.execute("UPDATE config SET value = '66' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, 'Unable to apply sick update 66, rollback issued. Error message : %s' % (str(msg)) diff --git a/dak/rm.py b/dak/rm.py index 0583755b..d0b46f62 100755 --- a/dak/rm.py +++ b/dak/rm.py @@ -51,7 +51,7 @@ from daklib.config import Config from daklib.dbconn import * from daklib import utils from daklib.dak_exceptions import * -from daklib.regexes import re_strip_source_version, re_build_dep_arch +from daklib.regexes import re_strip_source_version, re_build_dep_arch, re_bin_only_nmu import debianbts as bts ################################################################################ @@ -126,16 +126,11 @@ def reverse_depends_check(removals, suite, arches=None, session=None): params['arch_id'] = get_architecture(architecture, session).arch_id statement = ''' - create temp table suite_binaries ( - id integer primary key, - package text, - source integer, - file integer); - insert into suite_binaries - select b.id, b.package, b.source, b.file + WITH suite_binaries AS + (select b.id, b.package, b.source, b.file from binaries b WHERE b.id in (SELECT bin FROM bin_associations WHERE suite = :suite_id) - AND b.architecture in (:arch_id, :arch_all_id); + AND b.architecture in (:arch_id, :arch_all_id)) SELECT b.id, b.package, s.source, c.name as component, bmd.value as depends, bmp.value as provides FROM suite_binaries b @@ -671,6 +666,7 @@ def main (): Subst_close_other = Subst_common bcc = [] wnpp = utils.parse_wnpp_bug_file() + versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) if len(versions) == 1: Subst_close_other["__VERSION__"] = versions[0] else: diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 1fa7974f..63730187 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -81,6 +81,9 @@ import warnings warnings.filterwarnings('ignore', \ "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \ SADeprecationWarning) +warnings.filterwarnings('ignore', \ + "Predicate of partial index .* ignored during reflection", \ + SAWarning) ################################################################################