From: Ansgar Burchardt Date: Tue, 10 Sep 2013 21:23:12 +0000 (+0200) Subject: Merge remote-tracking branch 'origin/master' X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=36f2e4eed512ab916efac0dc2ecb6dc0d503397b;hp=d7b3341ac987031468fd7ba8020620d2fecaccd1;p=dak.git Merge remote-tracking branch 'origin/master' --- diff --git a/dak/admin.py b/dak/admin.py index 4923234b..a4f5f4bb 100755 --- a/dak/admin.py +++ b/dak/admin.py @@ -77,8 +77,17 @@ Perform administrative work on the dak database. If SUITELIST is given, add to each of the suites at the same time + component: + component list show a list of components + component rm COMPONENT remove a component (will only work if + empty) + component add NAME DESCRIPTION ORDERING + add component NAME with DESCRIPTION. + Ordered at ORDERING. + suite / s: - s list show a list of suites + s list [--print-archive] + show a list of suites s show SUITE show config details for a suite s add SUITE VERSION [ label=LABEL ] [ description=DESCRIPTION ] [ origin=ORIGIN ] [ codename=CODENAME ] @@ -98,6 +107,17 @@ Perform administrative work on the dak database. s-a rm SUITE ARCH remove ARCH from suite (will only work if no packages remain for the arch in the suite) + suite-component / s-c: + s-c list show the architectures for all suites + s-c list-suite COMPONENT + show the suites a COMPONENT is in + s-c list-component SUITE + show the components in a SUITE + s-c add SUITE COMPONENT + add COMPONENT to suite + s-c rm SUITE COMPONENT remove component from suite (will only work if + no packages remain for the component in the suite) + archive: archive list list all archives archive add NAME ROOT DESCRIPTION [primary-mirror=MIRROR] [tainted=1] @@ -193,10 +213,84 @@ dispatch['a'] = architecture ################################################################################ +def component_list(): + session = DBConn().session() + for component in session.query(Component).order_by(Component.component_name): + print "{0} ordering={1}".format(component.component_name, component.ordering) + +def component_add(args): + (name, description, ordering) = args[0:3] + + attributes = dict( + component_name=name, + description=description, + ordering=ordering, + ) + + for option in args[3:]: + (key, value) = option.split('=') + attributes[key] = value + + session = DBConn().session() + + component = Component() + for key, value in attributes.iteritems(): + setattr(component, key, value) + + session.add(component) + session.flush() + + if dryrun: + session.rollback() + else: + session.commit() + +def component_rm(name): + session = DBConn().session() + component = get_component(name, session) + session.delete(component) + session.flush() + + if dryrun: + session.rollback() + else: + session.commit() + +def component_rename(oldname, newname): + session = DBConn().session() + component = get_component(oldname, session) + component.component_name = newname + session.flush() + + if dryrun: + session.rollback() + else: + session.commit() + +def component(command): + mode = command[1] + if mode == 'list': + component_list() + elif mode == 'rename': + component_rename(command[2], command[3]) + elif mode == 'add': + component_add(command[2:]) + elif mode == 'rm': + component_rm(command[2]) + else: + die("E: component command unknown") + +dispatch['component'] = component + +################################################################################ + def __suite_list(d, args): s = d.session() - for j in s.query(Suite).order_by(Suite.suite_name).all(): - print j.suite_name + for j in s.query(Suite).join(Suite.archive).order_by(Archive.archive_name, Suite.suite_name).all(): + if len(args) > 2 and args[2] == "--print-archive": + print "{0} {1}".format(j.archive.archive_name, j.suite_name) + else: + print "{0}".format(j.suite_name) def __suite_show(d, args): if len(args) < 2: @@ -261,6 +355,23 @@ def __suite_add(d, args, addallarches=False): s.commit() +def __suite_rm(d, args): + die_arglen(args, 3, "E: removing a suite requires at least a name") + name = args[2] + print "Removing suite {0}".format(name) + if not dryrun: + try: + s = d.session() + su = get_suite(name.lower()) + if su is None: + die("E: Cannot find suite {0}".format(name)) + s.delete(su) + s.commit() + except IntegrityError as e: + die("E: Integrity error removing suite {0} (suite-arch entries probably still exist)".format(name)) + except SQLAlchemyError as e: + die("E: Error removing suite {0} ({1})".format(name, e)) + print "Suite {0} removed".format(name) def suite(command): args = [str(x) for x in command] @@ -275,6 +386,8 @@ def suite(command): __suite_list(d, args) elif mode == 'show': __suite_show(d, args) + elif mode == 'rm': + __suite_rm(d, args) elif mode == 'add': __suite_add(d, args, False) elif mode == 'add-all-arches': @@ -389,6 +502,108 @@ dispatch['s-a'] = suite_architecture ################################################################################ +def __suite_component_list(d, args): + s = d.session() + for j in s.query(Suite).order_by(Suite.suite_name): + components = j.components + print j.suite_name + ': ' + \ + ', '.join([c.component_name for c in components]) + + +def __suite_component_listcomponent(d, args): + die_arglen(args, 3, "E: suite-component list-component requires a suite") + suite = get_suite(args[2].lower(), d.session()) + if suite is None: + die('E: suite %s is invalid' % args[2].lower()) + for c in suite.components: + print c.component_name + + +def __suite_component_listsuite(d, args): + die_arglen(args, 3, "E: suite-component list-suite requires an component") + component = get_component(args[2].lower(), d.session()) + if component is None: + die("E: component %s is invalid" % args[2].lower()) + for s in component.suites: + print s.suite_name + + +def __suite_component_add(d, args): + if len(args) < 3: + die("E: adding a suite-component entry requires a suite and component") + + s = d.session() + + suite = get_suite(args[2].lower(), s) + if suite is None: die("E: Can't find suite %s" % args[2].lower()) + + component = get_component(args[3].lower(), s) + if component is None: die("E: Can't find component %s" % args[3].lower()) + + if not dryrun: + try: + suite.components.append(component) + s.commit() + except IntegrityError as e: + die("E: Can't add suite-component entry (%s, %s) - probably already exists" % (args[2].lower(), args[3].lower())) + except SQLAlchemyError as e: + die("E: Can't add suite-component entry (%s, %s) - %s" % (args[2].lower(), args[3].lower(), e)) + + print "Added suite-component entry for %s, %s" % (args[2].lower(), args[3].lower()) + + +def __suite_component_rm(d, args): + if len(args) < 3: + die("E: removing an suite-component entry requires a suite and component") + + s = d.session() + if not dryrun: + try: + suite_name = args[2].lower() + suite = get_suite(suite_name, s) + if suite is None: + die('E: no such suite %s' % suite_name) + component_string = args[3].lower() + component = get_component(arch_string, s) + if component not in suite.components: + die("E: component %s not found in suite %s" % (component_string, suite_name)) + suite.components.remove(component) + s.commit() + except IntegrityError as e: + die("E: Can't remove suite-component entry (%s, %s) - it's probably referenced" % (args[2].lower(), args[3].lower())) + except SQLAlchemyError as e: + die("E: Can't remove suite-component entry (%s, %s) - %s" % (args[2].lower(), args[3].lower(), e)) + + print "Removed suite-component entry for %s, %s" % (args[2].lower(), args[3].lower()) + + +def suite_component(command): + args = [str(x) for x in command] + Cnf = utils.get_conf() + d = DBConn() + + die_arglen(args, 2, "E: suite-component needs at least a command") + + mode = args[1].lower() + + if mode == 'list': + __suite_component_list(d, args) + elif mode == 'list-component': + __suite_component_listcomponent(d, args) + elif mode == 'list-suite': + __suite_component_listsuite(d, args) + elif mode == 'add': + __suite_component_add(d, args) + # elif mode == 'rm': + # __suite_architecture_rm(d, args) + else: + die("E: suite-component command unknown") + +dispatch['suite-component'] = suite_component +dispatch['s-c'] = suite_component + +################################################################################ + def archive_list(): session = DBConn().session() for archive in session.query(Archive).order_by(Archive.archive_name): @@ -423,7 +638,7 @@ def archive_add(args): def archive_rm(name): session = DBConn().session() - archive = session.query(Archive).filter_by(archive_name=name).one() + archive = get_archive(name, session) session.delete(archive) session.flush() diff --git a/dak/dakdb/update100.py b/dak/dakdb/update100.py new file mode 100644 index 00000000..85733cb6 --- /dev/null +++ b/dak/dakdb/update100.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Add a component - suite mapping to only expose certain components in certain suites + +@contact: Debian FTP Master +@copyright: 2012 Varnish Software AS +@author: Tollef Fog Heen +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError +from daklib.config import Config + +################################################################################ +def do_update(self): + print __doc__ + try: + cnf = Config() + + c = self.db.cursor() + + c.execute(""" + CREATE TABLE component_suite ( + component_id INTEGER NOT NULL REFERENCES component(id) ON DELETE CASCADE, + suite_id INTEGER NOT NULL REFERENCES suite(id) ON DELETE CASCADE, + PRIMARY KEY (component_id, suite_id) + ) + """) + # Set up default mappings for all components to all suites + c.execute("INSERT INTO component_suite(component_id, suite_id) SELECT component.id,suite.id from suite, component") + + c.execute("UPDATE config SET value = '100' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError as msg: + self.db.rollback() + raise DBUpdateError('Unable to apply sick update 100, rollback issued. Error message: {0}'.format(msg)) diff --git a/dak/dakdb/update99.py b/dak/dakdb/update99.py new file mode 100644 index 00000000..f701fb7c --- /dev/null +++ b/dak/dakdb/update99.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Add component ordering + +@contact: Debian FTP Master +@copyright: 2012 Varnish Software AS +@author: Tollef Fog Heen +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError +from daklib.config import Config + +statements = [ +""" +ALTER TABLE component +ADD COLUMN ordering INTEGER UNIQUE +""", + +""" +CREATE SEQUENCE component_ordering_seq +INCREMENT BY 10 +START WITH 100 +OWNED BY component.ordering +""", +] + +################################################################################ +def do_update(self): + print __doc__ + try: + cnf = Config() + + c = self.db.cursor() + + for stmt in statements: + c.execute(stmt) + + for component in ('main', 'contrib', 'non-free'): + c.execute("UPDATE component SET ordering = nextval('component_ordering_seq') WHERE name = '{0}'".format(component)) + c.execute("UPDATE component SET ordering = nextval('component_ordering_seq') WHERE ordering IS NULL") + c.execute("""ALTER TABLE component ALTER COLUMN ordering SET NOT NULL""") + c.execute("""ALTER TABLE component ALTER COLUMN ordering SET DEFAULT nextval('component_ordering_seq')""") + + c.execute("UPDATE config SET value = '99' WHERE name = 'db_revision'") + + self.db.commit() + + except psycopg2.ProgrammingError as msg: + self.db.rollback() + raise DBUpdateError('Unable to apply sick update 99, rollback issued. Error message: {0}'.format(msg)) diff --git a/dak/generate_packages_sources2.py b/dak/generate_packages_sources2.py index d941d984..2c69afce 100755 --- a/dak/generate_packages_sources2.py +++ b/dak/generate_packages_sources2.py @@ -364,7 +364,6 @@ def main(): force = Options.has_key("Force") and Options["Force"] - component_ids = [ c.component_id for c in session.query(Component).all() ] def parse_results(message): # Split out into (code, msg) @@ -377,6 +376,7 @@ def main(): logger.log(['E: ', msg]) for s in suites: + component_ids = [ c.component_id for c in s.components ] if s.untouchable and not force: import utils utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name) diff --git a/dak/generate_releases.py b/dak/generate_releases.py index c80e0d9e..a86c33ce 100755 --- a/dak/generate_releases.py +++ b/dak/generate_releases.py @@ -170,7 +170,7 @@ class ReleaseWriter(object): out.write("Architectures: %s\n" % (" ".join([a.arch_string for a in architectures]))) - components = [ c.component_name for c in session.query(Component) ] + components = [ c.component_name for c in suite.components ] out.write("Components: %s\n" % (" ".join(components))) diff --git a/dak/update_db.py b/dak/update_db.py index 5bbc40c5..61a10892 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -158,7 +158,7 @@ Updates dak's database schema to the lastest version. You should disable crontab database_revision = 0 dbfiles = glob(os.path.join(os.path.dirname(__file__), 'dakdb/update*.py')) - required_database_schema = int(max(findall('update(\d+).py', " ".join(dbfiles)))) + required_database_schema = max(map(int, findall('update(\d+).py', " ".join(dbfiles)))) print "dak database schema at %d" % database_revision print "dak version requires schema %d" % required_database_schema diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 4d551ad1..22d20a57 100644 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -311,7 +311,7 @@ class ORMObject(object): return object_session(self) def clone(self, session = None): - ''' + """ Clones the current object in a new session and returns the new clone. A fresh session is created if the optional session parameter is not provided. The function will fail if a session is provided and has @@ -324,8 +324,8 @@ class ORMObject(object): WARNING: Only persistent (committed) objects can be cloned. Changes made to the original object that are not committed yet will get lost. The session of the new object will always be rolled back to avoid - ressource leaks. - ''' + resource leaks. + """ if self.session() is None: raise RuntimeError( \ @@ -433,27 +433,6 @@ def get_architecture(architecture, session=None): __all__.append('get_architecture') -# TODO: should be removed because the implementation is too trivial -@session_wrapper -def get_architecture_suites(architecture, session=None): - """ - Returns list of Suite objects for given C{architecture} name - - @type architecture: str - @param architecture: Architecture name to search for - - @type session: Session - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) - - @rtype: list - @return: list of Suite objects for the given name (may be empty) - """ - - return get_architecture(architecture, session).suites - -__all__.append('get_architecture_suites') - ################################################################################ class Archive(object): @@ -2558,6 +2537,7 @@ class DBConn(object): 'changelogs_text', 'changes', 'component', + 'component_suite', 'config', 'dsc_files', 'external_overrides', @@ -2687,7 +2667,8 @@ class DBConn(object): mapper(Component, self.tbl_component, properties = dict(component_id = self.tbl_component.c.id, - component_name = self.tbl_component.c.name), + component_name = self.tbl_component.c.name, + suites = relation(Suite, secondary=self.tbl_component_suite)), extension = validator) mapper(DBConfig, self.tbl_config, @@ -2833,7 +2814,10 @@ class DBConn(object): srcformats = relation(SrcFormat, secondary=self.tbl_suite_src_formats, backref=backref('suites', lazy='dynamic')), archive = relation(Archive, backref='suites'), - acls = relation(ACL, secondary=self.tbl_suite_acl_map, collection_class=set)), + acls = relation(ACL, secondary=self.tbl_suite_acl_map, collection_class=set), + components = relation(Component, secondary=self.tbl_component_suite, + order_by=self.tbl_component.c.ordering, + backref=backref('suite'))), extension = validator) mapper(Uid, self.tbl_uid, diff --git a/daklib/policy.py b/daklib/policy.py index cdff56a7..aeed9a2c 100644 --- a/daklib/policy.py +++ b/daklib/policy.py @@ -279,13 +279,18 @@ class PolicyQueueUploadHandler(object): )) components.add(component) + source = self.upload.source source_component = '(unknown)' - for component in ('main', 'contrib', 'non-free'): + for component, in self.session.query(Component.component_name).order_by(Component.ordering): if component in components: source_component = component break + else: + if source is not None: + if self._source_override(component) is not None: + source_component = component + break - source = self.upload.source if source is not None: override = self._source_override(source_component) if override is None: diff --git a/tests/dbtest_packages.py b/tests/dbtest_packages.py index f2587709..73679b57 100755 --- a/tests/dbtest_packages.py +++ b/tests/dbtest_packages.py @@ -60,13 +60,6 @@ class PackageTestCase(DBDakTestCase): architectures = get_suite_architectures('lenny', skipall = True, session = self.session) self.assertEqual(3, len(architectures)) self.assertTrue(self.arch['all'] not in architectures) - # check the function get_architecture_suites() - suites = get_architecture_suites('i386', self.session) - self.assertEqual(3, len(suites)) - self.assertTrue(self.suite['lenny'] in suites) - suites = get_architecture_suites('kfreebsd-i386', self.session) - self.assertEqual(2, len(suites)) - self.assertTrue(self.suite['lenny'] not in suites) # check overrides self.assertEqual(0, self.suite['lenny'].overrides.count())