From: Mike O'Connor Date: Thu, 5 Nov 2009 00:53:31 +0000 (-0500) Subject: merge from ftp-master X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=ca19ea22806872ba8360086b121c468689fe98df;p=dak.git merge from ftp-master Signed-off-by: Mike O'Connor --- ca19ea22806872ba8360086b121c468689fe98df diff --cc dak/dakdb/update25.py index 00000000,00000000..a61deb61 new file mode 100644 --- /dev/null +++ b/dak/dakdb/update25.py @@@ -1,0 -1,0 +1,268 @@@ ++#!/usr/bin/env python ++# coding=utf8 ++ ++""" ++Adding a trainee field to the process-new notes ++ ++@contact: Debian FTP Master ++@copyright: 2009 Mike O'Connor ++@license: GNU General Public License version 2 or later ++""" ++ ++# This program is free software; you can redistribute it and/or modify ++# it under the terms of the GNU General Public License as published by ++# the Free Software Foundation; either version 2 of the License, or ++# (at your option) any later version. ++ ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++ ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, write to the Free Software ++# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ++ ++################################################################################ ++ ++ ++################################################################################ ++ ++import psycopg2 ++import time ++from daklib.dak_exceptions import DBUpdateError ++ ++################################################################################ ++ ++def suites(): ++ """ ++ return a list of suites to operate on ++ """ ++ if Config().has_key( "%s::%s" %(options_prefix,"Suite")): ++ suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")]) ++ else: ++ suites = [ 'unstable', 'testing' ] ++# suites = Config().SubTree("Suite").List() ++ ++ return suites ++ ++def arches(cursor, suite): ++ """ ++ return a list of archs to operate on ++ """ ++ arch_list = [] ++ cursor.execute("""SELECT s.architecture, a.arch_string ++ FROM suite_architectures s ++ JOIN architecture a ON (s.architecture=a.id) ++ WHERE suite = :suite""", {'suite' : suite }) ++ ++ while True: ++ r = cursor.fetchone() ++ if not r: ++ break ++ ++ if r[1] != "source" and r[1] != "all": ++ arch_list.append((r[0], r[1])) ++ ++ return arch_list ++ ++def do_update(self): ++ """ ++ Adding contents table as first step to maybe, finally getting rid ++ of apt-ftparchive ++ """ ++ ++ print __doc__ ++ ++ try: ++ c = self.db.cursor() ++ ++ c.execute("""CREATE TABLE pending_bin_contents ( ++ id serial NOT NULL, ++ package text NOT NULL, ++ version debversion NOT NULL, ++ arch int NOT NULL, ++ filename text NOT NULL, ++ type int NOT NULL, ++ PRIMARY KEY(id))""" ); ++ ++ c.execute("""CREATE TABLE deb_contents ( ++ filename text, ++ section text, ++ package text, ++ binary_id integer, ++ arch integer, ++ suite integer)""" ) ++ ++ c.execute("""CREATE TABLE udeb_contents ( ++ filename text, ++ section text, ++ package text, ++ binary_id integer, ++ suite integer, ++ arch integer)""" ) ++ ++ c.execute("""ALTER TABLE ONLY deb_contents ++ ADD CONSTRAINT deb_contents_arch_fkey ++ FOREIGN KEY (arch) REFERENCES architecture(id) ++ ON DELETE CASCADE;""") ++ ++ c.execute("""ALTER TABLE ONLY udeb_contents ++ ADD CONSTRAINT udeb_contents_arch_fkey ++ FOREIGN KEY (arch) REFERENCES architecture(id) ++ ON DELETE CASCADE;""") ++ ++ c.execute("""ALTER TABLE ONLY deb_contents ++ ADD CONSTRAINT deb_contents_pkey ++ PRIMARY KEY (filename,package,arch,suite);""") ++ ++ c.execute("""ALTER TABLE ONLY udeb_contents ++ ADD CONSTRAINT udeb_contents_pkey ++ PRIMARY KEY (filename,package,arch,suite);""") ++ ++ c.execute("""ALTER TABLE ONLY deb_contents ++ ADD CONSTRAINT deb_contents_suite_fkey ++ FOREIGN KEY (suite) REFERENCES suite(id) ++ ON DELETE CASCADE;""") ++ ++ c.execute("""ALTER TABLE ONLY udeb_contents ++ ADD CONSTRAINT udeb_contents_suite_fkey ++ FOREIGN KEY (suite) REFERENCES suite(id) ++ ON DELETE CASCADE;""") ++ ++ c.execute("""ALTER TABLE ONLY deb_contents ++ ADD CONSTRAINT deb_contents_binary_fkey ++ FOREIGN KEY (binary_id) REFERENCES binaries(id) ++ ON DELETE CASCADE;""") ++ ++ c.execute("""ALTER TABLE ONLY udeb_contents ++ ADD CONSTRAINT udeb_contents_binary_fkey ++ FOREIGN KEY (binary_id) REFERENCES binaries(id) ++ ON DELETE CASCADE;""") ++ ++ c.execute("""CREATE INDEX ind_deb_contents_binary ON deb_contents(binary_id);""" ) ++ ++ ++ suites = self.suites() ++ ++ for suite in [i.lower() for i in suites]: ++ suite_id = DBConn().get_suite_id(suite) ++ arch_list = arches(c, suite_id) ++ arch_list = arches(c, suite_id) ++ ++ for (arch_id,arch_str) in arch_list: ++ c.execute( "CREATE INDEX ind_deb_contents_%s_%s ON deb_contents (arch,suite) WHERE (arch=2 OR arch=%d) AND suite=$d"%(arch_str,suite,arch_id,suite_id) ) ++ ++ for section, sname in [("debian-installer","main"), ++ ("non-free/debian-installer", "nonfree")]: ++ c.execute( "CREATE INDEX ind_udeb_contents_%s_%s ON udeb_contents (section,suite) WHERE section=%s AND suite=$d"%(sname,suite,section,suite_id) ) ++ ++ ++ c.execute( """CREATE OR REPLACE FUNCTION update_contents_for_bin_a() RETURNS trigger AS $$ ++ event = TD["event"] ++ if event == "DELETE" or event == "UPDATE": ++ ++ plpy.execute(plpy.prepare("DELETE FROM deb_contents WHERE binary_id=$1 and suite=$2", ++ ["int","int"]), ++ [TD["old"]["bin"], TD["old"]["suite"]]) ++ ++ if event == "INSERT" or event == "UPDATE": ++ ++ content_data = plpy.execute(plpy.prepare( ++ """SELECT s.section, b.package, b.architecture, ot.type ++ FROM override o ++ JOIN override_type ot on o.type=ot.id ++ JOIN binaries b on b.package=o.package ++ JOIN files f on b.file=f.id ++ JOIN location l on l.id=f.location ++ JOIN section s on s.id=o.section ++ WHERE b.id=$1 ++ AND o.suite=$2 ++ """, ++ ["int", "int"]), ++ [TD["new"]["bin"], TD["new"]["suite"]])[0] ++ ++ tablename="%s_contents" % content_data['type'] ++ ++ plpy.execute(plpy.prepare("""DELETE FROM %s ++ WHERE package=$1 and arch=$2 and suite=$3""" % tablename, ++ ['text','int','int']), ++ [content_data['package'], ++ content_data['architecture'], ++ TD["new"]["suite"]]) ++ ++ filenames = plpy.execute(plpy.prepare( ++ "SELECT bc.file FROM bin_contents bc where bc.binary_id=$1", ++ ["int"]), ++ [TD["new"]["bin"]]) ++ ++ for filename in filenames: ++ plpy.execute(plpy.prepare( ++ """INSERT INTO %s ++ (filename,section,package,binary_id,arch,suite) ++ VALUES($1,$2,$3,$4,$5,$6)""" % tablename, ++ ["text","text","text","int","int","int"]), ++ [filename["file"], ++ content_data["section"], ++ content_data["package"], ++ TD["new"]["bin"], ++ content_data["architecture"], ++ TD["new"]["suite"]] ) ++$$ LANGUAGE plpythonu VOLATILE SECURITY DEFINER; ++""") ++ ++ ++ c.execute( """CREATE OR REPLACE FUNCTION update_contents_for_override() RETURNS trigger AS $$ ++ event = TD["event"] ++ if event == "UPDATE": ++ ++ otype = plpy.execute(plpy.prepare("SELECT type from override_type where id=$1",["int"]),[TD["new"]["type"]] )[0]; ++ if otype["type"].endswith("deb"): ++ section = plpy.execute(plpy.prepare("SELECT section from section where id=$1",["int"]),[TD["new"]["section"]] )[0]; ++ ++ table_name = "%s_contents" % otype["type"] ++ plpy.execute(plpy.prepare("UPDATE %s set section=$1 where package=$2 and suite=$3" % table_name, ++ ["text","text","int"]), ++ [section["section"], ++ TD["new"]["package"], ++ TD["new"]["suite"]]) ++ ++$$ LANGUAGE plpythonu VOLATILE SECURITY DEFINER; ++""") ++ ++ c.execute("""CREATE OR REPLACE FUNCTION update_contents_for_override() ++ RETURNS trigger AS $$ ++ event = TD["event"] ++ if event == "UPDATE" or event == "INSERT": ++ row = TD["new"] ++ r = plpy.execute(plpy.prepare( """SELECT 1 from suite_architectures sa ++ JOIN binaries b ON b.architecture = sa.architecture ++ WHERE b.id = $1 and sa.suite = $2""", ++ ["int", "int"]), ++ [row["bin"], row["suite"]]) ++ if not len(r): ++ plpy.error("Illegal architecture for this suite") ++ ++$$ LANGUAGE plpythonu VOLATILE;""") ++ ++ c.execute( """CREATE TRIGGER illegal_suite_arch_bin_associations_trigger ++ BEFORE INSERT OR UPDATE ON bin_associations ++ FOR EACH ROW EXECUTE PROCEDURE update_contents_for_override();""") ++ ++ c.execute( """CREATE TRIGGER bin_associations_contents_trigger ++ AFTER INSERT OR UPDATE OR DELETE ON bin_associations ++ FOR EACH ROW EXECUTE PROCEDURE update_contents_for_bin_a();""") ++ c.execute("""CREATE TRIGGER override_contents_trigger ++ AFTER UPDATE ON override ++ FOR EACH ROW EXECUTE PROCEDURE update_contents_for_override();""") ++ ++ ++ c.execute( "CREATE INDEX ind_deb_contents_name ON deb_contents(package);"); ++ c.execute( "CREATE INDEX ind_udeb_contents_name ON udeb_contents(package);"); ++ ++ self.db.commit() ++ ++ except psycopg2.ProgrammingError, msg: ++ self.db.rollback() ++ raise DBUpdateError, "Unable to apply process-new update 14, rollback issued. Error message : %s" % (str(msg)) ++ diff --cc daklib/binary.py index a70aadb9,c6ee96f8..a70aadb9 mode 100755,100644..100644 --- a/daklib/binary.py +++ b/daklib/binary.py diff --cc daklib/dbconn.py index 921f1daa,361dcf42..d05dd159 mode 100755,100644..100644 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@@ -50,10 -50,7 +50,9 @@@ from sqlalchemy import types as sqltype from sqlalchemy.exc import * from sqlalchemy.orm.exc import NoResultFound +# Only import Config until Queue stuff is changed to store its config +# in the database from config import Config - from singleton import Singleton from textutils import fix_maintainer ################################################################################ @@@ -2262,59 -2510,68 +2534,68 @@@ class DBConn(object) """ database module init. """ + __shared_state = {} + def __init__(self, *args, **kwargs): - super(DBConn, self).__init__(*args, **kwargs) + self.__dict__ = self.__shared_state - def _startup(self, *args, **kwargs): - self.debug = False - if kwargs.has_key('debug'): - self.debug = True - self.__createconn() + if not getattr(self, 'initialised', False): + self.initialised = True + self.debug = kwargs.has_key('debug') + self.__createconn() def __setuptables(self): - self.tbl_architecture = Table('architecture', self.db_meta, autoload=True) - self.tbl_archive = Table('archive', self.db_meta, autoload=True) - self.tbl_bin_contents = Table('bin_contents', self.db_meta, autoload=True) - self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True) - self.tbl_binaries = Table('binaries', self.db_meta, autoload=True) - self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True) - self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True) - self.tbl_component = Table('component', self.db_meta, autoload=True) - self.tbl_config = Table('config', self.db_meta, autoload=True) - self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True) - self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True) - self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True) - self.tbl_changes_pending_files = Table('changes_pending_files', self.db_meta, autoload=True) - self.tbl_changes_pool_files = Table('changes_pool_files', self.db_meta, autoload=True) - self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True) - self.tbl_deb_contents = Table('deb_contents', self.db_meta, autoload=True) - self.tbl_files = Table('files', self.db_meta, autoload=True) - self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True) - self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True) - self.tbl_known_changes = Table('known_changes', self.db_meta, autoload=True) - self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True) - self.tbl_location = Table('location', self.db_meta, autoload=True) - self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True) - self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True) - self.tbl_override = Table('override', self.db_meta, autoload=True) - self.tbl_override_type = Table('override_type', self.db_meta, autoload=True) - self.tbl_pending_bin_contents = Table('pending_bin_contents', self.db_meta, autoload=True) - self.tbl_priority = Table('priority', self.db_meta, autoload=True) - self.tbl_queue = Table('queue', self.db_meta, autoload=True) - self.tbl_queue_files = Table('queue_files', self.db_meta, autoload=True) - self.tbl_section = Table('section', self.db_meta, autoload=True) - self.tbl_source = Table('source', self.db_meta, autoload=True) - self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True) - self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True) - self.tbl_src_format = Table('src_format', self.db_meta, autoload=True) - self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True) - self.tbl_suite = Table('suite', self.db_meta, autoload=True) - self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True) - self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True) - self.tbl_suite_queue_copy = Table('suite_queue_copy', self.db_meta, autoload=True) - self.tbl_udeb_contents = Table('udeb_contents', self.db_meta, autoload=True) - self.tbl_uid = Table('uid', self.db_meta, autoload=True) - self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True) + tables = ( + 'architecture', + 'archive', + 'bin_associations', + 'binaries', + 'binary_acl', + 'binary_acl_map', ++ 'bin_contents' + 'build_queue', + 'build_queue_files', + 'component', + 'config', - 'content_associations', - 'content_file_names', - 'content_file_paths', + 'changes_pending_binaries', + 'changes_pending_files', + 'changes_pending_files_map', + 'changes_pending_source', + 'changes_pending_source_files', + 'changes_pool_files', ++ 'deb_contents', + 'dsc_files', + 'files', + 'fingerprint', + 'keyrings', + 'changes', + 'keyring_acl_map', + 'location', + 'maintainer', + 'new_comments', + 'override', + 'override_type', - 'pending_content_associations', ++ 'pending_bin_contents', + 'policy_queue', + 'priority', + 'section', + 'source', + 'source_acl', + 'src_associations', + 'src_format', + 'src_uploaders', + 'suite', + 'suite_architectures', + 'suite_src_formats', + 'suite_build_queue_copy', ++ 'udeb_contents', + 'uid', + 'upload_blocks', + ) + + for table_name in tables: + table = Table(table_name, self.db_meta, autoload=True) + setattr(self, 'tbl_%s' % table_name, table) def __setupmappers(self): mapper(Architecture, self.tbl_architecture, @@@ -2420,11 -2660,31 +2701,36 @@@ poolfiles = relation(PoolFile, secondary=self.tbl_changes_pool_files, backref="changeslinks"), + files = relation(ChangePendingFile, + secondary=self.tbl_changes_pending_files_map, + backref="changesfile"), + in_queue_id = self.tbl_changes.c.in_queue, + in_queue = relation(PolicyQueue, + primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)), + approved_for_id = self.tbl_changes.c.approved_for)) + + mapper(ChangePendingBinary, self.tbl_changes_pending_binaries, + properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id)) + + mapper(ChangePendingFile, self.tbl_changes_pending_files, + properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id)) + + mapper(ChangePendingSource, self.tbl_changes_pending_source, + properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id, + change = relation(DBChange), + maintainer = relation(Maintainer, + primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)), + changedby = relation(Maintainer, + primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)), + fingerprint = relation(Fingerprint), + source_files = relation(ChangePendingFile, + secondary=self.tbl_changes_pending_source_files, + backref="pending_sources"))) + files = relation(KnownChangePendingFile, backref="changesfile"))) + + mapper(KnownChangePendingFile, self.tbl_changes_pending_files, + properties = dict(known_change_pending_file_id = self.tbl_changes_pending_files.c.id)) + mapper(KeyringACLMap, self.tbl_keyring_acl_map, properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id, keyring = relation(Keyring, backref="keyring_acl_map"), @@@ -2464,16 -2726,8 +2773,9 @@@ mapper(Priority, self.tbl_priority, properties = dict(priority_id = self.tbl_priority.c.id)) - mapper(Queue, self.tbl_queue, - properties = dict(queue_id = self.tbl_queue.c.id)) - - mapper(QueueFile, self.tbl_queue_files, - properties = dict(queue = relation(Queue, backref='queuefiles'), - poolfile = relation(PoolFile, backref='queueinstances'))) - mapper(Section, self.tbl_section, - properties = dict(section_id = self.tbl_section.c.id)) + properties = dict(section_id = self.tbl_section.c.id, + section=self.tbl_section.c.section)) mapper(DBSource, self.tbl_source, properties = dict(source_id = self.tbl_source.c.id, diff --cc daklib/utils.py index c3e4dbb3,3cc40538..b7401751 mode 100755,100644..100644 --- a/daklib/utils.py +++ b/daklib/utils.py