From: Torsten Werner Date: Mon, 21 Mar 2011 08:24:12 +0000 (+0100) Subject: Remove old contents code and tables. X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=62a46fad36e7f656052e19dce6b3cf5a1552c1b2;p=dak.git Remove old contents code and tables. Signed-off-by: Torsten Werner --- diff --git a/dak/dakdb/update43.py b/dak/dakdb/update43.py new file mode 100755 index 00000000..b9c7f584 --- /dev/null +++ b/dak/dakdb/update43.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Remove old contents tables that are no longer needed by the current +implementation. + +@contact: Debian FTP Master +@copyright: 2011 Torsten Werner +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError +from socket import gethostname; + +################################################################################ +def do_update(self): + """ + Remove old contents tables that are no longer needed by the current + implementation. + """ + print __doc__ + try: + c = self.db.cursor() + + # remove useless type casts + for table in ('pending_bin_contents', 'deb_contents', 'udeb_contents'): + c.execute("DROP TABLE %s" % table) + + c.execute("UPDATE config SET value = '43' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, 'Unable to apply sick update 43, rollback issued. Error message : %s' % (str(msg)) diff --git a/dak/update_db.py b/dak/update_db.py index 928396c5..71bac955 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError ################################################################################ Cnf = None -required_database_schema = 42 +required_database_schema = 43 ################################################################################ diff --git a/daklib/binary.py b/daklib/binary.py deleted file mode 100644 index d1f78f3a..00000000 --- a/daklib/binary.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/python - -""" -Functions related debian binary packages - -@contact: Debian FTPMaster -@copyright: 2009 Mike O'Connor -@license: GNU General Public License version 2 or later -""" - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -################################################################################ - -# are we going the xorg way? -# a dak without a dak.conf? -# automatically detect the wrong settings at runtime? -# yes! -# well, we'll probably always need dak.conf (how do you get the database setting -# but removing most of the config into the database seems sane -# mhy: dont spoil the fun -# mhy: and i know how. we nmap localhost and check all open ports -# maybe one answers to sql -# we will discover projectb via avahi -# you're both sick -# really fucking sick - -################################################################################ - -import os -import sys -import shutil -import tarfile -import commands -import traceback -import atexit - -try: - # starting with squeeze - from debian import deb822 -except: - # up to lenny - from debian_bundle import deb822 - -from dbconn import * -from config import Config -import utils - -################################################################################ - -__all__ = [] - -################################################################################ - -class Binary(object): - def __init__(self, filename, reject=None): - """ - @type filename: string - @param filename: path of a .deb - - @type reject: function - @param reject: a function to log reject messages to - """ - self.filename = filename - self.tmpdir = None - self.chunks = None - self.wrapped_reject = reject - # Store rejects for later use - self.rejects = [] - - def reject(self, message): - """ - if we were given a reject function, send the reject message, - otherwise send it to stderr. - """ - print >> sys.stderr, message - self.rejects.append(message) - if self.wrapped_reject: - self.wrapped_reject(message) - - def __del__(self): - """ - make sure we cleanup when we are garbage collected. - """ - self._cleanup() - - def _cleanup(self): - """ - we need to remove the temporary directory, if we created one - """ - if self.tmpdir and os.path.exists(self.tmpdir): - shutil.rmtree(self.tmpdir) - self.tmpdir = None - - def __scan_ar(self): - # get a list of the ar contents - if not self.chunks: - - cmd = "ar t %s" % (self.filename) - (result, output) = commands.getstatusoutput(cmd) - if result != 0: - rejected = True - print("%s: 'ar t' invocation failed." % (self.filename)) - self.reject("%s: 'ar t' invocation failed." % (self.filename)) - self.reject(utils.prefix_multi_line_string(output, " [ar output:] ")) - self.chunks = output.split('\n') - - - - def __unpack(self): - # Internal function which extracts the contents of the .ar to - # a temporary directory - - if not self.tmpdir: - tmpdir = utils.temp_dirname() - cwd = os.getcwd() - try: - os.chdir( tmpdir ) - cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2]) - (result, output) = commands.getstatusoutput(cmd) - if result != 0: - print("%s: '%s' invocation failed." % (self.filename, cmd)) - self.reject("%s: '%s' invocation failed." % (self.filename, cmd)) - self.reject(utils.prefix_multi_line_string(output, " [ar output:] ")) - else: - self.tmpdir = tmpdir - atexit.register( self._cleanup ) - - finally: - os.chdir( cwd ) - - def valid_deb(self, relaxed=False): - """ - Check deb contents making sure the .deb contains: - 1. debian-binary - 2. control.tar.gz - 3. data.tar.gz or data.tar.bz2 - in that order, and nothing else. - """ - self.__scan_ar() - rejected = not self.chunks - if relaxed: - if len(self.chunks) < 3: - rejected = True - self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks))) - else: - if len(self.chunks) != 3: - rejected = True - self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks))) - if self.chunks[0] != "debian-binary": - rejected = True - self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0])) - if not rejected and self.chunks[1] != "control.tar.gz": - rejected = True - self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1])) - if not rejected and self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]: - rejected = True - self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2])) - - return not rejected - - def scan_package(self, bootstrap_id=0, relaxed=False, session=None): - """ - Unpack the .deb, do sanity checking, and gather info from it. - - Currently information gathering consists of getting the contents list. In - the hopefully near future, it should also include gathering info from the - control file. - - @type bootstrap_id: int - @param bootstrap_id: the id of the binary these packages - should be associated or zero meaning we are not bootstrapping - so insert into a temporary table - - @return: True if the deb is valid and contents were imported - """ - result = False - rejected = not self.valid_deb(relaxed) - if not rejected: - self.__unpack() - - - cwd = os.getcwd() - if not rejected and self.tmpdir: - try: - os.chdir(self.tmpdir) - if self.chunks[1] == "control.tar.gz": - control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz") - control.extract('./control', self.tmpdir ) - if self.chunks[2] == "data.tar.gz": - data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz") - elif self.chunks[2] == "data.tar.bz2": - data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2") - - if bootstrap_id: - result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session) - else: - pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control'))) - pkg = pkgs.next() - result = insert_pending_content_paths(pkg, - self.filename.endswith('.udeb'), - [tarinfo.name for tarinfo in data if not tarinfo.isdir()], - session) - - except: - traceback.print_exc() - - os.chdir(cwd) - self._cleanup() - return result - - def check_utf8_package(self, package): - """ - Unpack the .deb, do sanity checking, and gather info from it. - - Currently information gathering consists of getting the contents list. In - the hopefully near future, it should also include gathering info from the - control file. - - @type package: string - @param package: the name of the package to be checked - - @rtype: boolean - @return: True if the deb is valid and contents were imported - """ - rejected = not self.valid_deb(True) - self.__unpack() - - if not rejected and self.tmpdir: - cwd = os.getcwd() - try: - os.chdir(self.tmpdir) - if self.chunks[1] == "control.tar.gz": - control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz") - control.extract('control', self.tmpdir ) - if self.chunks[2] == "data.tar.gz": - data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz") - elif self.chunks[2] == "data.tar.bz2": - data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2") - - for tarinfo in data: - try: - unicode( tarinfo.name ) - except: - print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name) - - result = True - - except: - traceback.print_exc() - result = False - - os.chdir(cwd) - - return result - -__all__.append('Binary') - - -def copy_temporary_contents(binary, bin_association, reject, session=None): - """ - copy the previously stored contents from the temp table to the permanant one - - during process-unchecked, the deb should have been scanned and the - contents stored in pending_content_associations - """ - - cnf = Config() - - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True - - arch = get_architecture(archname, session=session) - - pending = session.query(PendingBinContents).filter_by(package=binary.package, - version=binary.version, - arch=binary.arch).first() - - if pending: - # This should NOT happen. We should have added contents - # during process-unchecked. if it did, log an error, and send - # an email. - subst = { - "__PACKAGE__": package, - "__VERSION__": version, - "__ARCH__": arch, - "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"], - "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] } - - message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents") - utils.send_mail(message) - - # rescan it now - exists = Binary(deb, reject).scan_package() - - if not exists: - # LOG? - return False - - component = binary.poolfile.location.component - override = session.query(Override).filter_by(package=binary.package, - suite=bin_association.suite, - component=component.id).first() - if not override: - # LOG? - return False - - - if not override.overridetype.type.endswith('deb'): - return True - - if override.overridetype.type == "udeb": - table = "udeb_contents" - elif override.overridetype.type == "deb": - table = "deb_contents" - else: - return False - - - if component.name == "main": - component_str = "" - else: - component_str = component.name + "/" - - vals = { 'package':binary.package, - 'version':binary.version, - 'arch':binary.architecture, - 'binary_id': binary.id, - 'component':component_str, - 'section':override.section.section - } - - session.execute( """INSERT INTO %s - (binary_id,package,version.component,arch,section,filename) - SELECT :binary_id, :package, :version, :component, :arch, :section - FROM pending_bin_contents pbc - WHERE pbc.package=:package - AND pbc.version=:version - AND pbc.arch=:arch""" % table, vals ) - - session.execute( """DELETE from pending_bin_contents package=:package - AND version=:version - AND arch=:arch""", vals ) - - if privatetrans: - session.commit() - session.close() - - return exists - -__all__.append('copy_temporary_contents') - - diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 9b757d13..4f6b02d8 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -77,9 +77,6 @@ import warnings warnings.filterwarnings('ignore', \ "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \ SADeprecationWarning) -# TODO: sqlalchemy needs some extra configuration to correctly reflect -# the ind_deb_contents_* indexes - we ignore the warnings at the moment -warnings.filterwarnings("ignore", 'Predicate of partial index', SAWarning) ################################################################################ @@ -1948,111 +1945,6 @@ __all__.append('get_override_type') ################################################################################ -class DebContents(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.package.package,self.file) - -__all__.append('DebContents') - - -class UdebContents(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.package.package,self.file) - -__all__.append('UdebContents') - -class PendingBinContents(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.contents_id - -__all__.append('PendingBinContents') - -def insert_pending_content_paths(package, - is_udeb, - fullpaths, - session=None): - """ - Make sure given paths are temporarily associated with given - package - - @type package: dict - @param package: the package to associate with should have been read in from the binary control file - @type fullpaths: list - @param fullpaths: the list of paths of the file being associated with the binary - @type session: SQLAlchemy session - @param session: Optional SQLAlchemy session. If this is passed, the caller - is responsible for ensuring a transaction has begun and committing the - results or rolling back based on the result code. If not passed, a commit - will be performed at the end of the function - - @return: True upon success, False if there is a problem - """ - - privatetrans = False - - if session is None: - session = DBConn().session() - privatetrans = True - - try: - arch = get_architecture(package['Architecture'], session) - arch_id = arch.arch_id - - # Remove any already existing recorded files for this package - q = session.query(PendingBinContents) - q = q.filter_by(package=package['Package']) - q = q.filter_by(version=package['Version']) - q = q.filter_by(architecture=arch_id) - q.delete() - - for fullpath in fullpaths: - - if fullpath.startswith( "./" ): - fullpath = fullpath[2:] - - pca = PendingBinContents() - pca.package = package['Package'] - pca.version = package['Version'] - pca.file = fullpath - pca.architecture = arch_id - - if isudeb: - pca.type = 8 # gross - else: - pca.type = 7 # also gross - session.add(pca) - - # Only commit if we set up the session ourself - if privatetrans: - session.commit() - session.close() - else: - session.flush() - - return True - except Exception, e: - traceback.print_exc() - - # Only rollback if we set up the session ourself - if privatetrans: - session.rollback() - session.close() - - return False - -__all__.append('insert_pending_content_paths') - -################################################################################ - class PolicyQueue(object): def __init__(self, *args, **kwargs): pass @@ -2940,7 +2832,6 @@ class DBConn(object): 'maintainer', 'new_comments', 'override_type', - 'pending_bin_contents', 'policy_queue', 'priority', 'section', @@ -2958,13 +2849,11 @@ class DBConn(object): 'changes_pending_files_map', 'changes_pending_source_files', 'changes_pool_files', - 'deb_contents', # TODO: the maintainer column in table override should be removed. 'override', 'suite_architectures', 'suite_src_formats', 'suite_build_queue_copy', - 'udeb_contents', ) views = ( @@ -3028,30 +2917,6 @@ class DBConn(object): properties = dict(archive_id = self.tbl_archive.c.id, archive_name = self.tbl_archive.c.name)) - mapper(PendingBinContents, self.tbl_pending_bin_contents, - properties = dict(contents_id =self.tbl_pending_bin_contents.c.id, - filename = self.tbl_pending_bin_contents.c.filename, - package = self.tbl_pending_bin_contents.c.package, - version = self.tbl_pending_bin_contents.c.version, - arch = self.tbl_pending_bin_contents.c.arch, - otype = self.tbl_pending_bin_contents.c.type)) - - mapper(DebContents, self.tbl_deb_contents, - properties = dict(binary_id=self.tbl_deb_contents.c.binary_id, - package=self.tbl_deb_contents.c.package, - suite=self.tbl_deb_contents.c.suite, - arch=self.tbl_deb_contents.c.arch, - section=self.tbl_deb_contents.c.section, - filename=self.tbl_deb_contents.c.filename)) - - mapper(UdebContents, self.tbl_udeb_contents, - properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id, - package=self.tbl_udeb_contents.c.package, - suite=self.tbl_udeb_contents.c.suite, - arch=self.tbl_udeb_contents.c.arch, - section=self.tbl_udeb_contents.c.section, - filename=self.tbl_udeb_contents.c.filename)) - mapper(BuildQueue, self.tbl_build_queue, properties = dict(queue_id = self.tbl_build_queue.c.id)) diff --git a/daklib/queue.py b/daklib/queue.py index a7e2d5a9..b3e26d7b 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -53,7 +53,6 @@ from dbconn import * from summarystats import SummaryStats from utils import parse_changes, check_dsc_files from textutils import fix_maintainer -from binary import Binary from lintian import parse_lintian_output, generate_reject_messages # suppress some deprecation warnings in squeeze related to apt_pkg @@ -866,13 +865,6 @@ class Upload(object): # Check the version and for file overwrites self.check_binary_against_db(f, session) - # Temporarily disable contents generation until we change the table storage layout - #b = Binary(f) - #b.scan_package() - #if len(b.rejects) > 0: - # for j in b.rejects: - # self.rejects.append(j) - def source_file_checks(self, f, session): entry = self.pkg.files[f]