From: Mark Hymers Date: Wed, 23 Mar 2011 18:33:54 +0000 (+0000) Subject: Merge remote branch 'ftpmaster/master' into psimport X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=b8690d6b1256a4a95198b874d9eb1aab62a825fb;hp=f265a8e0edb2e2cb097d237a62a58dab0598583e;p=dak.git Merge remote branch 'ftpmaster/master' into psimport Conflicts: daklib/dbconn.py Signed-off-by: Mark Hymers --- diff --git a/config/backports/dinstall.functions b/config/backports/dinstall.functions index 27854f8b..0dfe19ec 100644 --- a/config/backports/dinstall.functions +++ b/config/backports/dinstall.functions @@ -322,7 +322,7 @@ function mkfilesindices() { ARCHLIST=$(tempfile) - log "Querying $PGDATABASE..." + log "Querying postgres..." echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST includedirs () { diff --git a/config/debian-security/dak.conf b/config/debian-security/dak.conf index 58f77c11..b03e0f89 100644 --- a/config/debian-security/dak.conf +++ b/config/debian-security/dak.conf @@ -262,10 +262,13 @@ Dir DB { - Name "obscurity"; - Host ""; - Port -1; - + Service "obscurity"; + // PoolSize should be at least ThreadCount + 1 + PoolSize 5; + // MaxOverflow shouldn't exceed postgresql.conf's max_connections - PoolSize + MaxOverflow 13; + // should be false for encoding == SQL_ASCII + Unicode "false" }; Architectures diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index e4de479e..cac7c7c5 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -295,7 +295,7 @@ function mkfilesindices() { ARCHLIST=$(tempfile) - log "Querying $PGDATABASE..." + log "Querying postgres" echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST includedirs () { diff --git a/dak/dakdb/update48.py b/dak/dakdb/update48.py new file mode 100755 index 00000000..67ea8c5b --- /dev/null +++ b/dak/dakdb/update48.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Suite.version can be null + +@contact: Debian FTP Master +@copyright: 2011 Joerg Jaspert +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError +from socket import gethostname; + +################################################################################ +def do_update(self): + """ + Add table for source contents. + """ + print __doc__ + try: + c = self.db.cursor() + + c.execute("ALTER TABLE suite ALTER COLUMN version DROP NOT NULL") + c.execute("UPDATE suite SET version=NULL WHERE version='-'") + + c.execute("UPDATE config SET value = '48' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, 'Unable to apply sick update 48, rollback issued. Error message : %s' % (str(msg)) diff --git a/dak/make_changelog.py b/dak/make_changelog.py index 0a76bcc1..398c3526 100755 --- a/dak/make_changelog.py +++ b/dak/make_changelog.py @@ -52,12 +52,12 @@ Generate changelog entry between two suites import os import sys import apt_pkg -from commands import getstatusoutput from glob import glob from shutil import rmtree from daklib.dbconn import * from daklib import utils from daklib.config import Config +from daklib.contents import UnpackedSource from daklib.regexes import re_no_epoch ################################################################################ @@ -202,14 +202,11 @@ def export_files(session, pool, clpool, temppath): pass os.link(os.path.join(path, file), os.path.join(path, link)) - tempdir = utils.temp_dirname(parent=temppath) - os.rmdir(tempdir) - for p in unpack.keys(): package = os.path.splitext(os.path.basename(p))[0].split('_') - cmd = 'dpkg-source --no-check --no-copy -x %s %s' % (p, tempdir) - (result, output) = getstatusoutput(cmd) - if not result: + try: + unpacked = UnpackedSource(p) + tempdir = unpacked.get_root_directory() stats['unpack'] += 1 for file in files: for f in glob(os.path.join(tempdir, 'debian', '*%s' % file)): @@ -227,13 +224,11 @@ def export_files(session, pool, clpool, temppath): pass os.link(version, suite) stats['created'] += 1 - else: - print 'make-changelog: unable to unpack %s_%s: %s' \ - % (package[0], package[1], output) + unpacked.cleanup() + except Exception, e: + print 'make-changelog: unable to unpack %s\n%s' % (p, e) stats['errors'] += 1 - rmtree(tempdir) - for root, dirs, files in os.walk(clpool): if len(files): if root.split('/')[-1] not in sources.keys(): diff --git a/dak/update_db.py b/dak/update_db.py index 985051ec..3effa477 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -46,7 +46,7 @@ from daklib.daklog import Logger ################################################################################ Cnf = None -required_database_schema = 47 +required_database_schema = 48 ################################################################################ @@ -123,9 +123,12 @@ Updates dak's database schema to the lastest version. You should disable crontab try: # Build a connect string - connect_str = "dbname=%s"% (cnf["DB::Name"]) - if cnf["DB::Host"] != '': connect_str += " host=%s" % (cnf["DB::Host"]) - if cnf["DB::Port"] != '-1': connect_str += " port=%d" % (int(cnf["DB::Port"])) + if cnf["DB::Service"]: + connect_str = "service=%s" % cnf["DB::Service"] + else: + connect_str = "dbname=%s"% (cnf["DB::Name"]) + if cnf["DB::Host"] != '': connect_str += " host=%s" % (cnf["DB::Host"]) + if cnf["DB::Port"] != '-1': connect_str += " port=%d" % (int(cnf["DB::Port"])) self.db = psycopg2.connect(connect_str) diff --git a/daklib/contents.py b/daklib/contents.py index 4a0b3ae2..a158e8fc 100755 --- a/daklib/contents.py +++ b/daklib/contents.py @@ -29,7 +29,9 @@ from daklib.dbconn import * from daklib.config import Config from multiprocessing import Pool -from subprocess import Popen, PIPE +from shutil import rmtree +from subprocess import Popen, PIPE, check_call +from tempfile import mkdtemp import os.path @@ -313,3 +315,64 @@ def scan_helper(binary_id): ''' scanner = ContentsScanner(binary_id) scanner.scan() + + +class UnpackedSource(object): + ''' + UnpackedSource extracts a source package into a temporary location and + gives you some convinient function for accessing it. + ''' + def __init__(self, dscfilename): + ''' + The dscfilename is a name of a DSC file that will be extracted. + ''' + self.root_directory = os.path.join(mkdtemp(), 'root') + command = ('dpkg-source', '--no-copy', '--no-check', '-x', dscfilename, + self.root_directory) + # dpkg-source does not have a --quiet option + devnull = open(os.devnull, 'w') + check_call(command, stdout = devnull, stderr = devnull) + devnull.close() + + def get_root_directory(self): + ''' + Returns the name of the package's root directory which is the directory + where the debian subdirectory is located. + ''' + return self.root_directory + + def get_changelog_file(self): + ''' + Returns a file object for debian/changelog or None if no such file exists. + ''' + changelog_name = os.path.join(self.root_directory, 'debian', 'changelog') + try: + return open(changelog_name) + except IOError: + return None + + def get_all_filenames(self): + ''' + Returns an iterator over all filenames. The filenames will be relative + to the root directory. + ''' + skip = len(self.root_directory) + 1 + for root, _, files in os.walk(self.root_directory): + for name in files: + yield os.path.join(root[skip:], name) + + def cleanup(self): + ''' + Removes all temporary files. + ''' + if self.root_directory is None: + return + parent_directory = os.path.dirname(self.root_directory) + rmtree(parent_directory) + self.root_directory = None + + def __del__(self): + ''' + Enforce cleanup. + ''' + self.cleanup() diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 79e0621b..4d30e663 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -61,6 +61,8 @@ from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \ from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \ backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers from sqlalchemy import types as sqltypes +from sqlalchemy.orm.collections import attribute_mapped_collection +from sqlalchemy.ext.associationproxy import association_proxy # Don't remove this, we re-export the exceptions to scripts which import us from sqlalchemy.exc import * @@ -499,6 +501,8 @@ class DBBinary(ORMObject): return ['package', 'version', 'maintainer', 'source', 'poolfile', \ 'binarytype'] + metadata = association_proxy('key', 'value') + def get_component_name(self): return self.poolfile.location.component.component_name @@ -2205,6 +2209,8 @@ class DBSource(ORMObject): fields = Deb822(open(self.poolfile.fullpath, 'r')) return fields + metadata = association_proxy('key', 'value') + __all__.append('DBSource') @session_wrapper @@ -2850,10 +2856,10 @@ __all__.append('MetadataKey') ################################################################################ class BinaryMetadata(ORMObject): - def __init__(self, binary = None, key = None, value = None): - self.binary = binary + def __init__(self, key = None, value = None, binary = None): self.key = key self.value = value + self.binary = binary def properties(self): return ['binary', 'key', 'value'] @@ -2866,10 +2872,10 @@ __all__.append('BinaryMetadata') ################################################################################ class SourceMetadata(ORMObject): - def __init__(self, source = None, key = None, value = None): - self.source = source + def __init__(self, key = None, value = None, source = None): self.key = key self.value = value + self.source = source def properties(self): return ['source', 'key', 'value'] @@ -3018,7 +3024,9 @@ class DBConn(object): suites = relation(Suite, secondary=self.tbl_bin_associations, backref=backref('binaries', lazy='dynamic')), extra_sources = relation(DBSource, secondary=self.tbl_extra_src_references, - backref=backref('extra_binary_references', lazy='dynamic'))), + backref=backref('extra_binary_references', lazy='dynamic')), + key = relation(BinaryMetadata, cascade='all', + collection_class=attribute_mapped_collection('key'))), extension = validator) mapper(BinaryACL, self.tbl_binary_acl, @@ -3188,7 +3196,9 @@ class DBConn(object): primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)), suites = relation(Suite, secondary=self.tbl_src_associations, backref=backref('sources', lazy='dynamic')), - srcuploaders = relation(SrcUploader)), + srcuploaders = relation(SrcUploader), + key = relation(SourceMetadata, cascade='all', + collection_class=attribute_mapped_collection('key'))), extension = validator) mapper(SourceACL, self.tbl_source_acl, diff --git a/daklib/queue.py b/daklib/queue.py index dfbe3685..b4c62d38 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -54,6 +54,7 @@ from summarystats import SummaryStats from utils import parse_changes, check_dsc_files from textutils import fix_maintainer from lintian import parse_lintian_output, generate_reject_messages +from contents import UnpackedSource # suppress some deprecation warnings in squeeze related to apt_pkg # module @@ -1265,11 +1266,10 @@ class Upload(object): os.symlink(self.pkg.orig_files[orig_file]["path"], dest) # Extract the source - cmd = "dpkg-source -sn -x %s" % (dsc_filename) - (result, output) = commands.getstatusoutput(cmd) - if (result != 0): - self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result)) - self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] ")) + try: + unpacked = UnpackedSource(dsc_filename) + except: + self.rejects.append("'dpkg-source -x' failed for %s." % dsc_filename) return if not cnf.Find("Dir::Queue::BTSVersionTrack"): @@ -1281,19 +1281,19 @@ class Upload(object): upstr_version = re_strip_revision.sub('', upstr_version) # Ensure the changelog file exists - changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version) - if not os.path.exists(changelog_filename): + changelog_file = unpacked.get_changelog_file() + if changelog_file is None: self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename)) return # Parse the changelog self.pkg.dsc["bts changelog"] = "" - changelog_file = utils.open_file(changelog_filename) for line in changelog_file.readlines(): m = re_changelog_versions.match(line) if m: self.pkg.dsc["bts changelog"] += line changelog_file.close() + unpacked.cleanup() # Check we found at least one revision in the changelog if not self.pkg.dsc["bts changelog"]: diff --git a/tests/dbtest_contents.py b/tests/dbtest_contents.py index 158ec892..90fe4966 100755 --- a/tests/dbtest_contents.py +++ b/tests/dbtest_contents.py @@ -1,12 +1,13 @@ #!/usr/bin/env python -from db_test import DBDakTestCase +from db_test import DBDakTestCase, fixture from daklib.dbconn import * -from daklib.contents import ContentsWriter, ContentsScanner +from daklib.contents import ContentsWriter, ContentsScanner, UnpackedSource from os.path import normpath from sqlalchemy.exc import FlushError, IntegrityError +from subprocess import CalledProcessError import unittest class ContentsTestCase(DBDakTestCase): @@ -172,6 +173,21 @@ class ContentsTestCase(DBDakTestCase): self.assertEqual('usr/bin/hello', bin_contents_list[0].file) self.assertEqual('usr/share/doc/hello/copyright', bin_contents_list[1].file) + def test_unpack(self): + ''' + Tests the UnpackedSource class. + ''' + self.setup_poolfiles() + dscfilename = fixture('ftp/pool/' + self.file['hello_2.2-1.dsc'].filename) + unpacked = UnpackedSource(dscfilename) + self.assertTrue(len(unpacked.get_root_directory()) > 0) + self.assertEqual('hello (2.2-1) unstable; urgency=low\n', + unpacked.get_changelog_file().readline()) + all_filenames = set(unpacked.get_all_filenames()) + self.assertEqual(8, len(all_filenames)) + self.assertTrue('debian/rules' in all_filenames) + self.assertRaises(CalledProcessError, lambda: UnpackedSource('invalidname')) + def classes_to_clean(self): return [Override, Suite, BinContents, DBBinary, DBSource, Architecture, Section, \ OverrideType, Maintainer, Component, Priority, PoolFile] diff --git a/tests/dbtest_metadata.py b/tests/dbtest_metadata.py index f1f30098..1b698b85 100755 --- a/tests/dbtest_metadata.py +++ b/tests/dbtest_metadata.py @@ -11,33 +11,83 @@ class MetadataTestCase(DBDakTestCase): This TestCase checks the metadata handling. """ + def setup_metadata(self): + ''' + Setup the metadata objects. + ''' + self.setup_binaries() + self.depends = MetadataKey('Depends') + self.session.add(self.depends) + self.session.flush() + self.bin_hello = self.binary['hello_2.2-1_i386'] + self.src_hello = self.bin_hello.source + self.session.add(self.bin_hello) + self.session.add(self.src_hello) + self.hello_dep = BinaryMetadata(self.depends, 'foobar (>= 1.0)', self.bin_hello) + self.session.add(self.hello_dep) + self.recommends = MetadataKey('Recommends') + self.bin_hello.key[self.recommends] = BinaryMetadata(self.recommends, 'goodbye') + self.build_dep = MetadataKey('Build-Depends') + self.hello_bd = SourceMetadata(self.build_dep, 'foobar-dev', self.src_hello) + self.session.add(self.hello_bd) + self.homepage = MetadataKey('Homepage') + self.src_hello.key[self.homepage] = SourceMetadata(self.homepage, 'http://debian.org') + self.session.flush() + def test_mappers(self): ''' Tests the mapper configuration. ''' - self.setup_binaries() + self.setup_metadata() # MetadataKey - depends = MetadataKey(key = 'Depends') - self.session.add(depends) - self.session.flush() - self.assertTrue(depends.key_id is not None) + self.assertTrue(self.depends.key_id is not None) # BinaryMetadata - hello_dep = BinaryMetadata(binary = self.binary['hello_2.2-1_i386'], - key = depends, value = 'foobar (>= 1.0)') - self.session.add(hello_dep) - self.session.flush() - self.assertEqual('hello', hello_dep.binary.package) - self.assertEqual('Depends', hello_dep.key.key) - self.assertEqual('foobar (>= 1.0)', hello_dep.value) + self.assertEqual('hello', self.hello_dep.binary.package) + self.assertEqual('Depends', self.hello_dep.key.key) + self.assertEqual('foobar (>= 1.0)', self.hello_dep.value) # SourceMetadata - build_dep = MetadataKey(key = 'Build-Depends') - hello_bd = SourceMetadata(source = self.binary['hello_2.2-1_i386'].source, - key = build_dep, value = 'foobar-dev') - self.session.add(hello_bd) + self.assertEqual('hello', self.hello_bd.source.source) + self.assertEqual('Build-Depends', self.hello_bd.key.key) + self.assertEqual('foobar-dev', self.hello_bd.value) + # DBBinary relation + self.assertEqual(self.hello_dep, self.bin_hello.key[self.depends]) + self.assertEqual('goodbye', self.bin_hello.key[self.recommends].value) + # DBSource relation + self.assertEqual(self.hello_bd, self.src_hello.key[self.build_dep]) + self.assertEqual('http://debian.org', self.src_hello.key[self.homepage].value) + + def test_association_proxy(self): + ''' + Test the association proxies 'metadata' in DBBinary and DBSource. + ''' + self.setup_metadata() + # DBBinary association proxy + essential = MetadataKey('Essential') + self.bin_hello.metadata[essential] = 'yes' + self.session.flush() + self.assertEqual('yes', self.bin_hello.metadata[essential]) + self.assertEqual('foobar (>= 1.0)', self.bin_hello.metadata[self.depends]) + self.assertTrue(self.recommends in self.bin_hello.metadata) + # DBSource association proxy + std_version = MetadataKey('Standards-Version') + self.src_hello.metadata[std_version] = '3.9.1' + self.session.flush() + self.assertEqual('3.9.1', self.src_hello.metadata[std_version]) + self.assertEqual('http://debian.org', self.src_hello.metadata[self.homepage]) + self.assertTrue(self.depends not in self.src_hello.metadata) + + def test_delete(self): + ''' + Tests the delete / cascading behaviour. + ''' + self.setup_metadata() + self.session.delete(self.bin_hello) + # Remove associated binaries because we have no cascading rule for + # them. + for binary in self.src_hello.binaries: + self.session.delete(binary) + self.session.delete(self.src_hello) self.session.flush() - self.assertEqual('hello', hello_bd.source.source) - self.assertEqual('Build-Depends', hello_bd.key.key) - self.assertEqual('foobar-dev', hello_bd.value) if __name__ == '__main__': unittest.main() diff --git a/tests/fixtures/ftp/pool/main/h/hello/hello_2.2-1.debian.tar.gz b/tests/fixtures/ftp/pool/main/h/hello/hello_2.2-1.debian.tar.gz new file mode 100644 index 00000000..c185f1bb Binary files /dev/null and b/tests/fixtures/ftp/pool/main/h/hello/hello_2.2-1.debian.tar.gz differ diff --git a/tests/fixtures/ftp/pool/main/h/hello/hello_2.2-1.dsc b/tests/fixtures/ftp/pool/main/h/hello/hello_2.2-1.dsc new file mode 100644 index 00000000..f564ce44 --- /dev/null +++ b/tests/fixtures/ftp/pool/main/h/hello/hello_2.2-1.dsc @@ -0,0 +1,13 @@ +Format: 3.0 (quilt) +Source: hello +Version: 2.2-1 +Maintainer: Mr. Me +Checksums-Sha1: + 9613ac479ddb6bca7f3ec5436b27ab983733b963 147 hello_2.2.orig.tar.gz + 97cfabb792685ac19c1ddc03f7d4aa1022f626e1 462 hello_2.2-1.debian.tar.gz +Checksums-Sha256: + b041547e956f091a46030f133b6e47af15bc836771540118fec98d0913602ce0 147 hello_2.2.orig.tar.gz + da28e21cbae014b915abc8afc4be1c0b8e5148b78802dce815d5342e80cd52e7 462 hello_2.2-1.debian.tar.gz +Files: + cc4b081e2697fca88c87986b1cad905f 147 hello_2.2.orig.tar.gz + d7bdb277cbdbaad4ab700c6d5cee9b54 462 hello_2.2-1.debian.tar.gz diff --git a/tests/fixtures/ftp/pool/main/h/hello/hello_2.2.orig.tar.gz b/tests/fixtures/ftp/pool/main/h/hello/hello_2.2.orig.tar.gz new file mode 100644 index 00000000..f3fbc183 Binary files /dev/null and b/tests/fixtures/ftp/pool/main/h/hello/hello_2.2.orig.tar.gz differ