ARCHLIST=$(tempfile)
- log "Querying $PGDATABASE..."
+ log "Querying postgres..."
echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
includedirs () {
DB
{
- Name "obscurity";
- Host "";
- Port -1;
-
+ Service "obscurity";
+ // PoolSize should be at least ThreadCount + 1
+ PoolSize 5;
+ // MaxOverflow shouldn't exceed postgresql.conf's max_connections - PoolSize
+ MaxOverflow 13;
+ // should be false for encoding == SQL_ASCII
+ Unicode "false"
};
Architectures
ARCHLIST=$(tempfile)
- log "Querying $PGDATABASE..."
+ log "Querying postgres"
echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
includedirs () {
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Suite.version can be null
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2011 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+import psycopg2
+from daklib.dak_exceptions import DBUpdateError
+from socket import gethostname;
+
+################################################################################
+def do_update(self):
+ """
+ Add table for source contents.
+ """
+ print __doc__
+ try:
+ c = self.db.cursor()
+
+ c.execute("ALTER TABLE suite ALTER COLUMN version DROP NOT NULL")
+ c.execute("UPDATE suite SET version=NULL WHERE version='-'")
+
+ c.execute("UPDATE config SET value = '48' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, 'Unable to apply sick update 48, rollback issued. Error message : %s' % (str(msg))
import os
import sys
import apt_pkg
-from commands import getstatusoutput
from glob import glob
from shutil import rmtree
from daklib.dbconn import *
from daklib import utils
from daklib.config import Config
+from daklib.contents import UnpackedSource
from daklib.regexes import re_no_epoch
################################################################################
pass
os.link(os.path.join(path, file), os.path.join(path, link))
- tempdir = utils.temp_dirname(parent=temppath)
- os.rmdir(tempdir)
-
for p in unpack.keys():
package = os.path.splitext(os.path.basename(p))[0].split('_')
- cmd = 'dpkg-source --no-check --no-copy -x %s %s' % (p, tempdir)
- (result, output) = getstatusoutput(cmd)
- if not result:
+ try:
+ unpacked = UnpackedSource(p)
+ tempdir = unpacked.get_root_directory()
stats['unpack'] += 1
for file in files:
for f in glob(os.path.join(tempdir, 'debian', '*%s' % file)):
pass
os.link(version, suite)
stats['created'] += 1
- else:
- print 'make-changelog: unable to unpack %s_%s: %s' \
- % (package[0], package[1], output)
+ unpacked.cleanup()
+ except Exception, e:
+ print 'make-changelog: unable to unpack %s\n%s' % (p, e)
stats['errors'] += 1
- rmtree(tempdir)
-
for root, dirs, files in os.walk(clpool):
if len(files):
if root.split('/')[-1] not in sources.keys():
################################################################################
Cnf = None
-required_database_schema = 47
+required_database_schema = 48
################################################################################
try:
# Build a connect string
- connect_str = "dbname=%s"% (cnf["DB::Name"])
- if cnf["DB::Host"] != '': connect_str += " host=%s" % (cnf["DB::Host"])
- if cnf["DB::Port"] != '-1': connect_str += " port=%d" % (int(cnf["DB::Port"]))
+ if cnf["DB::Service"]:
+ connect_str = "service=%s" % cnf["DB::Service"]
+ else:
+ connect_str = "dbname=%s"% (cnf["DB::Name"])
+ if cnf["DB::Host"] != '': connect_str += " host=%s" % (cnf["DB::Host"])
+ if cnf["DB::Port"] != '-1': connect_str += " port=%d" % (int(cnf["DB::Port"]))
self.db = psycopg2.connect(connect_str)
from daklib.config import Config
from multiprocessing import Pool
-from subprocess import Popen, PIPE
+from shutil import rmtree
+from subprocess import Popen, PIPE, check_call
+from tempfile import mkdtemp
import os.path
'''
scanner = ContentsScanner(binary_id)
scanner.scan()
+
+
+class UnpackedSource(object):
+ '''
+ UnpackedSource extracts a source package into a temporary location and
+ gives you some convinient function for accessing it.
+ '''
+ def __init__(self, dscfilename):
+ '''
+ The dscfilename is a name of a DSC file that will be extracted.
+ '''
+ self.root_directory = os.path.join(mkdtemp(), 'root')
+ command = ('dpkg-source', '--no-copy', '--no-check', '-x', dscfilename,
+ self.root_directory)
+ # dpkg-source does not have a --quiet option
+ devnull = open(os.devnull, 'w')
+ check_call(command, stdout = devnull, stderr = devnull)
+ devnull.close()
+
+ def get_root_directory(self):
+ '''
+ Returns the name of the package's root directory which is the directory
+ where the debian subdirectory is located.
+ '''
+ return self.root_directory
+
+ def get_changelog_file(self):
+ '''
+ Returns a file object for debian/changelog or None if no such file exists.
+ '''
+ changelog_name = os.path.join(self.root_directory, 'debian', 'changelog')
+ try:
+ return open(changelog_name)
+ except IOError:
+ return None
+
+ def get_all_filenames(self):
+ '''
+ Returns an iterator over all filenames. The filenames will be relative
+ to the root directory.
+ '''
+ skip = len(self.root_directory) + 1
+ for root, _, files in os.walk(self.root_directory):
+ for name in files:
+ yield os.path.join(root[skip:], name)
+
+ def cleanup(self):
+ '''
+ Removes all temporary files.
+ '''
+ if self.root_directory is None:
+ return
+ parent_directory = os.path.dirname(self.root_directory)
+ rmtree(parent_directory)
+ self.root_directory = None
+
+ def __del__(self):
+ '''
+ Enforce cleanup.
+ '''
+ self.cleanup()
from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers
from sqlalchemy import types as sqltypes
+from sqlalchemy.orm.collections import attribute_mapped_collection
+from sqlalchemy.ext.associationproxy import association_proxy
# Don't remove this, we re-export the exceptions to scripts which import us
from sqlalchemy.exc import *
return ['package', 'version', 'maintainer', 'source', 'poolfile', \
'binarytype']
+ metadata = association_proxy('key', 'value')
+
def get_component_name(self):
return self.poolfile.location.component.component_name
fields = Deb822(open(self.poolfile.fullpath, 'r'))
return fields
+ metadata = association_proxy('key', 'value')
+
__all__.append('DBSource')
@session_wrapper
################################################################################
class BinaryMetadata(ORMObject):
- def __init__(self, binary = None, key = None, value = None):
- self.binary = binary
+ def __init__(self, key = None, value = None, binary = None):
self.key = key
self.value = value
+ self.binary = binary
def properties(self):
return ['binary', 'key', 'value']
################################################################################
class SourceMetadata(ORMObject):
- def __init__(self, source = None, key = None, value = None):
- self.source = source
+ def __init__(self, key = None, value = None, source = None):
self.key = key
self.value = value
+ self.source = source
def properties(self):
return ['source', 'key', 'value']
suites = relation(Suite, secondary=self.tbl_bin_associations,
backref=backref('binaries', lazy='dynamic')),
extra_sources = relation(DBSource, secondary=self.tbl_extra_src_references,
- backref=backref('extra_binary_references', lazy='dynamic'))),
+ backref=backref('extra_binary_references', lazy='dynamic')),
+ key = relation(BinaryMetadata, cascade='all',
+ collection_class=attribute_mapped_collection('key'))),
extension = validator)
mapper(BinaryACL, self.tbl_binary_acl,
primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
suites = relation(Suite, secondary=self.tbl_src_associations,
backref=backref('sources', lazy='dynamic')),
- srcuploaders = relation(SrcUploader)),
+ srcuploaders = relation(SrcUploader),
+ key = relation(SourceMetadata, cascade='all',
+ collection_class=attribute_mapped_collection('key'))),
extension = validator)
mapper(SourceACL, self.tbl_source_acl,
from utils import parse_changes, check_dsc_files
from textutils import fix_maintainer
from lintian import parse_lintian_output, generate_reject_messages
+from contents import UnpackedSource
# suppress some deprecation warnings in squeeze related to apt_pkg
# module
os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
# Extract the source
- cmd = "dpkg-source -sn -x %s" % (dsc_filename)
- (result, output) = commands.getstatusoutput(cmd)
- if (result != 0):
- self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
- self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "))
+ try:
+ unpacked = UnpackedSource(dsc_filename)
+ except:
+ self.rejects.append("'dpkg-source -x' failed for %s." % dsc_filename)
return
if not cnf.Find("Dir::Queue::BTSVersionTrack"):
upstr_version = re_strip_revision.sub('', upstr_version)
# Ensure the changelog file exists
- changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
- if not os.path.exists(changelog_filename):
+ changelog_file = unpacked.get_changelog_file()
+ if changelog_file is None:
self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
return
# Parse the changelog
self.pkg.dsc["bts changelog"] = ""
- changelog_file = utils.open_file(changelog_filename)
for line in changelog_file.readlines():
m = re_changelog_versions.match(line)
if m:
self.pkg.dsc["bts changelog"] += line
changelog_file.close()
+ unpacked.cleanup()
# Check we found at least one revision in the changelog
if not self.pkg.dsc["bts changelog"]:
#!/usr/bin/env python
-from db_test import DBDakTestCase
+from db_test import DBDakTestCase, fixture
from daklib.dbconn import *
-from daklib.contents import ContentsWriter, ContentsScanner
+from daklib.contents import ContentsWriter, ContentsScanner, UnpackedSource
from os.path import normpath
from sqlalchemy.exc import FlushError, IntegrityError
+from subprocess import CalledProcessError
import unittest
class ContentsTestCase(DBDakTestCase):
self.assertEqual('usr/bin/hello', bin_contents_list[0].file)
self.assertEqual('usr/share/doc/hello/copyright', bin_contents_list[1].file)
+ def test_unpack(self):
+ '''
+ Tests the UnpackedSource class.
+ '''
+ self.setup_poolfiles()
+ dscfilename = fixture('ftp/pool/' + self.file['hello_2.2-1.dsc'].filename)
+ unpacked = UnpackedSource(dscfilename)
+ self.assertTrue(len(unpacked.get_root_directory()) > 0)
+ self.assertEqual('hello (2.2-1) unstable; urgency=low\n',
+ unpacked.get_changelog_file().readline())
+ all_filenames = set(unpacked.get_all_filenames())
+ self.assertEqual(8, len(all_filenames))
+ self.assertTrue('debian/rules' in all_filenames)
+ self.assertRaises(CalledProcessError, lambda: UnpackedSource('invalidname'))
+
def classes_to_clean(self):
return [Override, Suite, BinContents, DBBinary, DBSource, Architecture, Section, \
OverrideType, Maintainer, Component, Priority, PoolFile]
This TestCase checks the metadata handling.
"""
+ def setup_metadata(self):
+ '''
+ Setup the metadata objects.
+ '''
+ self.setup_binaries()
+ self.depends = MetadataKey('Depends')
+ self.session.add(self.depends)
+ self.session.flush()
+ self.bin_hello = self.binary['hello_2.2-1_i386']
+ self.src_hello = self.bin_hello.source
+ self.session.add(self.bin_hello)
+ self.session.add(self.src_hello)
+ self.hello_dep = BinaryMetadata(self.depends, 'foobar (>= 1.0)', self.bin_hello)
+ self.session.add(self.hello_dep)
+ self.recommends = MetadataKey('Recommends')
+ self.bin_hello.key[self.recommends] = BinaryMetadata(self.recommends, 'goodbye')
+ self.build_dep = MetadataKey('Build-Depends')
+ self.hello_bd = SourceMetadata(self.build_dep, 'foobar-dev', self.src_hello)
+ self.session.add(self.hello_bd)
+ self.homepage = MetadataKey('Homepage')
+ self.src_hello.key[self.homepage] = SourceMetadata(self.homepage, 'http://debian.org')
+ self.session.flush()
+
def test_mappers(self):
'''
Tests the mapper configuration.
'''
- self.setup_binaries()
+ self.setup_metadata()
# MetadataKey
- depends = MetadataKey(key = 'Depends')
- self.session.add(depends)
- self.session.flush()
- self.assertTrue(depends.key_id is not None)
+ self.assertTrue(self.depends.key_id is not None)
# BinaryMetadata
- hello_dep = BinaryMetadata(binary = self.binary['hello_2.2-1_i386'],
- key = depends, value = 'foobar (>= 1.0)')
- self.session.add(hello_dep)
- self.session.flush()
- self.assertEqual('hello', hello_dep.binary.package)
- self.assertEqual('Depends', hello_dep.key.key)
- self.assertEqual('foobar (>= 1.0)', hello_dep.value)
+ self.assertEqual('hello', self.hello_dep.binary.package)
+ self.assertEqual('Depends', self.hello_dep.key.key)
+ self.assertEqual('foobar (>= 1.0)', self.hello_dep.value)
# SourceMetadata
- build_dep = MetadataKey(key = 'Build-Depends')
- hello_bd = SourceMetadata(source = self.binary['hello_2.2-1_i386'].source,
- key = build_dep, value = 'foobar-dev')
- self.session.add(hello_bd)
+ self.assertEqual('hello', self.hello_bd.source.source)
+ self.assertEqual('Build-Depends', self.hello_bd.key.key)
+ self.assertEqual('foobar-dev', self.hello_bd.value)
+ # DBBinary relation
+ self.assertEqual(self.hello_dep, self.bin_hello.key[self.depends])
+ self.assertEqual('goodbye', self.bin_hello.key[self.recommends].value)
+ # DBSource relation
+ self.assertEqual(self.hello_bd, self.src_hello.key[self.build_dep])
+ self.assertEqual('http://debian.org', self.src_hello.key[self.homepage].value)
+
+ def test_association_proxy(self):
+ '''
+ Test the association proxies 'metadata' in DBBinary and DBSource.
+ '''
+ self.setup_metadata()
+ # DBBinary association proxy
+ essential = MetadataKey('Essential')
+ self.bin_hello.metadata[essential] = 'yes'
+ self.session.flush()
+ self.assertEqual('yes', self.bin_hello.metadata[essential])
+ self.assertEqual('foobar (>= 1.0)', self.bin_hello.metadata[self.depends])
+ self.assertTrue(self.recommends in self.bin_hello.metadata)
+ # DBSource association proxy
+ std_version = MetadataKey('Standards-Version')
+ self.src_hello.metadata[std_version] = '3.9.1'
+ self.session.flush()
+ self.assertEqual('3.9.1', self.src_hello.metadata[std_version])
+ self.assertEqual('http://debian.org', self.src_hello.metadata[self.homepage])
+ self.assertTrue(self.depends not in self.src_hello.metadata)
+
+ def test_delete(self):
+ '''
+ Tests the delete / cascading behaviour.
+ '''
+ self.setup_metadata()
+ self.session.delete(self.bin_hello)
+ # Remove associated binaries because we have no cascading rule for
+ # them.
+ for binary in self.src_hello.binaries:
+ self.session.delete(binary)
+ self.session.delete(self.src_hello)
self.session.flush()
- self.assertEqual('hello', hello_bd.source.source)
- self.assertEqual('Build-Depends', hello_bd.key.key)
- self.assertEqual('foobar-dev', hello_bd.value)
if __name__ == '__main__':
unittest.main()
--- /dev/null
+Format: 3.0 (quilt)
+Source: hello
+Version: 2.2-1
+Maintainer: Mr. Me <me@somewhere.world>
+Checksums-Sha1:
+ 9613ac479ddb6bca7f3ec5436b27ab983733b963 147 hello_2.2.orig.tar.gz
+ 97cfabb792685ac19c1ddc03f7d4aa1022f626e1 462 hello_2.2-1.debian.tar.gz
+Checksums-Sha256:
+ b041547e956f091a46030f133b6e47af15bc836771540118fec98d0913602ce0 147 hello_2.2.orig.tar.gz
+ da28e21cbae014b915abc8afc4be1c0b8e5148b78802dce815d5342e80cd52e7 462 hello_2.2-1.debian.tar.gz
+Files:
+ cc4b081e2697fca88c87986b1cad905f 147 hello_2.2.orig.tar.gz
+ d7bdb277cbdbaad4ab700c6d5cee9b54 462 hello_2.2-1.debian.tar.gz
projects / dak.git / commitdiff