################################################################################
+import apt_pkg
import os
from os.path import normpath
import re
import psycopg2
import traceback
import commands
+import signal
try:
# python >= 2.6
# in the database
from config import Config
from textutils import fix_maintainer
-from dak_exceptions import DBUpdateError, NoSourceFieldError
+from dak_exceptions import DBUpdateError, NoSourceFieldError, FileExistsError
# suppress some deprecation warnings in squeeze related to sqlalchemy
import warnings
################################################################################
+def subprocess_setup():
+ # Python installs a SIGPIPE handler by default. This is usually not what
+ # non-Python subprocesses expect.
+ signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
class DBBinary(ORMObject):
def __init__(self, package = None, source = None, version = None, \
maintainer = None, architecture = None, poolfile = None, \
package does not contain any regular file.
'''
fullpath = self.poolfile.fullpath
- dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE)
+ dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE,
+ preexec_fn = subprocess_setup)
tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
for member in tar.getmembers():
if not member.isdir():
# Check if we have a file of this name or this ID already
for f in self.queuefiles:
- if f.fileid is not None and f.fileid == poolfile.file_id or \
- f.poolfile.filename == poolfile_basename:
+ if (f.fileid is not None and f.fileid == poolfile.file_id) or \
+ (f.poolfile is not None and f.poolfile.filename == poolfile_basename):
# In this case, update the BuildQueueFile entry so we
# don't remove it too early
f.lastused = datetime.now()
else:
os.symlink(targetpath, queuepath)
qf.fileid = poolfile.file_id
+ except FileExistsError:
+ if not poolfile.identical_to(queuepath):
+ raise
except OSError:
return None
# Always copy files from policy queues as they might move around.
import utils
utils.copy(source, target)
+ except FileExistsError:
+ if not policyqueuefile.identical_to(target):
+ raise
except OSError:
return None
def __repr__(self):
return '<ChangePendingFile %s>' % self.change_pending_file_id
+ def identical_to(self, filename):
+ """
+ compare size and hash with the given file
+
+ @rtype: bool
+ @return: true if the given file has the same size and hash as this object; false otherwise
+ """
+ st = os.stat(filename)
+ if self.size != st.st_size:
+ return False
+
+ f = open(filename, "r")
+ sha256sum = apt_pkg.sha256sum(f)
+ if sha256sum != self.sha256sum:
+ return False
+
+ return True
+
__all__.append('ChangePendingFile')
################################################################################
################################################################################
+class ExternalOverride(ORMObject):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<ExternalOverride %s = %s: %s>' % (self.package, self.key, self.value)
+
+__all__.append('ExternalOverride')
+
+################################################################################
+
class PoolFile(ORMObject):
def __init__(self, filename = None, location = None, filesize = -1, \
md5sum = None):
def not_null_constraints(self):
return ['filename', 'md5sum', 'location']
+ def identical_to(self, filename):
+ """
+ compare size and hash with the given file
+
+ @rtype: bool
+ @return: true if the given file has the same size and hash as this object; false otherwise
+ """
+ st = os.stat(filename)
+ if self.filesize != st.st_size:
+ return False
+
+ f = open(filename, "r")
+ sha256sum = apt_pkg.sha256sum(f)
+ if sha256sum != self.sha256sum:
+ return False
+
+ return True
+
__all__.append('PoolFile')
@session_wrapper
q = session.query(DBSource).filter_by(source=source). \
filter(DBSource.version.in_([source_version, orig_source_version]))
if suite != "any":
- # source must exist in suite X, or in some other suite that's
- # mapped to X, recursively... silent-maps are counted too,
- # unreleased-maps aren't.
- maps = cnf.ValueList("SuiteMappings")[:]
- maps.reverse()
- maps = [ m.split() for m in maps ]
- maps = [ (x[1], x[2]) for x in maps
- if x[0] == "map" or x[0] == "silent-map" ]
- s = [suite]
- for (from_, to) in maps:
- if from_ in s and to not in s:
- s.append(to)
-
- q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
+ # source must exist in 'suite' or a suite that is enhanced by 'suite'
+ s = get_suite(suite, session)
+ enhances_vcs = session.query(VersionCheck).filter(VersionCheck.suite==s).filter_by(check='Enhances')
+ considered_suites = [ vc.reference for vc in enhances_vcs ]
+ considered_suites.append(s)
+
+ q = q.filter(DBSource.suites.any(Suite.suite_id.in_([s.suite_id for s in considered_suites])))
if q.count() > 0:
continue
################################################################################
+def split_uploaders(uploaders_list):
+ '''
+ Split the Uploaders field into the individual uploaders and yield each of
+ them. Beware: email addresses might contain commas.
+ '''
+ import re
+ for uploader in re.sub(">[ ]*,", ">\t", uploaders_list).split("\t"):
+ yield uploader.strip()
+
@session_wrapper
def add_dsc_to_db(u, filename, session=None):
entry = u.pkg.files[filename]
source.source = u.pkg.dsc["source"]
source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
- source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
+ # If Changed-By isn't available, fall back to maintainer
+ if u.pkg.changes.has_key("changed-by"):
+ source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
+ else:
+ source.changedby_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
source.install_date = datetime.now().date()
session.add(df)
# Add the src_uploaders to the DB
+ session.flush()
+ session.refresh(source)
source.uploaders = [source.maintainer]
if u.pkg.dsc.has_key("uploaders"):
- for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
- up = up.strip()
+ for up in split_uploaders(u.pkg.dsc["uploaders"]):
source.uploaders.append(get_or_set_maintainer(up, session))
session.flush()
return session.query(DBSource).filter_by(source = source). \
with_parent(self)
+ def get_overridesuite(self):
+ if self.overridesuite is None:
+ return self
+ else:
+ return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one()
+
__all__.append('Suite')
@session_wrapper
def get_version_checks(suite_name, check = None, session = None):
suite = get_suite(suite_name, session)
if not suite:
- return None
+ # Make sure that what we return is iterable so that list comprehensions
+ # involving this don't cause a traceback
+ return []
q = session.query(VersionCheck).filter_by(suite=suite)
if check:
q = q.filter_by(check=check)
'changes_pending_source_files',
'changes_pool_files',
'dsc_files',
+ 'external_overrides',
'extra_src_references',
'files',
'fingerprint',
'almost_obsolete_all_associations',
'almost_obsolete_src_associations',
'any_associations_source',
- 'bin_assoc_by_arch',
'bin_associations_binaries',
'binaries_suite_arch',
'binfiles_suite_component_arch',
poolfile_id = self.tbl_dsc_files.c.file,
poolfile = relation(PoolFile)))
+ mapper(ExternalOverride, self.tbl_external_overrides,
+ properties = dict(
+ suite_id = self.tbl_external_overrides.c.suite,
+ suite = relation(Suite),
+ component_id = self.tbl_external_overrides.c.component,
+ component = relation(Component)))
+
mapper(PoolFile, self.tbl_files,
properties = dict(file_id = self.tbl_files.c.id,
filesize = self.tbl_files.c.size,
self.__setupmappers()
self.pid = os.getpid()
- def session(self):
+ def session(self, work_mem = 0):
+ '''
+ Returns a new session object. If a work_mem parameter is provided a new
+ transaction is started and the work_mem parameter is set for this
+ transaction. The work_mem parameter is measured in MB. A default value
+ will be used if the parameter is not set.
+ '''
# reinitialize DBConn in new processes
if self.pid != os.getpid():
clear_mappers()
self.__createconn()
- return self.db_smaker()
+ session = self.db_smaker()
+ if work_mem > 0:
+ session.execute("SET LOCAL work_mem TO '%d MB'" % work_mem)
+ return session
__all__.append('DBConn')
projects / dak.git / blobdiff