################################################################################
import apt_pkg
+import daklib.daksubprocess
import os
from os.path import normpath
import re
import psycopg2
+import subprocess
import traceback
-import commands
-import signal
try:
# python >= 2.6
from datetime import datetime, timedelta
from errno import ENOENT
from tempfile import mkstemp, mkdtemp
-from subprocess import Popen, PIPE
from tarfile import TarFile
from inspect import getargspec
return None
sa_major_version = sqlalchemy.__version__[0:3]
-if sa_major_version in ["0.5", "0.6", "0.7"]:
+if sa_major_version in ["0.5", "0.6", "0.7", "0.8", "0.9"]:
from sqlalchemy.databases import postgres
postgres.ischema_names['debversion'] = DebVersion
else:
- raise Exception("dak only ported to SQLA versions 0.5 to 0.7. See daklib/dbconn.py")
+ raise Exception("dak only ported to SQLA versions 0.5 to 0.9. See daklib/dbconn.py")
################################################################################
return object_session(self)
def clone(self, session = None):
- '''
+ """
Clones the current object in a new session and returns the new clone. A
fresh session is created if the optional session parameter is not
provided. The function will fail if a session is provided and has
WARNING: Only persistent (committed) objects can be cloned. Changes
made to the original object that are not committed yet will get lost.
The session of the new object will always be rolled back to avoid
- ressource leaks.
- '''
+ resource leaks.
+ """
if self.session() is None:
raise RuntimeError( \
__all__.append('get_architecture')
-# TODO: should be removed because the implementation is too trivial
-@session_wrapper
-def get_architecture_suites(architecture, session=None):
- """
- Returns list of Suite objects for given C{architecture} name
-
- @type architecture: str
- @param architecture: Architecture name to search for
-
- @type session: Session
- @param session: Optional SQL session object (a temporary one will be
- generated if not supplied)
-
- @rtype: list
- @return: list of Suite objects for the given name (may be empty)
- """
-
- return get_architecture(architecture, session).suites
-
-__all__.append('get_architecture_suites')
-
################################################################################
class Archive(object):
################################################################################
-def subprocess_setup():
- # Python installs a SIGPIPE handler by default. This is usually not what
- # non-Python subprocesses expect.
- signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-
class DBBinary(ORMObject):
def __init__(self, package = None, source = None, version = None, \
maintainer = None, architecture = None, poolfile = None, \
package does not contain any regular file.
'''
fullpath = self.poolfile.fullpath
- dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE,
- preexec_fn = subprocess_setup)
+ dpkg_cmd = ('dpkg-deb', '--fsys-tarfile', fullpath)
+ dpkg = daklib.daksubprocess.Popen(dpkg_cmd, stdout=subprocess.PIPE)
tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
for member in tar.getmembers():
if not member.isdir():
'''
import utils
fullpath = self.poolfile.fullpath
- deb_file = open(fullpath, 'r')
- stanza = utils.deb_extract_control(deb_file)
- deb_file.close()
-
- return stanza
+ with open(fullpath, 'r') as deb_file:
+ return utils.deb_extract_control(deb_file)
def read_control_fields(self):
'''
@rtype: dict
@return: fields of the control section as a dictionary.
'''
- import apt_pkg
stanza = self.read_control()
return apt_pkg.TagSection(stanza)
+ @property
+ def proxy(self):
+ session = object_session(self)
+ query = session.query(BinaryMetadata).filter_by(binary=self)
+ return MetadataProxy(session, query)
+
__all__.append('DBBinary')
@session_wrapper
__all__.append('get_component')
+def get_mapped_component_name(component_name):
+ cnf = Config()
+ for m in cnf.value_list("ComponentMappings"):
+ (src, dst) = m.split()
+ if component_name == src:
+ component_name = dst
+ return component_name
+
+__all__.append('get_mapped_component_name')
+
@session_wrapper
def get_mapped_component(component_name, session=None):
"""get component after mappings
@rtype: L{daklib.dbconn.Component} or C{None}
@return: component after applying maps or C{None}
"""
- cnf = Config()
- for m in cnf.value_list("ComponentMappings"):
- (src, dst) = m.split()
- if component_name == src:
- component_name = dst
+ component_name = get_mapped_component_name(component_name)
component = session.query(Component).filter_by(component_name=component_name).first()
return component
################################################################################
-@session_wrapper
-def get_or_set_contents_file_id(filename, session=None):
- """
- Returns database id for given filename.
-
- If no matching file is found, a row is inserted.
-
- @type filename: string
- @param filename: The filename
- @type session: SQLAlchemy
- @param session: Optional SQL session object (a temporary one will be
- generated if not supplied). If not passed, a commit will be performed at
- the end of the function, otherwise the caller is responsible for commiting.
-
- @rtype: int
- @return: the database id for the given component
- """
-
- q = session.query(ContentFilename).filter_by(filename=filename)
-
- try:
- ret = q.one().cafilename_id
- except NoResultFound:
- cf = ContentFilename()
- cf.filename = filename
- session.add(cf)
- session.commit_or_flush()
- ret = cf.cafilename_id
-
- return ret
-
-__all__.append('get_or_set_contents_file_id')
-
-@session_wrapper
-def get_contents(suite, overridetype, section=None, session=None):
- """
- Returns contents for a suite / overridetype combination, limiting
- to a section if not None.
-
- @type suite: Suite
- @param suite: Suite object
-
- @type overridetype: OverrideType
- @param overridetype: OverrideType object
-
- @type section: Section
- @param section: Optional section object to limit results to
-
- @type session: SQLAlchemy
- @param session: Optional SQL session object (a temporary one will be
- generated if not supplied)
-
- @rtype: ResultsProxy
- @return: ResultsProxy object set up to return tuples of (filename, section,
- package, arch_id)
- """
-
- # find me all of the contents for a given suite
- contents_q = """SELECT (p.path||'/'||n.file) AS fn,
- s.section,
- b.package,
- b.architecture
- FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
- JOIN content_file_names n ON (c.filename=n.id)
- JOIN binaries b ON (b.id=c.binary_pkg)
- JOIN override o ON (o.package=b.package)
- JOIN section s ON (s.id=o.section)
- WHERE o.suite = :suiteid AND o.type = :overridetypeid
- AND b.type=:overridetypename"""
-
- vals = {'suiteid': suite.suite_id,
- 'overridetypeid': overridetype.overridetype_id,
- 'overridetypename': overridetype.overridetype}
-
- if section is not None:
- contents_q += " AND s.id = :sectionid"
- vals['sectionid'] = section.section_id
-
- contents_q += " ORDER BY fn"
-
- return session.execute(contents_q, vals)
-
-__all__.append('get_contents')
-
-################################################################################
-
-class ContentFilepath(object):
- def __init__(self, *args, **kwargs):
- pass
-
- def __repr__(self):
- return '<ContentFilepath %s>' % self.filepath
-
-__all__.append('ContentFilepath')
-
-@session_wrapper
-def get_or_set_contents_path_id(filepath, session=None):
- """
- Returns database id for given path.
-
- If no matching file is found, a row is inserted.
-
- @type filepath: string
- @param filepath: The filepath
-
- @type session: SQLAlchemy
- @param session: Optional SQL session object (a temporary one will be
- generated if not supplied). If not passed, a commit will be performed at
- the end of the function, otherwise the caller is responsible for commiting.
-
- @rtype: int
- @return: the database id for the given path
- """
-
- q = session.query(ContentFilepath).filter_by(filepath=filepath)
-
- try:
- ret = q.one().cafilepath_id
- except NoResultFound:
- cf = ContentFilepath()
- cf.filepath = filepath
- session.add(cf)
- session.commit_or_flush()
- ret = cf.cafilepath_id
-
- return ret
-
-__all__.append('get_or_set_contents_path_id')
-
-################################################################################
-
-class ContentAssociation(object):
- def __init__(self, *args, **kwargs):
- pass
-
- def __repr__(self):
- return '<ContentAssociation %s>' % self.ca_id
-
-__all__.append('ContentAssociation')
-
-def insert_content_paths(binary_id, fullpaths, session=None):
- """
- Make sure given path is associated with given binary id
-
- @type binary_id: int
- @param binary_id: the id of the binary
- @type fullpaths: list
- @param fullpaths: the list of paths of the file being associated with the binary
- @type session: SQLAlchemy session
- @param session: Optional SQLAlchemy session. If this is passed, the caller
- is responsible for ensuring a transaction has begun and committing the
- results or rolling back based on the result code. If not passed, a commit
- will be performed at the end of the function, otherwise the caller is
- responsible for commiting.
-
- @return: True upon success
- """
-
- privatetrans = False
- if session is None:
- session = DBConn().session()
- privatetrans = True
-
- try:
- # Insert paths
- def generate_path_dicts():
- for fullpath in fullpaths:
- if fullpath.startswith( './' ):
- fullpath = fullpath[2:]
-
- yield {'filename':fullpath, 'id': binary_id }
-
- for d in generate_path_dicts():
- session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
- d )
-
- session.commit()
- if privatetrans:
- session.close()
- return True
-
- except:
- traceback.print_exc()
-
- # Only rollback if we set up the session ourself
- if privatetrans:
- session.rollback()
- session.close()
-
- return False
-
-__all__.append('insert_content_paths')
-
-################################################################################
-
class DSCFile(object):
def __init__(self, *args, **kwargs):
pass
@property
def fullpath(self):
session = DBConn().session().object_session(self)
- af = session.query(ArchiveFile).join(Archive).filter(ArchiveFile.file == self).first()
+ af = session.query(ArchiveFile).join(Archive) \
+ .filter(ArchiveFile.file == self) \
+ .order_by(Archive.tainted.desc()).first()
return af.path
@property
__all__.append('PoolFile')
-@session_wrapper
-def get_poolfile_like_name(filename, session=None):
- """
- Returns an array of PoolFile objects which are like the given name
-
- @type filename: string
- @param filename: the filename of the file to check against the DB
-
- @rtype: array
- @return: array of PoolFile objects
- """
-
- # TODO: There must be a way of properly using bind parameters with %FOO%
- q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
-
- return q.all()
-
-__all__.append('get_poolfile_like_name')
-
################################################################################
class Fingerprint(ORMObject):
################################################################################
class Keyring(object):
- gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
- " --with-colons --fingerprint --fingerprint"
-
keys = {}
fpr_lookup = {}
if not self.keyring_id:
raise Exception('Must be initialized with database information')
- k = os.popen(self.gpg_invocation % keyring, "r")
+ cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
+ "--with-colons", "--fingerprint", "--fingerprint"]
+ p = daklib.daksubprocess.Popen(cmd, stdout=subprocess.PIPE)
+
key = None
- signingkey = False
+ need_fingerprint = False
- for line in k:
+ for line in p.stdout:
field = line.split(":")
if field[0] == "pub":
key = field[4]
if "@" in addr:
self.keys[key]["email"] = addr
self.keys[key]["name"] = name
- self.keys[key]["fingerprints"] = []
- signingkey = True
- elif key and field[0] == "sub" and len(field) >= 12:
- signingkey = ("s" in field[11])
+ need_fingerprint = True
elif key and field[0] == "uid":
(name, addr) = self.parse_address(field[9])
if "email" not in self.keys[key] and "@" in addr:
self.keys[key]["email"] = addr
self.keys[key]["name"] = name
- elif signingkey and field[0] == "fpr":
- self.keys[key]["fingerprints"].append(field[9])
+ elif need_fingerprint and field[0] == "fpr":
+ self.keys[key]["fingerprints"] = [field[9]]
self.fpr_lookup[field[9]] = key
+ need_fingerprint = False
+
+ r = p.wait()
+ if r != 0:
+ raise subprocess.CalledProcessError(r, cmd)
def import_users_from_ldap(self, session):
import ldap
LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
+ ca_cert_file = cnf.get('Import-LDAP-Fingerprints::CACertFile')
l = ldap.open(LDAPServer)
+
+ if ca_cert_file:
+ l.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_HARD)
+ l.set_option(ldap.OPT_X_TLS_CACERTFILE, ca_cert_file)
+ l.set_option(ldap.OPT_X_TLS_NEWCTX, True)
+ l.start_tls_s()
+
l.simple_bind_s("","")
Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
"(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
__all__.append('get_active_keyring_paths')
-@session_wrapper
-def get_primary_keyring_path(session=None):
- """
- Get the full path to the highest priority active keyring
-
- @rtype: str or None
- @return: path to the active keyring with the highest priority or None if no
- keyring is configured
- """
- keyrings = get_active_keyring_paths()
-
- if len(keyrings) > 0:
- return keyrings[0]
- else:
- return None
-
-__all__.append('get_primary_keyring_path')
-
################################################################################
class DBChange(object):
__all__.append('NewComment')
@session_wrapper
-def has_new_comment(package, version, session=None):
+def has_new_comment(policy_queue, package, version, session=None):
"""
Returns true if the given combination of C{package}, C{version} has a comment.
@return: true/false
"""
- q = session.query(NewComment)
+ q = session.query(NewComment).filter_by(policy_queue=policy_queue)
q = q.filter_by(package=package)
q = q.filter_by(version=version)
__all__.append('has_new_comment')
@session_wrapper
-def get_new_comments(package=None, version=None, comment_id=None, session=None):
+def get_new_comments(policy_queue, package=None, version=None, comment_id=None, session=None):
"""
Returns (possibly empty) list of NewComment objects for the given
parameters
@return: A (possibly empty) list of NewComment objects will be returned
"""
- q = session.query(NewComment)
+ q = session.query(NewComment).filter_by(policy_queue=policy_queue)
if package is not None: q = q.filter_by(package=package)
if version is not None: q = q.filter_by(version=version)
if comment_id is not None: q = q.filter_by(comment_id=comment_id)
self.contents_sha1 = signed_file.contents_sha1()
return self
+ def query(self, session):
+ return session.query(SignatureHistory).filter_by(fingerprint=self.fingerprint, signature_timestamp=self.signature_timestamp, contents_sha1=self.contents_sha1).first()
+
__all__.append('SignatureHistory')
################################################################################
fileset.add(name)
return fileset
-__all__.append('DBSource')
-
-@session_wrapper
-def source_exists(source, source_version, suites = ["any"], session=None):
- """
- Ensure that source exists somewhere in the archive for the binary
- upload being processed.
- 1. exact match => 1.0-3
- 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
-
- @type source: string
- @param source: source name
-
- @type source_version: string
- @param source_version: expected source version
-
- @type suites: list
- @param suites: list of suites to check in, default I{any}
-
- @type session: Session
- @param session: Optional SQLA session object (a temporary one will be
- generated if not supplied)
-
- @rtype: int
- @return: returns 1 if a source with expected version is found, otherwise 0
-
- """
-
- cnf = Config()
- ret = True
-
- from daklib.regexes import re_bin_only_nmu
- orig_source_version = re_bin_only_nmu.sub('', source_version)
-
- for suite in suites:
- q = session.query(DBSource).filter_by(source=source). \
- filter(DBSource.version.in_([source_version, orig_source_version]))
- if suite != "any":
- # source must exist in 'suite' or a suite that is enhanced by 'suite'
- s = get_suite(suite, session)
- if s:
- enhances_vcs = session.query(VersionCheck).filter(VersionCheck.suite==s).filter_by(check='Enhances')
- considered_suites = [ vc.reference for vc in enhances_vcs ]
- considered_suites.append(s)
-
- q = q.filter(DBSource.suites.any(Suite.suite_id.in_([s.suite_id for s in considered_suites])))
-
- if q.count() > 0:
- continue
-
- # No source found so return not ok
- ret = False
-
- return ret
+ @property
+ def proxy(self):
+ session = object_session(self)
+ query = session.query(SourceMetadata).filter_by(source=self)
+ return MetadataProxy(session, query)
-__all__.append('source_exists')
+__all__.append('DBSource')
@session_wrapper
def get_suites_source_in(source, session=None):
__all__.append('get_suites_source_in')
-@session_wrapper
-def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
- """
- Returns list of DBSource objects for given C{source} name and other parameters
-
- @type source: str
- @param source: DBSource package name to search for
-
- @type version: str or None
- @param version: DBSource version name to search for or None if not applicable
-
- @type dm_upload_allowed: bool
- @param dm_upload_allowed: If None, no effect. If True or False, only
- return packages with that dm_upload_allowed setting
-
- @type session: Session
- @param session: Optional SQL session object (a temporary one will be
- generated if not supplied)
-
- @rtype: list
- @return: list of DBSource objects for the given name (may be empty)
- """
-
- q = session.query(DBSource).filter_by(source=source)
-
- if version is not None:
- q = q.filter_by(version=version)
-
- if dm_upload_allowed is not None:
- q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
-
- return q.all()
-
-__all__.append('get_sources_from_name')
-
# FIXME: This function fails badly if it finds more than 1 source package and
# its implementation is trivial enough to be inlined.
@session_wrapper
-def get_source_in_suite(source, suite, session=None):
+def get_source_in_suite(source, suite_name, session=None):
"""
- Returns a DBSource object for a combination of C{source} and C{suite}.
+ Returns a DBSource object for a combination of C{source} and C{suite_name}.
- B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
- - B{suite} - a suite name, eg. I{unstable}
+ - B{suite_name} - a suite name, eg. I{unstable}
@type source: string
@param source: source package name
- @type suite: string
+ @type suite_name: string
@param suite: the suite name
@rtype: string
@return: the version for I{source} in I{suite}
"""
-
- q = get_suite(suite, session).get_sources(source)
+ suite = get_suite(suite_name, session)
+ if suite is None:
+ return None
try:
- return q.one()
+ return suite.get_sources(source).one()
except NoResultFound:
return None
def path(self):
return os.path.join(self.archive.path, 'dists', self.suite_name)
+ @property
+ def release_suite_output(self):
+ if self.release_suite is not None:
+ return self.release_suite
+ return self.suite_name
+
__all__.append('Suite')
@session_wrapper
@return: Suite object for the requested suite name (None if not present)
"""
+ # Start by looking for the dak internal name
q = session.query(Suite).filter_by(suite_name=suite)
+ try:
+ return q.one()
+ except NoResultFound:
+ pass
+ # Now try codename
+ q = session.query(Suite).filter_by(codename=suite)
+ try:
+ return q.one()
+ except NoResultFound:
+ pass
+
+ # Finally give release_suite a try
+ q = session.query(Suite).filter_by(release_suite=suite)
try:
return q.one()
except NoResultFound:
################################################################################
+class MetadataProxy(object):
+ def __init__(self, session, query):
+ self.session = session
+ self.query = query
+
+ def _get(self, key):
+ metadata_key = self.session.query(MetadataKey).filter_by(key=key).first()
+ if metadata_key is None:
+ return None
+ metadata = self.query.filter_by(key=metadata_key).first()
+ return metadata
+
+ def __contains__(self, key):
+ if self._get(key) is not None:
+ return True
+ return False
+
+ def __getitem__(self, key):
+ metadata = self._get(key)
+ if metadata is None:
+ raise KeyError
+ return metadata.value
+
+ def get(self, key, default=None):
+ try:
+ return self[key]
+ except KeyError:
+ return default
+
+################################################################################
+
class VersionCheck(ORMObject):
def __init__(self, *args, **kwargs):
pass
'changelogs_text',
'changes',
'component',
+ 'component_suite',
'config',
'dsc_files',
'external_overrides',
'obsolete_any_associations',
'obsolete_any_by_all_associations',
'obsolete_src_associations',
+ 'package_list',
'source_suite',
'src_associations_bin',
'src_associations_src',
extension = validator)
mapper(NewComment, self.tbl_new_comments,
- properties = dict(comment_id = self.tbl_new_comments.c.id))
+ properties = dict(comment_id = self.tbl_new_comments.c.id,
+ policy_queue = relation(PolicyQueue)))
mapper(Override, self.tbl_override,
properties = dict(suite_id = self.tbl_override.c.suite,
properties = dict(suite_id = self.tbl_suite.c.id,
policy_queue = relation(PolicyQueue, primaryjoin=(self.tbl_suite.c.policy_queue_id == self.tbl_policy_queue.c.id)),
new_queue = relation(PolicyQueue, primaryjoin=(self.tbl_suite.c.new_queue_id == self.tbl_policy_queue.c.id)),
+ debug_suite = relation(Suite, remote_side=[self.tbl_suite.c.id]),
copy_queues = relation(BuildQueue,
secondary=self.tbl_suite_build_queue_copy),
srcformats = relation(SrcFormat, secondary=self.tbl_suite_src_formats,
backref=backref('suites', lazy='dynamic')),
archive = relation(Archive, backref='suites'),
- acls = relation(ACL, secondary=self.tbl_suite_acl_map, collection_class=set)),
+ acls = relation(ACL, secondary=self.tbl_suite_acl_map, collection_class=set),
+ components = relation(Component, secondary=self.tbl_component_suite,
+ order_by=self.tbl_component.c.ordering,
+ backref=backref('suites'))),
extension = validator)
mapper(Uid, self.tbl_uid,
engine_args['pool_size'] = int(cnf['DB::PoolSize'])
if cnf.has_key('DB::MaxOverflow'):
engine_args['max_overflow'] = int(cnf['DB::MaxOverflow'])
- if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \
+ if sa_major_version != '0.5' and cnf.has_key('DB::Unicode') and \
cnf['DB::Unicode'] == 'false':
engine_args['use_native_unicode'] = False
return session
__all__.append('DBConn')
-
-
projects / dak.git / blobdiff