X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=72dbaad0f615c8ceda12b13b2163b095bdf2362f;hb=0f00e503e04dd38e7fa14bb4086e1e15d3c9a8dc;hp=5028b7cdd3015818eea5fe866325e71bd5587205;hpb=475051efae41a30723cdc1ab82c521cd1accf75b;p=dak.git

diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 5028b7cd..72dbaad0 100755
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -33,6 +33,7 @@
 
 ################################################################################
 
+import apt_pkg
 import os
 from os.path import normpath
 import re
@@ -73,13 +74,16 @@ from sqlalchemy.orm.exc import NoResultFound
 # in the database
 from config import Config
 from textutils import fix_maintainer
-from dak_exceptions import DBUpdateError, NoSourceFieldError
+from dak_exceptions import DBUpdateError, NoSourceFieldError, FileExistsError
 
 # suppress some deprecation warnings in squeeze related to sqlalchemy
 import warnings
 warnings.filterwarnings('ignore', \
     "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
     SADeprecationWarning)
+warnings.filterwarnings('ignore', \
+    "Predicate of partial index .* ignored during reflection", \
+    SAWarning)
 
 
 ################################################################################
@@ -106,11 +110,11 @@ class DebVersion(UserDefinedType):
         return None
 
 sa_major_version = sqlalchemy.__version__[0:3]
-if sa_major_version in ["0.5", "0.6"]:
+if sa_major_version in ["0.5", "0.6", "0.7"]:
     from sqlalchemy.databases import postgres
     postgres.ischema_names['debversion'] = DebVersion
 else:
-    raise Exception("dak only ported to SQLA versions 0.5 and 0.6.  See daklib/dbconn.py")
+    raise Exception("dak only ported to SQLA versions 0.5 to 0.7.  See daklib/dbconn.py")
 
 ################################################################################
 
@@ -496,7 +500,7 @@ def subprocess_setup():
 class DBBinary(ORMObject):
     def __init__(self, package = None, source = None, version = None, \
         maintainer = None, architecture = None, poolfile = None, \
-        binarytype = 'deb'):
+        binarytype = 'deb', fingerprint=None):
         self.package = package
         self.source = source
         self.version = version
@@ -504,6 +508,7 @@ class DBBinary(ORMObject):
         self.architecture = architecture
         self.poolfile = poolfile
         self.binarytype = binarytype
+        self.fingerprint = fingerprint
 
     @property
     def pkid(self):
@@ -554,10 +559,10 @@ class DBBinary(ORMObject):
         @rtype: text
         @return: stanza text of the control section.
         '''
-        import apt_inst
+        import utils
         fullpath = self.poolfile.fullpath
         deb_file = open(fullpath, 'r')
-        stanza = apt_inst.debExtractControl(deb_file)
+        stanza = utils.deb_extract_control(deb_file)
         deb_file.close()
 
         return stanza
@@ -751,7 +756,7 @@ class BuildQueue(object):
             # Crude hack with open and append, but this whole section is and should be redone.
             if self.notautomatic:
                 release=open("Release", "a")
-                release.write("NotAutomatic: yes")
+                release.write("NotAutomatic: yes\n")
                 release.close()
 
             # Sign if necessary
@@ -814,7 +819,7 @@ class BuildQueue(object):
                     Logger.log(["I: Removing %s from the queue" % o.fullpath])
                     os.unlink(o.fullpath)
                     killdb = True
-            except OSError, e:
+            except OSError as e:
                 # If it wasn't there, don't worry
                 if e.errno == ENOENT:
                     killdb = True
@@ -874,7 +879,6 @@ class BuildQueue(object):
         # Prepare BuildQueueFile object
         qf = BuildQueueFile()
         qf.build_queue_id = self.queue_id
-        qf.lastused = datetime.now()
         qf.filename = poolfile_basename
 
         targetpath = poolfile.fullpath
@@ -890,6 +894,9 @@ class BuildQueue(object):
             else:
                 os.symlink(targetpath, queuepath)
                 qf.fileid = poolfile.file_id
+        except FileExistsError:
+            if not poolfile.identical_to(queuepath):
+                raise
         except OSError:
             return None
 
@@ -948,6 +955,9 @@ class BuildQueue(object):
             # Always copy files from policy queues as they might move around.
             import utils
             utils.copy(source, target)
+        except FileExistsError:
+            if not policyqueuefile.identical_to(target):
+                raise
         except OSError:
             return None
 
@@ -1043,6 +1053,24 @@ class ChangePendingFile(object):
     def __repr__(self):
         return '<ChangePendingFile %s>' % self.change_pending_file_id
 
+    def identical_to(self, filename):
+        """
+        compare size and hash with the given file
+
+        @rtype: bool
+        @return: true if the given file has the same size and hash as this object; false otherwise
+        """
+        st = os.stat(filename)
+        if self.size != st.st_size:
+            return False
+
+        f = open(filename, "r")
+        sha256sum = apt_pkg.sha256sum(f)
+        if sha256sum != self.sha256sum:
+            return False
+
+        return True
+
 __all__.append('ChangePendingFile')
 
 ################################################################################
@@ -1107,6 +1135,19 @@ def get_component(component, session=None):
 
 __all__.append('get_component')
 
+@session_wrapper
+def get_component_names(session=None):
+    """
+    Returns list of strings of component names.
+
+    @rtype: list
+    @return: list of strings of component names
+    """
+
+    return [ x.component_name for x in session.query(Component).all() ]
+
+__all__.append('get_component_names')
+
 ################################################################################
 
 class DBConfig(object):
@@ -1382,6 +1423,10 @@ class PoolFile(ORMObject):
     def fullpath(self):
         return os.path.join(self.location.path, self.filename)
 
+    @property
+    def basename(self):
+        return os.path.basename(self.filename)
+
     def is_valid(self, filesize = -1, md5sum = None):
         return self.filesize == long(filesize) and self.md5sum == md5sum
 
@@ -1392,6 +1437,24 @@ class PoolFile(ORMObject):
     def not_null_constraints(self):
         return ['filename', 'md5sum', 'location']
 
+    def identical_to(self, filename):
+        """
+        compare size and hash with the given file
+
+        @rtype: bool
+        @return: true if the given file has the same size and hash as this object; false otherwise
+        """
+        st = os.stat(filename)
+        if self.filesize != st.st_size:
+            return False
+
+        f = open(filename, "r")
+        sha256sum = apt_pkg.sha256sum(f)
+        if sha256sum != self.sha256sum:
+            return False
+
+        return True
+
 __all__.append('PoolFile')
 
 @session_wrapper
@@ -1627,7 +1690,7 @@ class Keyring(object):
         key = None
         signingkey = False
 
-        for line in k.xreadlines():
+        for line in k:
             field = line.split(":")
             if field[0] == "pub":
                 key = field[4]
@@ -1734,6 +1797,34 @@ def get_keyring(keyring, session=None):
 
 __all__.append('get_keyring')
 
+@session_wrapper
+def get_active_keyring_paths(session=None):
+    """
+    @rtype: list
+    @return: list of active keyring paths
+    """
+    return [ x.keyring_name for x in session.query(Keyring).filter(Keyring.active == True).order_by(desc(Keyring.priority)).all() ]
+
+__all__.append('get_active_keyring_paths')
+
+@session_wrapper
+def get_primary_keyring_path(session=None):
+    """
+    Get the full path to the highest priority active keyring
+
+    @rtype: str or None
+    @return: path to the active keyring with the highest priority or None if no
+             keyring is configured
+    """
+    keyrings = get_active_keyring_paths()
+
+    if len(keyrings) > 0:
+        return keyrings[0]
+    else:
+        return None
+
+__all__.append('get_primary_keyring_path')
+
 ################################################################################
 
 class KeyringACLMap(object):
@@ -2373,13 +2464,14 @@ class Dak822(Deb822):
 
 class DBSource(ORMObject):
     def __init__(self, source = None, version = None, maintainer = None, \
-        changedby = None, poolfile = None, install_date = None):
+        changedby = None, poolfile = None, install_date = None, fingerprint = None):
         self.source = source
         self.version = version
         self.maintainer = maintainer
         self.changedby = changedby
         self.poolfile = poolfile
         self.install_date = install_date
+        self.fingerprint = fingerprint
 
     @property
     def pkid(self):
@@ -2392,7 +2484,7 @@ class DBSource(ORMObject):
 
     def not_null_constraints(self):
         return ['source', 'version', 'install_date', 'maintainer', \
-            'changedby', 'poolfile', 'install_date']
+            'changedby', 'poolfile']
 
     def read_control_fields(self):
         '''
@@ -2407,6 +2499,9 @@ class DBSource(ORMObject):
 
     metadata = association_proxy('key', 'value')
 
+    def get_component_name(self):
+        return self.poolfile.location.component.component_name
+
     def scan_contents(self):
         '''
         Returns a set of names for non directories. The path names are
@@ -2464,20 +2559,14 @@ def source_exists(source, source_version, suites = ["any"], session=None):
         q = session.query(DBSource).filter_by(source=source). \
             filter(DBSource.version.in_([source_version, orig_source_version]))
         if suite != "any":
-            # source must exist in suite X, or in some other suite that's
-            # mapped to X, recursively... silent-maps are counted too,
-            # unreleased-maps aren't.
-            maps = cnf.ValueList("SuiteMappings")[:]
-            maps.reverse()
-            maps = [ m.split() for m in maps ]
-            maps = [ (x[1], x[2]) for x in maps
-                            if x[0] == "map" or x[0] == "silent-map" ]
-            s = [suite]
-            for (from_, to) in maps:
-                if from_ in s and to not in s:
-                    s.append(to)
-
-            q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
+            # source must exist in 'suite' or a suite that is enhanced by 'suite'
+            s = get_suite(suite, session)
+            if s:
+                enhances_vcs = session.query(VersionCheck).filter(VersionCheck.suite==s).filter_by(check='Enhances')
+                considered_suites = [ vc.reference for vc in enhances_vcs ]
+                considered_suites.append(s)
+
+                q = q.filter(DBSource.suites.any(Suite.suite_id.in_([s.suite_id for s in considered_suites])))
 
         if q.count() > 0:
             continue
@@ -2599,6 +2688,15 @@ __all__.append('import_metadata_into_db')
 
 ################################################################################
 
+def split_uploaders(uploaders_list):
+    '''
+    Split the Uploaders field into the individual uploaders and yield each of
+    them. Beware: email addresses might contain commas.
+    '''
+    import re
+    for uploader in re.sub(">[ ]*,", ">\t", uploaders_list).split("\t"):
+        yield uploader.strip()
+
 @session_wrapper
 def add_dsc_to_db(u, filename, session=None):
     entry = u.pkg.files[filename]
@@ -2608,7 +2706,11 @@ def add_dsc_to_db(u, filename, session=None):
     source.source = u.pkg.dsc["source"]
     source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
     source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
-    source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
+    # If Changed-By isn't available, fall back to maintainer
+    if u.pkg.changes.has_key("changed-by"):
+        source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
+    else:
+        source.changedby_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
     source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
     source.install_date = datetime.now().date()
 
@@ -2685,8 +2787,7 @@ def add_dsc_to_db(u, filename, session=None):
     session.refresh(source)
     source.uploaders = [source.maintainer]
     if u.pkg.dsc.has_key("uploaders"):
-        for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
-            up = up.strip()
+        for up in split_uploaders(u.pkg.dsc["uploaders"]):
             source.uploaders.append(get_or_set_maintainer(up, session))
 
     session.flush()
@@ -2728,10 +2829,25 @@ def add_deb_to_db(u, filename, session=None):
 
     # Find source id
     bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
+
+    # If we couldn't find anything and the upload contains Arch: source,
+    # fall back to trying the source package, source version uploaded
+    # This maintains backwards compatibility with previous dak behaviour
+    # and deals with slightly broken binary debs which don't properly
+    # declare their source package name
+    if len(bin_sources) == 0:
+        if u.pkg.changes["architecture"].has_key("source") \
+           and u.pkg.dsc.has_key("source") and u.pkg.dsc.has_key("version"):
+            bin_sources = get_sources_from_name(u.pkg.dsc["source"], u.pkg.dsc["version"], session=session)
+
+    # If we couldn't find a source here, we reject
+    # TODO: Fix this so that it doesn't kill process-upload and instead just
+    #       performs a reject.  To be honest, we should probably spot this
+    #       *much* earlier than here
     if len(bin_sources) != 1:
-        raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
+        raise NoSourceFieldError("Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
                                   (bin.package, bin.version, entry["architecture"],
-                                   filename, bin.binarytype, u.pkg.changes["fingerprint"])
+                                   filename, bin.binarytype, u.pkg.changes["fingerprint"]))
 
     bin.source_id = bin_sources[0].source_id
 
@@ -2739,9 +2855,9 @@ def add_deb_to_db(u, filename, session=None):
         for srcname, version in entry["built-using"]:
             exsources = get_sources_from_name(srcname, version, session=session)
             if len(exsources) != 1:
-                raise NoSourceFieldError, "Unable to find source package (%s = %s) in Built-Using for %s (%s), %s, file %s, type %s, signed by %s" % \
+                raise NoSourceFieldError("Unable to find source package (%s = %s) in Built-Using for %s (%s), %s, file %s, type %s, signed by %s" % \
                                           (srcname, version, bin.package, bin.version, entry["architecture"],
-                                           filename, bin.binarytype, u.pkg.changes["fingerprint"])
+                                           filename, bin.binarytype, u.pkg.changes["fingerprint"]))
 
             bin.extra_sources.append(exsources[0])
 
@@ -2887,6 +3003,10 @@ class Suite(ORMObject):
         else:
             return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one()
 
+    @property
+    def path(self):
+        return os.path.join(self.archive.path, 'dists', self.suite_name)
+
 __all__.append('Suite')
 
 @session_wrapper
@@ -2916,11 +3036,11 @@ __all__.append('get_suite')
 
 ################################################################################
 
-# TODO: should be removed because the implementation is too trivial
 @session_wrapper
 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
     """
-    Returns list of Architecture objects for given C{suite} name
+    Returns list of Architecture objects for given C{suite} name. The list is
+    empty if suite does not exist.
 
     @type suite: str
     @param suite: Suite name to search for
@@ -2941,48 +3061,15 @@ def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
     @return: list of Architecture objects for the given name (may be empty)
     """
 
-    return get_suite(suite, session).get_architectures(skipsrc, skipall)
+    try:
+        return get_suite(suite, session).get_architectures(skipsrc, skipall)
+    except AttributeError:
+        return []
 
 __all__.append('get_suite_architectures')
 
 ################################################################################
 
-class SuiteSrcFormat(object):
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def __repr__(self):
-        return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
-
-__all__.append('SuiteSrcFormat')
-
-@session_wrapper
-def get_suite_src_formats(suite, session=None):
-    """
-    Returns list of allowed SrcFormat for C{suite}.
-
-    @type suite: str
-    @param suite: Suite name to search for
-
-    @type session: Session
-    @param session: Optional SQL session object (a temporary one will be
-    generated if not supplied)
-
-    @rtype: list
-    @return: the list of allowed source formats for I{suite}
-    """
-
-    q = session.query(SrcFormat)
-    q = q.join(SuiteSrcFormat)
-    q = q.join(Suite).filter_by(suite_name=suite)
-    q = q.order_by('format_name')
-
-    return q.all()
-
-__all__.append('get_suite_src_formats')
-
-################################################################################
-
 class Uid(ORMObject):
     def __init__(self, uid = None, name = None):
         self.uid = uid
@@ -3280,8 +3367,8 @@ class DBConn(object):
         mapper(Architecture, self.tbl_architecture,
             properties = dict(arch_id = self.tbl_architecture.c.id,
                suites = relation(Suite, secondary=self.tbl_suite_architectures,
-                   order_by='suite_name',
-                   backref=backref('architectures', order_by='arch_string'))),
+                   order_by=self.tbl_suite.c.suite_name,
+                   backref=backref('architectures', order_by=self.tbl_architecture.c.arch_string))),
             extension = validator)
 
         mapper(Archive, self.tbl_archive,
@@ -3515,15 +3602,12 @@ class DBConn(object):
                properties = dict(suite_id = self.tbl_suite.c.id,
                                  policy_queue = relation(PolicyQueue),
                                  copy_queues = relation(BuildQueue,
-                                     secondary=self.tbl_suite_build_queue_copy)),
+                                     secondary=self.tbl_suite_build_queue_copy),
+                                 srcformats = relation(SrcFormat, secondary=self.tbl_suite_src_formats,
+                                     backref=backref('suites', lazy='dynamic')),
+                                 archive = relation(Archive, backref='suites')),
                 extension = validator)
 
-        mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
-               properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
-                                 suite = relation(Suite, backref='suitesrcformats'),
-                                 src_format_id = self.tbl_suite_src_formats.c.src_format,
-                                 src_format = relation(SrcFormat)))
-
         mapper(Uid, self.tbl_uid,
                properties = dict(uid_id = self.tbl_uid.c.id,
                                  fingerprint = relation(Fingerprint)),
@@ -3615,15 +3699,21 @@ class DBConn(object):
 
         sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak
 
-        self.db_pg   = create_engine(connstr, **engine_args)
-        self.db_meta = MetaData()
-        self.db_meta.bind = self.db_pg
-        self.db_smaker = sessionmaker(bind=self.db_pg,
-                                      autoflush=True,
-                                      autocommit=False)
+        try:
+            self.db_pg   = create_engine(connstr, **engine_args)
+            self.db_meta = MetaData()
+            self.db_meta.bind = self.db_pg
+            self.db_smaker = sessionmaker(bind=self.db_pg,
+                                          autoflush=True,
+                                          autocommit=False)
+
+            self.__setuptables()
+            self.__setupmappers()
+
+        except OperationalError as e:
+            import utils
+            utils.fubar("Cannot connect to database (%s)" % str(e))
 
-        self.__setuptables()
-        self.__setupmappers()
         self.pid = os.getpid()
 
     def session(self, work_mem = 0):