X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdatabase.py;h=94e0a7cc51f81273d1d61f953653b6df7f5012d2;hb=fdf3c42445b4f11f4cd71634dd2b57cb7d7a4f36;hp=095b13980ca5696d0f2984d4fbb7076487ef4102;hpb=74c5a1256401a709d74c496b972cc0bd38decaa8;p=dak.git

diff --git a/daklib/database.py b/daklib/database.py
index 095b1398..94e0a7cc 100755
--- a/daklib/database.py
+++ b/daklib/database.py
@@ -1,7 +1,17 @@
 #!/usr/bin/env python
 
-""" DB access fucntions """
-# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006  James Troup <james@nocrew.org>
+""" DB access functions
+@group readonly: get_suite_id, get_section_id, get_priority_id, get_override_type_id,
+                 get_architecture_id, get_archive_id, get_component_id, get_location_id,
+                 get_source_id, get_suite_version, get_files_id, get_maintainer, get_suites,
+                 get_suite_architectures
+@group read/write: get_or_set*, set_files_id
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2004, 2006  James Troup <james@nocrew.org>
+@copyright: 2009  Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,45 +35,62 @@ import types
 
 ################################################################################
 
-Cnf = None
-projectB = None
-suite_id_cache = {}
-section_id_cache = {}
-priority_id_cache = {}
-override_type_id_cache = {}
-architecture_id_cache = {}
-archive_id_cache = {}
-component_id_cache = {}
-location_id_cache = {}
-maintainer_id_cache = {}
-keyring_id_cache = {}
-source_id_cache = {}
-files_id_cache = {}
-maintainer_cache = {}
-fingerprint_id_cache = {}
-queue_id_cache = {}
-uid_id_cache = {}
-suite_version_cache = {}
+Cnf = None                    #: Configuration, apt_pkg.Configuration
+projectB = None               #: database connection, pgobject
+suite_id_cache = {}           #: cache for suites
+section_id_cache = {}         #: cache for sections
+priority_id_cache = {}        #: cache for priorities
+override_type_id_cache = {}   #: cache for overrides
+architecture_id_cache = {}    #: cache for architectures
+archive_id_cache = {}         #: cache for archives
+component_id_cache = {}       #: cache for components
+location_id_cache = {}        #: cache for locations
+maintainer_id_cache = {}      #: cache for maintainers
+keyring_id_cache = {}         #: cache for keyrings
+source_id_cache = {}          #: cache for sources
+
+files_id_cache = {}           #: cache for files
+maintainer_cache = {}         #: cache for maintainer names
+fingerprint_id_cache = {}     #: cache for fingerprints
+queue_id_cache = {}           #: cache for queues
+uid_id_cache = {}             #: cache for uids
+suite_version_cache = {}      #: cache for suite_versions (packages)
+suite_bin_version_cache = {}
+cache_preloaded = False
 
 ################################################################################
 
 def init (config, sql):
-    """ database module init. Just sets two variables"""
+    """
+    database module init.
+
+    @type config: apt_pkg.Configuration
+    @param config: apt config, see U{http://apt.alioth.debian.org/python-apt-doc/apt_pkg/cache.html#Configuration}
+
+    @type sql: pgobject
+    @param sql: database connection
+
+    """
     global Cnf, projectB
 
     Cnf = config
     projectB = sql
 
 
-def do_query(q):
+def do_query(query):
     """
-    Executes a database query q. Writes statistics to stderr and returns
-    the result.
+    Executes a database query. Writes statistics / timing to stderr.
+
+    @type query: string
+    @param query: database query string, passed unmodified
 
+    @return: db result
+
+    @warning: The query is passed B{unmodified}, so be careful what you use this for.
     """
-    sys.stderr.write("query: \"%s\" ... " % (q))
+    sys.stderr.write("query: \"%s\" ... " % (query))
     before = time.time()
-    r = projectB.query(q)
+    r = projectB.query(query)
     time_diff = time.time()-before
     sys.stderr.write("took %.3f seconds.\n" % (time_diff))
     if type(r) is int:
@@ -77,7 +104,17 @@ def do_query(q):
 ################################################################################
 
 def get_suite_id (suite):
-    """ Returns database suite_id for given suite, caches result. """
+    """
+    Returns database id for given C{suite}.
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type suite: string
+    @param suite: The name of the suite
+
+    @rtype: int
+    @return: the database id for the given suite
+
+    """
     global suite_id_cache
 
     if suite_id_cache.has_key(suite):
@@ -94,7 +131,17 @@ def get_suite_id (suite):
     return suite_id
 
 def get_section_id (section):
-    """ Returns database section_id for given section, caches result. """
+    """
+    Returns database id for given C{section}.
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type section: string
+    @param section: The name of the section
+
+    @rtype: int
+    @return: the database id for the given section
+
+    """
     global section_id_cache
 
     if section_id_cache.has_key(section):
@@ -111,7 +158,17 @@ def get_section_id (section):
     return section_id
 
 def get_priority_id (priority):
-    """ Returns database priority_id for given priority, caches result. """
+    """
+    Returns database id for given C{priority}.
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type priority: string
+    @param priority: The name of the priority
+
+    @rtype: int
+    @return: the database id for the given priority
+
+    """
     global priority_id_cache
 
     if priority_id_cache.has_key(priority):
@@ -128,7 +185,17 @@ def get_priority_id (priority):
     return priority_id
 
 def get_override_type_id (type):
-    """ Returns database override_id for given override_type type, caches result. """
+    """
+    Returns database id for given override C{type}.
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type type: string
+    @param type: The name of the override type
+
+    @rtype: int
+    @return: the database id for the given override type
+
+    """
     global override_type_id_cache
 
     if override_type_id_cache.has_key(type):
@@ -145,7 +212,17 @@ def get_override_type_id (type):
     return override_type_id
 
 def get_architecture_id (architecture):
-    """ Returns database architecture_id for given architecture, caches result. """
+    """
+    Returns database id for given C{architecture}.
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type architecture: string
+    @param architecture: The name of the override type
+
+    @rtype: int
+    @return: the database id for the given architecture
+
+    """
     global architecture_id_cache
 
     if architecture_id_cache.has_key(architecture):
@@ -162,7 +239,17 @@ def get_architecture_id (architecture):
     return architecture_id
 
 def get_archive_id (archive):
-    """ Returns database archive_id for given archive, caches result. """
+    """
+    Returns database id for given C{archive}.
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type archive: string
+    @param archive: The name of the override type
+
+    @rtype: int
+    @return: the database id for the given archive
+
+    """
     global archive_id_cache
 
     archive = archive.lower()
@@ -181,7 +268,17 @@ def get_archive_id (archive):
     return archive_id
 
 def get_component_id (component):
-    """ Returns database component_id for given component, caches result. """
+    """
+    Returns database id for given C{component}.
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type component: string
+    @param component: The name of the component
+
+    @rtype: int
+    @return: the database id for the given component
+
+    """
     global component_id_cache
 
     component = component.lower()
@@ -201,15 +298,23 @@ def get_component_id (component):
 
 def get_location_id (location, component, archive):
     """
-    Returns database location_id for given combination of
-    location
-    component
-    archive.
+    Returns database id for the location behind the given combination of
+      - B{location} - the path of the location, eg. I{/srv/ftp.debian.org/ftp/pool/}
+      - B{component} - the id of the component as returned by L{get_component_id}
+      - B{archive} - the id of the archive as returned by L{get_archive_id}
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type location: string
+    @param location: the path of the location
+
+    @type component: int
+    @param component: the id of the component
 
-    The 3 parameters are the database ids returned by the respective
-    "get_foo_id" functions.
+    @type archive: int
+    @param archive: the id of the archive
 
-    The result will be cached.
+    @rtype: int
+    @return: the database id for the location
 
     """
     global location_id_cache
@@ -235,7 +340,22 @@ def get_location_id (location, component, archive):
     return location_id
 
 def get_source_id (source, version):
-    """ Returns database source_id for given combination of source and version, caches result. """
+    """
+    Returns database id for the combination of C{source} and C{version}
+      - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
+      - B{version}
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type source: string
+    @param source: source package name
+
+    @type version: string
+    @param version: the source version
+
+    @rtype: int
+    @return: the database id for the source
+
+    """
     global source_id_cache
 
     cache_key = source + '_' + version + '_'
@@ -253,7 +373,25 @@ def get_source_id (source, version):
     return source_id
 
 def get_suite_version(source, suite):
-    """ Returns database version for a given source in a given suite, caches result. """
+    """
+    Returns database id for a combination of C{source} and C{suite}.
+
+      - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
+      - B{suite} - a suite name, eg. I{unstable}
+
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type source: string
+    @param source: source package name
+
+    @type suite: string
+    @param suite: the suite name
+
+    @rtype: string
+    @return: the version for I{source} in I{suite}
+
+    """
+
     global suite_version_cache
     cache_key = "%s_%s" % (source, suite)
 
@@ -276,15 +414,91 @@ def get_suite_version(source, suite):
 
     return version
 
+def get_latest_binary_version_id(binary, section, suite, arch):
+    global suite_bin_version_cache
+    cache_key = "%s_%s_%s_%s" % (binary, section, suite, arch)
+    cache_key_all = "%s_%s_%s_%s" % (binary, section, suite, get_architecture_id("all"))
+
+    # Check for the cache hit for its arch, then arch all
+    if suite_bin_version_cache.has_key(cache_key):
+        return suite_bin_version_cache[cache_key]
+    if suite_bin_version_cache.has_key(cache_key_all):
+        return suite_bin_version_cache[cache_key_all]
+    if cache_preloaded == True:
+        return # package does not exist
+
+    q = projectB.query("SELECT DISTINCT b.id FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN override o ON (o.package=b.package) WHERE b.package = '%s' AND b.architecture = '%d' AND ba.suite = '%d' AND o.section = '%d'" % (binary, int(arch), int(suite), int(section)))
+
+    if not q.getresult():
+        return False
+
+    highest_bid = q.getresult()[0][0]
+
+    suite_bin_version_cache[cache_key] = highest_bid
+    return highest_bid
+
+def preload_binary_id_cache():
+    global suite_bin_version_cache, cache_preloaded
+
+    # Get suite info
+    q = projectB.query("SELECT id FROM suite")
+    suites = q.getresult()
+
+    # Get arch mappings
+    q = projectB.query("SELECT id FROM architecture")
+    arches = q.getresult()
+
+    for suite in suites:
+        for arch in arches:
+            q = projectB.query("SELECT DISTINCT b.id, b.package, o.section FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN override o ON (o.package=b.package) WHERE b.architecture = '%d' AND ba.suite = '%d'" % (int(arch[0]), int(suite[0])))
+
+            for bi in q.getresult():
+                cache_key = "%s_%s_%s_%s" % (bi[1], bi[2], suite[0], arch[0])
+                suite_bin_version_cache[cache_key] = int(bi[0])
+
+    cache_preloaded = True
+
+def get_suite_architectures(suite):
+    """
+    Returns list of architectures for C{suite}.
+
+    @type suite: string, int
+    @param suite: the suite name or the suite_id
+
+    @rtype: list
+    @return: the list of architectures for I{suite}
+    """
+
+    suite_id = None
+    if type(suite) == str:
+        suite_id = get_suite_id(suite)
+    elif type(suite) == int:
+        suite_id = suite
+    else:
+        return None
+
+    sql = """ SELECT a.arch_string FROM suite_architectures sa
+              JOIN architecture a ON (a.id = sa.architecture)
+              WHERE suite='%s' """ % (suite_id)
+
+    q = projectB.query(sql)
+    return map(lambda x: x[0], q.getresult())
+
 ################################################################################
 
 def get_or_set_maintainer_id (maintainer):
     """
-    If maintainer does not have an entry in the maintainer table yet, create one
-    and return its id.
-    If maintainer already has an entry, simply return its id.
+    If C{maintainer} does not have an entry in the maintainer table yet, create one
+    and return the new id.
+    If C{maintainer} already has an entry, simply return the existing id.
+
+    Results are kept in a cache during runtime to minimize database queries.
 
-    Result is cached.
+    @type maintainer: string
+    @param maintainer: the maintainer name
+
+    @rtype: int
+    @return: the database id for the maintainer
 
     """
     global maintainer_id_cache
@@ -305,11 +519,17 @@ def get_or_set_maintainer_id (maintainer):
 
 def get_or_set_keyring_id (keyring):
     """
-    If keyring does not have an entry in the keyring table yet, create one
-    and return its id.
-    If keyring already has an entry, simply return its id.
+    If C{keyring} does not have an entry in the C{keyrings} table yet, create one
+    and return the new id.
+    If C{keyring} already has an entry, simply return the existing id.
+
+    Results are kept in a cache during runtime to minimize database queries.
 
-    Result is cached.
+    @type keyring: string
+    @param keyring: the keyring name
+
+    @rtype: int
+    @return: the database id for the keyring
 
     """
     global keyring_id_cache
@@ -330,13 +550,20 @@ def get_or_set_keyring_id (keyring):
 
 def get_or_set_uid_id (uid):
     """
-    If uid does not have an entry in the uid table yet, create one
-    and return its id.
-    If uid already has an entry, simply return its id.
+    If C{uid} does not have an entry in the uid table yet, create one
+    and return the new id.
+    If C{uid} already has an entry, simply return the existing id.
+
+    Results are kept in a cache during runtime to minimize database queries.
 
-    Result is cached.
+    @type uid: string
+    @param uid: the uid.
+
+    @rtype: int
+    @return: the database id for the uid
 
     """
+
     global uid_id_cache
 
     if uid_id_cache.has_key(uid):
@@ -355,11 +582,17 @@ def get_or_set_uid_id (uid):
 
 def get_or_set_fingerprint_id (fingerprint):
     """
-    If fingerprintd does not have an entry in the fingerprint table yet, create one
-    and return its id.
-    If fingerprint already has an entry, simply return its id.
+    If C{fingerprint} does not have an entry in the fingerprint table yet, create one
+    and return the new id.
+    If C{fingerprint} already has an entry, simply return the existing id.
 
-    Result is cached.
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type fingerprint: string
+    @param fingerprint: the fingerprint
+
+    @rtype: int
+    @return: the database id for the fingerprint
 
     """
     global fingerprint_id_cache
@@ -380,22 +613,35 @@ def get_or_set_fingerprint_id (fingerprint):
 
 def get_files_id (filename, size, md5sum, location_id):
     """
-    Returns -1, -2 or the file_id for a given combination of
-    filename
-    size
-    md5sum
-    location_id.
+    Returns -1, -2 or the file_id for filename, if its C{size} and C{md5sum} match an
+    existing copy.
+
+    The database is queried using the C{filename} and C{location_id}. If a file does exist
+    at that location, the existing size and md5sum are checked against the provided
+    parameters. A size or checksum mismatch returns -2. If more than one entry is
+    found within the database, a -1 is returned, no result returns None, otherwise
+    the file id.
 
-    The database is queried using filename and location_id, size and md5sum are for
-    extra checks.
+    Results are kept in a cache during runtime to minimize database queries.
 
-    Return values:
-    -1 - The given combination of arguments result in more (or less) than
-         one result from the database
-    -2 - The given size and md5sum do not match the values in the database
-    anything else is a file_id
+    @type filename: string
+    @param filename: the filename of the file to check against the DB
 
-    Result is cached.
+    @type size: int
+    @param size: the size of the file to check against the DB
+
+    @type md5sum: string
+    @param md5sum: the md5sum of the file to check against the DB
+
+    @type location_id: int
+    @param location_id: the id of the location as returned by L{get_location_id}
+
+    @rtype: int / None
+    @return: Various return values are possible:
+               - -2: size/checksum error
+               - -1: more than one file found in database
+               - None: no file found in database
+               - int: file id
 
     """
     global files_id_cache
@@ -425,11 +671,17 @@ def get_files_id (filename, size, md5sum, location_id):
 
 def get_or_set_queue_id (queue):
     """
-    If queue does not have an entry in the queue_name table yet, create one
-    and return its id.
-    If queue already has an entry, simply return its id.
+    If C{queue} does not have an entry in the queue table yet, create one
+    and return the new id.
+    If C{queue} already has an entry, simply return the existing id.
+
+    Results are kept in a cache during runtime to minimize database queries.
 
-    Result is cached.
+    @type queue: string
+    @param queue: the queue name (no full path)
+
+    @rtype: int
+    @return: the database id for the queue
 
     """
     global queue_id_cache
@@ -450,9 +702,28 @@ def get_or_set_queue_id (queue):
 
 def set_files_id (filename, size, md5sum, sha1sum, sha256sum, location_id):
     """
-    Insert a new entry into the files table.
+    Insert a new entry into the files table and return its id.
+
+    @type filename: string
+    @param filename: the filename
+
+    @type size: int
+    @param size: the size in bytes
+
+    @type md5sum: string
+    @param md5sum: md5sum of the file
 
-    Returns the new file_id
+    @type sha1sum: string
+    @param sha1sum: sha1sum of the file
+
+    @type sha256sum: string
+    @param sha256sum: sha256sum of the file
+
+    @type location_id: int
+    @param location_id: the id of the location as returned by L{get_location_id}
+
+    @rtype: int
+    @return: the database id for the new file
 
     """
     global files_id_cache
@@ -474,7 +745,18 @@ def set_files_id (filename, size, md5sum, sha1sum, sha256sum, location_id):
 ################################################################################
 
 def get_maintainer (maintainer_id):
-    """ Return the name of the maintainer behind maintainer_id """
+    """
+    Return the name of the maintainer behind C{maintainer_id}.
+
+    Results are kept in a cache during runtime to minimize database queries.
+
+    @type maintainer_id: int
+    @param maintainer_id: the id of the maintainer, eg. from L{get_or_set_maintainer_id}
+
+    @rtype: string
+    @return: the name of the maintainer
+
+    """
     global maintainer_cache
 
     if not maintainer_cache.has_key(maintainer_id):
@@ -486,7 +768,20 @@ def get_maintainer (maintainer_id):
 ################################################################################
 
 def get_suites(pkgname, src=False):
-    """ Return the suites in which pkgname is. If src is True, query for source package, else binary. """
+    """
+    Return the suites in which C{pkgname} can be found. If C{src} is True query for source
+    package, else binary package.
+
+    @type pkgname: string
+    @param pkgname: name of the package
+
+    @type src: bool
+    @param src: if True look for source packages, false (default) looks for binary.
+
+    @rtype: list
+    @return: list of suites, or empty list if no match
+
+    """
     if src:
         sql = """
         SELECT suite_name
@@ -510,3 +805,47 @@ def get_suites(pkgname, src=False):
 
     q = projectB.query(sql)
     return map(lambda x: x[0], q.getresult())
+
+
+################################################################################
+
+def copy_temporary_contents(package, version, deb):
+    """
+    copy the previously stored contents from the temp table to the permanant one
+
+    during process-unchecked, the deb should have been scanned and the
+    contents stored in temp_content_associations
+    """
+
+    # first see if contents exist:
+
+    exists = projectB.query("""SELECT 1 FROM temp_content_associations
+                               WHERE package='%s' LIMIT 1""" % package ).getresult()
+
+    if not exists:
+        # This should NOT happen.  We should have added contents
+        # during process-unchecked.  if it did, log an error, and send
+        # an email.
+        subst = {
+            "__PACKAGE__": package,
+            "__VERSION__": version,
+            "__TO_ADDRESS__": Cnf["Dinstall::MyAdminAddress"]
+            "__DAK_ADDRESS__": Cnf["Dinstall::MyEmailAddress"]
+            }
+
+        message = utils.TemplateSubst(Subst, Cnf["Dir::Templates"]+"/missing-contents")
+        utils.send_mail( message )
+
+        exists = DBConn().insert_content_path(package, version, deb)
+
+    if exists:
+        sql = """INSERT INTO content_associations(binary_pkg,filepath,filename)
+                 SELECT currval('binaries_id_seq'), filepath, filename FROM temp_content_associations
+                 WHERE package='%s'
+                     AND version='%s'""" % (package, version)
+        projectB.query(sql)
+        projectB.query("""DELETE from temp_content_associations
+                          WHERE package='%s'
+                            AND version='%s'""" % (package, version))
+
+    return exists