From: Mike O'Connor Date: Mon, 9 Feb 2009 07:46:48 +0000 (-0500) Subject: Merge branch 'psycopg2' into content_generation X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=commitdiff_plain;h=cd5b29ddfd8de263c085f494b9573d683913f6f3 Merge branch 'psycopg2' into content_generation Conflicts: daklib/database.py Signed-off-by: Mike O'Connor --- cd5b29ddfd8de263c085f494b9573d683913f6f3 diff --cc dak/dak.py index d04eebc2,a08f20e0..fc22ec5c --- a/dak/dak.py +++ b/dak/dak.py @@@ -103,10 -108,10 +108,12 @@@ def init() ("make-suite-file-list", "Generate lists of packages per suite for apt-ftparchive"), + ("make-pkg-file-mapping", + "Generate package <-> file mapping"), ("generate-releases", "Generate Release files"), + ("generate-contents", + "Generate contest files"), ("generate-index-diffs", "Generate .diff/Index files"), ("clean-suites", diff --cc daklib/database.py index 9cefc381,1882ad8b..3cbb67b7 --- a/daklib/database.py +++ b/daklib/database.py @@@ -23,30 -34,25 +34,30 @@@ import type ################################################################################ - Cnf = None - projectB = None - suite_id_cache = {} - section_id_cache = {} - priority_id_cache = {} - override_type_id_cache = {} - architecture_id_cache = {} - archive_id_cache = {} - component_id_cache = {} - location_id_cache = {} - maintainer_id_cache = {} - keyring_id_cache = {} - source_id_cache = {} - files_id_cache = {} - maintainer_cache = {} - fingerprint_id_cache = {} - queue_id_cache = {} - uid_id_cache = {} - suite_version_cache = {} + Cnf = None #: Configuration, apt_pkg.Configuration + projectB = None #: database connection, pgobject + suite_id_cache = {} #: cache for suites + section_id_cache = {} #: cache for sections + priority_id_cache = {} #: cache for priorities + override_type_id_cache = {} #: cache for overrides + architecture_id_cache = {} #: cache for architectures + archive_id_cache = {} #: cache for archives + component_id_cache = {} #: cache for components + location_id_cache = {} #: cache for locations + maintainer_id_cache = {} #: cache for maintainers + keyring_id_cache = {} #: cache for keyrings + source_id_cache = {} #: cache for sources + files_id_cache = {} #: cache for files + maintainer_cache = {} #: cache for maintainer names + fingerprint_id_cache = {} #: cache for fingerprints + queue_id_cache = {} #: cache for queues + uid_id_cache = {} #: cache for uids + suite_version_cache = {} #: cache for suite_versions (packages) +suite_bin_version_cache = {} +content_path_id_cache = {} +content_file_id_cache = {} +insert_contents_file_cache = {} +cache_preloaded = False ################################################################################ @@@ -229,7 -368,25 +373,26 @@@ def get_source_id (source, version) return source_id - def get_suite_version(source, suite, arch): + def get_suite_version(source, suite): + """ + Returns database id for a combination of C{source} and C{suite}. + + - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc} + - B{suite} - a suite name, eg. I{unstable} + + Results are kept in a cache during runtime to minimize database queries. + + @type source: string + @param source: source package name + + @type suite: string + @param suite: the suite name + + @rtype: string + @return: the version for I{source} in I{suite} + + """ ++ global suite_version_cache cache_key = "%s_%s" % (source, suite) @@@ -440,65 -693,40 +743,96 @@@ def get_maintainer (maintainer_id) ################################################################################ def get_suites(pkgname, src=False): + """ + Return the suites in which C{pkgname} can be found. If C{src} is True query for source + package, else binary package. + + @type pkgname: string + @param pkgname: name of the package + + @type src: bool + @param src: if True look for source packages, false (default) looks for binary. + + @rtype: list + @return: list of suites, or empty list if no match + + """ if src: - sql = "select suite_name from source, src_associations,suite where source.id=src_associations.source and source.source='%s' and src_associations.suite = suite.id"%pkgname + sql = """ + SELECT suite_name + FROM source, + src_associations, + suite + WHERE source.id = src_associations.source + AND source.source = '%s' + AND src_associations.suite = suite.id + """ % (pkgname) else: - sql = "select suite_name from binaries, bin_associations,suite where binaries.id=bin_associations.bin and package='%s' and bin_associations.suite = suite.id"%pkgname + sql = """ + SELECT suite_name + FROM binaries, + bin_associations, + suite + WHERE binaries.id = bin_associations.bin + AND package = '%s' + AND bin_associations.suite = suite.id + """ % (pkgname) + q = projectB.query(sql) return map(lambda x: x[0], q.getresult()) + +################################################################################ + +def get_or_set_contents_file_id(file): + global content_file_id_cache + + if not content_file_id_cache.has_key(file): + sql_select = "SELECT id FROM content_file_names WHERE file = '%s'" % file + q = projectB.query(sql_select) + if not q.getresult(): + # since this can be called within a transaction, we can't use currval + q = projectB.query("INSERT INTO content_file_names VALUES (DEFAULT, '%s') RETURNING id" % (file)) + content_file_id_cache[file] = int(q.getresult()[0][0]) + return content_file_id_cache[file] + +################################################################################ + +def get_or_set_contents_path_id(path): + global content_path_id_cache + + if not content_path_id_cache.has_key(path): + sql_select = "SELECT id FROM content_file_paths WHERE path = '%s'" % path + q = projectB.query(sql_select) + if not q.getresult(): + # since this can be called within a transaction, we can't use currval + q = projectB.query("INSERT INTO content_file_paths VALUES (DEFAULT, '%s') RETURNING id" % (path)) + content_path_id_cache[path] = int(q.getresult()[0][0]) + return content_path_id_cache[path] + +################################################################################ + +def insert_content_path(bin_id, fullpath): + global insert_contents_file_cache + cache_key = "%s_%s" % (bin_id, fullpath) + + # have we seen this contents before? + # probably only revelant during package import + if insert_contents_file_cache.has_key(cache_key): + return + + # split the path into basename, and pathname + (path, file) = os.path.split(fullpath) + + # Get the necessary IDs ... + file_id = get_or_set_contents_file_id(file) + path_id = get_or_set_contents_path_id(path) + + # Determine if we're inserting a duplicate row + q = projectB.query("SELECT 1 FROM content_associations WHERE binary_pkg = '%d' AND filepath = '%d' AND filename = '%d'" % (int(bin_id), path_id, file_id)) + if q.getresult(): + # Yes we are, return without doing the insert + return + + # Put them into content_assiocations + projectB.query("INSERT INTO content_associations VALUES (DEFAULT, '%d', '%d', '%d')" % (bin_id, path_id, file_id)) + return