X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fimport_archive.py;h=34dc61bc66664cb817ee73c9b6be0920e8438e11;hb=6c92853af8a7ca65fad19cb0f1a0ffaecec48fb7;hp=a23abb706f51803a34e4967c24d1c1b086825e08;hpb=59fd5aa2a8be3b76dbc968429c457f096adfa472;p=dak.git diff --git a/dak/import_archive.py b/dak/import_archive.py index a23abb70..34dc61bc 100755 --- a/dak/import_archive.py +++ b/dak/import_archive.py @@ -1,8 +1,7 @@ #!/usr/bin/env python # Populate the DB -# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: neve,v 1.20 2004-06-17 14:59:57 troup Exp $ +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,9 +36,10 @@ ############################################################################### -import commands, os, pg, re, sys, time; -import apt_pkg; -import db_access, utils; +import commands, os, pg, re, sys, time +import apt_pkg +import daklib.database +import daklib.utils ############################################################################### @@ -47,36 +47,36 @@ re_arch_from_filename = re.compile(r"binary-[^/]+") ############################################################################### -Cnf = None; -projectB = None; -files_id_cache = {}; -source_cache = {}; -arch_all_cache = {}; -binary_cache = {}; -location_path_cache = {}; +Cnf = None +projectB = None +files_id_cache = {} +source_cache = {} +arch_all_cache = {} +binary_cache = {} +location_path_cache = {} # -files_id_serial = 0; -source_id_serial = 0; -src_associations_id_serial = 0; -dsc_files_id_serial = 0; -files_query_cache = None; -source_query_cache = None; -src_associations_query_cache = None; -dsc_files_query_cache = None; -orig_tar_gz_cache = {}; +files_id_serial = 0 +source_id_serial = 0 +src_associations_id_serial = 0 +dsc_files_id_serial = 0 +files_query_cache = None +source_query_cache = None +src_associations_query_cache = None +dsc_files_query_cache = None +orig_tar_gz_cache = {} # -binaries_id_serial = 0; -binaries_query_cache = None; -bin_associations_id_serial = 0; -bin_associations_query_cache = None; +binaries_id_serial = 0 +binaries_query_cache = None +bin_associations_id_serial = 0 +bin_associations_query_cache = None # -source_cache_for_binaries = {}; -reject_message = ""; +source_cache_for_binaries = {} +reject_message = "" ################################################################################ def usage(exit_code=0): - print """Usage: neve + print """Usage: dak import-archive Initializes a projectB database from an existing archive -a, --action actually perform the initalization @@ -86,124 +86,124 @@ Initializes a projectB database from an existing archive ############################################################################### def reject (str, prefix="Rejected: "): - global reject_message; + global reject_message if str: - reject_message += prefix + str + "\n"; + reject_message += prefix + str + "\n" ############################################################################### def check_signature (filename): - if not utils.re_taint_free.match(os.path.basename(filename)): - reject("!!WARNING!! tainted filename: '%s'." % (filename)); - return None; + if not daklib.utils.re_taint_free.match(os.path.basename(filename)): + reject("!!WARNING!! tainted filename: '%s'." % (filename)) + return None - status_read, status_write = os.pipe(); - cmd = "gpgv --status-fd %s --keyring %s --keyring %s %s" \ - % (status_write, Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename); - (output, status, exit_status) = utils.gpgv_get_status_output(cmd, status_read, status_write); + status_read, status_write = os.pipe() + cmd = "gpgv --status-fd %s %s %s" \ + % (status_write, daklib.utils.gpg_keyring_args(), filename) + (output, status, exit_status) = daklib.utils.gpgv_get_status_output(cmd, status_read, status_write) # Process the status-fd output - keywords = {}; - bad = internal_error = ""; + keywords = {} + bad = internal_error = "" for line in status.split('\n'): - line = line.strip(); + line = line.strip() if line == "": - continue; - split = line.split(); + continue + split = line.split() if len(split) < 2: - internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line); - continue; - (gnupg, keyword) = split[:2]; + internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line) + continue + (gnupg, keyword) = split[:2] if gnupg != "[GNUPG:]": - internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg); - continue; - args = split[2:]; + internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg) + continue + args = split[2:] if keywords.has_key(keyword) and keyword != "NODATA" and keyword != "SIGEXPIRED": - internal_error += "found duplicate status token ('%s').\n" % (keyword); - continue; + internal_error += "found duplicate status token ('%s').\n" % (keyword) + continue else: - keywords[keyword] = args; + keywords[keyword] = args # If we failed to parse the status-fd output, let's just whine and bail now if internal_error: - reject("internal error while performing signature check on %s." % (filename)); - reject(internal_error, ""); - reject("Please report the above errors to the Archive maintainers by replying to this mail.", ""); - return None; + reject("internal error while performing signature check on %s." % (filename)) + reject(internal_error, "") + reject("Please report the above errors to the Archive maintainers by replying to this mail.", "") + return None # Now check for obviously bad things in the processed output if keywords.has_key("SIGEXPIRED"): - utils.warn("%s: signing key has expired." % (filename)); + daklib.utils.warn("%s: signing key has expired." % (filename)) if keywords.has_key("KEYREVOKED"): - reject("key used to sign %s has been revoked." % (filename)); - bad = 1; + reject("key used to sign %s has been revoked." % (filename)) + bad = 1 if keywords.has_key("BADSIG"): - reject("bad signature on %s." % (filename)); - bad = 1; + reject("bad signature on %s." % (filename)) + bad = 1 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"): - reject("failed to check signature on %s." % (filename)); - bad = 1; + reject("failed to check signature on %s." % (filename)) + bad = 1 if keywords.has_key("NO_PUBKEY"): - args = keywords["NO_PUBKEY"]; + args = keywords["NO_PUBKEY"] if len(args) < 1: - reject("internal error while checking signature on %s." % (filename)); - bad = 1; + reject("internal error while checking signature on %s." % (filename)) + bad = 1 else: - fingerprint = args[0]; + fingerprint = args[0] if keywords.has_key("BADARMOR"): - reject("ascii armour of signature was corrupt in %s." % (filename)); - bad = 1; + reject("ascii armour of signature was corrupt in %s." % (filename)) + bad = 1 if keywords.has_key("NODATA"): - utils.warn("no signature found for %s." % (filename)); - return "NOSIG"; - #reject("no signature found in %s." % (filename)); - #bad = 1; + daklib.utils.warn("no signature found for %s." % (filename)) + return "NOSIG" + #reject("no signature found in %s." % (filename)) + #bad = 1 if bad: - return None; + return None # Next check gpgv exited with a zero return code if exit_status and not keywords.has_key("NO_PUBKEY"): - reject("gpgv failed while checking %s." % (filename)); + reject("gpgv failed while checking %s." % (filename)) if status.strip(): - reject(utils.prefix_multi_line_string(status, " [GPG status-fd output:] "), ""); + reject(daklib.utils.prefix_multi_line_string(status, " [GPG status-fd output:] "), "") else: - reject(utils.prefix_multi_line_string(output, " [GPG output:] "), ""); - return None; + reject(daklib.utils.prefix_multi_line_string(output, " [GPG output:] "), "") + return None # Sanity check the good stuff we expect if not keywords.has_key("VALIDSIG"): if not keywords.has_key("NO_PUBKEY"): - reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename)); - bad = 1; + reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename)) + bad = 1 else: - args = keywords["VALIDSIG"]; + args = keywords["VALIDSIG"] if len(args) < 1: - reject("internal error while checking signature on %s." % (filename)); - bad = 1; + reject("internal error while checking signature on %s." % (filename)) + bad = 1 else: - fingerprint = args[0]; + fingerprint = args[0] if not keywords.has_key("GOODSIG") and not keywords.has_key("NO_PUBKEY"): - reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename)); - bad = 1; + reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename)) + bad = 1 if not keywords.has_key("SIG_ID") and not keywords.has_key("NO_PUBKEY"): - reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename)); - bad = 1; + reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename)) + bad = 1 # Finally ensure there's not something we don't recognise - known_keywords = utils.Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", + known_keywords = daklib.utils.Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="", - NODATA=""); + NODATA="") for keyword in keywords.keys(): if not known_keywords.has_key(keyword): - reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], filename)); - bad = 1; + reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], filename)) + bad = 1 if bad: - return None; + return None else: - return fingerprint; + return fingerprint ################################################################################ @@ -211,36 +211,36 @@ def check_signature (filename): def poolify (s, sub): for i in xrange(len(sub)): if sub[i:] == s[0:len(sub)-i]: - return s[len(sub)-i:]; - return s; + return s[len(sub)-i:] + return s def update_archives (): projectB.query("DELETE FROM archive") for archive in Cnf.SubTree("Archive").List(): - SubSec = Cnf.SubTree("Archive::%s" % (archive)); + SubSec = Cnf.SubTree("Archive::%s" % (archive)) projectB.query("INSERT INTO archive (name, origin_server, description) VALUES ('%s', '%s', '%s')" - % (archive, SubSec["OriginServer"], SubSec["Description"])); + % (archive, SubSec["OriginServer"], SubSec["Description"])) def update_components (): projectB.query("DELETE FROM component") for component in Cnf.SubTree("Component").List(): - SubSec = Cnf.SubTree("Component::%s" % (component)); + SubSec = Cnf.SubTree("Component::%s" % (component)) projectB.query("INSERT INTO component (name, description, meets_dfsg) VALUES ('%s', '%s', '%s')" % - (component, SubSec["Description"], SubSec["MeetsDFSG"])); + (component, SubSec["Description"], SubSec["MeetsDFSG"])) def update_locations (): projectB.query("DELETE FROM location") for location in Cnf.SubTree("Location").List(): - SubSec = Cnf.SubTree("Location::%s" % (location)); - archive_id = db_access.get_archive_id(SubSec["archive"]); - type = SubSec.Find("type"); + SubSec = Cnf.SubTree("Location::%s" % (location)) + archive_id = daklib.database.get_archive_id(SubSec["archive"]) + type = SubSec.Find("type") if type == "legacy-mixed": - projectB.query("INSERT INTO location (path, archive, type) VALUES ('%s', %d, '%s')" % (location, archive_id, SubSec["type"])); + projectB.query("INSERT INTO location (path, archive, type) VALUES ('%s', %d, '%s')" % (location, archive_id, SubSec["type"])) else: for component in Cnf.SubTree("Component").List(): - component_id = db_access.get_component_id(component); + component_id = daklib.database.get_component_id(component) projectB.query("INSERT INTO location (path, component, archive, type) VALUES ('%s', %d, %d, '%s')" % - (location, component_id, archive_id, SubSec["type"])); + (location, component_id, archive_id, SubSec["type"])) def update_architectures (): projectB.query("DELETE FROM architecture") @@ -251,66 +251,66 @@ def update_suites (): projectB.query("DELETE FROM suite") for suite in Cnf.SubTree("Suite").List(): SubSec = Cnf.SubTree("Suite::%s" %(suite)) - projectB.query("INSERT INTO suite (suite_name) VALUES ('%s')" % suite.lower()); + projectB.query("INSERT INTO suite (suite_name) VALUES ('%s')" % suite.lower()) for i in ("Version", "Origin", "Description"): if SubSec.has_key(i): projectB.query("UPDATE suite SET %s = '%s' WHERE suite_name = '%s'" % (i.lower(), SubSec[i], suite.lower())) for architecture in Cnf.ValueList("Suite::%s::Architectures" % (suite)): - architecture_id = db_access.get_architecture_id (architecture); - projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id)); + architecture_id = daklib.database.get_architecture_id (architecture) + projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id)) def update_override_type(): - projectB.query("DELETE FROM override_type"); + projectB.query("DELETE FROM override_type") for type in Cnf.ValueList("OverrideType"): - projectB.query("INSERT INTO override_type (type) VALUES ('%s')" % (type)); + projectB.query("INSERT INTO override_type (type) VALUES ('%s')" % (type)) def update_priority(): - projectB.query("DELETE FROM priority"); + projectB.query("DELETE FROM priority") for priority in Cnf.SubTree("Priority").List(): - projectB.query("INSERT INTO priority (priority, level) VALUES ('%s', %s)" % (priority, Cnf["Priority::%s" % (priority)])); + projectB.query("INSERT INTO priority (priority, level) VALUES ('%s', %s)" % (priority, Cnf["Priority::%s" % (priority)])) def update_section(): - projectB.query("DELETE FROM section"); + projectB.query("DELETE FROM section") for component in Cnf.SubTree("Component").List(): - if Cnf["Natalie::ComponentPosition"] == "prefix": - suffix = ""; + if Cnf["Control-Overrides::ComponentPosition"] == "prefix": + suffix = "" if component != 'main': - prefix = component + '/'; + prefix = component + '/' else: - prefix = ""; + prefix = "" else: - prefix = ""; - component = component.replace("non-US/", ""); + prefix = "" + component = component.replace("non-US/", "") if component != 'main': - suffix = '/' + component; + suffix = '/' + component else: - suffix = ""; + suffix = "" for section in Cnf.ValueList("Section"): - projectB.query("INSERT INTO section (section) VALUES ('%s%s%s')" % (prefix, section, suffix)); + projectB.query("INSERT INTO section (section) VALUES ('%s%s%s')" % (prefix, section, suffix)) def get_location_path(directory): - global location_path_cache; + global location_path_cache if location_path_cache.has_key(directory): - return location_path_cache[directory]; + return location_path_cache[directory] - q = projectB.query("SELECT DISTINCT path FROM location WHERE path ~ '%s'" % (directory)); + q = projectB.query("SELECT DISTINCT path FROM location WHERE path ~ '%s'" % (directory)) try: - path = q.getresult()[0][0]; + path = q.getresult()[0][0] except: - utils.fubar("[neve] get_location_path(): Couldn't get path for %s" % (directory)); - location_path_cache[directory] = path; - return path; + daklib.utils.fubar("[import-archive] get_location_path(): Couldn't get path for %s" % (directory)) + location_path_cache[directory] = path + return path ################################################################################ def get_or_set_files_id (filename, size, md5sum, location_id): - global files_id_cache, files_id_serial, files_query_cache; + global files_id_cache, files_id_serial, files_query_cache - cache_key = "~".join((filename, size, md5sum, repr(location_id))); + cache_key = "_".join((filename, size, md5sum, repr(location_id))) if not files_id_cache.has_key(cache_key): files_id_serial += 1 - files_query_cache.write("%d\t%s\t%s\t%s\t%d\t\\N\n" % (files_id_serial, filename, size, md5sum, location_id)); + files_query_cache.write("%d\t%s\t%s\t%s\t%d\t\\N\n" % (files_id_serial, filename, size, md5sum, location_id)) files_id_cache[cache_key] = files_id_serial return files_id_cache[cache_key] @@ -318,300 +318,300 @@ def get_or_set_files_id (filename, size, md5sum, location_id): ############################################################################### def process_sources (filename, suite, component, archive): - global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache, reject_message; + global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache, reject_message - suite = suite.lower(); - suite_id = db_access.get_suite_id(suite); + suite = suite.lower() + suite_id = daklib.database.get_suite_id(suite) try: - file = utils.open_file (filename); - except utils.cant_open_exc: - utils.warn("can't open '%s'" % (filename)); - return; - Scanner = apt_pkg.ParseTagFile(file); + file = daklib.utils.open_file (filename) + except daklib.utils.cant_open_exc: + daklib.utils.warn("can't open '%s'" % (filename)) + return + Scanner = apt_pkg.ParseTagFile(file) while Scanner.Step() != 0: - package = Scanner.Section["package"]; - version = Scanner.Section["version"]; - directory = Scanner.Section["directory"]; - dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))); + package = Scanner.Section["package"] + version = Scanner.Section["version"] + directory = Scanner.Section["directory"] + dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, daklib.utils.re_no_epoch.sub('', version))) # Sometimes the Directory path is a lie; check in the pool if not os.path.exists(dsc_file): if directory.split('/')[0] == "dists": - directory = Cnf["Dir::PoolRoot"] + utils.poolify(package, component); - dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))); + directory = Cnf["Dir::PoolRoot"] + daklib.utils.poolify(package, component) + dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, daklib.utils.re_no_epoch.sub('', version))) if not os.path.exists(dsc_file): - utils.fubar("%s not found." % (dsc_file)); - install_date = time.strftime("%Y-%m-%d", time.localtime(os.path.getmtime(dsc_file))); - fingerprint = check_signature(dsc_file); - fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint); + daklib.utils.fubar("%s not found." % (dsc_file)) + install_date = time.strftime("%Y-%m-%d", time.localtime(os.path.getmtime(dsc_file))) + fingerprint = check_signature(dsc_file) + fingerprint_id = daklib.database.get_or_set_fingerprint_id(fingerprint) if reject_message: - utils.fubar("%s: %s" % (dsc_file, reject_message)); + daklib.utils.fubar("%s: %s" % (dsc_file, reject_message)) maintainer = Scanner.Section["maintainer"] - maintainer = maintainer.replace("'", "\\'"); - maintainer_id = db_access.get_or_set_maintainer_id(maintainer); - location = get_location_path(directory.split('/')[0]); - location_id = db_access.get_location_id (location, component, archive); + maintainer = maintainer.replace("'", "\\'") + maintainer_id = daklib.database.get_or_set_maintainer_id(maintainer) + location = get_location_path(directory.split('/')[0]) + location_id = daklib.database.get_location_id (location, component, archive) if not directory.endswith("/"): - directory += '/'; - directory = poolify (directory, location); + directory += '/' + directory = poolify (directory, location) if directory != "" and not directory.endswith("/"): - directory += '/'; - no_epoch_version = utils.re_no_epoch.sub('', version); + directory += '/' + no_epoch_version = daklib.utils.re_no_epoch.sub('', version) # Add all files referenced by the .dsc to the files table - ids = []; + ids = [] for line in Scanner.Section["files"].split('\n'): - id = None; - (md5sum, size, filename) = line.strip().split(); + id = None + (md5sum, size, filename) = line.strip().split() # Don't duplicate .orig.tar.gz's if filename.endswith(".orig.tar.gz"): - cache_key = "%s~%s~%s" % (filename, size, md5sum); + cache_key = "%s_%s_%s" % (filename, size, md5sum) if orig_tar_gz_cache.has_key(cache_key): - id = orig_tar_gz_cache[cache_key]; + id = orig_tar_gz_cache[cache_key] else: - id = get_or_set_files_id (directory + filename, size, md5sum, location_id); - orig_tar_gz_cache[cache_key] = id; + id = get_or_set_files_id (directory + filename, size, md5sum, location_id) + orig_tar_gz_cache[cache_key] = id else: - id = get_or_set_files_id (directory + filename, size, md5sum, location_id); - ids.append(id); + id = get_or_set_files_id (directory + filename, size, md5sum, location_id) + ids.append(id) # If this is the .dsc itself; save the ID for later. if filename.endswith(".dsc"): - files_id = id; + files_id = id filename = directory + package + '_' + no_epoch_version + '.dsc' - cache_key = "%s~%s" % (package, version); + cache_key = "%s_%s" % (package, version) if not source_cache.has_key(cache_key): - nasty_key = "%s~%s" % (package, version) - source_id_serial += 1; + nasty_key = "%s_%s" % (package, version) + source_id_serial += 1 if not source_cache_for_binaries.has_key(nasty_key): - source_cache_for_binaries[nasty_key] = source_id_serial; - tmp_source_id = source_id_serial; - source_cache[cache_key] = source_id_serial; + source_cache_for_binaries[nasty_key] = source_id_serial + tmp_source_id = source_id_serial + source_cache[cache_key] = source_id_serial source_query_cache.write("%d\t%s\t%s\t%d\t%d\t%s\t%s\n" % (source_id_serial, package, version, maintainer_id, files_id, install_date, fingerprint_id)) for id in ids: - dsc_files_id_serial += 1; - dsc_files_query_cache.write("%d\t%d\t%d\n" % (dsc_files_id_serial, tmp_source_id,id)); + dsc_files_id_serial += 1 + dsc_files_query_cache.write("%d\t%d\t%d\n" % (dsc_files_id_serial, tmp_source_id,id)) else: - tmp_source_id = source_cache[cache_key]; + tmp_source_id = source_cache[cache_key] - src_associations_id_serial += 1; + src_associations_id_serial += 1 src_associations_query_cache.write("%d\t%d\t%d\n" % (src_associations_id_serial, suite_id, tmp_source_id)) - file.close(); + file.close() ############################################################################### def process_packages (filename, suite, component, archive): - global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache, reject_message; + global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache, reject_message - count_total = 0; - count_bad = 0; - suite = suite.lower(); - suite_id = db_access.get_suite_id(suite); + count_total = 0 + count_bad = 0 + suite = suite.lower() + suite_id = daklib.database.get_suite_id(suite) try: - file = utils.open_file (filename); - except utils.cant_open_exc: - utils.warn("can't open '%s'" % (filename)); - return; - Scanner = apt_pkg.ParseTagFile(file); + file = daklib.utils.open_file (filename) + except daklib.utils.cant_open_exc: + daklib.utils.warn("can't open '%s'" % (filename)) + return + Scanner = apt_pkg.ParseTagFile(file) while Scanner.Step() != 0: package = Scanner.Section["package"] version = Scanner.Section["version"] maintainer = Scanner.Section["maintainer"] maintainer = maintainer.replace("'", "\\'") - maintainer_id = db_access.get_or_set_maintainer_id(maintainer); + maintainer_id = daklib.database.get_or_set_maintainer_id(maintainer) architecture = Scanner.Section["architecture"] - architecture_id = db_access.get_architecture_id (architecture); - fingerprint = "NOSIG"; - fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint); + architecture_id = daklib.database.get_architecture_id (architecture) + fingerprint = "NOSIG" + fingerprint_id = daklib.database.get_or_set_fingerprint_id(fingerprint) if not Scanner.Section.has_key("source"): source = package else: source = Scanner.Section["source"] source_version = "" if source.find("(") != -1: - m = utils.re_extract_src_version.match(source) + m = daklib.utils.re_extract_src_version.match(source) source = m.group(1) source_version = m.group(2) if not source_version: source_version = version filename = Scanner.Section["filename"] - location = get_location_path(filename.split('/')[0]); - location_id = db_access.get_location_id (location, component, archive) + location = get_location_path(filename.split('/')[0]) + location_id = daklib.database.get_location_id (location, component, archive) filename = poolify (filename, location) if architecture == "all": - filename = re_arch_from_filename.sub("binary-all", filename); - cache_key = "%s~%s" % (source, source_version); - source_id = source_cache_for_binaries.get(cache_key, None); - size = Scanner.Section["size"]; - md5sum = Scanner.Section["md5sum"]; - files_id = get_or_set_files_id (filename, size, md5sum, location_id); + filename = re_arch_from_filename.sub("binary-all", filename) + cache_key = "%s_%s" % (source, source_version) + source_id = source_cache_for_binaries.get(cache_key, None) + size = Scanner.Section["size"] + md5sum = Scanner.Section["md5sum"] + files_id = get_or_set_files_id (filename, size, md5sum, location_id) type = "deb"; # FIXME - cache_key = "%s~%s~%s~%d~%d~%d~%d" % (package, version, repr(source_id), architecture_id, location_id, files_id, suite_id); + cache_key = "%s_%s_%s_%d_%d_%d_%d" % (package, version, repr(source_id), architecture_id, location_id, files_id, suite_id) if not arch_all_cache.has_key(cache_key): - arch_all_cache[cache_key] = 1; - cache_key = "%s~%s~%s~%d" % (package, version, repr(source_id), architecture_id); + arch_all_cache[cache_key] = 1 + cache_key = "%s_%s_%s_%d" % (package, version, repr(source_id), architecture_id) if not binary_cache.has_key(cache_key): if not source_id: - source_id = "\N"; - count_bad += 1; + source_id = "\N" + count_bad += 1 else: - source_id = repr(source_id); - binaries_id_serial += 1; - binaries_query_cache.write("%d\t%s\t%s\t%d\t%s\t%d\t%d\t%s\t%s\n" % (binaries_id_serial, package, version, maintainer_id, source_id, architecture_id, files_id, type, fingerprint_id)); - binary_cache[cache_key] = binaries_id_serial; - tmp_binaries_id = binaries_id_serial; + source_id = repr(source_id) + binaries_id_serial += 1 + binaries_query_cache.write("%d\t%s\t%s\t%d\t%s\t%d\t%d\t%s\t%s\n" % (binaries_id_serial, package, version, maintainer_id, source_id, architecture_id, files_id, type, fingerprint_id)) + binary_cache[cache_key] = binaries_id_serial + tmp_binaries_id = binaries_id_serial else: - tmp_binaries_id = binary_cache[cache_key]; + tmp_binaries_id = binary_cache[cache_key] - bin_associations_id_serial += 1; - bin_associations_query_cache.write("%d\t%d\t%d\n" % (bin_associations_id_serial, suite_id, tmp_binaries_id)); - count_total += 1; + bin_associations_id_serial += 1 + bin_associations_query_cache.write("%d\t%d\t%d\n" % (bin_associations_id_serial, suite_id, tmp_binaries_id)) + count_total += 1 - file.close(); + file.close() if count_bad != 0: - print "%d binary packages processed; %d with no source match which is %.2f%%" % (count_total, count_bad, (float(count_bad)/count_total)*100); + print "%d binary packages processed; %d with no source match which is %.2f%%" % (count_total, count_bad, (float(count_bad)/count_total)*100) else: - print "%d binary packages processed; 0 with no source match which is 0%%" % (count_total); + print "%d binary packages processed; 0 with no source match which is 0%%" % (count_total) ############################################################################### def do_sources(sources, suite, component, server): - temp_filename = utils.temp_filename(); - (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (sources, temp_filename)); + temp_filename = daklib.utils.temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (sources, temp_filename)) if (result != 0): - utils.fubar("Gunzip invocation failed!\n%s" % (output), result); - print 'Processing '+sources+'...'; - process_sources (temp_filename, suite, component, server); - os.unlink(temp_filename); + daklib.utils.fubar("Gunzip invocation failed!\n%s" % (output), result) + print 'Processing '+sources+'...' + process_sources (temp_filename, suite, component, server) + os.unlink(temp_filename) ############################################################################### def do_da_do_da (): - global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache; + global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache - Cnf = utils.get_conf(); - Arguments = [('a', "action", "Neve::Options::Action"), - ('h', "help", "Neve::Options::Help")]; + Cnf = daklib.utils.get_conf() + Arguments = [('a', "action", "Import-Archive::Options::Action"), + ('h', "help", "Import-Archive::Options::Help")] for i in [ "action", "help" ]: - if not Cnf.has_key("Neve::Options::%s" % (i)): - Cnf["Neve::Options::%s" % (i)] = ""; + if not Cnf.has_key("Import-Archive::Options::%s" % (i)): + Cnf["Import-Archive::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) - Options = Cnf.SubTree("Neve::Options") + Options = Cnf.SubTree("Import-Archive::Options") if Options["Help"]: - usage(); + usage() if not Options["Action"]: - utils.warn("""no -a/--action given; not doing anything. + daklib.utils.warn("""no -a/--action given; not doing anything. Please read the documentation before running this script. -"""); - usage(1); +""") + usage(1) print "Re-Creating DB..." - (result, output) = commands.getstatusoutput("psql -f init_pool.sql template1"); + (result, output) = commands.getstatusoutput("psql -f init_pool.sql template1") if (result != 0): - utils.fubar("psql invocation failed!\n", result); - print output; + daklib.utils.fubar("psql invocation failed!\n", result) + print output - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) - db_access.init (Cnf, projectB); + daklib.database.init (Cnf, projectB) print "Adding static tables from conf file..." - projectB.query("BEGIN WORK"); - update_architectures(); - update_components(); - update_archives(); - update_locations(); - update_suites(); - update_override_type(); - update_priority(); - update_section(); - projectB.query("COMMIT WORK"); - - files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"files","w"); - source_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"source","w"); - src_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"src_associations","w"); - dsc_files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"dsc_files","w"); - binaries_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"binaries","w"); - bin_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"bin_associations","w"); - - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") + update_architectures() + update_components() + update_archives() + update_locations() + update_suites() + update_override_type() + update_priority() + update_section() + projectB.query("COMMIT WORK") + + files_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"files","w") + source_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"source","w") + src_associations_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"src_associations","w") + dsc_files_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"dsc_files","w") + binaries_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"binaries","w") + bin_associations_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"bin_associations","w") + + projectB.query("BEGIN WORK") # Process Sources files to popoulate `source' and friends for location in Cnf.SubTree("Location").List(): - SubSec = Cnf.SubTree("Location::%s" % (location)); - server = SubSec["Archive"]; - type = Cnf.Find("Location::%s::Type" % (location)); + SubSec = Cnf.SubTree("Location::%s" % (location)) + server = SubSec["Archive"] + type = Cnf.Find("Location::%s::Type" % (location)) if type == "legacy-mixed": - sources = location + 'Sources.gz'; - suite = Cnf.Find("Location::%s::Suite" % (location)); - do_sources(sources, suite, "", server); + sources = location + 'Sources.gz' + suite = Cnf.Find("Location::%s::Suite" % (location)) + do_sources(sources, suite, "", server) elif type == "legacy" or type == "pool": for suite in Cnf.ValueList("Location::%s::Suites" % (location)): for component in Cnf.SubTree("Component").List(): - sources = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/source/' + 'Sources.gz'; - do_sources(sources, suite, component, server); + sources = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/source/' + 'Sources.gz' + do_sources(sources, suite, component, server) else: - utils.fubar("Unknown location type ('%s')." % (type)); + daklib.utils.fubar("Unknown location type ('%s')." % (type)) # Process Packages files to populate `binaries' and friends for location in Cnf.SubTree("Location").List(): - SubSec = Cnf.SubTree("Location::%s" % (location)); - server = SubSec["Archive"]; - type = Cnf.Find("Location::%s::Type" % (location)); + SubSec = Cnf.SubTree("Location::%s" % (location)) + server = SubSec["Archive"] + type = Cnf.Find("Location::%s::Type" % (location)) if type == "legacy-mixed": - packages = location + 'Packages'; - suite = Cnf.Find("Location::%s::Suite" % (location)); - print 'Processing '+location+'...'; - process_packages (packages, suite, "", server); + packages = location + 'Packages' + suite = Cnf.Find("Location::%s::Suite" % (location)) + print 'Processing '+location+'...' + process_packages (packages, suite, "", server) elif type == "legacy" or type == "pool": for suite in Cnf.ValueList("Location::%s::Suites" % (location)): for component in Cnf.SubTree("Component").List(): - architectures = filter(utils.real_arch, - Cnf.ValueList("Suite::%s::Architectures" % (suite))); + architectures = filter(daklib.utils.real_arch, + Cnf.ValueList("Suite::%s::Architectures" % (suite))) for architecture in architectures: packages = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/binary-' + architecture + '/Packages' - print 'Processing '+packages+'...'; - process_packages (packages, suite, component, server); - - files_query_cache.close(); - source_query_cache.close(); - src_associations_query_cache.close(); - dsc_files_query_cache.close(); - binaries_query_cache.close(); - bin_associations_query_cache.close(); - print "Writing data to `files' table..."; - projectB.query("COPY files FROM '%s'" % (Cnf["Neve::ExportDir"]+"files")); - print "Writing data to `source' table..."; - projectB.query("COPY source FROM '%s'" % (Cnf["Neve::ExportDir"]+"source")); - print "Writing data to `src_associations' table..."; - projectB.query("COPY src_associations FROM '%s'" % (Cnf["Neve::ExportDir"]+"src_associations")); - print "Writing data to `dsc_files' table..."; - projectB.query("COPY dsc_files FROM '%s'" % (Cnf["Neve::ExportDir"]+"dsc_files")); - print "Writing data to `binaries' table..."; - projectB.query("COPY binaries FROM '%s'" % (Cnf["Neve::ExportDir"]+"binaries")); - print "Writing data to `bin_associations' table..."; - projectB.query("COPY bin_associations FROM '%s'" % (Cnf["Neve::ExportDir"]+"bin_associations")); - print "Committing..."; - projectB.query("COMMIT WORK"); + print 'Processing '+packages+'...' + process_packages (packages, suite, component, server) + + files_query_cache.close() + source_query_cache.close() + src_associations_query_cache.close() + dsc_files_query_cache.close() + binaries_query_cache.close() + bin_associations_query_cache.close() + print "Writing data to `files' table..." + projectB.query("COPY files FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"files")) + print "Writing data to `source' table..." + projectB.query("COPY source FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"source")) + print "Writing data to `src_associations' table..." + projectB.query("COPY src_associations FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"src_associations")) + print "Writing data to `dsc_files' table..." + projectB.query("COPY dsc_files FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"dsc_files")) + print "Writing data to `binaries' table..." + projectB.query("COPY binaries FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"binaries")) + print "Writing data to `bin_associations' table..." + projectB.query("COPY bin_associations FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"bin_associations")) + print "Committing..." + projectB.query("COMMIT WORK") # Add the constraints and otherwise generally clean up the database. # See add_constraints.sql for more details... - print "Running add_constraints.sql..."; - (result, output) = commands.getstatusoutput("psql %s < add_constraints.sql" % (Cnf["DB::Name"])); + print "Running add_constraints.sql..." + (result, output) = commands.getstatusoutput("psql %s < add_constraints.sql" % (Cnf["DB::Name"])) print output if (result != 0): - utils.fubar("psql invocation failed!\n%s" % (output), result); + daklib.utils.fubar("psql invocation failed!\n%s" % (output), result) - return; + return ################################################################################ def main(): - utils.try_with_debug(do_da_do_da); + daklib.utils.try_with_debug(do_da_do_da) ################################################################################ if __name__ == '__main__': - main(); + main()