X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fimport_archive.py;h=4432ff1ac602c6b8008a2efcff154cbd9e9a3d23;hb=f8996e240d9d0278bce098e23be63db0bcc6fbee;hp=86bcbec510c05ee633b9c7860fae84843ad6f911;hpb=30413cf0ff7bc21b8d2b8b4346406357fe55dc19;p=dak.git diff --git a/dak/import_archive.py b/dak/import_archive.py index 86bcbec5..4432ff1a 100755 --- a/dak/import_archive.py +++ b/dak/import_archive.py @@ -1,8 +1,7 @@ #!/usr/bin/env python # Populate the DB -# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: neve,v 1.20 2004-06-17 14:59:57 troup Exp $ +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,7 +38,9 @@ import commands, os, pg, re, sys, time import apt_pkg -import db_access, utils +from daklib import database +from daklib import utils +from daklib.dak_exceptions import * ############################################################################### @@ -76,7 +77,7 @@ reject_message = "" ################################################################################ def usage(exit_code=0): - print """Usage: neve + print """Usage: dak import-archive Initializes a projectB database from an existing archive -a, --action actually perform the initalization @@ -98,8 +99,8 @@ def check_signature (filename): return None status_read, status_write = os.pipe() - cmd = "gpgv --status-fd %s --keyring %s --keyring %s %s" \ - % (status_write, Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename) + cmd = "gpgv --status-fd %s %s %s" \ + % (status_write, utils.gpg_keyring_args(), filename) (output, status, exit_status) = utils.gpgv_get_status_output(cmd, status_read, status_write) # Process the status-fd output @@ -232,13 +233,13 @@ def update_locations (): projectB.query("DELETE FROM location") for location in Cnf.SubTree("Location").List(): SubSec = Cnf.SubTree("Location::%s" % (location)) - archive_id = db_access.get_archive_id(SubSec["archive"]) + archive_id = database.get_archive_id(SubSec["archive"]) type = SubSec.Find("type") if type == "legacy-mixed": projectB.query("INSERT INTO location (path, archive, type) VALUES ('%s', %d, '%s')" % (location, archive_id, SubSec["type"])) else: for component in Cnf.SubTree("Component").List(): - component_id = db_access.get_component_id(component) + component_id = database.get_component_id(component) projectB.query("INSERT INTO location (path, component, archive, type) VALUES ('%s', %d, %d, '%s')" % (location, component_id, archive_id, SubSec["type"])) @@ -256,7 +257,7 @@ def update_suites (): if SubSec.has_key(i): projectB.query("UPDATE suite SET %s = '%s' WHERE suite_name = '%s'" % (i.lower(), SubSec[i], suite.lower())) for architecture in Cnf.ValueList("Suite::%s::Architectures" % (suite)): - architecture_id = db_access.get_architecture_id (architecture) + architecture_id = database.get_architecture_id (architecture) projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id)) def update_override_type(): @@ -272,7 +273,7 @@ def update_priority(): def update_section(): projectB.query("DELETE FROM section") for component in Cnf.SubTree("Component").List(): - if Cnf["Natalie::ComponentPosition"] == "prefix": + if Cnf["Control-Overrides::ComponentPosition"] == "prefix": suffix = "" if component != 'main': prefix = component + '/' @@ -280,7 +281,6 @@ def update_section(): prefix = "" else: prefix = "" - component = component.replace("non-US/", "") if component != 'main': suffix = '/' + component else: @@ -298,7 +298,7 @@ def get_location_path(directory): try: path = q.getresult()[0][0] except: - utils.fubar("[neve] get_location_path(): Couldn't get path for %s" % (directory)) + utils.fubar("[import-archive] get_location_path(): Couldn't get path for %s" % (directory)) location_path_cache[directory] = path return path @@ -307,7 +307,7 @@ def get_location_path(directory): def get_or_set_files_id (filename, size, md5sum, location_id): global files_id_cache, files_id_serial, files_query_cache - cache_key = "~".join((filename, size, md5sum, repr(location_id))) + cache_key = "_".join((filename, size, md5sum, repr(location_id))) if not files_id_cache.has_key(cache_key): files_id_serial += 1 files_query_cache.write("%d\t%s\t%s\t%s\t%d\t\\N\n" % (files_id_serial, filename, size, md5sum, location_id)) @@ -321,10 +321,10 @@ def process_sources (filename, suite, component, archive): global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache, reject_message suite = suite.lower() - suite_id = db_access.get_suite_id(suite) + suite_id = database.get_suite_id(suite) try: file = utils.open_file (filename) - except utils.cant_open_exc: + except CantOpenError: utils.warn("can't open '%s'" % (filename)) return Scanner = apt_pkg.ParseTagFile(file) @@ -342,14 +342,14 @@ def process_sources (filename, suite, component, archive): utils.fubar("%s not found." % (dsc_file)) install_date = time.strftime("%Y-%m-%d", time.localtime(os.path.getmtime(dsc_file))) fingerprint = check_signature(dsc_file) - fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint) + fingerprint_id = database.get_or_set_fingerprint_id(fingerprint) if reject_message: utils.fubar("%s: %s" % (dsc_file, reject_message)) maintainer = Scanner.Section["maintainer"] maintainer = maintainer.replace("'", "\\'") - maintainer_id = db_access.get_or_set_maintainer_id(maintainer) + maintainer_id = database.get_or_set_maintainer_id(maintainer) location = get_location_path(directory.split('/')[0]) - location_id = db_access.get_location_id (location, component, archive) + location_id = database.get_location_id (location, component, archive) if not directory.endswith("/"): directory += '/' directory = poolify (directory, location) @@ -363,7 +363,7 @@ def process_sources (filename, suite, component, archive): (md5sum, size, filename) = line.strip().split() # Don't duplicate .orig.tar.gz's if filename.endswith(".orig.tar.gz"): - cache_key = "%s~%s~%s" % (filename, size, md5sum) + cache_key = "%s_%s_%s" % (filename, size, md5sum) if orig_tar_gz_cache.has_key(cache_key): id = orig_tar_gz_cache[cache_key] else: @@ -376,9 +376,9 @@ def process_sources (filename, suite, component, archive): if filename.endswith(".dsc"): files_id = id filename = directory + package + '_' + no_epoch_version + '.dsc' - cache_key = "%s~%s" % (package, version) + cache_key = "%s_%s" % (package, version) if not source_cache.has_key(cache_key): - nasty_key = "%s~%s" % (package, version) + nasty_key = "%s_%s" % (package, version) source_id_serial += 1 if not source_cache_for_binaries.has_key(nasty_key): source_cache_for_binaries[nasty_key] = source_id_serial @@ -404,10 +404,10 @@ def process_packages (filename, suite, component, archive): count_total = 0 count_bad = 0 suite = suite.lower() - suite_id = db_access.get_suite_id(suite) + suite_id = database.get_suite_id(suite) try: file = utils.open_file (filename) - except utils.cant_open_exc: + except CantOpenError: utils.warn("can't open '%s'" % (filename)) return Scanner = apt_pkg.ParseTagFile(file) @@ -416,11 +416,11 @@ def process_packages (filename, suite, component, archive): version = Scanner.Section["version"] maintainer = Scanner.Section["maintainer"] maintainer = maintainer.replace("'", "\\'") - maintainer_id = db_access.get_or_set_maintainer_id(maintainer) + maintainer_id = database.get_or_set_maintainer_id(maintainer) architecture = Scanner.Section["architecture"] - architecture_id = db_access.get_architecture_id (architecture) + architecture_id = database.get_architecture_id (architecture) fingerprint = "NOSIG" - fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint) + fingerprint_id = database.get_or_set_fingerprint_id(fingerprint) if not Scanner.Section.has_key("source"): source = package else: @@ -433,21 +433,24 @@ def process_packages (filename, suite, component, archive): if not source_version: source_version = version filename = Scanner.Section["filename"] + if filename.endswith(".deb"): + type = "deb" + else: + type = "udeb" location = get_location_path(filename.split('/')[0]) - location_id = db_access.get_location_id (location, component, archive) + location_id = database.get_location_id (location, component.replace("/debian-installer", ""), archive) filename = poolify (filename, location) if architecture == "all": filename = re_arch_from_filename.sub("binary-all", filename) - cache_key = "%s~%s" % (source, source_version) + cache_key = "%s_%s" % (source, source_version) source_id = source_cache_for_binaries.get(cache_key, None) size = Scanner.Section["size"] md5sum = Scanner.Section["md5sum"] files_id = get_or_set_files_id (filename, size, md5sum, location_id) - type = "deb"; # FIXME - cache_key = "%s~%s~%s~%d~%d~%d~%d" % (package, version, repr(source_id), architecture_id, location_id, files_id, suite_id) + cache_key = "%s_%s_%s_%d_%d_%d_%d" % (package, version, repr(source_id), architecture_id, location_id, files_id, suite_id) if not arch_all_cache.has_key(cache_key): arch_all_cache[cache_key] = 1 - cache_key = "%s~%s~%s~%d" % (package, version, repr(source_id), architecture_id) + cache_key = "%s_%s_%s_%d" % (package, version, repr(source_id), architecture_id) if not binary_cache.has_key(cache_key): if not source_id: source_id = "\N" @@ -488,17 +491,17 @@ def do_da_do_da (): global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache Cnf = utils.get_conf() - Arguments = [('a', "action", "Neve::Options::Action"), - ('h', "help", "Neve::Options::Help")] + Arguments = [('a', "action", "Import-Archive::Options::Action"), + ('h', "help", "Import-Archive::Options::Help")] for i in [ "action", "help" ]: - if not Cnf.has_key("Neve::Options::%s" % (i)): - Cnf["Neve::Options::%s" % (i)] = "" + if not Cnf.has_key("Import-Archive::Options::%s" % (i)): + Cnf["Import-Archive::Options::%s" % (i)] = "" apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) - Options = Cnf.SubTree("Neve::Options") + Options = Cnf.SubTree("Import-Archive::Options") if Options["Help"]: - usage() + usage() if not Options["Action"]: utils.warn("""no -a/--action given; not doing anything. @@ -514,7 +517,7 @@ Please read the documentation before running this script. projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) - db_access.init (Cnf, projectB) + database.init (Cnf, projectB) print "Adding static tables from conf file..." projectB.query("BEGIN WORK") @@ -528,12 +531,12 @@ Please read the documentation before running this script. update_section() projectB.query("COMMIT WORK") - files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"files","w") - source_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"source","w") - src_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"src_associations","w") - dsc_files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"dsc_files","w") - binaries_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"binaries","w") - bin_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"bin_associations","w") + files_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"files","w") + source_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"source","w") + src_associations_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"src_associations","w") + dsc_files_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"dsc_files","w") + binaries_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"binaries","w") + bin_associations_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"bin_associations","w") projectB.query("BEGIN WORK") # Process Sources files to popoulate `source' and friends @@ -566,7 +569,9 @@ Please read the documentation before running this script. process_packages (packages, suite, "", server) elif type == "legacy" or type == "pool": for suite in Cnf.ValueList("Location::%s::Suites" % (location)): - for component in Cnf.SubTree("Component").List(): + udeb_components = map(lambda x: x+"/debian-installer", + Cnf.ValueList("Suite::%s::UdebComponents" % suite)) + for component in Cnf.SubTree("Component").List() + udeb_components: architectures = filter(utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite))) for architecture in architectures: @@ -581,17 +586,17 @@ Please read the documentation before running this script. binaries_query_cache.close() bin_associations_query_cache.close() print "Writing data to `files' table..." - projectB.query("COPY files FROM '%s'" % (Cnf["Neve::ExportDir"]+"files")) + projectB.query("COPY files FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"files")) print "Writing data to `source' table..." - projectB.query("COPY source FROM '%s'" % (Cnf["Neve::ExportDir"]+"source")) + projectB.query("COPY source FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"source")) print "Writing data to `src_associations' table..." - projectB.query("COPY src_associations FROM '%s'" % (Cnf["Neve::ExportDir"]+"src_associations")) + projectB.query("COPY src_associations FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"src_associations")) print "Writing data to `dsc_files' table..." - projectB.query("COPY dsc_files FROM '%s'" % (Cnf["Neve::ExportDir"]+"dsc_files")) + projectB.query("COPY dsc_files FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"dsc_files")) print "Writing data to `binaries' table..." - projectB.query("COPY binaries FROM '%s'" % (Cnf["Neve::ExportDir"]+"binaries")) + projectB.query("COPY binaries FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"binaries")) print "Writing data to `bin_associations' table..." - projectB.query("COPY bin_associations FROM '%s'" % (Cnf["Neve::ExportDir"]+"bin_associations")) + projectB.query("COPY bin_associations FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"bin_associations")) print "Committing..." projectB.query("COMMIT WORK")