X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;ds=sidebyside;f=neve;h=e206da0d7a784741d4e51ce87bae44415bb9664c;hb=5ac1ebcc6c8514198de83318de2decbc1deeec9e;hp=59758d1ca5e649501c2b83b1c3a21b8f6dd83b69;hpb=c846e77a848d60dd115f00faa0d9a854161d99eb;p=dak.git diff --git a/neve b/neve index 59758d1c..e206da0d 100755 --- a/neve +++ b/neve @@ -2,7 +2,7 @@ # Populate the DB # Copyright (C) 2000, 2001, 2002 James Troup -# $Id: neve,v 1.14 2002-10-16 02:47:32 troup Exp $ +# $Id: neve,v 1.17 2003-01-02 18:15:07 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,7 +37,7 @@ ############################################################################### -import commands, os, pg, re, select, tempfile, time; +import commands, os, pg, re, select, sys, tempfile, time; import apt_pkg; import db_access, utils; @@ -53,6 +53,7 @@ files_id_cache = {}; source_cache = {}; arch_all_cache = {}; binary_cache = {}; +location_path_cache = {}; # files_id_serial = 0; source_id_serial = 0; @@ -72,6 +73,16 @@ bin_associations_query_cache = None; source_cache_for_binaries = {}; reject_message = ""; +################################################################################ + +def usage(exit_code=0): + print """Usage: neve +Initializes a projectB database from an existing archive + + -a, --action actually perform the initalization + -h, --help show this help and exit.""" + sys.exit(exit_code) + ############################################################################### # Our very own version of commands.getouputstatus(), hacked to support @@ -258,7 +269,7 @@ def check_signature (filename): else: return fingerprint; -######################################################################################### +################################################################################ # Prepares a filename or directory (s) to be file.filename by stripping any part of the location (sub) from it. def poolify (s, sub): @@ -322,7 +333,40 @@ def update_priority(): for priority in Cnf.SubTree("Priority").List(): projectB.query("INSERT INTO priority (priority, level) VALUES ('%s', %s)" % (priority, Cnf["Priority::%s" % (priority)])); -############################################################################### +def update_section(): + projectB.query("DELETE FROM section"); + for component in Cnf.SubTree("Component").List(): + if Cnf["Natalie::ComponentPosition"] == "prefix": + suffix = ""; + if component != 'main': + prefix = component + '/'; + else: + prefix = ""; + else: + prefix = ""; + component = component.replace("non-US/", ""); + if component != 'main': + suffix = '/' + component; + else: + suffix = ""; + for section in Cnf.ValueList("Section"): + projectB.query("INSERT INTO section (section) VALUES ('%s%s%s')" % (prefix, section, suffix)); + +def get_location_path(directory): + global location_path_cache; + + if location_path_cache.has_key(directory): + return location_path_cache[directory]; + + q = projectB.query("SELECT DISTINCT path FROM location WHERE path ~ '%s'" % (directory)); + try: + path = q.getresult()[0][0]; + except: + utils.fubar("[neve] get_location_path(): Couldn't get path for %s" % (directory)); + location_path_cache[directory] = path; + return path; + +################################################################################ def get_or_set_files_id (filename, size, md5sum, location_id): global files_id_cache, files_id_serial, files_query_cache; @@ -337,7 +381,7 @@ def get_or_set_files_id (filename, size, md5sum, location_id): ############################################################################### -def process_sources (location, filename, suite, component, archive, dsc_dir): +def process_sources (filename, suite, component, archive): global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache, reject_message; suite = suite.lower(); @@ -345,13 +389,21 @@ def process_sources (location, filename, suite, component, archive, dsc_dir): try: file = utils.open_file (filename); except utils.cant_open_exc: - print "WARNING: can't open '%s'" % (filename); + utils.warn("can't open '%s'" % (filename)); return; Scanner = apt_pkg.ParseTagFile(file); while Scanner.Step() != 0: package = Scanner.Section["package"]; version = Scanner.Section["version"]; - dsc_file = os.path.join(dsc_dir, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))); + directory = Scanner.Section["directory"]; + dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))); + # Sometimes the Directory path is a lie; check in the pool + if not os.path.exists(dsc_file): + if directory.split('/')[0] == "dists": + directory = Cnf["Dir::PoolRoot"] + utils.poolify(package, component); + dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))); + if not os.path.exists(dsc_file): + utils.fubar("%s not found." % (dsc_file)); install_date = time.strftime("%Y-%m-%d", time.localtime(os.path.getmtime(dsc_file))); fingerprint = check_signature(dsc_file); fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint); @@ -360,7 +412,7 @@ def process_sources (location, filename, suite, component, archive, dsc_dir): maintainer = Scanner.Section["maintainer"] maintainer = maintainer.replace("'", "\\'"); maintainer_id = db_access.get_or_set_maintainer_id(maintainer); - directory = Scanner.Section["directory"]; + location = get_location_path(directory.split('/')[0]); location_id = db_access.get_location_id (location, component, archive); if not directory.endswith("/"): directory += '/'; @@ -410,7 +462,7 @@ def process_sources (location, filename, suite, component, archive, dsc_dir): ############################################################################### -def process_packages (location, filename, suite, component, archive): +def process_packages (filename, suite, component, archive): global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache, reject_message; count_total = 0; @@ -420,7 +472,7 @@ def process_packages (location, filename, suite, component, archive): try: file = utils.open_file (filename); except utils.cant_open_exc: - print "WARNING: can't open '%s'" % (filename); + utils.warn("can't open '%s'" % (filename)); return; Scanner = apt_pkg.ParseTagFile(file); while Scanner.Step() != 0: @@ -445,6 +497,7 @@ def process_packages (location, filename, suite, component, archive): if not source_version: source_version = version filename = Scanner.Section["filename"] + location = get_location_path(filename.split('/')[0]); location_id = db_access.get_location_id (location, component, archive) filename = poolify (filename, location) if architecture == "all": @@ -455,7 +508,7 @@ def process_packages (location, filename, suite, component, archive): md5sum = Scanner.Section["md5sum"]; files_id = get_or_set_files_id (filename, size, md5sum, location_id); type = "deb"; # FIXME - cache_key = "%s~%s~%s~%d~%d~%d" % (package, version, repr(source_id), architecture_id, location_id, files_id); + cache_key = "%s~%s~%s~%d~%d~%d~%d" % (package, version, repr(source_id), architecture_id, location_id, files_id, suite_id); if not arch_all_cache.has_key(cache_key): arch_all_cache[cache_key] = 1; cache_key = "%s~%s~%s~%d" % (package, version, repr(source_id), architecture_id); @@ -484,24 +537,40 @@ def process_packages (location, filename, suite, component, archive): ############################################################################### -def do_sources(location, prefix, suite, component, server): +def do_sources(sources, suite, component, server): temp_filename = tempfile.mktemp(); fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700); os.close(fd); - sources = location + prefix + 'Sources.gz'; (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (sources, temp_filename)); if (result != 0): utils.fubar("Gunzip invocation failed!\n%s" % (output), result); print 'Processing '+sources+'...'; - process_sources (location, temp_filename, suite, component, server, os.path.dirname(sources)); + process_sources (temp_filename, suite, component, server); os.unlink(temp_filename); ############################################################################### -def main (): +def do_da_do_da (): global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache; Cnf = utils.get_conf(); + Arguments = [('a', "action", "Neve::Options::Action"), + ('h', "help", "Neve::Options::Help")]; + for i in [ "action", "help" ]: + if not Cnf.has_key("Neve::Options::%s" % (i)): + Cnf["Neve::Options::%s" % (i)] = ""; + + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + + Options = Cnf.SubTree("Neve::Options") + if Options["Help"]: + usage(); + + if not Options["Action"]: + utils.warn("""no -a/--action given; not doing anything. +Please read the documentation before running this script. +"""); + usage(1); print "Re-Creating DB..." (result, output) = commands.getstatusoutput("psql -f init_pool.sql template1"); @@ -522,6 +591,7 @@ def main (): update_suites(); update_override_type(); update_priority(); + update_section(); projectB.query("COMMIT WORK"); files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"files","w"); @@ -538,19 +608,14 @@ def main (): server = SubSec["Archive"]; type = Cnf.Find("Location::%s::Type" % (location)); if type == "legacy-mixed": - prefix = '' + sources = location + 'Sources.gz'; suite = Cnf.Find("Location::%s::Suite" % (location)); - do_sources(location, prefix, suite, "", server); - elif type == "legacy": + do_sources(sources, suite, "", server); + elif type == "legacy" or type == "pool": for suite in Cnf.ValueList("Location::%s::Suites" % (location)): for component in Cnf.SubTree("Component").List(): - prefix = Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/source/' - do_sources(location, prefix, suite, component, server); - elif type == "pool": - continue; -# for component in Cnf.SubTree("Component").List(): -# prefix = component + '/' -# do_sources(location, prefix); + sources = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/source/' + 'Sources.gz'; + do_sources(sources, suite, component, server); else: utils.fubar("Unknown location type ('%s')." % (type)); @@ -564,18 +629,16 @@ def main (): packages = location + 'Packages'; suite = Cnf.Find("Location::%s::Suite" % (location)); print 'Processing '+location+'...'; - process_packages (location, packages, suite, "", server); - elif type == "legacy": + process_packages (packages, suite, "", server); + elif type == "legacy" or type == "pool": for suite in Cnf.ValueList("Location::%s::Suites" % (location)): for component in Cnf.SubTree("Component").List(): architectures = filter(utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite))); for architecture in architectures: - packages = location + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/binary-' + architecture + '/Packages' + packages = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/binary-' + architecture + '/Packages' print 'Processing '+packages+'...'; - process_packages (location, packages, suite, component, server); - elif type == "pool": - continue; + process_packages (packages, suite, component, server); files_query_cache.close(); source_query_cache.close(); @@ -609,5 +672,12 @@ def main (): return; +################################################################################ + +def main(): + utils.try_with_debug(do_da_do_da); + +################################################################################ + if __name__ == '__main__': main();