X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=neve;h=eb94e2e3e72c2ff9ff3a20d4858060df8cce2e81;hb=cabe09b4d162c7f680b07863cc043359545dabdd;hp=100de615ab60a673588c08260b88f7929c1b8f9b;hpb=74b5880be1419dac1f6a539c0af41547fb743793;p=dak.git diff --git a/neve b/neve index 100de615..eb94e2e3 100755 --- a/neve +++ b/neve @@ -1,8 +1,8 @@ #!/usr/bin/env python # Populate the DB -# Copyright (C) 2000, 2001 James Troup -# $Id: neve,v 1.6 2001-11-04 22:41:31 troup Exp $ +# Copyright (C) 2000, 2001, 2002 James Troup +# $Id: neve,v 1.13 2002-06-09 17:33:46 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,11 +18,11 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -################################################################################ +############################################################################### # 04:36| elmo: you're making me waste 5 seconds per architecture!!!!!! YOU BASTARD!!!!! -################################################################################ +############################################################################### # This code is a horrible mess for two reasons: @@ -35,17 +35,17 @@ # script again in a hurry, and I don't want to spend any more time # on it than absolutely necessary. -############################################################################################################### +############################################################################### -import commands, os, pg, re, sys, string, tempfile -import apt_pkg -import db_access, utils +import commands, os, pg, re, select, string, tempfile, time; +import apt_pkg; +import db_access, utils; -############################################################################################################### +############################################################################### re_arch_from_filename = re.compile(r"binary-[^/]+") -############################################################################################################### +############################################################################### Cnf = None; projectB = None; @@ -70,8 +70,195 @@ bin_associations_id_serial = 0; bin_associations_query_cache = None; # source_cache_for_binaries = {}; +reject_message = ""; + +############################################################################### + +# Our very own version of commands.getouputstatus(), hacked to support +# gpgv's status fd. +def get_status_output(cmd, status_read, status_write): + cmd = ['/bin/sh', '-c', cmd]; + p2cread, p2cwrite = os.pipe(); + c2pread, c2pwrite = os.pipe(); + errout, errin = os.pipe(); + pid = os.fork(); + if pid == 0: + # Child + os.close(0); + os.close(1); + os.dup(p2cread); + os.dup(c2pwrite); + os.close(2); + os.dup(errin); + for i in range(3, 256): + if i != status_write: + try: + os.close(i); + except: + pass; + try: + os.execvp(cmd[0], cmd); + finally: + os._exit(1); + + # parent + os.close(p2cread) + os.dup2(c2pread, c2pwrite); + os.dup2(errout, errin); + + output = status = ""; + while 1: + i, o, e = select.select([c2pwrite, errin, status_read], [], []); + more_data = []; + for fd in i: + r = os.read(fd, 8196); + if len(r) > 0: + more_data.append(fd); + if fd == c2pwrite or fd == errin: + output = output + r; + elif fd == status_read: + status = status + r; + else: + utils.fubar("Unexpected file descriptor [%s] returned from select\n" % (fd)); + if not more_data: + pid, exit_status = os.waitpid(pid, 0) + try: + os.close(status_write); + os.close(status_read); + os.close(c2pwrite); + os.close(p2cwrite); + os.close(errin); + except: + pass; + break; + + return output, status, exit_status; + +############################################################################### + +def Dict(**dict): return dict + +def reject (str, prefix="Rejected: "): + global reject_message; + if str: + reject_message = reject_message + prefix + str + "\n"; + +############################################################################### + +def check_signature (filename): + if not utils.re_taint_free.match(os.path.basename(filename)): + reject("!!WARNING!! tainted filename: '%s'." % (filename)); + return None; + + status_read, status_write = os.pipe(); + cmd = "gpgv --status-fd %s --keyring %s --keyring %s %s" \ + % (status_write, Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename); + (output, status, exit_status) = get_status_output(cmd, status_read, status_write); + + # Process the status-fd output + keywords = {}; + bad = internal_error = ""; + for line in string.split(status, '\n'): + line = string.strip(line); + if line == "": + continue; + split = string.split(line); + if len(split) < 2: + internal_error = internal_error + "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line); + continue; + (gnupg, keyword) = split[:2]; + if gnupg != "[GNUPG:]": + internal_error = internal_error + "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg); + continue; + args = split[2:]; + if keywords.has_key(keyword) and keyword != "NODATA" and keyword != "SIGEXPIRED": + internal_error = internal_error + "found duplicate status token ('%s')." % (keyword); + continue; + else: + keywords[keyword] = args; + + # If we failed to parse the status-fd output, let's just whine and bail now + if internal_error: + reject("internal error while performing signature check on %s." % (filename)); + reject(internal_error, ""); + reject("Please report the above errors to the Archive maintainers by replying to this mail.", ""); + return None; + + # Now check for obviously bad things in the processed output + if keywords.has_key("SIGEXPIRED"): + utils.warn("%s: signing key has expired." % (filename)); + if keywords.has_key("KEYREVOKED"): + reject("key used to sign %s has been revoked." % (filename)); + bad = 1; + if keywords.has_key("BADSIG"): + reject("bad signature on %s." % (filename)); + bad = 1; + if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"): + reject("failed to check signature on %s." % (filename)); + bad = 1; + if keywords.has_key("NO_PUBKEY"): + args = keywords["NO_PUBKEY"]; + if len(args) < 1: + reject("internal error while checking signature on %s." % (filename)); + bad = 1; + else: + fingerprint = args[0]; + if keywords.has_key("BADARMOR"): + reject("ascii armour of signature was corrupt in %s." % (filename)); + bad = 1; + if keywords.has_key("NODATA"): + utils.warn("no signature found for %s." % (filename)); + return "NOSIG"; + #reject("no signature found in %s." % (filename)); + #bad = 1; + + if bad: + return None; + + # Next check gpgv exited with a zero return code + if exit_status and not keywords.has_key("NO_PUBKEY"): + reject("gpgv failed while checking %s." % (filename)); + if string.strip(status): + reject(utils.prefix_multi_line_string(status, " [GPG status-fd output:] "), ""); + else: + reject(utils.prefix_multi_line_string(output, " [GPG output:] "), ""); + return None; + + # Sanity check the good stuff we expect + if not keywords.has_key("VALIDSIG"): + if not keywords.has_key("NO_PUBKEY"): + reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename)); + bad = 1; + else: + args = keywords["VALIDSIG"]; + if len(args) < 1: + reject("internal error while checking signature on %s." % (filename)); + bad = 1; + else: + fingerprint = args[0]; + if not keywords.has_key("GOODSIG") and not keywords.has_key("NO_PUBKEY"): + reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename)); + bad = 1; + if not keywords.has_key("SIG_ID") and not keywords.has_key("NO_PUBKEY"): + reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename)); + bad = 1; + + # Finally ensure there's not something we don't recognise + known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", + SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="", + NODATA=""); + + for keyword in keywords.keys(): + if not known_keywords.has_key(keyword): + reject("found unknown status token '%s' from gpgv with args '%s' in %s." % (keyword, repr(keywords[keyword]), filename)); + bad = 1; + + if bad: + return None; + else: + return fingerprint; -############################################################################################################### +######################################################################################### # Prepares a filename or directory (s) to be file.filename by stripping any part of the location (sub) from it. def poolify (s, sub): @@ -121,11 +308,21 @@ def update_suites (): for i in ("Version", "Origin", "Description"): if SubSec.has_key(i): projectB.query("UPDATE suite SET %s = '%s' WHERE suite_name = '%s'" % (string.lower(i), SubSec[i], string.lower(suite))) - for architecture in Cnf.SubTree("Suite::%s::Architectures" % (suite)).List(): + for architecture in Cnf.ValueList("Suite::%s::Architectures" % (suite)): architecture_id = db_access.get_architecture_id (architecture); projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id)); -############################################################################################################## +def update_override_type(): + projectB.query("DELETE FROM override_type"); + for type in Cnf.ValueList("OverrideType"): + projectB.query("INSERT INTO override_type (type) VALUES ('%s')" % (type)); + +def update_priority(): + projectB.query("DELETE FROM priority"); + for priority in Cnf.SubTree("Priority").List(): + projectB.query("INSERT INTO priority (priority, level) VALUES ('%s', %s)" % (priority, Cnf["Priority::%s" % (priority)])); + +############################################################################### def get_or_set_files_id (filename, size, md5sum, location_id): global files_id_cache, files_id_serial, files_query_cache; @@ -138,35 +335,39 @@ def get_or_set_files_id (filename, size, md5sum, location_id): return files_id_cache[cache_key] -############################################################################################################## +############################################################################### -def process_sources (location, filename, suite, component, archive): - global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache; +def process_sources (location, filename, suite, component, archive, dsc_dir): + global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache, reject_message; - suite = string.lower(suite) + suite = string.lower(suite); suite_id = db_access.get_suite_id(suite); - if suite == 'stable': - testing_id = db_access.get_suite_id("testing"); try: file = utils.open_file (filename); except utils.cant_open_exc: print "WARNING: can't open '%s'" % (filename); return; - Scanner = apt_pkg.ParseTagFile(file) + Scanner = apt_pkg.ParseTagFile(file); while Scanner.Step() != 0: - package = Scanner.Section["package"] - version = Scanner.Section["version"] + package = Scanner.Section["package"]; + version = Scanner.Section["version"]; + dsc_file = os.path.join(dsc_dir, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))); + install_date = time.strftime("%Y-%m-%d", time.localtime(os.path.getmtime(dsc_file))); + fingerprint = check_signature(dsc_file); + fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint); + if reject_message: + utils.fubar("%s: %s" % (dsc_file, reject_message)); maintainer = Scanner.Section["maintainer"] - maintainer = string.replace(maintainer, "'", "\\'") + maintainer = string.replace(maintainer, "'", "\\'"); maintainer_id = db_access.get_or_set_maintainer_id(maintainer); - directory = Scanner.Section["directory"] - location_id = db_access.get_location_id (location, component, archive) + directory = Scanner.Section["directory"]; + location_id = db_access.get_location_id (location, component, archive); if directory[-1:] != "/": directory = directory + '/'; directory = poolify (directory, location); if directory != "" and directory[-1:] != "/": directory = directory + '/'; - no_epoch_version = utils.re_no_epoch.sub('', version) + no_epoch_version = utils.re_no_epoch.sub('', version); # Add all files referenced by the .dsc to the files table ids = []; for line in string.split(Scanner.Section["files"],'\n'): @@ -187,7 +388,7 @@ def process_sources (location, filename, suite, component, archive): if filename[-4:] == ".dsc": files_id = id; filename = directory + package + '_' + no_epoch_version + '.dsc' - cache_key = "%s~%s" % (package, version) + cache_key = "%s~%s" % (package, version); if not source_cache.has_key(cache_key): nasty_key = "%s~%s" % (package, version) source_id_serial = source_id_serial + 1; @@ -195,7 +396,7 @@ def process_sources (location, filename, suite, component, archive): source_cache_for_binaries[nasty_key] = source_id_serial; tmp_source_id = source_id_serial; source_cache[cache_key] = source_id_serial; - source_query_cache.write("%d\t%s\t%s\t%d\t%d\n" % (source_id_serial, package, version, maintainer_id, files_id)) + source_query_cache.write("%d\t%s\t%s\t%d\t%d\t%s\t%s\n" % (source_id_serial, package, version, maintainer_id, files_id, install_date, fingerprint_id)) for id in ids: dsc_files_id_serial = dsc_files_id_serial + 1; dsc_files_query_cache.write("%d\t%d\t%d\n" % (dsc_files_id_serial, tmp_source_id,id)); @@ -204,24 +405,18 @@ def process_sources (location, filename, suite, component, archive): src_associations_id_serial = src_associations_id_serial + 1; src_associations_query_cache.write("%d\t%d\t%d\n" % (src_associations_id_serial, suite_id, tmp_source_id)) - # populate 'testing' with a mirror of 'stable' - if suite == "stable": - src_associations_id_serial = src_associations_id_serial + 1; - src_associations_query_cache.write("%d\t%d\t%d\n" % (src_associations_id_serial, testing_id, tmp_source_id)) - file.close() + file.close(); -############################################################################################################## +############################################################################### def process_packages (location, filename, suite, component, archive): - global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache; + global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache, reject_message; count_total = 0; count_bad = 0; suite = string.lower(suite); suite_id = db_access.get_suite_id(suite); - if suite == "stable": - testing_id = db_access.get_suite_id("testing"); try: file = utils.open_file (filename); except utils.cant_open_exc: @@ -236,6 +431,8 @@ def process_packages (location, filename, suite, component, archive): maintainer_id = db_access.get_or_set_maintainer_id(maintainer); architecture = Scanner.Section["architecture"] architecture_id = db_access.get_architecture_id (architecture); + fingerprint = "NOSIG"; + fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint); if not Scanner.Section.has_key("source"): source = package else: @@ -269,7 +466,7 @@ def process_packages (location, filename, suite, component, archive): else: source_id = repr(source_id); binaries_id_serial = binaries_id_serial + 1; - binaries_query_cache.write("%d\t%s\t%s\t%d\t%s\t%d\t%d\t%s\n" % (binaries_id_serial, package, version, maintainer_id, source_id, architecture_id, files_id, type)); + binaries_query_cache.write("%d\t%s\t%s\t%d\t%s\t%d\t%d\t%s\t%s\n" % (binaries_id_serial, package, version, maintainer_id, source_id, architecture_id, files_id, type, fingerprint_id)); binary_cache[cache_key] = binaries_id_serial; tmp_binaries_id = binaries_id_serial; else: @@ -277,9 +474,6 @@ def process_packages (location, filename, suite, component, archive): bin_associations_id_serial = bin_associations_id_serial + 1; bin_associations_query_cache.write("%d\t%d\t%d\n" % (bin_associations_id_serial, suite_id, tmp_binaries_id)); - if suite == "stable": - bin_associations_id_serial = bin_associations_id_serial + 1; - bin_associations_query_cache.write("%d\t%d\t%d\n" % (bin_associations_id_serial, testing_id, tmp_binaries_id)); count_total = count_total +1; file.close(); @@ -288,7 +482,7 @@ def process_packages (location, filename, suite, component, archive): else: print "%d binary packages processed; 0 with no source match which is 0%%" % (count_total); -############################################################################################################## +############################################################################### def do_sources(location, prefix, suite, component, server): temp_filename = tempfile.mktemp(); @@ -299,26 +493,23 @@ def do_sources(location, prefix, suite, component, server): if (result != 0): utils.fubar("Gunzip invocation failed!\n%s" % (output), result); print 'Processing '+sources+'...'; - process_sources (location, temp_filename, suite, component, server); + process_sources (location, temp_filename, suite, component, server, os.path.dirname(sources)); os.unlink(temp_filename); -############################################################################################################## +############################################################################### def main (): global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache; - apt_pkg.init(); - - Cnf = apt_pkg.newConfiguration(); - apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); + Cnf = utils.get_conf(); print "Re-Creating DB..." - (result, output) = commands.getstatusoutput("psql -f init_pool.sql") + (result, output) = commands.getstatusoutput("psql -f init_pool.sql template1"); if (result != 0): utils.fubar("psql invocation failed!\n", result); - print output + print output; - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]), None, None, 'postgres') + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); db_access.init (Cnf, projectB); @@ -329,6 +520,8 @@ def main (): update_archives(); update_locations(); update_suites(); + update_override_type(); + update_priority(); projectB.query("COMMIT WORK"); files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"files","w"); @@ -349,9 +542,9 @@ def main (): suite = Cnf.Find("Location::%s::Suite" % (location)); do_sources(location, prefix, suite, "", server); elif type == "legacy": - for suite in Cnf.SubTree("Location::%s::Suites" % (location)).List(): + for suite in Cnf.ValueList("Location::%s::Suites" % (location)): for component in Cnf.SubTree("Component").List(): - prefix = Cnf.Find("Suite::%s::CodeName" % (suite)) + '/' + component + '/source/' + prefix = Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/source/' do_sources(location, prefix, suite, component, server); elif type == "pool": continue; @@ -373,12 +566,12 @@ def main (): print 'Processing '+location+'...'; process_packages (location, packages, suite, "", server); elif type == "legacy": - for suite in Cnf.SubTree("Location::%s::Suites" % (location)).List(): + for suite in Cnf.ValueList("Location::%s::Suites" % (location)): for component in Cnf.SubTree("Component").List(): - for architecture in Cnf.SubTree("Suite::%s::Architectures" % (suite)).List(): - if architecture == "source" or architecture == "all": - continue; - packages = location + Cnf.Find("Suite::%s::CodeName" % (suite)) + '/' + component + '/binary-' + architecture + '/Packages' + architectures = filter(utils.real_arch, + Cnf.ValueList("Suite::%s::Architectures" % (suite))); + for architecture in architectures: + packages = location + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/binary-' + architecture + '/Packages' print 'Processing '+packages+'...'; process_packages (location, packages, suite, component, server); elif type == "pool": @@ -409,7 +602,7 @@ def main (): # See add_constraints.sql for more details... print "Running add_constraints.sql..."; - (result, output) = commands.getstatusoutput("psql projectb < add_constraints.sql"); + (result, output) = commands.getstatusoutput("psql %s < add_constraints.sql" % (Cnf["DB::Name"])); print output if (result != 0): utils.fubar("psql invocation failed!\n%s" % (output), result); @@ -417,4 +610,4 @@ def main (): return; if __name__ == '__main__': - main() + main();