--- /dev/null
+#!/usr/bin/env python
+
+# Populate the DB
+# Copyright (C) 2000 James Troup <james@nocrew.org>
+# $Id: fix.1,v 1.1 2000-11-24 00:20:11 troup Exp $
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# 04:36|<aj> elmo: you're making me waste 5 seconds per architecture!!!!!! YOU BASTARD!!!!!
+
+################################################################################
+
+# This code is a horrible mess for two reasons:
+
+# (o) For Debian's usage, it's doing something like 160k INSERTs,
+# even on auric, that makes the program unusable unless we get
+# involed in sorts of silly optimization games (local dicts to avoid
+# redundant SELECTS, using COPY FROM rather than INSERTS etc.)
+
+# (o) It's very site specific, because I don't expect to use this
+# script again in a hurry, and I don't want to spend any more time
+# on it than absolutely necessary.
+
+###############################################################################################################
+
+import commands, os, pg, re, sys, string, tempfile
+import apt_pkg
+import db_access, utils
+
+###############################################################################################################
+
+re_arch_from_filename = re.compile(r"binary-[^/]+")
+
+###############################################################################################################
+
+Cnf = None;
+projectB = None;
+files_id_cache = {};
+source_cache = {};
+arch_all_cache = {};
+binary_cache = {};
+#
+files_id_serial = 0;
+source_id_serial = 0;
+src_associations_id_serial = 0;
+dsc_files_id_serial = 0;
+files_query_cache = None;
+source_query_cache = None;
+src_associations_query_cache = None;
+dsc_files_query_cache = None;
+orig_tar_gz_cache = {};
+#
+binaries_id_serial = 0;
+binaries_query_cache = None;
+bin_associations_id_serial = 0;
+bin_associations_query_cache = None;
+#
+source_cache_for_binaries = {};
+
+###############################################################################################################
+
+# Prepares a filename or directory (s) to be file.filename by stripping any part of the location (sub) from it.
+def poolify (s, sub):
+ for i in xrange(len(sub)):
+ if sub[i:] == s[0:len(sub)-i]:
+ return s[len(sub)-i:];
+ return s;
+
+##############################################################################################################
+
+def get_or_set_files_id (filename, size, md5sum, location_id):
+ global files_id_cache, files_id_serial, files_query_cache;
+
+ cache_key = string.join((filename, size, md5sum, repr(location_id)), '~')
+ if not files_id_cache.has_key(cache_key):
+ files_id_serial = files_id_serial + 1
+ files_query_cache.write("%d\t%s\t%s\t%s\t%d\n" % (files_id_serial, filename, size, md5sum, location_id));
+ files_id_cache[cache_key] = files_id_serial
+
+ return files_id_cache[cache_key]
+
+##############################################################################################################
+def process_packages (location, filename, suite, component, archive):
+ global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache;
+
+ apt_pkg.init();
+
+ Cnf = apt_pkg.newConfiguration();
+ apt_pkg.ReadConfigFileISC(Cnf,'/home/troup/katie/katie.conf');
+
+ projectB = pg.connect('projectb', 'localhost', -1, None, None, 'postgres')
+
+ db_access.init (Cnf, projectB);
+
+ count_total = 0;
+ count_bad = 0;
+ suite = string.lower(suite);
+ suite_id = db_access.get_suite_id(suite);
+ if suite == "stable":
+ testing_id = db_access.get_suite_id("testing");
+ suite_codename = Cnf["Suite::%s::CodeName" % (suite)];
+ try:
+ file = utils.open_file (filename, "r")
+ except utils.cant_open_exc:
+ print "WARNING: can't open '%s'" % (filename);
+ return;
+ Scanner = apt_pkg.ParseTagFile(file);
+ while Scanner.Step() != 0:
+ package = Scanner.Section["package"]
+ version = Scanner.Section["version"]
+ maintainer = Scanner.Section["maintainer"]
+ maintainer = string.replace(maintainer, "'", "\\'")
+ maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
+ architecture = Scanner.Section["architecture"]
+ architecture_id = db_access.get_architecture_id (architecture);
+ if not Scanner.Section.has_key("source"):
+ source = package
+ else:
+ source = Scanner.Section["source"]
+ source_version = ""
+ if string.find(source, "(") != -1:
+ m = utils.re_extract_src_version.match(source)
+ source = m.group(1)
+ source_version = m.group(2)
+ if not source_version:
+ source_version = version
+ xfilename = Scanner.Section["filename"]
+ filename = xfilename
+ location_id = db_access.get_location_id (location, component, archive)
+ filename = poolify (filename, location)
+ if architecture == "all":
+ filename = re_arch_from_filename.sub("binary-all", filename);
+ cache_key = "%s~%s" % (source, source_version);
+ source_id = source_cache_for_binaries.get(cache_key, None);
+ size = Scanner.Section["size"];
+ md5sum = Scanner.Section["md5sum"];
+ files_id = get_or_set_files_id (filename, size, md5sum, location_id);
+ cache_key = "%s~%s~%s~%d~%d~%d" % (package, version, repr(source_id), architecture_id, location_id, files_id);
+ if not arch_all_cache.has_key(cache_key):
+ arch_all_cache[cache_key] = 1;
+ cache_key = "%s~%s~%d" % (package, version, architecture_id);
+ if not binary_cache.has_key(cache_key):
+ binary_cache[cache_key] = (size, md5sum);
+ else:
+ (oldsize, oldmd5sum) = binary_cache[cache_key];
+ if oldsize != size or oldmd5sum != md5sum:
+ #print "/org/ftp.debian.org/ftp/%s" % (xfilename);
+ print "%s: %s vs. %s and %s vs. %s" % (xfilename, oldsize, size, oldmd5sum, md5sum);
+ #count_bad = count_bad + 1;
+
+ count_total = count_total +1;
+
+ file.close();
+ if count_bad != 0:
+ print "Found %d bad." % (count_bad)
+
+##############################################################################################################
+
+def main ():
+ global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache;
+
+ apt_pkg.init();
+
+ Cnf = apt_pkg.newConfiguration();
+ apt_pkg.ReadConfigFileISC(Cnf,'/home/troup/katie/katie.conf');
+
+ files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"files","w");
+ source_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"source","w");
+ src_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"src_associations","w");
+ dsc_files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"dsc_files","w");
+ binaries_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"binaries","w");
+ bin_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"bin_associations","w");
+
+ # Process Packages files to populate `binaries' and friends
+
+ for location in Cnf.SubTree("Location").List():
+ SubSec = Cnf.SubTree("Location::%s" % (location));
+ server = SubSec["Archive"];
+ if server != "ftp-master": # FIXME, don't hard code
+ continue;
+ type = Cnf.Find("Location::%s::Type" % (location));
+ if type == "legacy-mixed":
+ packages = location + 'Packages';
+ suite = Cnf.Find("Location::%s::Suite" % (location));
+ process_packages (location, packages, suite, "", server);
+ elif type == "legacy":
+ for suite in Cnf.SubTree("Location::%s::Suites" % (location)).List():
+ for component in Cnf.SubTree("Component").List():
+ for architecture in Cnf.SubTree("Suite::%s::Architectures" % (suite)).List():
+ if architecture == "source" or architecture == "all":
+ continue;
+ packages = location + Cnf.Find("Suite::%s::CodeName" % (suite)) + '/' + component + '/binary-' + architecture + '/Packages'
+ process_packages (location, packages, suite, component, server);
+ elif type == "pool":
+ continue;
+
+ files_query_cache.close();
+ source_query_cache.close();
+ src_associations_query_cache.close();
+ dsc_files_query_cache.close();
+ binaries_query_cache.close();
+ bin_associations_query_cache.close();
+
+ return;
+
+if __name__ == '__main__':
+ main()
projects / dak.git / blobdiff