#!/usr/bin/env python

# Populate the DB
# Copyright (C) 2000  James Troup <james@nocrew.org>
# $Id: fix.1,v 1.1 2000-11-24 00:20:11 troup Exp $

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

################################################################################

# 04:36|<aj> elmo: you're making me waste 5 seconds per architecture!!!!!! YOU BASTARD!!!!!

################################################################################

# This code is a horrible mess for two reasons:

#   (o) For Debian's usage, it's doing something like 160k INSERTs,
#   even on auric, that makes the program unusable unless we get
#   involed in sorts of silly optimization games (local dicts to avoid
#   redundant SELECTS, using COPY FROM rather than INSERTS etc.)

#   (o) It's very site specific, because I don't expect to use this
#   script again in a hurry, and I don't want to spend any more time
#   on it than absolutely necessary.

###############################################################################################################

import commands, os, pg, re, sys, string, tempfile
import apt_pkg
import db_access, utils

###############################################################################################################

re_arch_from_filename = re.compile(r"binary-[^/]+")

###############################################################################################################

Cnf = None;
projectB = None;
files_id_cache = {};
source_cache = {};
arch_all_cache = {};
binary_cache = {};
#
files_id_serial = 0;
source_id_serial = 0;
src_associations_id_serial = 0;
dsc_files_id_serial = 0;
files_query_cache = None;
source_query_cache = None;
src_associations_query_cache = None;
dsc_files_query_cache = None;
orig_tar_gz_cache = {};
#
binaries_id_serial = 0;
binaries_query_cache = None;
bin_associations_id_serial = 0;
bin_associations_query_cache = None;
#
source_cache_for_binaries = {};

###############################################################################################################

# Prepares a filename or directory (s) to be file.filename by stripping any part of the location (sub) from it.
def poolify (s, sub):
    for i in xrange(len(sub)):
        if sub[i:] == s[0:len(sub)-i]:
            return s[len(sub)-i:];
    return s;

##############################################################################################################

def get_or_set_files_id (filename, size, md5sum, location_id):
    global files_id_cache, files_id_serial, files_query_cache;

    cache_key = string.join((filename, size, md5sum, repr(location_id)), '~')
    if not files_id_cache.has_key(cache_key):
        files_id_serial = files_id_serial + 1
        files_query_cache.write("%d\t%s\t%s\t%s\t%d\n" % (files_id_serial, filename, size, md5sum, location_id));
        files_id_cache[cache_key] = files_id_serial
        
    return files_id_cache[cache_key]

##############################################################################################################
def process_packages (location, filename, suite, component, archive):
    global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache;
    
    apt_pkg.init();
    
    Cnf = apt_pkg.newConfiguration();
    apt_pkg.ReadConfigFileISC(Cnf,'/home/troup/katie/katie.conf');

    projectB = pg.connect('projectb', 'localhost', -1, None, None, 'postgres')
      
    db_access.init (Cnf, projectB);

    count_total = 0;
    count_bad = 0;
    suite = string.lower(suite);
    suite_id = db_access.get_suite_id(suite);
    if suite == "stable":
        testing_id = db_access.get_suite_id("testing");
    suite_codename = Cnf["Suite::%s::CodeName" % (suite)];
    try:
        file = utils.open_file (filename, "r")
    except utils.cant_open_exc:
        print "WARNING: can't open '%s'" % (filename);
        return;
    Scanner = apt_pkg.ParseTagFile(file);
    while Scanner.Step() != 0:
        package = Scanner.Section["package"]
        version = Scanner.Section["version"]
        maintainer = Scanner.Section["maintainer"]
        maintainer = string.replace(maintainer, "'", "\\'")
        maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
        architecture = Scanner.Section["architecture"]
        architecture_id = db_access.get_architecture_id (architecture);
        if not Scanner.Section.has_key("source"):
            source = package
        else:
            source = Scanner.Section["source"]
        source_version = ""
        if string.find(source, "(") != -1:
            m = utils.re_extract_src_version.match(source)
            source = m.group(1)
            source_version = m.group(2)
        if not source_version:
            source_version = version
        xfilename = Scanner.Section["filename"]
        filename = xfilename
        location_id = db_access.get_location_id (location, component, archive)
        filename = poolify (filename, location)
        if architecture == "all":
            filename = re_arch_from_filename.sub("binary-all", filename);
        cache_key = "%s~%s" % (source, source_version);
        source_id = source_cache_for_binaries.get(cache_key, None);
        size = Scanner.Section["size"];
        md5sum = Scanner.Section["md5sum"];
        files_id = get_or_set_files_id (filename, size, md5sum, location_id);
        cache_key = "%s~%s~%s~%d~%d~%d" % (package, version, repr(source_id), architecture_id, location_id, files_id);
        if not arch_all_cache.has_key(cache_key):
            arch_all_cache[cache_key] = 1;
            cache_key = "%s~%s~%d" % (package, version, architecture_id);
            if not binary_cache.has_key(cache_key):
                binary_cache[cache_key] = (size, md5sum);
            else:
                (oldsize, oldmd5sum) = binary_cache[cache_key];
                if oldsize != size or oldmd5sum != md5sum:
                    #print "/org/ftp.debian.org/ftp/%s" % (xfilename);
                    print "%s: %s vs. %s and %s vs. %s" % (xfilename, oldsize, size, oldmd5sum, md5sum);
                    #count_bad = count_bad + 1;

            count_total = count_total +1;

    file.close();
    if count_bad != 0:
        print "Found %d bad." % (count_bad)

##############################################################################################################

def main ():
    global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache;

    apt_pkg.init();
    
    Cnf = apt_pkg.newConfiguration();
    apt_pkg.ReadConfigFileISC(Cnf,'/home/troup/katie/katie.conf');

    files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"files","w");
    source_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"source","w");
    src_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"src_associations","w");
    dsc_files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"dsc_files","w");
    binaries_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"binaries","w");
    bin_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"bin_associations","w");

    # Process Packages files to populate `binaries' and friends

    for location in Cnf.SubTree("Location").List():
        SubSec = Cnf.SubTree("Location::%s" % (location));
        server = SubSec["Archive"];
        if server != "ftp-master": # FIXME, don't hard code
            continue;
        type = Cnf.Find("Location::%s::Type" % (location));
        if type == "legacy-mixed":
            packages = location + 'Packages';
            suite = Cnf.Find("Location::%s::Suite" % (location));
            process_packages (location, packages, suite, "", server);
        elif type == "legacy":
            for suite in Cnf.SubTree("Location::%s::Suites" % (location)).List():
                for component in Cnf.SubTree("Component").List():
                    for architecture in Cnf.SubTree("Suite::%s::Architectures" % (suite)).List():
                        if architecture == "source" or architecture == "all":
                            continue;
                        packages = location + Cnf.Find("Suite::%s::CodeName" % (suite)) + '/' + component + '/binary-' + architecture + '/Packages'
                        process_packages (location, packages, suite, component, server);
        elif type == "pool":
            continue;
            
    files_query_cache.close();
    source_query_cache.close();
    src_associations_query_cache.close();
    dsc_files_query_cache.close();
    binaries_query_cache.close();
    bin_associations_query_cache.close();
   
    return;
            
if __name__ == '__main__':
    main()