#!/usr/bin/env python
-# Populate the DB
+""" Populate the DB """
# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
# This program is free software; you can redistribute it and/or modify
###############################################################################
-import commands, os, pg, re, sys, time
+import commands, os, pg, sys, time
import apt_pkg
-import daklib.database
-import daklib.utils
-
-###############################################################################
-
-re_arch_from_filename = re.compile(r"binary-[^/]+")
+from daklib import database
+from daklib import utils
+from daklib.dak_exceptions import *
+from daklib.regexes import re_arch_from_filename, re_taint_free, re_no_epoch, \
+ re_extract_src_version
###############################################################################
###############################################################################
def check_signature (filename):
- if not daklib.utils.re_taint_free.match(os.path.basename(filename)):
+ if not re_taint_free.match(os.path.basename(filename)):
reject("!!WARNING!! tainted filename: '%s'." % (filename))
return None
status_read, status_write = os.pipe()
cmd = "gpgv --status-fd %s %s %s" \
- % (status_write, daklib.utils.gpg_keyring_args(), filename)
- (output, status, exit_status) = daklib.utils.gpgv_get_status_output(cmd, status_read, status_write)
+ % (status_write, utils.gpg_keyring_args(), filename)
+ (output, status, exit_status) = utils.gpgv_get_status_output(cmd, status_read, status_write)
# Process the status-fd output
keywords = {}
# Now check for obviously bad things in the processed output
if keywords.has_key("SIGEXPIRED"):
- daklib.utils.warn("%s: signing key has expired." % (filename))
+ utils.warn("%s: signing key has expired." % (filename))
if keywords.has_key("KEYREVOKED"):
reject("key used to sign %s has been revoked." % (filename))
bad = 1
reject("ascii armour of signature was corrupt in %s." % (filename))
bad = 1
if keywords.has_key("NODATA"):
- daklib.utils.warn("no signature found for %s." % (filename))
+ utils.warn("no signature found for %s." % (filename))
return "NOSIG"
#reject("no signature found in %s." % (filename))
#bad = 1
if exit_status and not keywords.has_key("NO_PUBKEY"):
reject("gpgv failed while checking %s." % (filename))
if status.strip():
- reject(daklib.utils.prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
+ reject(utils.prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
else:
- reject(daklib.utils.prefix_multi_line_string(output, " [GPG output:] "), "")
+ reject(utils.prefix_multi_line_string(output, " [GPG output:] "), "")
return None
# Sanity check the good stuff we expect
bad = 1
# Finally ensure there's not something we don't recognise
- known_keywords = daklib.utils.Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
+ known_keywords = utils.Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
NODATA="")
projectB.query("DELETE FROM location")
for location in Cnf.SubTree("Location").List():
SubSec = Cnf.SubTree("Location::%s" % (location))
- archive_id = daklib.database.get_archive_id(SubSec["archive"])
+ archive_id = database.get_archive_id(SubSec["archive"])
type = SubSec.Find("type")
- if type == "legacy-mixed":
- projectB.query("INSERT INTO location (path, archive, type) VALUES ('%s', %d, '%s')" % (location, archive_id, SubSec["type"]))
- else:
- for component in Cnf.SubTree("Component").List():
- component_id = daklib.database.get_component_id(component)
- projectB.query("INSERT INTO location (path, component, archive, type) VALUES ('%s', %d, %d, '%s')" %
- (location, component_id, archive_id, SubSec["type"]))
+ for component in Cnf.SubTree("Component").List():
+ component_id = database.get_component_id(component)
+ projectB.query("INSERT INTO location (path, component, archive, type) VALUES ('%s', %d, %d, '%s')" %
+ (location, component_id, archive_id, SubSec["type"]))
def update_architectures ():
projectB.query("DELETE FROM architecture")
for i in ("Version", "Origin", "Description"):
if SubSec.has_key(i):
projectB.query("UPDATE suite SET %s = '%s' WHERE suite_name = '%s'" % (i.lower(), SubSec[i], suite.lower()))
- for architecture in Cnf.ValueList("Suite::%s::Architectures" % (suite)):
- architecture_id = daklib.database.get_architecture_id (architecture)
+ for architecture in database.get_suite_architectures(suite):
+ architecture_id = database.get_architecture_id (architecture)
projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id))
def update_override_type():
prefix = ""
else:
prefix = ""
- component = component.replace("non-US/", "")
if component != 'main':
suffix = '/' + component
else:
try:
path = q.getresult()[0][0]
except:
- daklib.utils.fubar("[import-archive] get_location_path(): Couldn't get path for %s" % (directory))
+ utils.fubar("[import-archive] get_location_path(): Couldn't get path for %s" % (directory))
location_path_cache[directory] = path
return path
global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache, reject_message
suite = suite.lower()
- suite_id = daklib.database.get_suite_id(suite)
+ suite_id = database.get_suite_id(suite)
try:
- file = daklib.utils.open_file (filename)
- except daklib.utils.cant_open_exc:
- daklib.utils.warn("can't open '%s'" % (filename))
+ file = utils.open_file (filename)
+ except CantOpenError:
+ utils.warn("can't open '%s'" % (filename))
return
Scanner = apt_pkg.ParseTagFile(file)
while Scanner.Step() != 0:
package = Scanner.Section["package"]
version = Scanner.Section["version"]
directory = Scanner.Section["directory"]
- dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, daklib.utils.re_no_epoch.sub('', version)))
+ dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, re_no_epoch.sub('', version)))
# Sometimes the Directory path is a lie; check in the pool
if not os.path.exists(dsc_file):
if directory.split('/')[0] == "dists":
- directory = Cnf["Dir::PoolRoot"] + daklib.utils.poolify(package, component)
- dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, daklib.utils.re_no_epoch.sub('', version)))
+ directory = Cnf["Dir::PoolRoot"] + utils.poolify(package, component)
+ dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, re_no_epoch.sub('', version)))
if not os.path.exists(dsc_file):
- daklib.utils.fubar("%s not found." % (dsc_file))
+ utils.fubar("%s not found." % (dsc_file))
install_date = time.strftime("%Y-%m-%d", time.localtime(os.path.getmtime(dsc_file)))
fingerprint = check_signature(dsc_file)
- fingerprint_id = daklib.database.get_or_set_fingerprint_id(fingerprint)
+ fingerprint_id = database.get_or_set_fingerprint_id(fingerprint)
if reject_message:
- daklib.utils.fubar("%s: %s" % (dsc_file, reject_message))
+ utils.fubar("%s: %s" % (dsc_file, reject_message))
maintainer = Scanner.Section["maintainer"]
maintainer = maintainer.replace("'", "\\'")
- maintainer_id = daklib.database.get_or_set_maintainer_id(maintainer)
+ maintainer_id = database.get_or_set_maintainer_id(maintainer)
location = get_location_path(directory.split('/')[0])
- location_id = daklib.database.get_location_id (location, component, archive)
+ location_id = database.get_location_id (location, component, archive)
if not directory.endswith("/"):
directory += '/'
directory = poolify (directory, location)
if directory != "" and not directory.endswith("/"):
directory += '/'
- no_epoch_version = daklib.utils.re_no_epoch.sub('', version)
+ no_epoch_version = re_no_epoch.sub('', version)
# Add all files referenced by the .dsc to the files table
ids = []
for line in Scanner.Section["files"].split('\n'):
count_total = 0
count_bad = 0
suite = suite.lower()
- suite_id = daklib.database.get_suite_id(suite)
+ suite_id = database.get_suite_id(suite)
try:
- file = daklib.utils.open_file (filename)
- except daklib.utils.cant_open_exc:
- daklib.utils.warn("can't open '%s'" % (filename))
+ file = utils.open_file (filename)
+ except CantOpenError:
+ utils.warn("can't open '%s'" % (filename))
return
Scanner = apt_pkg.ParseTagFile(file)
while Scanner.Step() != 0:
version = Scanner.Section["version"]
maintainer = Scanner.Section["maintainer"]
maintainer = maintainer.replace("'", "\\'")
- maintainer_id = daklib.database.get_or_set_maintainer_id(maintainer)
+ maintainer_id = database.get_or_set_maintainer_id(maintainer)
architecture = Scanner.Section["architecture"]
- architecture_id = daklib.database.get_architecture_id (architecture)
+ architecture_id = database.get_architecture_id (architecture)
fingerprint = "NOSIG"
- fingerprint_id = daklib.database.get_or_set_fingerprint_id(fingerprint)
+ fingerprint_id = database.get_or_set_fingerprint_id(fingerprint)
if not Scanner.Section.has_key("source"):
source = package
else:
source = Scanner.Section["source"]
source_version = ""
if source.find("(") != -1:
- m = daklib.utils.re_extract_src_version.match(source)
+ m = re_extract_src_version.match(source)
source = m.group(1)
source_version = m.group(2)
if not source_version:
source_version = version
filename = Scanner.Section["filename"]
+ if filename.endswith(".deb"):
+ type = "deb"
+ else:
+ type = "udeb"
location = get_location_path(filename.split('/')[0])
- location_id = daklib.database.get_location_id (location, component, archive)
+ location_id = database.get_location_id (location, component.replace("/debian-installer", ""), archive)
filename = poolify (filename, location)
if architecture == "all":
filename = re_arch_from_filename.sub("binary-all", filename)
size = Scanner.Section["size"]
md5sum = Scanner.Section["md5sum"]
files_id = get_or_set_files_id (filename, size, md5sum, location_id)
- type = "deb"; # FIXME
cache_key = "%s_%s_%s_%d_%d_%d_%d" % (package, version, repr(source_id), architecture_id, location_id, files_id, suite_id)
if not arch_all_cache.has_key(cache_key):
arch_all_cache[cache_key] = 1
###############################################################################
def do_sources(sources, suite, component, server):
- temp_filename = daklib.utils.temp_filename()
+ (fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (sources, temp_filename))
if (result != 0):
- daklib.utils.fubar("Gunzip invocation failed!\n%s" % (output), result)
+ utils.fubar("Gunzip invocation failed!\n%s" % (output), result)
print 'Processing '+sources+'...'
process_sources (temp_filename, suite, component, server)
os.unlink(temp_filename)
def do_da_do_da ():
global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache
- Cnf = daklib.utils.get_conf()
+ Cnf = utils.get_conf()
Arguments = [('a', "action", "Import-Archive::Options::Action"),
('h', "help", "Import-Archive::Options::Help")]
for i in [ "action", "help" ]:
- if not Cnf.has_key("Import-Archive::Options::%s" % (i)):
- Cnf["Import-Archive::Options::%s" % (i)] = ""
+ if not Cnf.has_key("Import-Archive::Options::%s" % (i)):
+ Cnf["Import-Archive::Options::%s" % (i)] = ""
apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
Options = Cnf.SubTree("Import-Archive::Options")
if Options["Help"]:
- usage()
+ usage()
if not Options["Action"]:
- daklib.utils.warn("""no -a/--action given; not doing anything.
+ utils.warn("""no -a/--action given; not doing anything.
Please read the documentation before running this script.
""")
usage(1)
print "Re-Creating DB..."
(result, output) = commands.getstatusoutput("psql -f init_pool.sql template1")
if (result != 0):
- daklib.utils.fubar("psql invocation failed!\n", result)
+ utils.fubar("psql invocation failed!\n", result)
print output
projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- daklib.database.init (Cnf, projectB)
+ database.init (Cnf, projectB)
print "Adding static tables from conf file..."
projectB.query("BEGIN WORK")
update_section()
projectB.query("COMMIT WORK")
- files_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"files","w")
- source_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"source","w")
- src_associations_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"src_associations","w")
- dsc_files_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"dsc_files","w")
- binaries_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"binaries","w")
- bin_associations_query_cache = daklib.utils.open_file(Cnf["Import-Archive::ExportDir"]+"bin_associations","w")
+ files_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"files","w")
+ source_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"source","w")
+ src_associations_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"src_associations","w")
+ dsc_files_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"dsc_files","w")
+ binaries_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"binaries","w")
+ bin_associations_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"bin_associations","w")
projectB.query("BEGIN WORK")
# Process Sources files to popoulate `source' and friends
SubSec = Cnf.SubTree("Location::%s" % (location))
server = SubSec["Archive"]
type = Cnf.Find("Location::%s::Type" % (location))
- if type == "legacy-mixed":
- sources = location + 'Sources.gz'
- suite = Cnf.Find("Location::%s::Suite" % (location))
- do_sources(sources, suite, "", server)
- elif type == "legacy" or type == "pool":
+ if type == "pool":
for suite in Cnf.ValueList("Location::%s::Suites" % (location)):
for component in Cnf.SubTree("Component").List():
sources = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/source/' + 'Sources.gz'
do_sources(sources, suite, component, server)
else:
- daklib.utils.fubar("Unknown location type ('%s')." % (type))
+ utils.fubar("Unknown location type ('%s')." % (type))
# Process Packages files to populate `binaries' and friends
SubSec = Cnf.SubTree("Location::%s" % (location))
server = SubSec["Archive"]
type = Cnf.Find("Location::%s::Type" % (location))
- if type == "legacy-mixed":
- packages = location + 'Packages'
- suite = Cnf.Find("Location::%s::Suite" % (location))
- print 'Processing '+location+'...'
- process_packages (packages, suite, "", server)
- elif type == "legacy" or type == "pool":
+ if type == "pool":
for suite in Cnf.ValueList("Location::%s::Suites" % (location)):
- for component in Cnf.SubTree("Component").List():
- architectures = filter(daklib.utils.real_arch,
- Cnf.ValueList("Suite::%s::Architectures" % (suite)))
+ udeb_components = map(lambda x: x+"/debian-installer",
+ Cnf.ValueList("Suite::%s::UdebComponents" % suite))
+ for component in Cnf.SubTree("Component").List() + udeb_components:
+ architectures = filter(utils.real_arch, database.get_suite_architectures(suite))
for architecture in architectures:
packages = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/binary-' + architecture + '/Packages'
print 'Processing '+packages+'...'
(result, output) = commands.getstatusoutput("psql %s < add_constraints.sql" % (Cnf["DB::Name"]))
print output
if (result != 0):
- daklib.utils.fubar("psql invocation failed!\n%s" % (output), result)
+ utils.fubar("psql invocation failed!\n%s" % (output), result)
return
################################################################################
def main():
- daklib.utils.try_with_debug(do_da_do_da)
+ utils.try_with_debug(do_da_do_da)
################################################################################