From 6c68616c83b577e687f91a6ee75a8139757f213f Mon Sep 17 00:00:00 2001 From: James Troup Date: Thu, 25 Jan 2001 06:00:07 +0000 Subject: [PATCH] claire section/component fix. --- TODO | 14 +++++++-- claire.py | 90 +++++++++++++++++++++++++++++++++++++++++++++---------- jenna | 10 +++---- utils.py | 43 +++++++++++++++----------- 4 files changed, 118 insertions(+), 39 deletions(-) diff --git a/TODO b/TODO index abff3f33..4be2fbed 100644 --- a/TODO +++ b/TODO @@ -5,14 +5,24 @@ Urgent ------ o Claire needs to know about sections on auric + o maintainers file needs overrides - o jenna needs to use order by [aj] - o seperate apt.conf for stable... [jgg] + o need to have process_it run a big try: loop so that individual package screws up don't kill the daily cron run. + o Add an 'add' ability to melanie (? separate prog maybe) + o Make experimental work + o fix katie's handling of source packages in different directories; + e.g. if a package moves from main to contrib, the .orig.tar.gz + needs to what? copy or be moved and old source removed; one of the + two. + + o fix ssh appearing in 3 packages files + + Less Urgent ----------- diff --git a/claire.py b/claire.py index 1c3784c6..7a8dc908 100755 --- a/claire.py +++ b/claire.py @@ -2,7 +2,7 @@ # 'Fix' stable to make debian-cd and dpkg -BORGiE users happy # Copyright (C) 2000 James Troup -# $Id: claire.py,v 1.1 2000-12-05 04:27:48 troup Exp $ +# $Id: claire.py,v 1.2 2001-01-25 06:00:07 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,15 +24,19 @@ ################################################################################ -import pg, sys, os, string +import os, pg, re, string, sys import utils, db_access import apt_pkg; ################################################################################ +re_strip_section_prefix = re.compile(r'.*/'); + Cnf = None; projectB = None; +################################################################################ + # Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'. # Returns fixed 'src' def clean_symlink (src, dest, root): @@ -46,28 +50,84 @@ def clean_symlink (src, dest, root): ################################################################################ +def fix_component_section (component, section): + if component == "": + (None, component) = utils.extract_component_from_section(section); + + # FIXME: ugly hacks to work around override brain damage + section = re_strip_section_prefix.sub('', section); + section = string.replace(string.lower(section), 'non-us', ''); + if section != '': + section = section + '/'; + + return (component, section); + +################################################################################ + def find_dislocated_stable(Cnf, projectB): dislocated_files = {} # Source - q = projectB.query("SELECT su.suite_name, c.name, s.id FROM suite su, src_associations sa, source s, files f, component c, location l WHERE su.suite_name = 'stable' AND sa.suite = su.id AND sa.source = s.id AND f.id = s.file AND f.location = l.id AND (l.component = c.id OR (l.component = NULL AND c.name = 'non-US/main')) AND NOT (f.filename ~ '^potato/');") + q = projectB.query(""" +SELECT DISTINCT ON (f.id) c.name, sec.section, l.path, f.filename, f.id + FROM component c, override o, section sec, source s, files f, location l, + dsc_files df, suite su, src_associations sa, files f2, location l2 + WHERE su.suite_name = 'stable' AND sa.suite = su.id AND sa.source = s.id + AND f2.id = s.file AND f2.location = l2.id AND df.source = s.id + AND f.id = df.file AND f.location = l.id AND o.package = s.source + AND sec.id = o.section AND NOT (f.filename ~ '^potato/') + AND l.component = c.id +UNION SELECT DISTINCT ON (f.id) null, sec.section, l.path, f.filename, f.id + FROM component c, override o, section sec, source s, files f, location l, + dsc_files df, suite su, src_associations sa, files f2, location l2 + WHERE su.suite_name = 'stable' AND sa.suite = su.id AND sa.source = s.id + AND f2.id = s.file AND f2.location = l2.id AND df.source = s.id + AND f.id = df.file AND f.location = l.id AND o.package = s.source + AND sec.id = o.section AND NOT (f.filename ~ '^potato/') + AND NOT EXISTS (SELECT l.path FROM location l WHERE l.component IS NOT NULL AND f.location = l.id); +"""); for i in q.getresult(): - q = projectB.query("SELECT l.path, f.filename, f.id FROM source s, files f, location l, dsc_files df WHERE s.id = %d AND df.source = %d AND f.id = df.file AND f.location = l.id AND NOT (f.filename ~ '^potato/')" % (i[2], i[2])); - for j in q.getresult(): - src = j[0]+j[1] - dest = Cnf["Dir::RootDir"]+'dists/'+i[0]+'/'+i[1]+'/source/'+os.path.basename(j[1]); - src = clean_symlink(src, dest, Cnf["Dir::RootDir"]); - if not os.path.exists(dest): - if Cnf.Find("Claire::Options::Verbose"): - print src+' -> '+dest - os.symlink(src, dest); - dislocated_files[j[2]] = dest; + src = i[2]+i[3] + (component, section) = fix_component_section(i[0], i[1]); + dest = "%sdists/stable/%s/source/%s%s" % (Cnf["Dir::RootDir"], component, section, os.path.basename(i[3])); + src = clean_symlink(src, dest, Cnf["Dir::RootDir"]); + if not os.path.exists(dest): + if Cnf.Find("Claire::Options::Verbose"): + print src+' -> '+dest + os.symlink(src, dest); + dislocated_files[i[4]] = dest; + + return dislocated_files; + # TODO later when there's something to test it with! # Binary - q = projectB.query("SELECT su.suite_name, c.name, a.arch_string, b.package, b.version, l.path, f.filename, f.id FROM suite su, bin_associations ba, binaries b, files f, component c, architecture a, location l WHERE ba.suite = su.id AND su.suite_name = 'stable' AND ba.bin = b.id AND f.id = b.file AND f.location = l.id AND (l.component = c.id OR (l.component = NULL and c.name = 'non-US/main')) AND b.architecture = a.id AND NOT (f.filename ~ '^potato/');"); + q = projectB.query(""" +SELECT DISTINCT ON (f.id) c.name, a.arch_string, sec.section, b.package, + b.version, l.path, f.filename, f.id + FROM architecture a, bin_associations ba, binaries b, component c, files f, + location l, override o, section sec, suite su + WHERE su.suite_name = 'stable' AND ba.suite = su.id AND ba.bin = b.id + AND f.id = b.file AND f.location = l.id AND o.package = b.package + AND sec.id = o.section AND NOT (f.filename ~ '^potato/') + AND b.architecture = a.id AND l.component = c.id +UNION SELECT DISTINCT ON (f.id) null, a.arch_string, sec.section, b.package, + b.version, l.path, f.filename, f.id + FROM architecture a, bin_associations ba, binaries b, component c, files f, + location l, override o, section sec, suite su + WHERE su.suite_name = 'stable' AND ba.suite = su.id AND ba.bin = b.id + AND f.id = b.file AND f.location = l.id AND o.package = b.package + AND sec.id = o.section AND NOT (f.filename ~ '^potato/') + AND b.architecture = a.id AND NOT EXISTS + (SELECT l.path FROM location l WHERE l.component IS NOT NULL AND f.location = l.id); +"""); for i in q.getresult(): + (component, section) = fix_component_section(i[0], i[2]); + architecture = i[1]; + package = i[3] + version = utils.re_no_epoch.sub('', i[4]); src = i[5]+i[6] - dest = Cnf["Dir::RootDir"]+'dists/'+i[0]+'/'+i[1]+'/binary-'+i[2]+'/'+i[3]+'_'+utils.re_no_epoch.sub('', i[4])+'.deb' + + dest = "%sdists/stable/%s/binary-%s/%s%s_%s.deb" % (Cnf["Dir::RootDir"], component, architecture, section, package, version); src = clean_symlink(src, dest, Cnf["Dir::RootDir"]); if not os.path.exists(dest): if Cnf.Find("Claire::Options::Verbose"): diff --git a/jenna b/jenna index d2c8e17c..d84ffb29 100755 --- a/jenna +++ b/jenna @@ -2,7 +2,7 @@ # Generate file list which is then fed to apt-ftparchive to generate Packages and Sources files # Copyright (C) 2000 James Troup -# $Id: jenna,v 1.3 2001-01-16 21:52:37 troup Exp $ +# $Id: jenna,v 1.4 2001-01-25 06:00:07 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,10 +39,10 @@ def generate_src_list(suite, component, output, dislocated_files): suite_id = db_access.get_suite_id(suite); if component == "-": - q = projectB.query("SELECT s.source, s.version, l.path, f.filename, s.id, f.id FROM source s, src_associations sa, location l, files f WHERE sa.source = s.id AND sa.suite = '%d' AND l.id = f.location AND s.file = f.id" + q = projectB.query("SELECT s.source, s.version, l.path, f.filename, s.id, f.id FROM source s, src_associations sa, location l, files f WHERE sa.source = s.id AND sa.suite = '%d' AND l.id = f.location AND s.file = f.id ORDER BY s.source, s.version" % (suite_id)); else: - q = projectB.query("SELECT s.source, s.version, l.path, f.filename, s.id, f.id FROM source s, src_associations sa, location l, component c, files f WHERE lower(c.name) = '%s' AND (c.id = l.component OR l.component = NULL) AND sa.source = s.id AND sa.suite = '%d' AND l.id = f.location AND s.file = f.id" + q = projectB.query("SELECT s.source, s.version, l.path, f.filename, s.id, f.id FROM source s, src_associations sa, location l, component c, files f WHERE lower(c.name) = '%s' AND (c.id = l.component OR l.component = NULL) AND sa.source = s.id AND sa.suite = '%d' AND l.id = f.location AND s.file = f.id ORDER BY s.source, s.version" % (component, suite_id)); entries = q.getresult(); for entry in entries: @@ -90,9 +90,9 @@ def generate_bin_list(suite, component, architecture, output, type, dislocated_f suite_id = db_access.get_suite_id(suite); if component == "-": - q = projectB.query("SELECT b.package, b.version, l.path, f.filename, b.id, f.id FROM architecture a, binaries b, bin_associations ba, location l, files f WHERE ( a.arch_string = '%s' OR a.arch_string = 'all' ) AND a.id = b.architecture AND ba.bin = b.id AND ba.suite = '%d' AND l.id = f.location AND b.file = f.id AND b.type = '%s'" % (architecture, suite_id, type)); + q = projectB.query("SELECT b.package, b.version, l.path, f.filename, b.id, f.id FROM architecture a, binaries b, bin_associations ba, location l, files f WHERE ( a.arch_string = '%s' OR a.arch_string = 'all' ) AND a.id = b.architecture AND ba.bin = b.id AND ba.suite = '%d' AND l.id = f.location AND b.file = f.id AND b.type = '%s' ORDER BY b.package, b.version, a.arch_string" % (architecture, suite_id, type)); else: - q = projectB.query("SELECT b.package, b.version, l.path, f.filename, b.id, f.id FROM architecture a, binaries b, bin_associations ba, location l, component c, files f WHERE lower(c.name) = '%s' AND (c.id = l.component OR l.component = NULL) AND (a.arch_string = '%s' OR a.arch_string = 'all') AND a.id = b.architecture AND ba.bin = b.id AND ba.suite = '%d' AND l.id = f.location AND b.file = f.id AND b.type = '%s'" % (component, architecture, suite_id, type)); + q = projectB.query("SELECT b.package, b.version, l.path, f.filename, b.id, f.id FROM architecture a, binaries b, bin_associations ba, location l, component c, files f WHERE lower(c.name) = '%s' AND (c.id = l.component OR l.component = NULL) AND (a.arch_string = '%s' OR a.arch_string = 'all') AND a.id = b.architecture AND ba.bin = b.id AND ba.suite = '%d' AND l.id = f.location AND b.file = f.id AND b.type = '%s' ORDER BY b.package, b.version, a.arch_string" % (component, architecture, suite_id, type)); entries = q.getresult(); for entry in entries: package = entry[0] diff --git a/utils.py b/utils.py index 8a8c8e93..8058d3be 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,6 @@ # Utility functions # Copyright (C) 2000 James Troup -# $Id: utils.py,v 1.11 2001-01-16 21:52:37 troup Exp $ +# $Id: utils.py,v 1.12 2001-01-25 06:00:07 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -54,6 +54,29 @@ def our_raw_input(): ###################################################################################### +# What a mess. FIXME +def extract_component_from_section(section): + component = ""; + + if string.find(section, '/') != -1: + component = string.split(section, '/')[0]; + if string.lower(component) == "non-us" and string.count(section, '/') > 0: + s = string.split(section, '/')[1]; + if s == "main" or s == "non-free" or s == "contrib": # Avoid e.g. non-US/libs + component = string.split(section, '/')[0]+ '/' + string.split(section, '/')[1]; + + if string.lower(section) == "non-us": + component = "non-US/main"; + + if component == "": + component = "main"; + elif string.lower(component) == "non-us": + component = "non-US/main"; + + return (section, component); + +###################################################################################### + def parse_changes(filename): changes_in = open_file(filename,'r'); error = "" @@ -104,7 +127,7 @@ def build_file_list(changes, dsc): if i == "": break s = string.split(i) - section = priority = component = "" + section = priority = ""; try: if dsc != "": (md5, size, name) = s @@ -116,21 +139,7 @@ def build_file_list(changes, dsc): if section == "": section = "-" if priority == "": priority = "-" - # What a mess. FIXME - if string.find(section, '/') != -1: - component = string.split(section, '/')[0] - if string.lower(component) == "non-us" and string.count(section, '/') > 0: - s = string.split(section, '/')[1] - if s == "main" or s == "non-free" or s == "contrib": # Avoid e.g. non-US/libs - component = string.split(section, '/')[0]+ '/' + string.split(section, '/')[1] - - if string.lower(section) == "non-us": - component = "non-US/main"; - - if component == "": - component = "main"; - elif string.lower(component) == "non-us": - component = "non-US/main"; + (section, component) = extract_component_from_section(section); files[name] = { "md5sum" : md5, "size" : size, -- 2.39.5