From 53f1d77f290ed84db4b4436bfd2748d63e0e6892 Mon Sep 17 00:00:00 2001 From: James Troup Date: Wed, 10 Jan 2001 06:08:03 +0000 Subject: [PATCH] sync --- THANKS | 1 + TODO | 4 ++ db_access.py | 99 +++++++++++++++++++++++++++++-------- heidi | 38 +++++++-------- katie | 135 +++++++++++++++++++++++++++++++-------------------- rhona | 22 ++++++--- 6 files changed, 201 insertions(+), 98 deletions(-) diff --git a/THANKS b/THANKS index 182916b3..cee3b67d 100644 --- a/THANKS +++ b/THANKS @@ -14,6 +14,7 @@ Drake Diedrich Guy Maor Jason Gunthorpe Joey Hess +Mark Brown Michael Beattie Michael Mattice Robert Bihlmeyer diff --git a/TODO b/TODO index 07d03fb8..28567aa2 100644 --- a/TODO +++ b/TODO @@ -31,6 +31,10 @@ Less Urgent o jenna's untochable flag doesn't stop adding new packages o db_access' get_files needs to use exceptions not this None, > 0, < 0 return val BS o Fix 'mixed' component handling + o Fix component handling in overrides (aj) + o Fix lack of entires in source overrides (aj) + o Fix potato files to not regen and sort by in jenna + o Fix rhona to unconditionally read override file on startup =================================================================================================== diff --git a/db_access.py b/db_access.py index 0b038862..c62cde95 100644 --- a/db_access.py +++ b/db_access.py @@ -1,6 +1,6 @@ # DB access fucntions # Copyright (C) 2000 James Troup -# $Id: db_access.py,v 1.4 2000-12-18 07:11:25 troup Exp $ +# $Id: db_access.py,v 1.5 2001-01-10 06:08:03 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,16 +18,23 @@ import pg, string -Cnf = None -projectB = None -suite_id_cache = {} -architecture_id_cache = {} -archive_id_cache = {} -component_id_cache = {} -location_id_cache = {} -maintainer_id_cache = {} -source_id_cache = {} -files_id_cache = {} +############################################################################################ + +Cnf = None; +projectB = None; +suite_id_cache = {}; +section_id_cache = {}; +priority_id_cache = {}; +override_type_id_cache = {}; +architecture_id_cache = {}; +archive_id_cache = {}; +component_id_cache = {}; +location_id_cache = {}; +maintainer_id_cache = {}; +source_id_cache = {}; +files_id_cache = {}; + +############################################################################################ def init (config, sql): global Cnf, projectB @@ -48,22 +55,74 @@ def get_suite_id (suite): if ql == []: return -1; - suite_id = q.getresult()[0][0] + suite_id = ql[0][0]; suite_id_cache[suite] = suite_id return suite_id +def get_section_id (section): + global section_id_cache + + if section_id_cache.has_key(section): + return section_id_cache[section] + + q = projectB.query("SELECT id FROM section WHERE section = '%s'" % (section)) + ql = q.getresult(); + if ql == []: + return -1; + + section_id = ql[0][0]; + section_id_cache[section] = section_id + + return section_id + +def get_priority_id (priority): + global priority_id_cache + + if priority_id_cache.has_key(priority): + return priority_id_cache[priority] + + q = projectB.query("SELECT id FROM priority WHERE priority = '%s'" % (priority)) + ql = q.getresult(); + if ql == []: + return -1; + + priority_id = ql[0][0]; + priority_id_cache[priority] = priority_id + + return priority_id + +def get_override_type_id (type): + global override_type_id_cache; + + if override_type_id_cache.has_key(type): + return override_type_id_cache[type]; + + q = projectB.query("SELECT id FROM override_type WHERE type = '%s'" % (type)); + ql = q.getresult(); + if ql == []: + return -1; + + override_type_id = ql[0][0]; + override_type_id_cache[type] = override_type_id; + + return override_type_id; + def get_architecture_id (architecture): - global architecture_id_cache + global architecture_id_cache; if architecture_id_cache.has_key(architecture): - return architecture_id_cache[architecture] + return architecture_id_cache[architecture]; q = projectB.query("SELECT id FROM architecture WHERE arch_string = '%s'" % (architecture)) - architecture_id = q.getresult()[0][0] - architecture_id_cache[architecture] = architecture_id + ql = q.getresult(); + if ql == []: + return -1; + + architecture_id = ql[0][0]; + architecture_id_cache[architecture] = architecture_id; - return architecture_id + return architecture_id; def get_archive_id (archive): global archive_id_cache @@ -88,7 +147,7 @@ def get_component_id (component): if ql == []: return -1; - component_id = ql[0][0] + component_id = ql[0][0]; component_id_cache[component] = component_id return component_id @@ -162,12 +221,12 @@ def get_files_id (filename, size, md5sum, location_id): if ql: if len(ql) != 1: return -1; - ql = ql[0] + ql = ql[0]; orig_size = int(ql[1]); orig_md5sum = ql[2]; if orig_size != size or orig_md5sum != md5sum: return -2; - files_id_cache[cache_key] = ql[0] + files_id_cache[cache_key] = ql[0]; return files_id_cache[cache_key] else: return None diff --git a/heidi b/heidi index 2ce00e7b..309a4e94 100755 --- a/heidi +++ b/heidi @@ -2,7 +2,7 @@ # Manipulate suite tags # Copyright (C) 2000 James Troup -# $Id: heidi,v 1.2 2000-12-05 04:27:48 troup Exp $ +# $Id: heidi,v 1.3 2001-01-10 06:08:03 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,14 +56,14 @@ projectB = None; def process_file (file, suite_id, action): if action == "set": - projectB.query("DELETE FROM bin_associations WHERE suite = %d" % (suite_id)); - projectB.query("DELETE FROM src_associations WHERE suite = %d" % (suite_id)); + projectB.query("DELETE FROM bin_associations WHERE suite = %s" % (suite_id)); + projectB.query("DELETE FROM src_associations WHERE suite = %s" % (suite_id)); action = "add"; for line in file.readlines(): split_line = string.split(string.strip(line[:-1])); if len(split_line) != 3: - sys.stderr.write("W: '%s' does not break into 'package version architecture'.\n"); + sys.stderr.write("W: '%s' does not break into 'package version architecture'.\n" % (line[:-1])); continue; (package, version, architecture) = split_line; @@ -84,7 +84,7 @@ def process_file (file, suite_id, action): if architecture == "source": # Find the existing assoications ID, if any - q = projectB.query("SELECT id FROM src_associations WHERE suite = %d and source = %d" % (suite_id, id)); + q = projectB.query("SELECT id FROM src_associations WHERE suite = %s and source = %s" % (suite_id, id)); ql = q.getresult(); if ql == []: assoication_id = None; @@ -93,19 +93,19 @@ def process_file (file, suite_id, action): # Take action if action == "add": if assoication_id != None: - sys.stderr.write("W: '%s~%s~%s' already exists in suite %d.\n" % (package, version, architecture, suite_id)); + sys.stderr.write("W: '%s~%s~%s' already exists in suite %s.\n" % (package, version, architecture, suite_id)); continue; else: - q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, %d)" % (suite_id, id)); + q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%s, %s)" % (suite_id, id)); elif action == "remove": if assoication_id == None: - sys.stderr.write("W: '%s~%s~%s' doesn't exist in suite %d.\n" % (package, version, architecture, suite_id)); + sys.stderr.write("W: '%s~%s~%s' doesn't exist in suite %s.\n" % (package, version, architecture, suite_id)); continue; else: - q = projectB.query("DELETE FROM src_associations WHERE id = %d" % (assoication_id)); + q = projectB.query("DELETE FROM src_associations WHERE id = %s" % (assoication_id)); else: # Find the existing assoications ID, if any - q = projectB.query("SELECT id FROM bin_associations WHERE suite = %d and bin = %d" % (suite_id, id)); + q = projectB.query("SELECT id FROM bin_associations WHERE suite = %s and bin = %s" % (suite_id, id)); ql = q.getresult(); if ql == []: assoication_id = None; @@ -114,31 +114,31 @@ def process_file (file, suite_id, action): # Take action if action == "add": if assoication_id != None: - sys.stderr.write("W: '%s~%s~%s' already exists in suite %d.\n" % (package, version, architecture, suite_id)); + sys.stderr.write("W: '%s~%s~%s' already exists in suite %s.\n" % (package, version, architecture, suite_id)); continue; else: - q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, %d)" % (suite_id, id)); + q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%s, %s)" % (suite_id, id)); elif action == "remove": if assoication_id == None: - sys.stderr.write("W: '%s~%s~%s' doesn't exist in suite %d.\n" % (package, version, architecture, suite_id)); + sys.stderr.write("W: '%s~%s~%s' doesn't exist in suite %s.\n" % (package, version, architecture, suite_id)); continue; else: - q = projectB.query("DELETE FROM bin_associations WHERE id = %d" % (assoication_id)); + q = projectB.query("DELETE FROM bin_associations WHERE id = %s" % (assoication_id)); ####################################################################################### def get_list (suite_id): # List binaries - q = projectB.query("SELECT b.package, b.version, a.arch_string FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %d AND ba.bin = b.id AND b.architecture = a.id" % (suite_id)); + q = projectB.query("SELECT b.package, b.version, a.arch_string FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id" % (suite_id)); ql = q.getresult(); for i in ql: - print "%s %s %s" % (i[0], i[1], i[2]); + print string.join(i); # List source - q = projectB.query("SELECT s.source, s.version FROM source s, src_associations sa WHERE sa.suite = %d AND sa.source = s.id" % (suite_id)); + q = projectB.query("SELECT s.source, s.version FROM source s, src_associations sa WHERE sa.suite = %s AND sa.source = s.id" % (suite_id)); ql = q.getresult(); for i in ql: - print "%s %s source" % (i[0], i[1]); + print string.join(i) + " source"; ####################################################################################### @@ -194,7 +194,7 @@ def main (): else: if file_list != []: for file in file_list: - process_file(utils.open_file(file_list[0],'r'), suite_id, action); + process_file(utils.open_file(file,'r'), suite_id, action); else: process_file(sys.stdin, suite_id, action); diff --git a/katie b/katie index 8b820902..7ce746c5 100755 --- a/katie +++ b/katie @@ -2,7 +2,7 @@ # Installs Debian packaes # Copyright (C) 2000 James Troup -# $Id: katie,v 1.16 2000-12-20 08:25:56 troup Exp $ +# $Id: katie,v 1.17 2001-01-10 06:08:03 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,7 +32,7 @@ ######################################################################################### -import FCNTL, commands, fcntl, getopt, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time +import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time import apt_inst, apt_pkg import utils, db_access @@ -41,15 +41,10 @@ import utils, db_access re_isanum = re.compile (r'^\d+$'); re_isadeb = re.compile (r'.*\.u?deb$'); re_issource = re.compile (r'(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)'); -re_dpackage = re.compile (r'^package:\s*(.*)', re.IGNORECASE); -re_darchitecture = re.compile (r'^architecture:\s*(.*)', re.IGNORECASE); -re_dversion = re.compile (r'^version:\s*(.*)', re.IGNORECASE); -re_dsection = re.compile (r'^section:\s*(.*)', re.IGNORECASE); -re_dpriority = re.compile (r'^priority:\s*(.*)', re.IGNORECASE); re_changes = re.compile (r'changes$'); -re_override_package = re.compile(r'(\S*)\s+.*'); re_default_answer = re.compile(r"\[(.*)\]"); re_fdnic = re.compile("\n\n"); +re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null"); ############################################################################### @@ -85,11 +80,11 @@ files = {}; projectB = None; new_ack_new = {}; new_ack_old = {}; -overrides = {}; install_count = 0; install_bytes = 0.0; reprocess = 0; orig_tar_id = None; +legacy_source_untouchable = {}; ######################################################################################### @@ -116,53 +111,38 @@ def check_signature (filename): ##################################################################################################################### -def read_override_file (filename, suite, component, binary_type): - global overrides; - - file = utils.open_file(filename, 'r'); - for line in file.readlines(): - line = string.strip(utils.re_comments.sub('', line)) - override_package = re_override_package.sub(r'\1', line) - if override_package != "": - overrides[suite][component][binary_type][override_package] = 1 - file.close() - - -# See if a given package is in the override file. Caches and only loads override files on demand. +# See if a given package is in the override table def in_override_p (package, component, suite, binary_type): global overrides; - if binary_type == "" or binary_type == "deb": - binary_type = "-"; + if binary_type == "": # must be source + type = "dsc"; + else: + type = binary_type; # Avoid on unknown distributions - if db_access.get_suite_id(suite) == -1: + suite_id = db_access.get_suite_id(suite); + if suite_id == -1: return None; + component_id = db_access.get_component_id(component); + type_id = db_access.get_override_type_id(type); # FIXME: nasty non-US speficic hack if string.lower(component[:7]) == "non-us/": component = component[7:]; - if not overrides.has_key(suite) or not overrides[suite].has_key(component) or not overrides[suite][component].has_key(binary_type): - if not overrides.has_key(suite): - overrides[suite] = {} - if not overrides[suite].has_key(component): - overrides[suite][component] = {} - if not overrides[suite][component].has_key(binary_type): - overrides[suite][component][binary_type] = {} - if Cnf.has_key("Suite::%s::SingleOverrideFile" % (suite)): # legacy mixed suite (i.e. experimental) - override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)]; - read_override_file (override_filename, suite, component, binary_type); - else: # all others. - if binary_type == "udeb": - override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)] + '.debian-installer.' + component; - read_override_file (override_filename, suite, component, binary_type); - else: - for src in ("", ".src"): - override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)] + '.' + component + src; - read_override_file (override_filename, suite, component, binary_type); - return overrides[suite][component][binary_type].get(package, None); + q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" + % (package, suite_id, component_id, type_id)); + result = q.getresult(); + # If checking for a source package fall back on the binary override type + if type == "dsc" and not result: + type_id = db_access.get_override_type_id("deb"); + q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" + % (package, suite_id, component_id, type_id)); + result = q.getresult(); + + return result; ##################################################################################################################### @@ -442,7 +422,7 @@ def check_files(): ############################################################################### def check_dsc (): - global dsc, dsc_files, reject_message, reprocess, orig_tar_id; + global dsc, dsc_files, reject_message, reprocess, orig_tar_id, legacy_source_untouchable; for file in files.keys(): if files[file]["type"] == "dsc": @@ -470,23 +450,44 @@ def check_dsc (): found = "%s in incoming" % (dsc_file) # Check the file does not already exist in the archive if not changes.has_key("stable upload"): - q = projectB.query("SELECT f.id FROM files f, location l WHERE f.filename ~ '/%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); + q = projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file)); if q.getresult() != []: reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file) elif dsc_file[-12:] == ".orig.tar.gz": # Check in the pool - q = projectB.query("SELECT l.path, f.filename, l.type, f.id FROM files f, location l WHERE f.filename ~ '/%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); + q = projectB.query("SELECT l.path, f.filename, l.type, f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file)); ql = q.getresult(); - if len(ql) > 0: - old_file = ql[0][0] + ql[0][1]; + + + if ql != []: + # Unfortunately, we make get more than one match + # here if, for example, the package was in potato + # but had a -sa upload in woody. So we need to a) + # choose the right one and b) mark all wrong ones + # as excluded from the source poolification (to + # avoid file overwrites). + + x = ql[0]; # default to something sane in case we don't match any or have only one + + if len(ql) > 1: + for i in ql: + old_file = i[0] + i[1]; + actual_md5 = apt_pkg.md5sum(utils.open_file(old_file,"r")); + actual_size = os.stat(old_file)[stat.ST_SIZE]; + if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]): + x = i; + else: + legacy_source_untouchable[i[3]] = ""; + + old_file = x[0] + x[1]; actual_md5 = apt_pkg.md5sum(utils.open_file(old_file,"r")); actual_size = os.stat(old_file)[stat.ST_SIZE]; found = old_file; - suite_type = ql[0][2]; - dsc_files[dsc_file]["files id"] = ql[0][3]; # need this for updating dsc_files in install() + suite_type = x[2]; + dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install() # See install()... if suite_type == "legacy" or suite_type == "legacy-mixed": - orig_tar_id = ql[0][3]; + orig_tar_id = x[3]; else: # Not there? Check in Incoming... # [See comment above process_it() for explanation @@ -498,6 +499,7 @@ def check_dsc (): files[dsc_file]["section"] = files[file]["section"]; files[dsc_file]["priority"] = files[file]["priority"]; files[dsc_file]["component"] = files[file]["component"]; + files[dsc_file]["type"] = "orig.tar.gz"; reprocess = 1; return 1; else: @@ -518,6 +520,30 @@ def check_dsc (): ############################################################################### +# Some cunning stunt broke dpkg-source in dpkg 1.8{,.1}; detect the +# resulting bad source packages and reject them. + +# Even more amusingly the fix in 1.8.1.1 didn't actually fix the +# problem just changed the symptoms. + +def check_diff (): + global dsc, dsc_files, reject_message, reprocess, orig_tar_id; + + for filename in files.keys(): + if files[filename]["type"] == "diff.gz": + file = gzip.GzipFile(filename, 'r'); + for line in file.readlines(): + if re_bad_diff.search(line): + reject_message = reject_message + "Rejected: source package was produced by broken dpkg 1.8.1[.1]; please rebuild with later version.\n"; + break; + + if string.find(reject_message, "Rejected:") != -1: + return 0 + else: + return 1 + +############################################################################### + def check_md5sums (): global reject_message; @@ -713,6 +739,9 @@ def install (changes_filename, summary, short_summary): q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id)); qd = q.dictresult(); for qid in qd: + # Is this an old upload superseded by a newer -sa upload? (See check_dsc() for details) + if legacy_source_untouchable.has_key(qid["files_id"]): + continue; # First move the files to the new location legacy_filename = qid["path"]+qid["filename"]; pool_location = utils.poolify (changes["source"], files[file]["component"]); @@ -1080,6 +1109,7 @@ def process_it (changes_file): dsc_files = {}; files = {}; orig_tar_id = None; + legacy_source_untouchable = {}; # Absolutize the filename to avoid the requirement of being in the # same directory as the .changes file. @@ -1096,6 +1126,7 @@ def process_it (changes_file): check_files (); check_md5sums (); check_dsc (); + check_diff (); action(changes_file); diff --git a/rhona b/rhona index 3f3be031..fe887747 100755 --- a/rhona +++ b/rhona @@ -2,7 +2,7 @@ # rhona, cleans up unassociated binary and source packages # Copyright (C) 2000 James Troup -# $Id: rhona,v 1.6 2000-12-20 08:15:35 troup Exp $ +# $Id: rhona,v 1.7 2001-01-10 06:08:03 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -59,12 +59,16 @@ def in_override_p (package): if overrides == {}: filename = Cnf["Dir::OverrideDir"] + Cnf["Rhona::OverrideFilename"]; - file = utils.open_file(filename, 'r'); - for line in file.readlines(): - line = string.strip(utils.re_comments.sub('', line)) - if line != "": - overrides[line] = 1 - file.close() + try: + file = utils.open_file(filename, 'r'); + except utils.cant_open_exc: + pass; + else: + for line in file.readlines(): + line = string.strip(utils.re_comments.sub('', line)) + if line != "": + overrides[line] = 1 + file.close() return overrides.get(package, None); @@ -358,6 +362,10 @@ def main(): print "rhona version 0.0000000000"; usage(0); + override_filename = Cnf["Dir::OverrideDir"] + Cnf["Rhona::OverrideFilename"]; + if not os.access(override_filename, os.R_OK): + sys.stderr.write("W: Could not find source-only override file '%s'.\n" % (override_filename)); + delete_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time())); check_binaries(); -- 2.39.2