X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=katie;h=4858485042cc9b77fb76397060ee46e2d17e3bf8;hb=e4f2f33a24646716adedd63afa87a672cd6ed599;hp=80a064454d9ccd4b837c08f90e67cbfa1eb728b3;hpb=e8eacb00ee41d7ab974ba913fff40e5c082b8e1a;p=dak.git diff --git a/katie b/katie index 80a06445..48584850 100755 --- a/katie +++ b/katie @@ -2,7 +2,7 @@ # Installs Debian packaes # Copyright (C) 2000 James Troup -# $Id: katie,v 1.10 2000-12-13 03:18:50 troup Exp $ +# $Id: katie,v 1.29 2001-02-09 22:15:45 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,24 +32,17 @@ ######################################################################################### -import FCNTL, commands, fcntl, getopt, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time +import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time import apt_inst, apt_pkg import utils, db_access ############################################################################### re_isanum = re.compile (r'^\d+$'); -re_isadeb = re.compile (r'.*\.u?deb$'); -re_issource = re.compile (r'(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)'); -re_dpackage = re.compile (r'^package:\s*(.*)', re.IGNORECASE); -re_darchitecture = re.compile (r'^architecture:\s*(.*)', re.IGNORECASE); -re_dversion = re.compile (r'^version:\s*(.*)', re.IGNORECASE); -re_dsection = re.compile (r'^section:\s*(.*)', re.IGNORECASE); -re_dpriority = re.compile (r'^priority:\s*(.*)', re.IGNORECASE); re_changes = re.compile (r'changes$'); -re_override_package = re.compile(r'(\S*)\s+.*'); re_default_answer = re.compile(r"\[(.*)\]"); re_fdnic = re.compile("\n\n"); +re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null"); ############################################################################### @@ -85,24 +78,25 @@ files = {}; projectB = None; new_ack_new = {}; new_ack_old = {}; -overrides = {}; install_count = 0; install_bytes = 0.0; reprocess = 0; orig_tar_id = None; +legacy_source_untouchable = {}; ######################################################################################### def usage (exit_code): print """Usage: dinstall [OPTION]... [CHANGES]... -a, --automatic automatic run - -d, --debug=VALUE debug - -k, --ack-new acknowledge new packages + -D, --debug=VALUE turn on debugging + -h, --help show this help and exit. + -k, --ack-new acknowledge new packages !! for cron.daily only !! -m, --manual-reject=MSG manual reject with `msg' - -n, --dry-run don't do anything + -n, --no-action don't do anything -p, --no-lock don't check lockfile !! for cron.daily only !! - -r, --no-version-check override version check - -u, --distribution=DIST override distribution to `dist'""" + -u, --distribution=DIST override distribution to `dist' + -v, --version display the version number and exit""" sys.exit(exit_code) def check_signature (filename): @@ -110,83 +104,84 @@ def check_signature (filename): (result, output) = commands.getstatusoutput("gpg --emulate-md-encode-bug --batch --no-options --no-default-keyring --always-trust --keyring=%s --keyring=%s < %s >/dev/null" % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename)) if (result != 0): - reject_message = "Rejected: GPG signature check failed on `%s'.\n%s\n" % (filename, output) + reject_message = "Rejected: GPG signature check failed on `%s'.\n%s\n" % (os.path.basename(filename), output) return 0 return 1 ##################################################################################################################### -def read_override_file (filename, suite, component): - global overrides; +# See if a given package is in the override table - file = utils.open_file(filename, 'r'); - for line in file.readlines(): - line = string.strip(utils.re_comments.sub('', line)) - override_package = re_override_package.sub(r'\1', line) - if override_package != "": - overrides[suite][component][override_package] = 1 - file.close() - - -# See if a given package is in the override file. Caches and only loads override files on demand. +def in_override_p (package, component, suite, binary_type, file): + global files; + + if binary_type == "": # must be source + type = "dsc"; + else: + type = binary_type; -def in_override_p (package, component, suite, binary_type): - global overrides; + # Override suite name; used for example with proposed-updates + if Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "": + suite = Cnf["Suite::%s::OverrideSuite" % (suite)]; # Avoid on unknown distributions - if db_access.get_suite_id(suite) == -1: + suite_id = db_access.get_suite_id(suite); + if suite_id == -1: return None; + component_id = db_access.get_component_id(component); + type_id = db_access.get_override_type_id(type); # FIXME: nasty non-US speficic hack if string.lower(component[:7]) == "non-us/": component = component[7:]; - if not overrides.has_key(suite) or not overrides[suite].has_key(component): - if not overrides.has_key(suite): - overrides[suite] = {} - if not overrides[suite].has_key(component): - overrides[suite][component] = {} - if Cnf.has_key("Suite::%s::SingleOverrideFile" % (suite)): # legacy mixed suite (i.e. experimental) - override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)]; - read_override_file (override_filename, suite, component); - else: # all others. - if binary_type == "udeb": - override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)] + '.debian-installer.' + component; - read_override_file (override_filename, suite, component); - else: - for src in ("", ".src"): - override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)] + '.' + component + src; - read_override_file (override_filename, suite, component); - return overrides[suite][component].get(package, None); + q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id" + % (package, suite_id, component_id, type_id)); + result = q.getresult(); + # If checking for a source package fall back on the binary override type + if type == "dsc" and not result: + type_id = db_access.get_override_type_id("deb"); + q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id" + % (package, suite_id, component_id, type_id)); + result = q.getresult(); + + # Remember the section and priority so we can check them later if appropriate + if result != []: + files[file]["override section"] = result[0][0]; + files[file]["override priority"] = result[0][1]; + + return result; ##################################################################################################################### def check_changes(filename): global reject_message, changes, files - # Parse the .changes field into a dictionary [FIXME - need to trap errors, pass on to reject_message etc.] + # Default in case we bail out + changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"]; + + # Parse the .changes field into a dictionary try: - changes = utils.parse_changes(filename) + changes = utils.parse_changes(filename, 0) except utils.cant_open_exc: reject_message = "Rejected: can't read changes file '%s'.\n" % (filename) return 0; except utils.changes_parse_error_exc, line: reject_message = "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line) - changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"]; return 0; - # Parse the Files field from the .changes into another dictionary [FIXME need to trap errors as above] - files = utils.build_file_list(changes, "") + # Parse the Files field from the .changes into another dictionary + try: + files = utils.build_file_list(changes, ""); + except utils.changes_parse_error_exc, line: + reject_message = "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line); # Check for mandatory fields for i in ("source", "binary", "architecture", "version", "distribution","maintainer", "files"): if not changes.has_key(i): - reject_message = "Rejected: Missing field `%s' in changes file." % (i) + reject_message = "Rejected: Missing field `%s' in changes file.\n" % (i) return 0 # Avoid errors during later tests - # Fix the Maintainer: field to be RFC822 compatible - (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"]) - # Override the Distribution: field if appropriate if Cnf["Dinstall::Options::Override-Distribution"] != "": reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Cnf["Dinstall::Options::Override-Distribution"]) @@ -201,6 +196,17 @@ def check_changes(filename): for j in string.split(o): changes[i][j] = 1 + # Fix the Maintainer: field to be RFC822 compatible + (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"]) + + # Fix the Changed-By: field to be RFC822 compatible; if it exists. + (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by","")); + + # For source uploads the Changed-By field wins; otherwise Maintainer wins. + if changes["architecture"].has_key("source"): + changes["uploader822"] = "To: %s\nCc: %s" % (changes["changedby822"], changes["maintainer822"]); + # changes["uploadername"], changes["uploaderemail"]) = (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]); + # Ensure all the values in Closes: are numbers if changes.has_key("closes"): for i in changes["closes"].keys(): @@ -210,19 +216,31 @@ def check_changes(filename): # Map frozen to unstable if frozen doesn't exist if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"): del changes["distribution"]["frozen"] + changes["distribution"]["unstable"] = 1; reject_message = reject_message + "Mapping frozen to unstable.\n" + # Map testing to unstable + if changes["distribution"].has_key("testing"): + del changes["distribution"]["testing"] + changes["distribution"]["unstable"] = 1; + reject_message = reject_message + "Mapping testing to unstable.\n" + # Ensure target distributions exist for i in changes["distribution"].keys(): if not Cnf.has_key("Suite::%s" % (i)): reject_message = reject_message + "Rejected: Unknown distribution `%s'.\n" % (i) + # Ensure there _is_ a target distribution + if changes["distribution"].keys() == []: + reject_message = reject_message + "Rejected: huh? Distribution field is empty in changes file.\n"; + # Map unreleased arches from stable to unstable if changes["distribution"].has_key("stable"): for i in changes["architecture"].keys(): if not Cnf.has_key("Suite::Stable::Architectures::%s" % (i)): reject_message = reject_message + "Mapping stable to unstable for unreleased arch `%s'.\n" % (i) del changes["distribution"]["stable"] + changes["distribution"]["unstable"] = 1; # Map arches not being released from frozen to unstable if changes["distribution"].has_key("frozen"): @@ -230,6 +248,7 @@ def check_changes(filename): if not Cnf.has_key("Suite::Frozen::Architectures::%s" % (i)): reject_message = reject_message + "Mapping frozen to unstable for non-releasing arch `%s'.\n" % (i) del changes["distribution"]["frozen"] + changes["distribution"]["unstable"] = 1; # Handle uploads to stable if changes["distribution"].has_key("stable"): @@ -277,9 +296,16 @@ def check_files(): files[file]["byhand"] = 1; files[file]["type"] = "byhand"; # Checks for a binary package... - elif re_isadeb.match(file) != None: + elif utils.re_isadeb.match(file) != None: + files[file]["type"] = "deb"; + # Extract package information using dpkg-deb - control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file,"r"))) + try: + control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file,"r"))) + except: + reject_message = reject_message + "Rejected: %s: debExtractControl() raised %s.\n" % (file, sys.exc_type); + # Can't continue, none of the checks on control would work. + continue; # Check for mandatory fields if control.Find("Package") == None: @@ -318,14 +344,13 @@ def check_files(): files[file]["dbtype"] = "deb"; else: reject_message = reject_message + "Rejected: %s is neither a .deb or a .udeb.\n " % (file); - files[file]["type"] = "deb"; files[file]["fullname"] = "%s_%s_%s.deb" % (control.Find("Package", ""), epochless_version, control.Find("Architecture", "")) files[file]["source"] = control.Find("Source", ""); if files[file]["source"] == "": files[file]["source"] = files[file]["package"]; # Checks for a source package... else: - m = re_issource.match(file) + m = utils.re_issource.match(file) if m != None: files[file]["package"] = m.group(1) files[file]["version"] = m.group(2) @@ -357,7 +382,7 @@ def check_files(): else: files[file]["byhand"] = 1; files[file]["type"] = "byhand"; - + files[file]["oldfiles"] = {} for suite in changes["distribution"].keys(): # Skip byhand @@ -369,7 +394,7 @@ def check_files(): continue # See if the package is NEW - if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype","")): + if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): files[file]["new"] = 1 # Find any old binary packages @@ -381,11 +406,7 @@ def check_files(): files[file]["oldfiles"][suite] = oldfile # Check versions [NB: per-suite only; no cross-suite checking done (yet)] if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1: - if Cnf["Dinstall::Options::No-Version-Check"]: - reject_message = reject_message + "Overriden rejection" - else: - reject_message = reject_message + "Rejected" - reject_message = reject_message + ": %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"]) + reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"]) # Check for existing copies of the file if not changes.has_key("stable upload"): q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"])) @@ -406,11 +427,17 @@ def check_files(): if component_id == -1: reject_message = reject_message + "Rejected: file '%s' has unknown component '%s'.\n" % (file, component); continue; + + # Validate the priority + if string.find(files[file]["priority"],'/') != -1: + reject_message = reject_message + "Rejected: file '%s' has invalid priority '%s' [contains '/'].\n" % (file, files[file]["priority"]); # Check the md5sum & size against existing files (if any) location = Cnf["Dir::PoolDir"]; files[file]["location id"] = db_access.get_location_id (location, component, archive); - files_id = db_access.get_files_id(component + '/' + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]); + + files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]); + files_id = db_access.get_files_id(files[file]["pool name"] + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]); if files_id == -1: reject_message = reject_message + "Rejected: INTERNAL ERROR, get_files_id() returned multiple matches for %s.\n" % (file) elif files_id == -2: @@ -430,49 +457,87 @@ def check_files(): ############################################################################### def check_dsc (): - global dsc, dsc_files, reject_message, reprocess, orig_tar_id; + global dsc, dsc_files, reject_message, reprocess, orig_tar_id, legacy_source_untouchable; for file in files.keys(): if files[file]["type"] == "dsc": try: - dsc = utils.parse_changes(file) + dsc = utils.parse_changes(file, 1) except utils.cant_open_exc: reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename) return 0; except utils.changes_parse_error_exc, line: reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line) return 0; + except utils.invalid_dsc_format_exc, line: + reject_message = reject_message + "Rejected: syntax error in .dsc file '%s', line %s.\n" % (filename, line) + return 0; try: dsc_files = utils.build_file_list(dsc, 1) except utils.no_files_exc: reject_message = reject_message + "Rejected: no Files: field in .dsc file.\n"; continue; + except utils.changes_parse_error_exc, line: + reject_message = "Rejected: error parsing .dsc file '%s', can't grok: %s.\n" % (filename, line); + continue; # Try and find all files mentioned in the .dsc. This has # to work harder to cope with the multiple possible # locations of an .orig.tar.gz. for dsc_file in dsc_files.keys(): if files.has_key(dsc_file): - actual_md5 = files[dsc_file]["md5sum"] + actual_md5 = files[dsc_file]["md5sum"]; + actual_size = int(files[dsc_file]["size"]); found = "%s in incoming" % (dsc_file) # Check the file does not already exist in the archive if not changes.has_key("stable upload"): - q = projectB.query("SELECT f.id FROM files f, location l WHERE f.filename ~ '%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); + q = projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file)); + + # "It has not broken them. It has fixed a + # brokenness. Your crappy hack exploited a + # bug in the old dinstall. + # + # "(Come on! I thought it was always obvious + # that one just doesn't release different + # files with the same name and version.)" + # -- ajk@ on d-devel@l.d.o + if q.getresult() != []: reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file) elif dsc_file[-12:] == ".orig.tar.gz": # Check in the pool - q = projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename ~ '%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); + q = projectB.query("SELECT l.path, f.filename, l.type, f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file)); ql = q.getresult(); - if len(ql) > 0: - old_file = ql[0][0] + ql[0][1]; + + if ql != []: + # Unfortunately, we make get more than one match + # here if, for example, the package was in potato + # but had a -sa upload in woody. So we need to a) + # choose the right one and b) mark all wrong ones + # as excluded from the source poolification (to + # avoid file overwrites). + + x = ql[0]; # default to something sane in case we don't match any or have only one + + if len(ql) > 1: + for i in ql: + old_file = i[0] + i[1]; + actual_md5 = apt_pkg.md5sum(utils.open_file(old_file,"r")); + actual_size = os.stat(old_file)[stat.ST_SIZE]; + if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]): + x = i; + else: + legacy_source_untouchable[i[3]] = ""; + + old_file = x[0] + x[1]; actual_md5 = apt_pkg.md5sum(utils.open_file(old_file,"r")); + actual_size = os.stat(old_file)[stat.ST_SIZE]; found = old_file; - suite_type = ql[0][2]; - dsc_files[dsc_file]["location id"] = ql[0][4]; # need this for updating dsc_files in install() + suite_type = x[2]; + dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install() # See install()... if suite_type == "legacy" or suite_type == "legacy-mixed": - orig_tar_id = ql[0][3]; + orig_tar_id = x[3]; else: # Not there? Check in Incoming... # [See comment above process_it() for explanation @@ -484,16 +549,43 @@ def check_dsc (): files[dsc_file]["section"] = files[file]["section"]; files[dsc_file]["priority"] = files[file]["priority"]; files[dsc_file]["component"] = files[file]["component"]; + files[dsc_file]["type"] = "orig.tar.gz"; reprocess = 1; return 1; else: reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file); continue; else: - reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming." % (file, dsc_file); + reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming.\n" % (file, dsc_file); continue; if actual_md5 != dsc_files[dsc_file]["md5sum"]: - reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file) + reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file); + if actual_size != int(dsc_files[dsc_file]["size"]): + reject_message = reject_message + "Rejected: size for %s doesn't match %s.\n" % (found, file); + + if string.find(reject_message, "Rejected:") != -1: + return 0 + else: + return 1 + +############################################################################### + +# Some cunning stunt broke dpkg-source in dpkg 1.8{,.1}; detect the +# resulting bad source packages and reject them. + +# Even more amusingly the fix in 1.8.1.1 didn't actually fix the +# problem just changed the symptoms. + +def check_diff (): + global dsc, dsc_files, reject_message, reprocess, orig_tar_id; + + for filename in files.keys(): + if files[filename]["type"] == "diff.gz": + file = gzip.GzipFile(filename, 'r'); + for line in file.readlines(): + if re_bad_diff.search(line): + reject_message = reject_message + "Rejected: [dpkg-sucks] source package was produced by a broken version of dpkg-dev 1.8.x; please rebuild with >= 1.8.3 version installed.\n"; + break; if string.find(reject_message, "Rejected:") != -1: return 0 @@ -514,6 +606,53 @@ def check_md5sums (): if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]: reject_message = reject_message + "Rejected: md5sum check failed for %s.\n" % (file); +def check_override (): + # Only check section & priority on sourceful uploads + if not changes["architecture"].has_key("source"): + return; + + summary = "" + for file in files.keys(): + if not files[file].has_key("new") and files[file]["type"] == "deb": + section = files[file]["section"]; + override_section = files[file]["override section"]; + if section != override_section and section != "-": + # Ignore this; it's a common mistake and not worth whining about + if string.lower(section) == "non-us/main" and string.lower(override_section) == "non-us": + continue; + summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section); + priority = files[file]["priority"]; + override_priority = files[file]["override priority"]; + if priority != override_priority and priority != "-": + summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority); + + if summary == "": + return; + + mail_message = """Return-Path: %s +From: %s +To: %s +Bcc: troup@auric.debian.org +Subject: %s override disparity + +There are disparities between your recently installed upload and the +override file for the following file(s): + +%s +Either the package or the override file is incorrect. If you think +the override is correct and the package wrong please fix the package +so that this disparity is fixed in the next upload. If you feel the +override is incorrect then please reply to this mail and explain why. + +-- +Debian distribution maintenance software + +(This message was generated automatically; if you believe that there +is a problem with it please contact the archive administrators by +mailing ftpmaster@debian.org) +""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes["source"], summary); + utils.send_mail (mail_message, "") + ##################################################################################################################### def action (changes_filename): @@ -646,15 +785,14 @@ def install (changes_filename, summary, short_summary): projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"])); for dsc_file in dsc_files.keys(): filename = files[file]["pool name"] + dsc_file; - # use location id from dsc_files first if it exists as - # the .orig.tar.gz may still be in a legacy location - location_id = dsc_files[dsc_file].get("location id", None); - if location_id == None: - location_id = files[file]["location id"]; - files_id = db_access.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], location_id); + # If the .orig.tar.gz is already in the pool, it's + # files id is stored in dsc_files by check_dsc(). + files_id = dsc_files[dsc_file].get("files id", None); + if files_id == None: + files_id = db_access.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id); # FIXME: needs to check for -1/-2 and or handle exception if files_id == None: - files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], location_id); + files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id); projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id)); # Add the .deb files to the DB @@ -695,28 +833,20 @@ def install (changes_filename, summary, short_summary): # it, so that apt-get source (and anything else that goes by the # "Directory:" field in the Sources.gz file) works. if orig_tar_id != None: - q = projectB.query("SELECT l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location" % (orig_tar_id)); + q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id)); qd = q.dictresult(); for qid in qd: + # Is this an old upload superseded by a newer -sa upload? (See check_dsc() for details) + if legacy_source_untouchable.has_key(qid["files_id"]): + continue; # First move the files to the new location legacy_filename = qid["path"]+qid["filename"]; pool_location = utils.poolify (changes["source"], files[file]["component"]); pool_filename = pool_location + os.path.basename(qid["filename"]); destination = Cnf["Dir::PoolDir"] + pool_location utils.move(legacy_filename, destination); - # Update the DB: files table - new_files_id = db_access.set_files_id(pool_filename, qid["size"], qid["md5sum"], dsc_location_id); - # Update the DB: dsc_files table - projectB.query("INSERT INTO dsc_files (source, file) VALUES (%s, %s)" % (qid["source"], new_files_id)); - # Update the DB: source table - if legacy_filename[-4:] == ".dsc": - projectB.query("UPDATE source SET file = %s WHERE id = %d" % (new_files_id, qid["source"])); - - for qid in qd: - # Remove old data from the DB: dsc_files table - projectB.query("DELETE FROM dsc_files WHERE id = %s" % (qid["dsc_files_id"])); - # Remove old data from the DB: files table - projectB.query("DELETE FROM files WHERE id = %s" % (qid["files_id"])); + # Then Update the DB's files table + q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"])); # Install the files into the pool for file in files.keys(): @@ -752,6 +882,7 @@ Installing: %s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), reject_message, summary, installed_footer) utils.send_mail (mail_message, "") announce (short_summary, 1) + check_override (); ##################################################################################################################### @@ -813,7 +944,7 @@ def stable_install (changes_filename, summary, short_summary): for file in files.keys(): if files[file]["type"] == "deb": new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file)); - elif re_issource.match(file) != None: + elif utils.re_issource.match(file) != None: new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file)); else: new_changelog.write("%s\n" % (file)); @@ -853,11 +984,19 @@ def reject (changes_filename, manual_reject_mail_filename): reason_filename = re_changes.sub("reason", base_changes_filename); reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename); - # Move the .changes files and it's contents into REJECT/ - utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename)); + # Move the .changes files and it's contents into REJECT/ (if we can; errors are ignored) + try: + utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename)); + except: + sys.stderr.write("W: couldn't reject changes file '%s' [Got %s]" % (base_changes_filename, sys.exc_type)); + pass; for file in files.keys(): - if os.access(file,os.R_OK) == 0: - utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file)); + if os.path.exists(file): + try: + utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file)); + except: + sys.stderr.write("W: couldn't reject file '%s' [Got %s].\n" % (file, sys.exc_type)); + pass; # If this is not a manual rejection generate the .reason file and rejection mail message if manual_reject_mail_filename == "": @@ -1026,7 +1165,7 @@ administrators by mailing ftpmaster@debian.org) control_message = "" for bug in bugs: summary = summary + "%s " % (bug) - control_message = control_message + "severity %s fixed\n" % (bug) + control_message = control_message + "tag %s + fixed\n" % (bug) if action and control_message != "": mail_message = """Return-Path: %s From: %s @@ -1058,9 +1197,17 @@ non-maintainer upload. The .changes file follows. # into the .changes structure and reprocess the .changes file. def process_it (changes_file): - global reprocess, orig_tar_id; + global reprocess, orig_tar_id, changes, dsc, dsc_files, files, reject_message; + # Reset some globals reprocess = 1; + changes = {}; + dsc = {}; + dsc_files = {}; + files = {}; + orig_tar_id = None; + legacy_source_untouchable = {}; + reject_message = ""; orig_tar_id = None; # Absolutize the filename to avoid the requirement of being in the @@ -1070,7 +1217,7 @@ def process_it (changes_file): # And since handling of installs to stable munges with the CWD; # save and restore it. cwd = os.getcwd(); - + check_signature (changes_file); check_changes (changes_file); while reprocess: @@ -1078,6 +1225,7 @@ def process_it (changes_file): check_files (); check_md5sums (); check_dsc (); + check_diff (); action(changes_file); @@ -1087,7 +1235,7 @@ def process_it (changes_file): ############################################################################### def main(): - global Cnf, projectB, reject_message, install_bytes, new_ack_old + global Cnf, projectB, install_bytes, new_ack_old apt_pkg.init(); @@ -1101,7 +1249,6 @@ def main(): ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"), ('n',"no-action","Dinstall::Options::No-Action"), ('p',"no-lock", "Dinstall::Options::No-Lock"), - ('r',"no-version-check", "Dinstall::Options::No-Version-Check"), ('s',"no-mail", "Dinstall::Options::No-Mail"), ('u',"override-distribution", "Dinstall::Options::Override-Distribution", "HasArg"), ('v',"version","Dinstall::Options::Version")]; @@ -1148,22 +1295,14 @@ def main(): # Process the changes files for changes_file in changes_files: - reject_message = "" print "\n" + changes_file; process_it (changes_file); - install_mag = " b"; - if install_bytes > 10000: - install_bytes = install_bytes / 1000; - install_mag = " Kb"; - if install_bytes > 10000: - install_bytes = install_bytes / 1000; - install_mag = " Mb"; if install_count: sets = "set" if install_count > 1: sets = "sets" - sys.stderr.write("Installed %d package %s, %d%s.\n" % (install_count, sets, int(install_bytes), install_mag)) + sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes)))); # Write out the list of already-acknowledged NEW packages if Cnf["Dinstall::Options::Ack-New"]: