X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=utils.py;h=580ceda8d0920734dafd0a9ce65951b638a419fc;hb=9540d873fa78598454af57f5f8a4875969ed0439;hp=af196e572b3958d4b2f54dada403a4c188ecc5a3;hpb=9a6ed8b073b00ab14baa23cf1ee75c51c52a66c1;p=dak.git diff --git a/utils.py b/utils.py index af196e57..580ceda8 100644 --- a/utils.py +++ b/utils.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # Utility functions -# Copyright (C) 2000, 2001, 2002, 2003 James Troup -# $Id: utils.py,v 1.60 2003-11-17 17:59:29 troup Exp $ +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 James Troup +# $Id: utils.py,v 1.73 2005-03-18 05:24:38 troup Exp $ ################################################################################ @@ -22,15 +22,16 @@ ################################################################################ -import commands, os, pwd, re, select, socket, shutil, string, sys, tempfile, traceback; +import codecs, commands, email.Header, os, pwd, re, select, socket, shutil, \ + string, sys, tempfile, traceback; import apt_pkg; import db_access; ################################################################################ re_comments = re.compile(r"\#.*") -re_no_epoch = re.compile(r"^\d*\:") -re_no_revision = re.compile(r"\-[^-]*$") +re_no_epoch = re.compile(r"^\d+\:") +re_no_revision = re.compile(r"-[^-]+$") re_arch_from_filename = re.compile(r"/binary-[^/]+/") re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)") re_isadeb = re.compile (r"(.+?)_(.+?)_(.+)\.u?deb$"); @@ -38,15 +39,15 @@ re_issource = re.compile (r"(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)$"); re_single_line_field = re.compile(r"^(\S*)\s*:\s*(.*)"); re_multi_line_field = re.compile(r"^\s(.*)"); -re_taint_free = re.compile(r"^[-+~\.\w]+$"); +re_taint_free = re.compile(r"^[-+~/\.\w]+$"); -re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\> \t]+)\>"); +re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>"); changes_parse_error_exc = "Can't parse line in .changes file"; invalid_dsc_format_exc = "Invalid .dsc file"; nk_format_exc = "Unknown Format: in .changes file"; -no_files_exc = "No Files: field in .dsc file."; -cant_open_exc = "Can't read file."; +no_files_exc = "No Files: field in .dsc or .changes file."; +cant_open_exc = "Can't open file"; unknown_hostname_exc = "Unknown hostname"; cant_overwrite_exc = "Permission denied; can't overwrite existent file." file_exists_exc = "Destination file exists"; @@ -58,6 +59,23 @@ default_apt_config = "/etc/katie/apt.conf"; ################################################################################ +class Error(Exception): + """Base class for exceptions in this module.""" + pass; + +class ParseMaintError(Error): + """Exception raised for errors in parsing a maintainer field. + + Attributes: + message -- explanation of the error + """ + + def __init__(self, message): + self.args = message,; + self.message = message; + +################################################################################ + def open_file(filename, mode='r'): try: f = open(filename, mode); @@ -93,7 +111,7 @@ def extract_component_from_section(section): if section.find('/') != -1: component = section.split('/')[0]; - if component.lower() == "non-us" and section.count('/') > 0: + if component.lower() == "non-us" and section.find('/') != -1: s = component + '/' + section.split('/')[1]; if Cnf.has_key("Component::%s" % s): # Avoid e.g. non-US/libs component = s; @@ -118,24 +136,27 @@ def extract_component_from_section(section): ################################################################################ -# Parses a changes file and returns a dictionary where each field is a -# key. The mandatory first argument is the filename of the .changes -# file. +def parse_changes(filename, signing_rules=0): + """Parses a changes file and returns a dictionary where each field is a +key. The mandatory first argument is the filename of the .changes +file. -# dsc_whitespace_rules is an optional boolean argument which defaults -# to off. If true, it turns on strict format checking to avoid -# allowing in source packages which are unextracable by the -# inappropriately fragile dpkg-source. -# -# The rules are: -# -# o The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----" -# followed by any PGP header data and must end with a blank line. -# -# o The data section must end with a blank line and must be followed by -# "-----BEGIN PGP SIGNATURE-----". +signing_rules is an optional argument: + + o If signing_rules == -1, no signature is required. + o If signing_rules == 0 (the default), a signature is required. + o If signing_rules == 1, it turns on the same strict format checking + as dpkg-source. + +The rules for (signing_rules == 1)-mode are: + + o The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----" + followed by any PGP header data and must end with a blank line. + + o The data section must end with a blank line and must be followed by + "-----BEGIN PGP SIGNATURE-----". +""" -def parse_changes(filename, dsc_whitespace_rules=0): error = ""; changes = {}; @@ -162,7 +183,7 @@ def parse_changes(filename, dsc_whitespace_rules=0): index += 1; line = indexed_lines[index]; if line == "": - if dsc_whitespace_rules: + if signing_rules == 1: index += 1; if index > num_of_lines: raise invalid_dsc_format_exc, index; @@ -176,12 +197,15 @@ def parse_changes(filename, dsc_whitespace_rules=0): if line.startswith("-----BEGIN PGP SIGNATURE"): break; if line.startswith("-----BEGIN PGP SIGNED MESSAGE"): - if dsc_whitespace_rules: - inside_signature = 1; + inside_signature = 1; + if signing_rules == 1: while index < num_of_lines and line != "": index += 1; line = indexed_lines[index]; continue; + # If we're not inside the signed data, don't process anything + if signing_rules >= 0 and not inside_signature: + continue; slf = re_single_line_field.match(line); if slf: field = slf.groups()[0].lower(); @@ -202,7 +226,7 @@ def parse_changes(filename, dsc_whitespace_rules=0): continue; error += line; - if dsc_whitespace_rules and inside_signature: + if signing_rules == 1 and inside_signature: raise invalid_dsc_format_exc, index; changes_in.close(); @@ -259,24 +283,82 @@ def build_file_list(changes, is_a_dsc=0): ################################################################################ -# Fix the `Maintainer:' field to be an RFC822 compatible address. -# cf. Debian Policy Manual (D.2.4) -# -# 06:28| 'The standard sucks, but my tool is supposed to -# interoperate with it. I know - I'll fix the suckage -# and make things incompatible!' +def force_to_utf8(s): + """Forces a string to UTF-8. If the string isn't already UTF-8, +it's assumed to be ISO-8859-1.""" + try: + unicode(s, 'utf-8'); + return s; + except UnicodeError: + latin1_s = unicode(s,'iso8859-1'); + return latin1_s.encode('utf-8'); + +def rfc2047_encode(s): + """Encodes a (header) string per RFC2047 if necessary. If the +string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1.""" + try: + codecs.lookup('ascii')[1](s) + return s; + except UnicodeError: + pass; + try: + codecs.lookup('utf-8')[1](s) + h = email.Header.Header(s, 'utf-8', 998); + return str(h); + except UnicodeError: + h = email.Header.Header(s, 'iso-8859-1', 998); + return str(h); + +################################################################################ + +# 'The standard sucks, but my tool is supposed to interoperate +# with it. I know - I'll fix the suckage and make things +# incompatible!' def fix_maintainer (maintainer): - m = re_parse_maintainer.match(maintainer); - rfc822 = maintainer; - name = ""; - email = ""; - if m != None and len(m.groups()) == 2: + """Parses a Maintainer or Changed-By field and returns: + (1) an RFC822 compatible version, + (2) an RFC2047 compatible version, + (3) the name + (4) the email + +The name is forced to UTF-8 for both (1) and (3). If the name field +contains '.' or ',' (as allowed by Debian policy), (1) and (2) are +switched to 'email (name)' format.""" + maintainer = maintainer.strip() + if not maintainer: + return ('', '', '', ''); + + if maintainer.find("<") == -1: + email = maintainer; + name = ""; + elif (maintainer[0] == "<" and maintainer[-1:] == ">"): + email = maintainer[1:-1]; + name = ""; + else: + m = re_parse_maintainer.match(maintainer); + if not m: + raise ParseMaintError, "Doesn't parse as a valid Maintainer field." name = m.group(1); email = m.group(2); - if name.find(',') != -1 or name.find('.') != -1: - rfc822 = "%s (%s)" % (email, name); - return (rfc822, name, email) + + # Get an RFC2047 compliant version of the name + rfc2047_name = rfc2047_encode(name); + + # Force the name to be UTF-8 + name = force_to_utf8(name); + + if name.find(',') != -1 or name.find('.') != -1: + rfc822_maint = "%s (%s)" % (email, name); + rfc2047_maint = "%s (%s)" % (email, rfc2047_name); + else: + rfc822_maint = "%s <%s>" % (name, email); + rfc2047_maint = "%s <%s>" % (rfc2047_name, email); + + if email.find("@") == -1 and email.find("buildd_") != 0: + raise ParseMaintError, "No @ found in email address part." + + return (rfc822_maint, rfc2047_maint, name, email); ################################################################################ @@ -421,13 +503,13 @@ def whoami (): ################################################################################ def size_type (c): - t = " b"; - if c > 10000: - c = c / 1000; - t = " Kb"; - if c > 10000: - c = c / 1000; - t = " Mb"; + t = " B"; + if c > 10240: + c = c / 1024; + t = " KB"; + if c > 10240: + c = c / 1024; + t = " MB"; return ("%d%s" % (c, t)) ################################################################################ @@ -463,8 +545,8 @@ def changes_compare (a, b): return q; # Sort by source version - a_version = a_changes.get("version"); - b_version = b_changes.get("version"); + a_version = a_changes.get("version", "0"); + b_version = b_changes.get("version", "0"); q = apt_pkg.VersionCompare(a_version, b_version); if q: return q; @@ -518,30 +600,44 @@ def prefix_multi_line_string(str, prefix, include_blank_lines=0): ################################################################################ -def validate_changes_file_arg(file, fatal=1): +def validate_changes_file_arg(filename, require_changes=1): + """'filename' is either a .changes or .katie file. If 'filename' is a +.katie file, it's changed to be the corresponding .changes file. The +function then checks if the .changes file a) exists and b) is +readable and returns the .changes filename if so. If there's a +problem, the next action depends on the option 'require_changes' +argument: + + o If 'require_changes' == -1, errors are ignored and the .changes + filename is returned. + o If 'require_changes' == 0, a warning is given and 'None' is returned. + o If 'require_changes' == 1, a fatal error is raised. +""" error = None; - orig_filename = file - if file.endswith(".katie"): - file = file[:-6]+".changes"; + orig_filename = filename + if filename.endswith(".katie"): + filename = filename[:-6]+".changes"; - if not file.endswith(".changes"): + if not filename.endswith(".changes"): error = "invalid file type; not a changes file"; else: - if not os.access(file,os.R_OK): - if os.path.exists(file): + if not os.access(filename,os.R_OK): + if os.path.exists(filename): error = "permission denied"; else: error = "file not found"; if error: - if fatal: + if require_changes == 1: fubar("%s: %s." % (orig_filename, error)); - else: + elif require_changes == 0: warn("Skipping %s - %s" % (orig_filename, error)); return None; + else: # We only care about the .katie file + return filename; else: - return file; + return filename; ################################################################################ @@ -557,6 +653,19 @@ def join_with_commas_and(list): ################################################################################ +def pp_deps (deps): + pp_deps = []; + for atom in deps: + (pkg, version, constraint) = atom; + if constraint: + pp_dep = "%s (%s %s)" % (pkg, constraint, version); + else: + pp_dep = pkg; + pp_deps.append(pp_dep); + return " |".join(pp_deps); + +################################################################################ + def get_conf(): return Cnf; @@ -759,24 +868,38 @@ def gpgv_get_status_output(cmd, status_read, status_write): ############################################################ -def check_signature (filename, reject): +def check_signature (sig_filename, reject, data_filename="", keyrings=None): """Check the signature of a file and return the fingerprint if the signature is valid or 'None' if it's not. The first argument is the filename whose signature should be checked. The second argument is a reject function and is called when an error is found. The reject() function must allow for two arguments: the first is the error message, the second is an optional prefix string. It's possible for reject() -to be called more than once during an invocation of check_signature().""" +to be called more than once during an invocation of check_signature(). +The third argument is optional and is the name of the files the +detached signature applies to. The fourth argument is optional and is +a *list* of keyrings to use. +""" # Ensure the filename contains no shell meta-characters or other badness - if not re_taint_free.match(os.path.basename(filename)): - reject("!!WARNING!! tainted filename: '%s'." % (filename)); - return 0; + if not re_taint_free.match(sig_filename): + reject("!!WARNING!! tainted signature filename: '%s'." % (sig_filename)); + return None; + if data_filename and not re_taint_free.match(data_filename): + reject("!!WARNING!! tainted data filename: '%s'." % (data_filename)); + return None; + + if not keyrings: + keyrings = (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"]) + + # Build the command line + status_read, status_write = os.pipe(); + cmd = "gpgv --status-fd %s" % (status_write); + for keyring in keyrings: + cmd += " --keyring %s" % (keyring); + cmd += " %s %s" % (sig_filename, data_filename); # Invoke gpgv on the file - status_read, status_write = os.pipe(); - cmd = "gpgv --status-fd %s --keyring %s --keyring %s %s" \ - % (status_write, Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename); (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write); # Process the status-fd output @@ -803,35 +926,35 @@ to be called more than once during an invocation of check_signature().""" # If we failed to parse the status-fd output, let's just whine and bail now if internal_error: - reject("internal error while performing signature check on %s." % (filename)); + reject("internal error while performing signature check on %s." % (sig_filename)); reject(internal_error, ""); reject("Please report the above errors to the Archive maintainers by replying to this mail.", ""); return None; # Now check for obviously bad things in the processed output if keywords.has_key("SIGEXPIRED"): - reject("The key used to sign %s has expired." % (filename)); + reject("The key used to sign %s has expired." % (sig_filename)); bad = 1; if keywords.has_key("KEYREVOKED"): - reject("The key used to sign %s has been revoked." % (filename)); + reject("The key used to sign %s has been revoked." % (sig_filename)); bad = 1; if keywords.has_key("BADSIG"): - reject("bad signature on %s." % (filename)); + reject("bad signature on %s." % (sig_filename)); bad = 1; if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"): - reject("failed to check signature on %s." % (filename)); + reject("failed to check signature on %s." % (sig_filename)); bad = 1; if keywords.has_key("NO_PUBKEY"): args = keywords["NO_PUBKEY"]; if len(args) >= 1: key = args[0]; - reject("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, filename)); + reject("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename)); bad = 1; if keywords.has_key("BADARMOR"): - reject("ASCII armour of signature was corrupt in %s." % (filename)); + reject("ASCII armour of signature was corrupt in %s." % (sig_filename)); bad = 1; if keywords.has_key("NODATA"): - reject("no signature found in %s." % (filename)); + reject("no signature found in %s." % (sig_filename)); bad = 1; if bad: @@ -839,7 +962,7 @@ to be called more than once during an invocation of check_signature().""" # Next check gpgv exited with a zero return code if exit_status: - reject("gpgv failed while checking %s." % (filename)); + reject("gpgv failed while checking %s." % (sig_filename)); if status.strip(): reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), ""); else: @@ -848,20 +971,20 @@ to be called more than once during an invocation of check_signature().""" # Sanity check the good stuff we expect if not keywords.has_key("VALIDSIG"): - reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename)); + reject("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename)); bad = 1; else: args = keywords["VALIDSIG"]; if len(args) < 1: - reject("internal error while checking signature on %s." % (filename)); + reject("internal error while checking signature on %s." % (sig_filename)); bad = 1; else: fingerprint = args[0]; if not keywords.has_key("GOODSIG"): - reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename)); + reject("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename)); bad = 1; if not keywords.has_key("SIG_ID"): - reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename)); + reject("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename)); bad = 1; # Finally ensure there's not something we don't recognise @@ -871,7 +994,7 @@ to be called more than once during an invocation of check_signature().""" for keyword in keywords.keys(): if not known_keywords.has_key(keyword): - reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], filename)); + reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename)); bad = 1; if bad: @@ -925,6 +1048,29 @@ def clean_symlink (src, dest, root): ################################################################################ +def temp_filename(directory=None, dotprefix=None, perms=0700): + """Return a secure and unique filename by pre-creating it. +If 'directory' is non-null, it will be the directory the file is pre-created in. +If 'dotprefix' is non-null, the filename will be prefixed with a '.'.""" + + if directory: + old_tempdir = tempfile.tempdir; + tempfile.tempdir = directory; + + filename = tempfile.mktemp(); + + if dotprefix: + filename = "%s/.%s" % (os.path.dirname(filename), os.path.basename(filename)); + fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, perms); + os.close(fd); + + if directory: + tempfile.tempdir = old_tempdir; + + return filename; + +################################################################################ + apt_pkg.init(); Cnf = apt_pkg.newConfiguration();