#!/usr/bin/env python
+# vim:set et ts=4 sw=4:
-# Utility functions
-# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+"""Utility functions
-################################################################################
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-################################################################################
-
-import codecs, commands, email.Header, os, pwd, re, select, socket, shutil, \
- sys, tempfile, traceback
+import codecs
+import commands
+import email.Header
+import os
+import pwd
+import select
+import socket
+import shutil
+import sys
+import tempfile
+import traceback
+import stat
import apt_pkg
import database
+import time
+from dak_exceptions import *
+from regexes import re_html_escaping, html_escaping, re_single_line_field, \
+ re_multi_line_field, re_srchasver, re_verwithext, \
+ re_parse_maintainer, re_taint_free, re_gpg_uid
################################################################################
-re_comments = re.compile(r"\#.*")
-re_no_epoch = re.compile(r"^\d+\:")
-re_no_revision = re.compile(r"-[^-]+$")
-re_arch_from_filename = re.compile(r"/binary-[^/]+/")
-re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)")
-re_isadeb = re.compile (r"(.+?)_(.+?)_(.+)\.u?deb$")
-re_issource = re.compile (r"(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)$")
-
-re_single_line_field = re.compile(r"^(\S*)\s*:\s*(.*)")
-re_multi_line_field = re.compile(r"^\s(.*)")
-re_taint_free = re.compile(r"^[-+~/\.\w]+$")
-
-re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>")
-
-changes_parse_error_exc = "Can't parse line in .changes file"
-invalid_dsc_format_exc = "Invalid .dsc file"
-nk_format_exc = "Unknown Format: in .changes file"
-no_files_exc = "No Files: field in .dsc or .changes file."
-cant_open_exc = "Can't open file"
-unknown_hostname_exc = "Unknown hostname"
-cant_overwrite_exc = "Permission denied; can't overwrite existent file."
-file_exists_exc = "Destination file exists"
-sendmail_failed_exc = "Sendmail invocation failed"
-tried_too_hard_exc = "Tried too hard to find a free filename."
-
-default_config = "/etc/dak/dak.conf"
-default_apt_config = "/etc/dak/apt.conf"
+default_config = "/etc/dak/dak.conf" #: default dak config, defines host properties
+default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used
+
+alias_cache = None #: Cache for email alias checks
+key_uid_email_cache = {} #: Cache for email addresses from gpg key uids
+
+# (hashname, function, earliest_changes_version)
+known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
+ ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc
################################################################################
-class Error(Exception):
- """Base class for exceptions in this module."""
- pass
+def html_escape(s):
+ """ Escape html chars """
+ return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)
-class ParseMaintError(Error):
- """Exception raised for errors in parsing a maintainer field.
+################################################################################
- Attributes:
- message -- explanation of the error
+def open_file(filename, mode='r'):
"""
+ Open C{file}, return fileobject.
- def __init__(self, message):
- self.args = message,
- self.message = message
+ @type filename: string
+ @param filename: path/filename to open
-################################################################################
+ @type mode: string
+ @param mode: open mode
-def open_file(filename, mode='r'):
+ @rtype: fileobject
+ @return: open fileobject
+
+ @raise CantOpenError: If IOError is raised by open, reraise it as CantOpenError.
+
+ """
try:
- f = open(filename, mode)
+ f = open(filename, mode)
except IOError:
- raise cant_open_exc, filename
+ raise CantOpenError, filename
return f
################################################################################
if section.find('/') != -1:
component = section.split('/')[0]
- if component.lower() == "non-us" and section.find('/') != -1:
- s = component + '/' + section.split('/')[1]
- if Cnf.has_key("Component::%s" % s): # Avoid e.g. non-US/libs
- component = s
-
- if section.lower() == "non-us":
- component = "non-US/main"
-
- # non-US prefix is case insensitive
- if component.lower()[:6] == "non-us":
- component = "non-US"+component[6:]
# Expand default component
if component == "":
component = section
else:
component = "main"
- elif component == "non-US":
- component = "non-US/main"
return (section, component)
################################################################################
-def parse_changes(filename, signing_rules=0):
- """Parses a changes file and returns a dictionary where each field is a
-key. The mandatory first argument is the filename of the .changes
-file.
-
-signing_rules is an optional argument:
-
- o If signing_rules == -1, no signature is required.
- o If signing_rules == 0 (the default), a signature is required.
- o If signing_rules == 1, it turns on the same strict format checking
- as dpkg-source.
-
-The rules for (signing_rules == 1)-mode are:
-
- o The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
- followed by any PGP header data and must end with a blank line.
-
- o The data section must end with a blank line and must be followed by
- "-----BEGIN PGP SIGNATURE-----".
-"""
-
+def parse_deb822(contents, signing_rules=0):
error = ""
changes = {}
- changes_in = open_file(filename)
- lines = changes_in.readlines()
+ # Split the lines in the input, keeping the linebreaks.
+ lines = contents.splitlines(True)
- if not lines:
- raise changes_parse_error_exc, "[Empty changes file]"
+ if len(lines) == 0:
+ raise ParseChangesError, "[Empty changes file]"
# Reindex by line number so we can easily verify the format of
# .dsc files...
if signing_rules == 1:
index += 1
if index > num_of_lines:
- raise invalid_dsc_format_exc, index
+ raise InvalidDscError, index
line = indexed_lines[index]
if not line.startswith("-----BEGIN PGP SIGNATURE"):
- raise invalid_dsc_format_exc, index
+ raise InvalidDscError, index
inside_signature = 0
break
else:
if slf:
field = slf.groups()[0].lower()
changes[field] = slf.groups()[1]
- first = 1
+ first = 1
continue
if line == " .":
changes[field] += '\n'
mlf = re_multi_line_field.match(line)
if mlf:
if first == -1:
- raise changes_parse_error_exc, "'%s'\n [Multi-line field continuing on from nothing?]" % (line)
+ raise ParseChangesError, "'%s'\n [Multi-line field continuing on from nothing?]" % (line)
if first == 1 and changes[field] != "":
changes[field] += '\n'
first = 0
- changes[field] += mlf.groups()[0] + '\n'
+ changes[field] += mlf.groups()[0] + '\n'
continue
- error += line
+ error += line
if signing_rules == 1 and inside_signature:
- raise invalid_dsc_format_exc, index
+ raise InvalidDscError, index
- changes_in.close()
changes["filecontents"] = "".join(lines)
+ if changes.has_key("source"):
+ # Strip the source version in brackets from the source field,
+ # put it in the "source-version" field instead.
+ srcver = re_srchasver.search(changes["source"])
+ if srcver:
+ changes["source"] = srcver.group(1)
+ changes["source-version"] = srcver.group(2)
+
if error:
- raise changes_parse_error_exc, error
+ raise ParseChangesError, error
return changes
################################################################################
+def parse_changes(filename, signing_rules=0):
+ """
+ Parses a changes file and returns a dictionary where each field is a
+ key. The mandatory first argument is the filename of the .changes
+ file.
+
+ signing_rules is an optional argument:
+
+ - If signing_rules == -1, no signature is required.
+ - If signing_rules == 0 (the default), a signature is required.
+ - If signing_rules == 1, it turns on the same strict format checking
+ as dpkg-source.
+
+ The rules for (signing_rules == 1)-mode are:
+
+ - The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
+ followed by any PGP header data and must end with a blank line.
+
+ - The data section must end with a blank line and must be followed by
+ "-----BEGIN PGP SIGNATURE-----".
+ """
+
+ changes_in = open_file(filename)
+ content = changes_in.read()
+ changes_in.close()
+ return parse_deb822(content, signing_rules)
+
+################################################################################
+
+def hash_key(hashname):
+ return '%ssum' % hashname
+
+################################################################################
+
+def create_hash(where, files, hashname, hashfunc):
+ """
+ create_hash extends the passed files dict with the given hash by
+ iterating over all files on disk and passing them to the hashing
+ function given.
+ """
+
+ rejmsg = []
+ for f in files.keys():
+ try:
+ file_handle = open_file(f)
+ except CantOpenError:
+ rejmsg.append("Could not open file %s for checksumming" % (f))
+
+ files[f][hash_key(hashname)] = hashfunc(file_handle)
+
+ file_handle.close()
+ return rejmsg
+
+################################################################################
+
+def check_hash(where, files, hashname, hashfunc):
+ """
+ check_hash checks the given hash in the files dict against the actual
+ files on disk. The hash values need to be present consistently in
+ all file entries. It does not modify its input in any way.
+ """
+
+ rejmsg = []
+ for f in files.keys():
+ file_handle = None
+ try:
+ try:
+ file_handle = open_file(f)
+
+ # Check for the hash entry, to not trigger a KeyError.
+ if not files[f].has_key(hash_key(hashname)):
+ rejmsg.append("%s: misses %s checksum in %s" % (f, hashname,
+ where))
+ continue
+
+ # Actually check the hash for correctness.
+ if hashfunc(file_handle) != files[f][hash_key(hashname)]:
+ rejmsg.append("%s: %s check failed in %s" % (f, hashname,
+ where))
+ except CantOpenError:
+ # TODO: This happens when the file is in the pool.
+ # warn("Cannot open file %s" % f)
+ continue
+ finally:
+ if file_handle:
+ file_handle.close()
+ return rejmsg
+
+################################################################################
+
+def check_size(where, files):
+ """
+ check_size checks the file sizes in the passed files dict against the
+ files on disk.
+ """
+
+ rejmsg = []
+ for f in files.keys():
+ try:
+ entry = os.stat(f)
+ except OSError, exc:
+ if exc.errno == 2:
+ # TODO: This happens when the file is in the pool.
+ continue
+ raise
+
+ actual_size = entry[stat.ST_SIZE]
+ size = int(files[f]["size"])
+ if size != actual_size:
+ rejmsg.append("%s: actual file size (%s) does not match size (%s) in %s"
+ % (f, actual_size, size, where))
+ return rejmsg
+
+################################################################################
+
+def check_hash_fields(what, manifest):
+ """
+ check_hash_fields ensures that there are no checksum fields in the
+ given dict that we do not know about.
+ """
+
+ rejmsg = []
+ hashes = map(lambda x: x[0], known_hashes)
+ for field in manifest:
+ if field.startswith("checksums-"):
+ hashname = field.split("-",1)[1]
+ if hashname not in hashes:
+ rejmsg.append("Unsupported checksum field for %s "\
+ "in %s" % (hashname, what))
+ return rejmsg
+
+################################################################################
+
+def _ensure_changes_hash(changes, format, version, files, hashname, hashfunc):
+ if format >= version:
+ # The version should contain the specified hash.
+ func = check_hash
+
+ # Import hashes from the changes
+ rejmsg = parse_checksums(".changes", files, changes, hashname)
+ if len(rejmsg) > 0:
+ return rejmsg
+ else:
+ # We need to calculate the hash because it can't possibly
+ # be in the file.
+ func = create_hash
+ return func(".changes", files, hashname, hashfunc)
+
+# We could add the orig which might be in the pool to the files dict to
+# access the checksums easily.
+
+def _ensure_dsc_hash(dsc, dsc_files, hashname, hashfunc):
+ """
+ ensure_dsc_hashes' task is to ensure that each and every *present* hash
+ in the dsc is correct, i.e. identical to the changes file and if necessary
+ the pool. The latter task is delegated to check_hash.
+ """
+
+ rejmsg = []
+ if not dsc.has_key('Checksums-%s' % (hashname,)):
+ return rejmsg
+ # Import hashes from the dsc
+ parse_checksums(".dsc", dsc_files, dsc, hashname)
+ # And check it...
+ rejmsg.extend(check_hash(".dsc", dsc_files, hashname, hashfunc))
+ return rejmsg
+
+################################################################################
+
+def ensure_hashes(changes, dsc, files, dsc_files):
+ rejmsg = []
+
+ # Make sure we recognise the format of the Files: field in the .changes
+ format = changes.get("format", "0.0").split(".", 1)
+ if len(format) == 2:
+ format = int(format[0]), int(format[1])
+ else:
+ format = int(float(format[0])), 0
+
+ # We need to deal with the original changes blob, as the fields we need
+ # might not be in the changes dict serialised into the .dak anymore.
+ orig_changes = parse_deb822(changes['filecontents'])
+
+ # Copy the checksums over to the current changes dict. This will keep
+ # the existing modifications to it intact.
+ for field in orig_changes:
+ if field.startswith('checksums-'):
+ changes[field] = orig_changes[field]
+
+ # Check for unsupported hashes
+ rejmsg.extend(check_hash_fields(".changes", changes))
+ rejmsg.extend(check_hash_fields(".dsc", dsc))
+
+ # We have to calculate the hash if we have an earlier changes version than
+ # the hash appears in rather than require it exist in the changes file
+ for hashname, hashfunc, version in known_hashes:
+ rejmsg.extend(_ensure_changes_hash(changes, format, version, files,
+ hashname, hashfunc))
+ if "source" in changes["architecture"]:
+ rejmsg.extend(_ensure_dsc_hash(dsc, dsc_files, hashname,
+ hashfunc))
+
+ return rejmsg
+
+def parse_checksums(where, files, manifest, hashname):
+ rejmsg = []
+ field = 'checksums-%s' % hashname
+ if not field in manifest:
+ return rejmsg
+ for line in manifest[field].split('\n'):
+ if not line:
+ break
+ checksum, size, checkfile = line.strip().split(' ')
+ if not files.has_key(checkfile):
+ # TODO: check for the file's entry in the original files dict, not
+ # the one modified by (auto)byhand and other weird stuff
+ # rejmsg.append("%s: not present in files but in checksums-%s in %s" %
+ # (file, hashname, where))
+ continue
+ if not files[checkfile]["size"] == size:
+ rejmsg.append("%s: size differs for files and checksums-%s entry "\
+ "in %s" % (checkfile, hashname, where))
+ continue
+ files[checkfile][hash_key(hashname)] = checksum
+ for f in files.keys():
+ if not files[f].has_key(hash_key(hashname)):
+ rejmsg.append("%s: no entry in checksums-%s in %s" % (checkfile,
+ hashname, where))
+ return rejmsg
+
+################################################################################
+
# Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
-def build_file_list(changes, is_a_dsc=0):
+def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
files = {}
# Make sure we have a Files: field to parse...
- if not changes.has_key("files"):
- raise no_files_exc
+ if not changes.has_key(field):
+ raise NoFilesFieldError
# Make sure we recognise the format of the Files: field
- format = changes.get("format", "")
- if format != "":
- format = float(format)
- if not is_a_dsc and (format < 1.5 or format > 2.0):
- raise nk_format_exc, format
+ format = re_verwithext.search(changes.get("format", "0.0"))
+ if not format:
+ raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
+
+ format = format.groups()
+ if format[1] == None:
+ format = int(float(format[0])), 0, format[2]
+ else:
+ format = int(format[0]), int(format[1]), format[2]
+ if format[2] == None:
+ format = format[:2]
+
+ if is_a_dsc:
+ # format = (1,0) are the only formats we currently accept,
+ # format = (0,0) are missing format headers of which we still
+ # have some in the archive.
+ if format != (1,0) and format != (0,0):
+ raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
+ else:
+ if (format < (1,5) or format > (1,8)):
+ raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
+ if field != "files" and format < (1,8):
+ raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
+
+ includes_section = (not is_a_dsc) and field == "files"
# Parse each entry/line:
- for i in changes["files"].split('\n'):
+ for i in changes[field].split('\n'):
if not i:
break
s = i.split()
section = priority = ""
try:
- if is_a_dsc:
- (md5, size, name) = s
- else:
+ if includes_section:
(md5, size, section, priority, name) = s
+ else:
+ (md5, size, name) = s
except ValueError:
- raise changes_parse_error_exc, i
+ raise ParseChangesError, i
if section == "":
section = "-"
(section, component) = extract_component_from_section(section)
- files[name] = Dict(md5sum=md5, size=size, section=section,
+ files[name] = Dict(size=size, section=section,
priority=priority, component=component)
+ files[name][hashname] = md5
return files
################################################################################
def force_to_utf8(s):
- """Forces a string to UTF-8. If the string isn't already UTF-8,
-it's assumed to be ISO-8859-1."""
+ """
+ Forces a string to UTF-8. If the string isn't already UTF-8,
+ it's assumed to be ISO-8859-1.
+ """
try:
unicode(s, 'utf-8')
return s
return latin1_s.encode('utf-8')
def rfc2047_encode(s):
- """Encodes a (header) string per RFC2047 if necessary. If the
-string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1."""
+ """
+ Encodes a (header) string per RFC2047 if necessary. If the
+ string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1.
+ """
try:
codecs.lookup('ascii')[1](s)
return s
# incompatible!'
def fix_maintainer (maintainer):
- """Parses a Maintainer or Changed-By field and returns:
- (1) an RFC822 compatible version,
- (2) an RFC2047 compatible version,
- (3) the name
- (4) the email
-
-The name is forced to UTF-8 for both (1) and (3). If the name field
-contains '.' or ',' (as allowed by Debian policy), (1) and (2) are
-switched to 'email (name)' format."""
+ """
+ Parses a Maintainer or Changed-By field and returns:
+ 1. an RFC822 compatible version,
+ 2. an RFC2047 compatible version,
+ 3. the name
+ 4. the email
+
+ The name is forced to UTF-8 for both 1. and 3.. If the name field
+ contains '.' or ',' (as allowed by Debian policy), 1. and 2. are
+ switched to 'email (name)' format.
+
+ """
maintainer = maintainer.strip()
if not maintainer:
return ('', '', '', '')
################################################################################
-# sendmail wrapper, takes _either_ a message string or a file as arguments
def send_mail (message, filename=""):
- # If we've been passed a string dump it into a temporary file
- if message:
- filename = tempfile.mktemp()
- fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700)
- os.write (fd, message)
- os.close (fd)
-
- # Invoke sendmail
- (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
- if (result != 0):
- raise sendmail_failed_exc, output
-
- # Clean up any temporary files
- if message:
- os.unlink (filename)
+ """sendmail wrapper, takes _either_ a message string or a file as arguments"""
+
+ # If we've been passed a string dump it into a temporary file
+ if message:
+ (fd, filename) = tempfile.mkstemp()
+ os.write (fd, message)
+ os.close (fd)
+
+ # Invoke sendmail
+ (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
+ if (result != 0):
+ raise SendmailFailedError, output
+
+ # Clean up any temporary files
+ if message:
+ os.unlink (filename)
################################################################################
def poolify (source, component):
if component:
- component += '/'
- # FIXME: this is nasty
- component = component.lower().replace("non-us/", "non-US/")
+ component += '/'
if source[:3] == "lib":
- return component + source[:4] + '/' + source + '/'
+ return component + source[:4] + '/' + source + '/'
else:
- return component + source[:1] + '/' + source + '/'
+ return component + source[:1] + '/' + source + '/'
################################################################################
def move (src, dest, overwrite = 0, perms = 0664):
if os.path.exists(dest) and os.path.isdir(dest):
- dest_dir = dest
+ dest_dir = dest
else:
- dest_dir = os.path.dirname(dest)
+ dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
- umask = os.umask(00000)
- os.makedirs(dest_dir, 02775)
- os.umask(umask)
+ umask = os.umask(00000)
+ os.makedirs(dest_dir, 02775)
+ os.umask(umask)
#print "Moving %s to %s..." % (src, dest)
if os.path.exists(dest) and os.path.isdir(dest):
- dest += '/' + os.path.basename(src)
+ dest += '/' + os.path.basename(src)
# Don't overwrite unless forced to
if os.path.exists(dest):
if not overwrite:
def copy (src, dest, overwrite = 0, perms = 0664):
if os.path.exists(dest) and os.path.isdir(dest):
- dest_dir = dest
+ dest_dir = dest
else:
- dest_dir = os.path.dirname(dest)
+ dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
- umask = os.umask(00000)
- os.makedirs(dest_dir, 02775)
- os.umask(umask)
+ umask = os.umask(00000)
+ os.makedirs(dest_dir, 02775)
+ os.umask(umask)
#print "Copying %s to %s..." % (src, dest)
if os.path.exists(dest) and os.path.isdir(dest):
- dest += '/' + os.path.basename(src)
+ dest += '/' + os.path.basename(src)
# Don't overwrite unless forced to
if os.path.exists(dest):
if not overwrite:
- raise file_exists_exc
+ raise FileExistsError
else:
if not os.access(dest, os.W_OK):
- raise cant_overwrite_exc
+ raise CantOverwriteError
shutil.copy2(src, dest)
os.chmod(dest, perms)
res = socket.gethostbyaddr(socket.gethostname())
database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname")
if database_hostname:
- return database_hostname
+ return database_hostname
else:
return res[0]
def which_conf_file ():
res = socket.gethostbyaddr(socket.gethostname())
if Cnf.get("Config::" + res[0] + "::DakConfig"):
- return Cnf["Config::" + res[0] + "::DakConfig"]
+ return Cnf["Config::" + res[0] + "::DakConfig"]
else:
- return default_config
+ return default_config
def which_apt_conf_file ():
res = socket.gethostbyaddr(socket.gethostname())
if Cnf.get("Config::" + res[0] + "::AptConfig"):
- return Cnf["Config::" + res[0] + "::AptConfig"]
+ return Cnf["Config::" + res[0] + "::AptConfig"]
else:
- return default_apt_config
+ return default_apt_config
+
+def which_alias_file():
+ hostname = socket.gethostbyaddr(socket.gethostname())[0]
+ aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
+ if os.path.exists(aliasfn):
+ return aliasfn
+ else:
+ return None
################################################################################
################################################################################
-# Perform a substition of template
def TemplateSubst(map, filename):
- file = open_file(filename)
- template = file.read()
+ """ Perform a substition of template """
+ templatefile = open_file(filename)
+ template = templatefile.read()
for x in map.keys():
template = template.replace(x,map[x])
- file.close()
+ templatefile.close()
return template
################################################################################
for j in o.split():
changes["architecture"][j] = 1
-# Sort by source name, source version, 'have source', and then by filename
def changes_compare (a, b):
+ """ Sort by source name, source version, 'have source', and then by filename """
try:
a_changes = parse_changes(a)
except:
dest = orig_dest + '.' + repr(extra)
extra += 1
if extra >= too_many:
- raise tried_too_hard_exc
+ raise NoFreeFilenameError
return dest
################################################################################
def result_join (original, sep = '\t'):
- list = []
+ resultlist = []
for i in xrange(len(original)):
if original[i] == None:
- list.append("")
+ resultlist.append("")
else:
- list.append(original[i])
- return sep.join(list)
+ resultlist.append(original[i])
+ return sep.join(resultlist)
################################################################################
################################################################################
def validate_changes_file_arg(filename, require_changes=1):
- """'filename' is either a .changes or .dak file. If 'filename' is a
-.dak file, it's changed to be the corresponding .changes file. The
-function then checks if the .changes file a) exists and b) is
-readable and returns the .changes filename if so. If there's a
-problem, the next action depends on the option 'require_changes'
-argument:
-
- o If 'require_changes' == -1, errors are ignored and the .changes
- filename is returned.
- o If 'require_changes' == 0, a warning is given and 'None' is returned.
- o If 'require_changes' == 1, a fatal error is raised.
-"""
+ """
+ 'filename' is either a .changes or .dak file. If 'filename' is a
+ .dak file, it's changed to be the corresponding .changes file. The
+ function then checks if the .changes file a) exists and b) is
+ readable and returns the .changes filename if so. If there's a
+ problem, the next action depends on the option 'require_changes'
+ argument:
+
+ - If 'require_changes' == -1, errors are ignored and the .changes
+ filename is returned.
+ - If 'require_changes' == 0, a warning is given and 'None' is returned.
+ - If 'require_changes' == 1, a fatal error is raised.
+
+ """
error = None
orig_filename = filename
################################################################################
def join_with_commas_and(list):
- if len(list) == 0: return "nothing"
- if len(list) == 1: return list[0]
- return ", ".join(list[:-1]) + " and " + list[-1]
+ if len(list) == 0: return "nothing"
+ if len(list) == 1: return list[0]
+ return ", ".join(list[:-1]) + " and " + list[-1]
################################################################################
################################################################################
def get_conf():
- return Cnf
+ return Cnf
################################################################################
-# Handle -a, -c and -s arguments; returns them as SQL constraints
def parse_args(Options):
+ """ Handle -a, -c and -s arguments; returns them as SQL constraints """
# Process suite
if Options["Suite"]:
suite_ids_list = []
################################################################################
-# Function for use in sorting lists of architectures.
-# Sorts normally except that 'source' dominates all others.
-
def arch_compare_sw (a, b):
+ """
+ Function for use in sorting lists of architectures.
+
+ Sorts normally except that 'source' dominates all others.
+ """
+
if a == "source" and b == "source":
return 0
elif a == "source":
################################################################################
-# Split command line arguments which can be separated by either commas
-# or whitespace. If dwim is set, it will complain about string ending
-# in comma since this usually means someone did 'dak ls -a i386, m68k
-# foo' or something and the inevitable confusion resulting from 'm68k'
-# being treated as an argument is undesirable.
-
def split_args (s, dwim=1):
+ """
+ Split command line arguments which can be separated by either commas
+ or whitespace. If dwim is set, it will complain about string ending
+ in comma since this usually means someone did 'dak ls -a i386, m68k
+ foo' or something and the inevitable confusion resulting from 'm68k'
+ being treated as an argument is undesirable.
+ """
+
if s.find(",") == -1:
return s.split()
else:
########################################
-# Our very own version of commands.getouputstatus(), hacked to support
-# gpgv's status fd.
def gpgv_get_status_output(cmd, status_read, status_write):
+ """
+ Our very own version of commands.getouputstatus(), hacked to support
+ gpgv's status fd.
+ """
+
cmd = ['/bin/sh', '-c', cmd]
p2cread, p2cwrite = os.pipe()
c2pread, c2pwrite = os.pipe()
internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg)
continue
args = split[2:]
- if keywords.has_key(keyword) and (keyword != "NODATA" and keyword != "SIGEXPIRED"):
+ if keywords.has_key(keyword) and keyword not in [ "NODATA", "SIGEXPIRED", "KEYEXPIRED" ]:
internal_error += "found duplicate status token ('%s').\n" % (keyword)
continue
else:
################################################################################
def retrieve_key (filename, keyserver=None, keyring=None):
- """Retrieve the key that signed 'filename' from 'keyserver' and
-add it to 'keyring'. Returns nothing on success, or an error message
-on error."""
+ """
+ Retrieve the key that signed 'filename' from 'keyserver' and
+ add it to 'keyring'. Returns nothing on success, or an error message
+ on error.
+ """
# Defaults for keyserver and keyring
if not keyserver:
keyserver = Cnf["Dinstall::KeyServer"]
if not keyring:
- keyring = Cnf["Dinstall::GPGKeyring"]
+ keyring = Cnf.ValueList("Dinstall::GPGKeyring")[0]
# Ensure the filename contains no shell meta-characters or other badness
if not re_taint_free.match(filename):
return "%s: tainted filename" % (filename)
# Invoke gpgv on the file
- status_read, status_write = os.pipe();
+ status_read, status_write = os.pipe()
cmd = "gpgv --status-fd %s --keyring /dev/null %s" % (status_write, filename)
(_, status, _) = gpgv_get_status_output(cmd, status_read, status_write)
################################################################################
+def gpg_keyring_args(keyrings=None):
+ if not keyrings:
+ keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
+
+ return " ".join(["--keyring %s" % x for x in keyrings])
+
+################################################################################
+
def check_signature (sig_filename, reject, data_filename="", keyrings=None, autofetch=None):
- """Check the signature of a file and return the fingerprint if the
-signature is valid or 'None' if it's not. The first argument is the
-filename whose signature should be checked. The second argument is a
-reject function and is called when an error is found. The reject()
-function must allow for two arguments: the first is the error message,
-the second is an optional prefix string. It's possible for reject()
-to be called more than once during an invocation of check_signature().
-The third argument is optional and is the name of the files the
-detached signature applies to. The fourth argument is optional and is
-a *list* of keyrings to use. 'autofetch' can either be None, True or
-False. If None, the default behaviour specified in the config will be
-used."""
+ """
+ Check the signature of a file and return the fingerprint if the
+ signature is valid or 'None' if it's not. The first argument is the
+ filename whose signature should be checked. The second argument is a
+ reject function and is called when an error is found. The reject()
+ function must allow for two arguments: the first is the error message,
+ the second is an optional prefix string. It's possible for reject()
+ to be called more than once during an invocation of check_signature().
+ The third argument is optional and is the name of the files the
+ detached signature applies to. The fourth argument is optional and is
+ a *list* of keyrings to use. 'autofetch' can either be None, True or
+ False. If None, the default behaviour specified in the config will be
+ used.
+ """
# Ensure the filename contains no shell meta-characters or other badness
if not re_taint_free.match(sig_filename):
return None
if not keyrings:
- keyrings = (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"])
+ keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
# Autofetch the signing key if that's enabled
if autofetch == None:
return None
# Build the command line
- status_read, status_write = os.pipe();
- cmd = "gpgv --status-fd %s" % (status_write)
- for keyring in keyrings:
- cmd += " --keyring %s" % (keyring)
- cmd += " %s %s" % (sig_filename, data_filename)
+ status_read, status_write = os.pipe()
+ cmd = "gpgv --status-fd %s %s %s %s" % (
+ status_write, gpg_keyring_args(keyrings), sig_filename, data_filename)
+
# Invoke gpgv on the file
(output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
bad = ""
# Now check for obviously bad things in the processed output
- if keywords.has_key("SIGEXPIRED"):
- reject("The key used to sign %s has expired." % (sig_filename))
- bad = 1
if keywords.has_key("KEYREVOKED"):
reject("The key used to sign %s has been revoked." % (sig_filename))
bad = 1
if keywords.has_key("NODATA"):
reject("no signature found in %s." % (sig_filename))
bad = 1
+ if keywords.has_key("EXPKEYSIG"):
+ args = keywords["EXPKEYSIG"]
+ if len(args) >= 1:
+ key = args[0]
+ reject("Signature made by expired key 0x%s" % (key))
+ bad = 1
+ if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
+ args = keywords["KEYEXPIRED"]
+ expiredate=""
+ if len(args) >= 1:
+ timestamp = args[0]
+ if timestamp.count("T") == 0:
+ expiredate = time.strftime("%Y-%m-%d", time.gmtime(timestamp))
+ else:
+ expiredate = timestamp
+ reject("The key used to sign %s has expired on %s" % (sig_filename, expiredate))
+ bad = 1
if bad:
return None
# Finally ensure there's not something we don't recognise
known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
- NODATA="")
+ NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="")
for keyword in keywords.keys():
if not known_keywords.has_key(keyword):
################################################################################
+def gpg_get_key_addresses(fingerprint):
+ """retreive email addresses from gpg key uids for a given fingerprint"""
+ addresses = key_uid_email_cache.get(fingerprint)
+ if addresses != None:
+ return addresses
+ addresses = set()
+ cmd = "gpg --no-default-keyring %s --fingerprint %s" \
+ % (gpg_keyring_args(), fingerprint)
+ (result, output) = commands.getstatusoutput(cmd)
+ if result == 0:
+ for l in output.split('\n'):
+ m = re_gpg_uid.match(l)
+ if m:
+ addresses.add(m.group(1))
+ key_uid_email_cache[fingerprint] = addresses
+ return addresses
+
+################################################################################
+
# Inspired(tm) by http://www.zopelabs.com/cookbook/1022242603
def wrap(paragraph, max_length, prefix=""):
################################################################################
-# Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
-# Returns fixed 'src'
def clean_symlink (src, dest, root):
+ """
+ Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
+ Returns fixed 'src'
+ """
src = src.replace(root, '', 1)
dest = dest.replace(root, '', 1)
dest = os.path.dirname(dest)
################################################################################
-def temp_filename(directory=None, dotprefix=None, perms=0700):
- """Return a secure and unique filename by pre-creating it.
-If 'directory' is non-null, it will be the directory the file is pre-created in.
-If 'dotprefix' is non-null, the filename will be prefixed with a '.'."""
-
- if directory:
- old_tempdir = tempfile.tempdir
- tempfile.tempdir = directory
+def temp_filename(directory=None, prefix="dak", suffix=""):
+ """
+ Return a secure and unique filename by pre-creating it.
+ If 'directory' is non-null, it will be the directory the file is pre-created in.
+ If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
+ If 'suffix' is non-null, the filename will end with it.
- filename = tempfile.mktemp()
+ Returns a pair (fd, name).
+ """
- if dotprefix:
- filename = "%s/.%s" % (os.path.dirname(filename), os.path.basename(filename))
- fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, perms)
- os.close(fd)
+ return tempfile.mkstemp(suffix, prefix, directory)
- if directory:
- tempfile.tempdir = old_tempdir
+################################################################################
- return filename
+def is_email_alias(email):
+ """ checks if the user part of the email is listed in the alias file """
+ global alias_cache
+ if alias_cache == None:
+ aliasfn = which_alias_file()
+ alias_cache = set()
+ if aliasfn:
+ for l in open(aliasfn):
+ alias_cache.add(l.split(':')[0])
+ uid = email.split('@')[0]
+ return uid in alias_cache
################################################################################
apt_pkg.ReadConfigFileISC(Cnf,default_config)
if which_conf_file() != default_config:
- apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
+ apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
################################################################################
+
+def generate_contents_information(filename):
+ """
+ Generate a list of flies contained in a .deb
+
+ @type filename: string
+ @param filename: the path to a .deb
+
+ @rtype: list
+ @return: a list of files in the data.tar.* portion of the .deb
+ """
+ cmd = "ar t %s" % (filename)
+ (result, output) = commands.getstatusoutput(cmd)
+ if result != 0:
+ reject("%s: 'ar t' invocation failed." % (filename))
+ reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
+
+ # Ugh ... this is ugly ... Code ripped from process_unchecked.py
+ chunks = output.split('\n')
+
+ contents = []
+ try:
+ cmd = "ar x %s %s" % (filename, chunks[2])
+ (result, output) = commands.getstatusoutput(cmd)
+ if result != 0:
+ reject("%s: '%s' invocation failed." % (filename, cmd))
+ reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
+
+ # Got deb tarballs, now lets go through and determine what bits
+ # and pieces the deb had ...
+ if chunks[2] == "data.tar.gz":
+ data = tarfile.open("data.tar.gz", "r:gz")
+ elif data_tar == "data.tar.bz2":
+ data = tarfile.open("data.tar.bz2", "r:bz2")
+ else:
+ os.remove(chunks[2])
+ reject("couldn't find data.tar.*")
+
+ for tarinfo in data:
+ if not tarinfo.isdir():
+ contents.append(tarinfo.name[2:])
+
+ finally:
+ if os.path.exists( chunks[2] ):
+ os.remove( chunks[2] )
+
+ return contents
+
+###############################################################################
projects / dak.git / blobdiff