X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Futils.py;h=c0a3bdff2936333d87385f473f4b031df08de180;hb=c6c97ebf8447d3ac7399615b028a2c9d0407c805;hp=3305f695ac4faab639dad148b95f8493b83da8b7;hpb=1ef1804c272cf73da57a3e2d1cbf6c7610207ee9;p=dak.git diff --git a/daklib/utils.py b/daklib/utils.py index 3305f695..c0a3bdff 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -36,11 +36,10 @@ import stat import apt_pkg import time import re -import string import email as modemail import subprocess -from dbconn import DBConn, get_architecture, get_component, get_suite +from dbconn import DBConn, get_architecture, get_component, get_suite, get_override_type, Keyring from dak_exceptions import * from textutils import fix_maintainer from regexes import re_html_escaping, html_escaping, re_single_line_field, \ @@ -48,6 +47,7 @@ from regexes import re_html_escaping, html_escaping, re_single_line_field, \ re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource, \ re_is_orig_source +from formats import parse_format, validate_changes_format from srcformats import get_format_from_string from collections import defaultdict @@ -63,14 +63,19 @@ key_uid_email_cache = {} #: Cache for email addresses from gpg key uids known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)), ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc -# Monkeypatch commands.getstatusoutput as it returns a "0" exit code in -# all situations under lenny's Python. -import commands +# Monkeypatch commands.getstatusoutput as it may not return the correct exit +# code in lenny's Python. This also affects commands.getoutput and +# commands.getstatus. def dak_getstatusoutput(cmd): pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = "".join(pipe.stdout.readlines()) + output = pipe.stdout.read() + + pipe.wait() + + if output[-1:] == '\n': + output = output[:-1] ret = pipe.wait() if ret is None: @@ -113,7 +118,12 @@ def open_file(filename, mode='r'): def our_raw_input(prompt=""): if prompt: - sys.stdout.write(prompt) + while 1: + try: + sys.stdout.write(prompt) + break + except IOError: + pass sys.stdout.flush() try: ret = raw_input() @@ -231,7 +241,7 @@ def parse_deb822(contents, signing_rules=0): ################################################################################ -def parse_changes(filename, signing_rules=0): +def parse_changes(filename, signing_rules=0, dsc_file=0): """ Parses a changes file and returns a dictionary where each field is a key. The mandatory first argument is the filename of the .changes @@ -260,7 +270,23 @@ def parse_changes(filename, signing_rules=0): unicode(content, 'utf-8') except UnicodeError: raise ChangesUnicodeError, "Changes file not proper utf-8" - return parse_deb822(content, signing_rules) + changes = parse_deb822(content, signing_rules) + + + if not dsc_file: + # Finally ensure that everything needed for .changes is there + must_keywords = ('Format', 'Date', 'Source', 'Binary', 'Architecture', 'Version', + 'Distribution', 'Maintainer', 'Description', 'Changes', 'Files') + + missingfields=[] + for keyword in must_keywords: + if not changes.has_key(keyword.lower()): + missingfields.append(keyword) + + if len(missingfields): + raise ParseChangesError, "Missing mandantory field(s) in changes file (policy 5.5): %s" % (missingfields) + + return changes ################################################################################ @@ -304,13 +330,13 @@ def check_hash(where, files, hashname, hashfunc): try: try: file_handle = open_file(f) - + # Check for the hash entry, to not trigger a KeyError. if not files[f].has_key(hash_key(hashname)): rejmsg.append("%s: misses %s checksum in %s" % (f, hashname, where)) continue - + # Actually check the hash for correctness. if hashfunc(file_handle) != files[f][hash_key(hashname)]: rejmsg.append("%s: %s check failed in %s" % (f, hashname, @@ -372,7 +398,7 @@ def check_dsc_files(dsc_filename, dsc=None, dsc_files=None): # Parse the file if needed if dsc is None: - dsc = parse_changes(dsc_filename, signing_rules=1); + dsc = parse_changes(dsc_filename, signing_rules=1, dsc_file=1); if dsc_files is None: dsc_files = build_file_list(dsc, is_a_dsc=1) @@ -527,9 +553,9 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): if not changes.has_key(field): raise NoFilesFieldError - # Get SourceFormat object for this Format and validate it - format = get_format_from_string(changes['format']) - format.validate_format(is_a_dsc=is_a_dsc, field=field) + # Validate .changes Format: field + if not is_a_dsc: + validate_changes_format(parse_format(changes['format']), field) includes_section = (not is_a_dsc) and field == "files" @@ -554,7 +580,7 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): (section, component) = extract_component_from_section(section) - files[name] = Dict(size=size, section=section, + files[name] = dict(size=size, section=section, priority=priority, component=component) files[name][hashname] = md5 @@ -562,9 +588,53 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): ################################################################################ +# see http://bugs.debian.org/619131 +def build_package_set(dsc, session = None): + if not dsc.has_key("package-set"): + return {} + + packages = {} + + for line in dsc["package-set"].split("\n"): + if not line: + break + + (name, section, priority) = line.split() + (section, component) = extract_component_from_section(section) + + package_type = "deb" + if name.find(":") != -1: + (package_type, name) = name.split(":", 1) + if package_type == "src": + package_type = "dsc" + + # Validate type if we have a session + if session and get_override_type(package_type, session) is None: + # Maybe just warn and ignore? exit(1) might be a bit hard... + utils.fubar("invalid type (%s) in Package-Set." % (package_type)) + + if section == "": + section = "-" + if priority == "": + priority = "-" + + if package_type == "dsc": + priority = "source" + + if not packages.has_key(name) or packages[name]["type"] == "dsc": + packages[name] = dict(priority=priority, section=section, type=package_type, component=component, files=[]) + + return packages + +################################################################################ + def send_mail (message, filename=""): """sendmail wrapper, takes _either_ a message string or a file as arguments""" + # Check whether we're supposed to be sending mail + if Cnf.has_key("Dinstall::Options::No-Mail") and Cnf["Dinstall::Options::No-Mail"]: + return + # If we've been passed a string dump it into a temporary file if message: (fd, filename) = tempfile.mkstemp() @@ -612,7 +682,7 @@ def send_mail (message, filename=""): if len(match) == 0: del message_raw[field] else: - message_raw.replace_header(field, string.join(match, ", ")) + message_raw.replace_header(field, ', '.join(match)) # Change message fields in order if we don't have a To header if not message_raw.has_key("To"): @@ -702,17 +772,20 @@ def copy (src, dest, overwrite = 0, perms = 0664): ################################################################################ def where_am_i (): - res = socket.gethostbyaddr(socket.gethostname()) - database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname") + res = socket.getfqdn() + database_hostname = Cnf.get("Config::" + res + "::DatabaseHostname") if database_hostname: return database_hostname else: - return res[0] + return res def which_conf_file (): - res = socket.gethostbyaddr(socket.gethostname()) + if os.getenv('DAK_CONFIG'): + return os.getenv('DAK_CONFIG') + + res = socket.getfqdn() # In case we allow local config files per user, try if one exists - if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"): + if Cnf.FindB("Config::" + res + "::AllowLocalConfig"): homedir = os.getenv("HOME") confpath = os.path.join(homedir, "/etc/dak.conf") if os.path.exists(confpath): @@ -720,27 +793,27 @@ def which_conf_file (): # We are still in here, so there is no local config file or we do # not allow local files. Do the normal stuff. - if Cnf.get("Config::" + res[0] + "::DakConfig"): - return Cnf["Config::" + res[0] + "::DakConfig"] - else: - return default_config + if Cnf.get("Config::" + res + "::DakConfig"): + return Cnf["Config::" + res + "::DakConfig"] + + return default_config def which_apt_conf_file (): - res = socket.gethostbyaddr(socket.gethostname()) + res = socket.getfqdn() # In case we allow local config files per user, try if one exists - if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"): + if Cnf.FindB("Config::" + res + "::AllowLocalConfig"): homedir = os.getenv("HOME") confpath = os.path.join(homedir, "/etc/dak.conf") if os.path.exists(confpath): apt_pkg.ReadConfigFileISC(Cnf,default_config) - if Cnf.get("Config::" + res[0] + "::AptConfig"): - return Cnf["Config::" + res[0] + "::AptConfig"] + if Cnf.get("Config::" + res + "::AptConfig"): + return Cnf["Config::" + res + "::AptConfig"] else: return default_apt_config def which_alias_file(): - hostname = socket.gethostbyaddr(socket.gethostname())[0] + hostname = socket.getfqdn() aliasfn = '/var/lib/misc/'+hostname+'/forward-alias' if os.path.exists(aliasfn): return aliasfn @@ -749,12 +822,12 @@ def which_alias_file(): ################################################################################ -def TemplateSubst(map, filename): +def TemplateSubst(subst_map, filename): """ Perform a substition of template """ templatefile = open_file(filename) template = templatefile.read() - for x in map.keys(): - template = template.replace(x, str(map[x])) + for k, v in subst_map.iteritems(): + template = template.replace(k, str(v)) templatefile.close() return template @@ -1087,10 +1160,6 @@ def split_args (s, dwim=1): ################################################################################ -def Dict(**dict): return dict - -######################################## - def gpgv_get_status_output(cmd, status_read, status_write): """ Our very own version of commands.getouputstatus(), hacked to support @@ -1267,7 +1336,7 @@ def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=No return (None, rejects) if not keyrings: - keyrings = Cnf.ValueList("Dinstall::GPGKeyring") + keyrings = [ x.keyring_name for x in DBConn().session().query(Keyring).all() ] # Autofetch the signing key if that's enabled if autofetch == None: @@ -1338,9 +1407,9 @@ def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=No if exit_status: rejects.append("gpgv failed while checking %s." % (sig_filename)) if status.strip(): - rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "), "") + rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] ")) else: - rejects.append(prefix_multi_line_string(output, " [GPG output:] "), "") + rejects.append(prefix_multi_line_string(output, " [GPG output:] ")) return (None, rejects) # Sanity check the good stuff we expect @@ -1358,9 +1427,9 @@ def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=No rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename)) # Finally ensure there's not something we don't recognise - known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", + known_keywords = dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="", - NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="") + NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="",POLICY_URL="") for keyword in keywords.keys(): if not known_keywords.has_key(keyword): @@ -1480,7 +1549,7 @@ def is_email_alias(email): ################################################################################ -def get_changes_files(dir): +def get_changes_files(from_dir): """ Takes a directory and lists all .changes files in it (as well as chdir'ing to the directory; this is due to broken behaviour on the part of p-u/p-a @@ -1490,10 +1559,10 @@ def get_changes_files(dir): """ try: # Much of the rest of p-u/p-a depends on being in the right place - os.chdir(dir) - changes_files = [x for x in os.listdir(dir) if x.endswith('.changes')] + os.chdir(from_dir) + changes_files = [x for x in os.listdir(from_dir) if x.endswith('.changes')] except OSError, e: - fubar("Failed to read list from directory %s (%s)" % (dir, e)) + fubar("Failed to read list from directory %s (%s)" % (from_dir, e)) return changes_files @@ -1502,54 +1571,81 @@ def get_changes_files(dir): apt_pkg.init() Cnf = apt_pkg.newConfiguration() -apt_pkg.ReadConfigFileISC(Cnf,default_config) +if not os.getenv("DAK_TEST"): + apt_pkg.ReadConfigFileISC(Cnf,default_config) if which_conf_file() != default_config: apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()) -############################################################################### +################################################################################ -def ensure_orig_files(changes, dest_dir, session): +def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm"): """ - Ensure that dest_dir contains all the orig tarballs for the specified - changes. If it does not, symlink them into place. + Parses the wnpp bug list available at http://qa.debian.org/data/bts/wnpp_rm + Well, actually it parsed a local copy, but let's document the source + somewhere ;) - Returns a 2-tuple (already_exists, symlinked) containing a list of files - that were already there and a list of files that were symlinked into place. + returns a dict associating source package name with a list of open wnpp + bugs (Yes, there might be more than one) """ - exists, symlinked = [], [] - - for dsc_file in changes.dsc_files: + line = [] + try: + f = open(file) + lines = f.readlines() + except IOError, e: + print "Warning: Couldn't open %s; don't know about WNPP bugs, so won't close any." % file + lines = [] + wnpp = {} - # Skip all files that are not orig tarballs - if not re_is_orig_source.match(dsc_file): - continue + for line in lines: + splited_line = line.split(": ", 1) + if len(splited_line) > 1: + wnpp[splited_line[0]] = splited_line[1].split("|") + + for source in wnpp.keys(): + bugs = [] + for wnpp_bug in wnpp[source]: + bug_no = re.search("(\d)+", wnpp_bug).group() + if bug_no: + bugs.append(bug_no) + wnpp[source] = bugs + return wnpp - # Skip orig files not identified in the pool - if not (dsc_file in changes.orig_files and - 'id' in changes.orig_files[dsc_file]): - continue +################################################################################ - dest = os.path.join(dest_dir, dsc_file) +def get_packages_from_ftp(root, suite, component, architecture): + """ + Returns an object containing apt_pkg-parseable data collected by + aggregating Packages.gz files gathered for each architecture. - if os.path.exists(dest): - exists.append(dest) - continue + @type root: string + @param root: path to ftp archive root directory - orig_file_id = changes.orig_files[dsc_file]['id'] + @type suite: string + @param suite: suite to extract files from - c = session.execute( - 'SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id', - {'id': orig_file_id} - ) + @type component: string + @param component: component to extract files from - res = c.fetchone() - if not res: - return "[INTERNAL ERROR] Couldn't find id %s in files table." % orig_file_id + @type architecture: string + @param architecture: architecture to extract files from - src = os.path.join(res[0], res[1]) - os.symlink(src, dest) - symlinked.append(dest) + @rtype: TagFile + @return: apt_pkg class containing package data - return (exists, symlinked) + """ + filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (root, suite, component, architecture) + (fd, temp_file) = temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_file)) + if (result != 0): + fubar("Gunzip invocation failed!\n%s\n" % (output), result) + filename = "%s/dists/%s/%s/debian-installer/binary-%s/Packages.gz" % (root, suite, component, architecture) + if os.path.exists(filename): + (result, output) = commands.getstatusoutput("gunzip -c %s >> %s" % (filename, temp_file)) + if (result != 0): + fubar("Gunzip invocation failed!\n%s\n" % (output), result) + packages = open_file(temp_file) + Packages = apt_pkg.ParseTagFile(packages) + os.unlink(temp_file) + return Packages