X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Futils.py;h=084ac195bad036266d869de61b2933e752562612;hb=52eb03dccab0a9898bca4d36a0a96ebe377fb977;hp=7129aa32ed4f7d1f62aca4c094adfb5051380704;hpb=b43ed3ff3738940ce46caa836d88b6937a76582c;p=dak.git diff --git a/daklib/utils.py b/daklib/utils.py index 7129aa32..084ac195 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -22,11 +22,12 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import codecs import commands +import datetime import email.Header import os import pwd +import grp import select import socket import shutil @@ -34,16 +35,32 @@ import sys import tempfile import traceback import stat +import apt_inst import apt_pkg -import database import time import re -import string import email as modemail +import subprocess +import ldap + +import daklib.config as config +from dbconn import DBConn, get_architecture, get_component, get_suite, \ + get_override_type, Keyring, session_wrapper, \ + get_active_keyring_paths, get_primary_keyring_path, \ + get_suite_architectures, get_or_set_metadatakey, DBSource, \ + Component, Override, OverrideType +from sqlalchemy import desc from dak_exceptions import * +from gpg import SignedFile +from textutils import fix_maintainer from regexes import re_html_escaping, html_escaping, re_single_line_field, \ - re_multi_line_field, re_srchasver, re_verwithext, \ - re_parse_maintainer, re_taint_free, re_gpg_uid, re_re_mark + re_multi_line_field, re_srchasver, re_taint_free, \ + re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource, \ + re_is_orig_source, re_build_dep_arch + +from formats import parse_format, validate_changes_format +from srcformats import get_format_from_string +from collections import defaultdict ################################################################################ @@ -57,6 +74,27 @@ key_uid_email_cache = {} #: Cache for email addresses from gpg key uids known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)), ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc +# Monkeypatch commands.getstatusoutput as it may not return the correct exit +# code in lenny's Python. This also affects commands.getoutput and +# commands.getstatus. +def dak_getstatusoutput(cmd): + pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + output = pipe.stdout.read() + + pipe.wait() + + if output[-1:] == '\n': + output = output[:-1] + + ret = pipe.wait() + if ret is None: + ret = 0 + + return ret, output +commands.getstatusoutput = dak_getstatusoutput + ################################################################################ def html_escape(s): @@ -84,14 +122,19 @@ def open_file(filename, mode='r'): try: f = open(filename, mode) except IOError: - raise CantOpenError, filename + raise CantOpenError(filename) return f ################################################################################ def our_raw_input(prompt=""): if prompt: - sys.stdout.write(prompt) + while 1: + try: + sys.stdout.write(prompt) + break + except IOError: + pass sys.stdout.flush() try: ret = raw_input() @@ -102,7 +145,7 @@ def our_raw_input(prompt=""): ################################################################################ -def extract_component_from_section(section): +def extract_component_from_section(section, session=None): component = "" if section.find('/') != -1: @@ -110,16 +153,25 @@ def extract_component_from_section(section): # Expand default component if component == "": - if Cnf.has_key("Component::%s" % section): - component = section - else: + comp = get_component(section, session) + if comp is None: component = "main" + else: + component = comp.component_name return (section, component) ################################################################################ -def parse_deb822(contents, signing_rules=0): +def parse_deb822(armored_contents, signing_rules=0, keyrings=None, session=None): + require_signature = True + if keyrings == None: + keyrings = [] + require_signature = False + + signed_file = SignedFile(armored_contents, keyrings=keyrings, require_signature=require_signature) + contents = signed_file.contents + error = "" changes = {} @@ -127,7 +179,7 @@ def parse_deb822(contents, signing_rules=0): lines = contents.splitlines(True) if len(lines) == 0: - raise ParseChangesError, "[Empty changes file]" + raise ParseChangesError("[Empty changes file]") # Reindex by line number so we can easily verify the format of # .dsc files... @@ -137,38 +189,16 @@ def parse_deb822(contents, signing_rules=0): index += 1 indexed_lines[index] = line[:-1] - inside_signature = 0 - num_of_lines = len(indexed_lines.keys()) index = 0 first = -1 while index < num_of_lines: index += 1 line = indexed_lines[index] - if line == "": - if signing_rules == 1: - index += 1 - if index > num_of_lines: - raise InvalidDscError, index - line = indexed_lines[index] - if not line.startswith("-----BEGIN PGP SIGNATURE"): - raise InvalidDscError, index - inside_signature = 0 - break - else: - continue - if line.startswith("-----BEGIN PGP SIGNATURE"): + if line == "" and signing_rules == 1: + if index != num_of_lines: + raise InvalidDscError(index) break - if line.startswith("-----BEGIN PGP SIGNED MESSAGE"): - inside_signature = 1 - if signing_rules == 1: - while index < num_of_lines and line != "": - index += 1 - line = indexed_lines[index] - continue - # If we're not inside the signed data, don't process anything - if signing_rules >= 0 and not inside_signature: - continue slf = re_single_line_field.match(line) if slf: field = slf.groups()[0].lower() @@ -181,7 +211,7 @@ def parse_deb822(contents, signing_rules=0): mlf = re_multi_line_field.match(line) if mlf: if first == -1: - raise ParseChangesError, "'%s'\n [Multi-line field continuing on from nothing?]" % (line) + raise ParseChangesError("'%s'\n [Multi-line field continuing on from nothing?]" % (line)) if first == 1 and changes[field] != "": changes[field] += '\n' first = 0 @@ -189,10 +219,7 @@ def parse_deb822(contents, signing_rules=0): continue error += line - if signing_rules == 1 and inside_signature: - raise InvalidDscError, index - - changes["filecontents"] = "".join(lines) + changes["filecontents"] = armored_contents if changes.has_key("source"): # Strip the source version in brackets from the source field, @@ -203,13 +230,13 @@ def parse_deb822(contents, signing_rules=0): changes["source-version"] = srcver.group(2) if error: - raise ParseChangesError, error + raise ParseChangesError(error) return changes ################################################################################ -def parse_changes(filename, signing_rules=0): +def parse_changes(filename, signing_rules=0, dsc_file=0, keyrings=None): """ Parses a changes file and returns a dictionary where each field is a key. The mandatory first argument is the filename of the .changes @@ -237,8 +264,24 @@ def parse_changes(filename, signing_rules=0): try: unicode(content, 'utf-8') except UnicodeError: - raise ChangesUnicodeError, "Changes file not proper utf-8" - return parse_deb822(content, signing_rules) + raise ChangesUnicodeError("Changes file not proper utf-8") + changes = parse_deb822(content, signing_rules, keyrings=keyrings) + + + if not dsc_file: + # Finally ensure that everything needed for .changes is there + must_keywords = ('Format', 'Date', 'Source', 'Binary', 'Architecture', 'Version', + 'Distribution', 'Maintainer', 'Description', 'Changes', 'Files') + + missingfields=[] + for keyword in must_keywords: + if not changes.has_key(keyword.lower()): + missingfields.append(keyword) + + if len(missingfields): + raise ParseChangesError("Missing mandantory field(s) in changes file (policy 5.5): %s" % (missingfields)) + + return changes ################################################################################ @@ -282,13 +325,13 @@ def check_hash(where, files, hashname, hashfunc): try: try: file_handle = open_file(f) - + # Check for the hash entry, to not trigger a KeyError. if not files[f].has_key(hash_key(hashname)): rejmsg.append("%s: misses %s checksum in %s" % (f, hashname, where)) continue - + # Actually check the hash for correctness. if hashfunc(file_handle) != files[f][hash_key(hashname)]: rejmsg.append("%s: %s check failed in %s" % (f, hashname, @@ -314,7 +357,7 @@ def check_size(where, files): for f in files.keys(): try: entry = os.stat(f) - except OSError, exc: + except OSError as exc: if exc.errno == 2: # TODO: This happens when the file is in the pool. continue @@ -329,6 +372,79 @@ def check_size(where, files): ################################################################################ +def check_dsc_files(dsc_filename, dsc, dsc_files): + """ + Verify that the files listed in the Files field of the .dsc are + those expected given the announced Format. + + @type dsc_filename: string + @param dsc_filename: path of .dsc file + + @type dsc: dict + @param dsc: the content of the .dsc parsed by C{parse_changes()} + + @type dsc_files: dict + @param dsc_files: the file list returned by C{build_file_list()} + + @rtype: list + @return: all errors detected + """ + rejmsg = [] + + # Ensure .dsc lists proper set of source files according to the format + # announced + has = defaultdict(lambda: 0) + + ftype_lookup = ( + (r'orig.tar.gz', ('orig_tar_gz', 'orig_tar')), + (r'diff.gz', ('debian_diff',)), + (r'tar.gz', ('native_tar_gz', 'native_tar')), + (r'debian\.tar\.(gz|bz2|xz)', ('debian_tar',)), + (r'orig\.tar\.(gz|bz2|xz)', ('orig_tar',)), + (r'tar\.(gz|bz2|xz)', ('native_tar',)), + (r'orig-.+\.tar\.(gz|bz2|xz)', ('more_orig_tar',)), + ) + + for f in dsc_files: + m = re_issource.match(f) + if not m: + rejmsg.append("%s: %s in Files field not recognised as source." + % (dsc_filename, f)) + continue + + # Populate 'has' dictionary by resolving keys in lookup table + matched = False + for regex, keys in ftype_lookup: + if re.match(regex, m.group(3)): + matched = True + for key in keys: + has[key] += 1 + break + + # File does not match anything in lookup table; reject + if not matched: + reject("%s: unexpected source file '%s'" % (dsc_filename, f)) + + # Check for multiple files + for file_type in ('orig_tar', 'native_tar', 'debian_tar', 'debian_diff'): + if has[file_type] > 1: + rejmsg.append("%s: lists multiple %s" % (dsc_filename, file_type)) + + # Source format specific tests + try: + format = get_format_from_string(dsc['format']) + rejmsg.extend([ + '%s: %s' % (dsc_filename, x) for x in format.reject_msgs(has) + ]) + + except UnknownFormatError: + # Not an error here for now + pass + + return rejmsg + +################################################################################ + def check_hash_fields(what, manifest): """ check_hash_fields ensures that there are no checksum fields in the @@ -383,41 +499,6 @@ def _ensure_dsc_hash(dsc, dsc_files, hashname, hashfunc): ################################################################################ -def ensure_hashes(changes, dsc, files, dsc_files): - rejmsg = [] - - # Make sure we recognise the format of the Files: field in the .changes - format = changes.get("format", "0.0").split(".", 1) - if len(format) == 2: - format = int(format[0]), int(format[1]) - else: - format = int(float(format[0])), 0 - - # We need to deal with the original changes blob, as the fields we need - # might not be in the changes dict serialised into the .dak anymore. - orig_changes = parse_deb822(changes['filecontents']) - - # Copy the checksums over to the current changes dict. This will keep - # the existing modifications to it intact. - for field in orig_changes: - if field.startswith('checksums-'): - changes[field] = orig_changes[field] - - # Check for unsupported hashes - rejmsg.extend(check_hash_fields(".changes", changes)) - rejmsg.extend(check_hash_fields(".dsc", dsc)) - - # We have to calculate the hash if we have an earlier changes version than - # the hash appears in rather than require it exist in the changes file - for hashname, hashfunc, version in known_hashes: - rejmsg.extend(_ensure_changes_hash(changes, format, version, files, - hashname, hashfunc)) - if "source" in changes["architecture"]: - rejmsg.extend(_ensure_dsc_hash(dsc, dsc_files, hashname, - hashfunc)) - - return rejmsg - def parse_checksums(where, files, manifest, hashname): rejmsg = [] field = 'checksums-%s' % hashname @@ -426,7 +507,12 @@ def parse_checksums(where, files, manifest, hashname): for line in manifest[field].split('\n'): if not line: break - checksum, size, checkfile = line.strip().split(' ') + clist = line.strip().split(' ') + if len(clist) == 3: + checksum, size, checkfile = clist + else: + rejmsg.append("Cannot parse checksum line [%s]" % (line)) + continue if not files.has_key(checkfile): # TODO: check for the file's entry in the original files dict, not # the one modified by (auto)byhand and other weird stuff @@ -440,8 +526,7 @@ def parse_checksums(where, files, manifest, hashname): files[checkfile][hash_key(hashname)] = checksum for f in files.keys(): if not files[f].has_key(hash_key(hashname)): - rejmsg.append("%s: no entry in checksums-%s in %s" % (checkfile, - hashname, where)) + rejmsg.append("%s: no entry in checksums-%s in %s" % (f, hashname, where)) return rejmsg ################################################################################ @@ -455,30 +540,9 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): if not changes.has_key(field): raise NoFilesFieldError - # Make sure we recognise the format of the Files: field - format = re_verwithext.search(changes.get("format", "0.0")) - if not format: - raise UnknownFormatError, "%s" % (changes.get("format","0.0")) - - format = format.groups() - if format[1] == None: - format = int(float(format[0])), 0, format[2] - else: - format = int(format[0]), int(format[1]), format[2] - if format[2] == None: - format = format[:2] - - if is_a_dsc: - # format = (1,0) are the only formats we currently accept, - # format = (0,0) are missing format headers of which we still - # have some in the archive. - if format != (1,0) and format != (0,0): - raise UnknownFormatError, "%s" % (changes.get("format","0.0")) - else: - if (format < (1,5) or format > (1,8)): - raise UnknownFormatError, "%s" % (changes.get("format","0.0")) - if field != "files" and format < (1,8): - raise UnknownFormatError, "%s" % (changes.get("format","0.0")) + # Validate .changes Format: field + if not is_a_dsc: + validate_changes_format(parse_format(changes['format']), field) includes_section = (not is_a_dsc) and field == "files" @@ -494,7 +558,7 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): else: (md5, size, name) = s except ValueError: - raise ParseChangesError, i + raise ParseChangesError(i) if section == "": section = "-" @@ -503,7 +567,7 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): (section, component) = extract_component_from_section(section) - files[name] = Dict(size=size, section=section, + files[name] = dict(size=size, section=section, priority=priority, component=component) files[name][hashname] = md5 @@ -511,94 +575,57 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): ################################################################################ -def force_to_utf8(s): - """ - Forces a string to UTF-8. If the string isn't already UTF-8, - it's assumed to be ISO-8859-1. - """ - try: - unicode(s, 'utf-8') - return s - except UnicodeError: - latin1_s = unicode(s,'iso8859-1') - return latin1_s.encode('utf-8') - -def rfc2047_encode(s): - """ - Encodes a (header) string per RFC2047 if necessary. If the - string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1. - """ - try: - codecs.lookup('ascii')[1](s) - return s - except UnicodeError: - pass - try: - codecs.lookup('utf-8')[1](s) - h = email.Header.Header(s, 'utf-8', 998) - return str(h) - except UnicodeError: - h = email.Header.Header(s, 'iso-8859-1', 998) - return str(h) - -################################################################################ +# see http://bugs.debian.org/619131 +def build_package_list(dsc, session = None): + if not dsc.has_key("package-list"): + return {} -# 'The standard sucks, but my tool is supposed to interoperate -# with it. I know - I'll fix the suckage and make things -# incompatible!' + packages = {} -def fix_maintainer (maintainer): - """ - Parses a Maintainer or Changed-By field and returns: - 1. an RFC822 compatible version, - 2. an RFC2047 compatible version, - 3. the name - 4. the email + for line in dsc["package-list"].split("\n"): + if not line: + break - The name is forced to UTF-8 for both 1. and 3.. If the name field - contains '.' or ',' (as allowed by Debian policy), 1. and 2. are - switched to 'email (name)' format. + fields = line.split() + name = fields[0] + package_type = fields[1] + (section, component) = extract_component_from_section(fields[2]) + priority = fields[3] - """ - maintainer = maintainer.strip() - if not maintainer: - return ('', '', '', '') - - if maintainer.find("<") == -1: - email = maintainer - name = "" - elif (maintainer[0] == "<" and maintainer[-1:] == ">"): - email = maintainer[1:-1] - name = "" - else: - m = re_parse_maintainer.match(maintainer) - if not m: - raise ParseMaintError, "Doesn't parse as a valid Maintainer field." - name = m.group(1) - email = m.group(2) + # Validate type if we have a session + if session and get_override_type(package_type, session) is None: + # Maybe just warn and ignore? exit(1) might be a bit hard... + utils.fubar("invalid type (%s) in Package-List." % (package_type)) - # Get an RFC2047 compliant version of the name - rfc2047_name = rfc2047_encode(name) + if name not in packages or packages[name]["type"] == "dsc": + packages[name] = dict(priority=priority, section=section, type=package_type, component=component, files=[]) - # Force the name to be UTF-8 - name = force_to_utf8(name) + return packages - if name.find(',') != -1 or name.find('.') != -1: - rfc822_maint = "%s (%s)" % (email, name) - rfc2047_maint = "%s (%s)" % (email, rfc2047_name) - else: - rfc822_maint = "%s <%s>" % (name, email) - rfc2047_maint = "%s <%s>" % (rfc2047_name, email) +################################################################################ - if email.find("@") == -1 and email.find("buildd_") != 0: - raise ParseMaintError, "No @ found in email address part." +def send_mail (message, filename="", whitelists=None): + """sendmail wrapper, takes _either_ a message string or a file as arguments - return (rfc822_maint, rfc2047_maint, name, email) + @type whitelists: list of (str or None) + @param whitelists: path to whitelists. C{None} or an empty list whitelists + everything, otherwise an address is whitelisted if it is + included in any of the lists. + In addition a global whitelist can be specified in + Dinstall::MailWhiteList. + """ -################################################################################ + maildir = Cnf.get('Dir::Mail') + if maildir: + path = os.path.join(maildir, datetime.datetime.now().isoformat()) + path = find_next_free(path) + fh = open(path, 'w') + print >>fh, message, + fh.close() -def send_mail (message, filename=""): - """sendmail wrapper, takes _either_ a message string or a file as arguments""" + # Check whether we're supposed to be sending mail + if Cnf.has_key("Dinstall::Options::No-Mail") and Cnf["Dinstall::Options::No-Mail"]: + return # If we've been passed a string dump it into a temporary file if message: @@ -606,22 +633,24 @@ def send_mail (message, filename=""): os.write (fd, message) os.close (fd) - if Cnf.has_key("Dinstall::MailWhiteList") and \ - Cnf["Dinstall::MailWhiteList"] != "": + if whitelists is None or None in whitelists: + whitelists = [] + if Cnf.get('Dinstall::MailWhiteList', ''): + whitelists.append(Cnf['Dinstall::MailWhiteList']) + if len(whitelists) != 0: message_in = open_file(filename) message_raw = modemail.message_from_file(message_in) message_in.close(); whitelist = []; - whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"]) - try: + for path in whitelists: + with open_file(path, 'r') as whitelist_in: for line in whitelist_in: - if re_re_mark.match(line): - whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1))) - else: - whitelist.append(re.compile(re.escape(line.strip()))) - finally: - whitelist_in.close() + if not re_whitespace_comment.match(line): + if re_re_mark.match(line): + whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1))) + else: + whitelist.append(re.compile(re.escape(line.strip()))) # Fields to check. fields = ["To", "Bcc", "Cc"] @@ -638,7 +667,7 @@ def send_mail (message, filename=""): mail_whitelisted = 1 break if not mail_whitelisted: - print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"]) + print "Skipping {0} since it's not whitelisted".format(item) continue match.append(item) @@ -646,7 +675,7 @@ def send_mail (message, filename=""): if len(match) == 0: del message_raw[field] else: - message_raw.replace_header(field, string.join(match, ", ")) + message_raw.replace_header(field, ', '.join(match)) # Change message fields in order if we don't have a To header if not message_raw.has_key("To"): @@ -663,14 +692,14 @@ def send_mail (message, filename=""): os.unlink (filename); return; - fd = os.open(filename, os.O_RDWR|os.O_EXCL, 0700); + fd = os.open(filename, os.O_RDWR|os.O_EXCL, 0o700); os.write (fd, message_raw.as_string(True)); os.close (fd); # Invoke sendmail (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename)) if (result != 0): - raise SendmailFailedError, output + raise SendmailFailedError(output) # Clean up any temporary files if message: @@ -678,24 +707,22 @@ def send_mail (message, filename=""): ################################################################################ -def poolify (source, component): - if component: - component += '/' +def poolify (source, component=None): if source[:3] == "lib": - return component + source[:4] + '/' + source + '/' + return source[:4] + '/' + source + '/' else: - return component + source[:1] + '/' + source + '/' + return source[:1] + '/' + source + '/' ################################################################################ -def move (src, dest, overwrite = 0, perms = 0664): +def move (src, dest, overwrite = 0, perms = 0o664): if os.path.exists(dest) and os.path.isdir(dest): dest_dir = dest else: dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): umask = os.umask(00000) - os.makedirs(dest_dir, 02775) + os.makedirs(dest_dir, 0o2775) os.umask(umask) #print "Moving %s to %s..." % (src, dest) if os.path.exists(dest) and os.path.isdir(dest): @@ -711,14 +738,14 @@ def move (src, dest, overwrite = 0, perms = 0664): os.chmod(dest, perms) os.unlink(src) -def copy (src, dest, overwrite = 0, perms = 0664): +def copy (src, dest, overwrite = 0, perms = 0o664): if os.path.exists(dest) and os.path.isdir(dest): dest_dir = dest else: dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): umask = os.umask(00000) - os.makedirs(dest_dir, 02775) + os.makedirs(dest_dir, 0o2775) os.umask(umask) #print "Copying %s to %s..." % (src, dest) if os.path.exists(dest) and os.path.isdir(dest): @@ -736,29 +763,34 @@ def copy (src, dest, overwrite = 0, perms = 0664): ################################################################################ def where_am_i (): - res = socket.gethostbyaddr(socket.gethostname()) - database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname") + res = socket.getfqdn() + database_hostname = Cnf.get("Config::" + res + "::DatabaseHostname") if database_hostname: return database_hostname else: - return res[0] + return res def which_conf_file (): - res = socket.gethostbyaddr(socket.gethostname()) - if Cnf.get("Config::" + res[0] + "::DakConfig"): - return Cnf["Config::" + res[0] + "::DakConfig"] - else: - return default_config + if os.getenv('DAK_CONFIG'): + return os.getenv('DAK_CONFIG') -def which_apt_conf_file (): - res = socket.gethostbyaddr(socket.gethostname()) - if Cnf.get("Config::" + res[0] + "::AptConfig"): - return Cnf["Config::" + res[0] + "::AptConfig"] - else: - return default_apt_config + res = socket.getfqdn() + # In case we allow local config files per user, try if one exists + if Cnf.find_b("Config::" + res + "::AllowLocalConfig"): + homedir = os.getenv("HOME") + confpath = os.path.join(homedir, "/etc/dak.conf") + if os.path.exists(confpath): + apt_pkg.ReadConfigFileISC(Cnf,confpath) + + # We are still in here, so there is no local config file or we do + # not allow local files. Do the normal stuff. + if Cnf.get("Config::" + res + "::DakConfig"): + return Cnf["Config::" + res + "::DakConfig"] + + return default_config def which_alias_file(): - hostname = socket.gethostbyaddr(socket.gethostname())[0] + hostname = socket.getfqdn() aliasfn = '/var/lib/misc/'+hostname+'/forward-alias' if os.path.exists(aliasfn): return aliasfn @@ -767,22 +799,12 @@ def which_alias_file(): ################################################################################ -# Escape characters which have meaning to SQL's regex comparison operator ('~') -# (woefully incomplete) - -def regex_safe (s): - s = s.replace('+', '\\\\+') - s = s.replace('.', '\\\\.') - return s - -################################################################################ - -def TemplateSubst(map, filename): +def TemplateSubst(subst_map, filename): """ Perform a substition of template """ templatefile = open_file(filename) template = templatefile.read() - for x in map.keys(): - template = template.replace(x,map[x]) + for k, v in subst_map.iteritems(): + template = template.replace(k, str(v)) templatefile.close() return template @@ -802,6 +824,9 @@ def warn(msg): def whoami (): return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '') +def getusername (): + return pwd.getpwuid(os.getuid())[0] + ################################################################################ def size_type (c): @@ -849,7 +874,7 @@ def changes_compare (a, b): # Sort by source version a_version = a_changes.get("version", "0") b_version = b_changes.get("version", "0") - q = apt_pkg.VersionCompare(a_version, b_version) + q = apt_pkg.version_compare(a_version, b_version) if q: return q @@ -977,15 +1002,19 @@ def get_conf(): def parse_args(Options): """ Handle -a, -c and -s arguments; returns them as SQL constraints """ + # XXX: This should go away and everything which calls it be converted + # to use SQLA properly. For now, we'll just fix it not to use + # the old Pg interface though + session = DBConn().session() # Process suite if Options["Suite"]: suite_ids_list = [] - for suite in split_args(Options["Suite"]): - suite_id = database.get_suite_id(suite) - if suite_id == -1: - warn("suite '%s' not recognised." % (suite)) + for suitename in split_args(Options["Suite"]): + suite = get_suite(suitename, session=session) + if not suite or suite.suite_id is None: + warn("suite '%s' not recognised." % (suite and suite.suite_name or suitename)) else: - suite_ids_list.append(suite_id) + suite_ids_list.append(suite.suite_id) if suite_ids_list: con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ]) else: @@ -996,12 +1025,12 @@ def parse_args(Options): # Process component if Options["Component"]: component_ids_list = [] - for component in split_args(Options["Component"]): - component_id = database.get_component_id(component) - if component_id == -1: - warn("component '%s' not recognised." % (component)) + for componentname in split_args(Options["Component"]): + component = get_component(componentname, session=session) + if component is None: + warn("component '%s' not recognised." % (componentname)) else: - component_ids_list.append(component_id) + component_ids_list.append(component.component_id) if component_ids_list: con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ]) else: @@ -1011,18 +1040,18 @@ def parse_args(Options): # Process architecture con_architectures = "" + check_source = 0 if Options["Architecture"]: arch_ids_list = [] - check_source = 0 - for architecture in split_args(Options["Architecture"]): - if architecture == "source": + for archname in split_args(Options["Architecture"]): + if archname == "source": check_source = 1 else: - architecture_id = database.get_architecture_id(architecture) - if architecture_id == -1: - warn("architecture '%s' not recognised." % (architecture)) + arch = get_architecture(archname, session=session) + if arch is None: + warn("architecture '%s' not recognised." % (archname)) else: - arch_ids_list.append(architecture_id) + arch_ids_list.append(arch.arch_id) if arch_ids_list: con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ]) else: @@ -1035,43 +1064,6 @@ def parse_args(Options): ################################################################################ -# Inspired(tm) by Bryn Keller's print_exc_plus (See -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52215) - -def print_exc(): - tb = sys.exc_info()[2] - while tb.tb_next: - tb = tb.tb_next - stack = [] - frame = tb.tb_frame - while frame: - stack.append(frame) - frame = frame.f_back - stack.reverse() - traceback.print_exc() - for frame in stack: - print "\nFrame %s in %s at line %s" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno) - for key, value in frame.f_locals.items(): - print "\t%20s = " % key, - try: - print value - except: - print "" - -################################################################################ - -def try_with_debug(function): - try: - function() - except SystemExit: - raise - except: - print_exc() - -################################################################################ - def arch_compare_sw (a, b): """ Function for use in sorting lists of architectures. @@ -1108,10 +1100,6 @@ def split_args (s, dwim=1): ################################################################################ -def Dict(**dict): return dict - -######################################## - def gpgv_get_status_output(cmd, status_read, status_write): """ Our very own version of commands.getouputstatus(), hacked to support @@ -1217,7 +1205,7 @@ def retrieve_key (filename, keyserver=None, keyring=None): if not keyserver: keyserver = Cnf["Dinstall::KeyServer"] if not keyring: - keyring = Cnf.ValueList("Dinstall::GPGKeyring")[0] + keyring = get_primary_keyring_path() # Ensure the filename contains no shell meta-characters or other badness if not re_taint_free.match(filename): @@ -1254,13 +1242,13 @@ def retrieve_key (filename, keyserver=None, keyring=None): def gpg_keyring_args(keyrings=None): if not keyrings: - keyrings = Cnf.ValueList("Dinstall::GPGKeyring") + keyrings = get_active_keyring_paths() return " ".join(["--keyring %s" % x for x in keyrings]) ################################################################################ - -def check_signature (sig_filename, reject, data_filename="", keyrings=None, autofetch=None): +@session_wrapper +def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None, session=None): """ Check the signature of a file and return the fingerprint if the signature is valid or 'None' if it's not. The first argument is the @@ -1276,17 +1264,19 @@ def check_signature (sig_filename, reject, data_filename="", keyrings=None, auto used. """ + rejects = [] + # Ensure the filename contains no shell meta-characters or other badness if not re_taint_free.match(sig_filename): - reject("!!WARNING!! tainted signature filename: '%s'." % (sig_filename)) - return None + rejects.append("!!WARNING!! tainted signature filename: '%s'." % (sig_filename)) + return (None, rejects) if data_filename and not re_taint_free.match(data_filename): - reject("!!WARNING!! tainted data filename: '%s'." % (data_filename)) - return None + rejects.append("!!WARNING!! tainted data filename: '%s'." % (data_filename)) + return (None, rejects) if not keyrings: - keyrings = Cnf.ValueList("Dinstall::GPGKeyring") + keyrings = [ x.keyring_name for x in session.query(Keyring).filter(Keyring.active == True).all() ] # Autofetch the signing key if that's enabled if autofetch == None: @@ -1294,8 +1284,8 @@ def check_signature (sig_filename, reject, data_filename="", keyrings=None, auto if autofetch: error_msg = retrieve_key(sig_filename) if error_msg: - reject(error_msg) - return None + rejects.append(error_msg) + return (None, rejects) # Build the command line status_read, status_write = os.pipe() @@ -1310,40 +1300,32 @@ def check_signature (sig_filename, reject, data_filename="", keyrings=None, auto # If we failed to parse the status-fd output, let's just whine and bail now if internal_error: - reject("internal error while performing signature check on %s." % (sig_filename)) - reject(internal_error, "") - reject("Please report the above errors to the Archive maintainers by replying to this mail.", "") - return None + rejects.append("internal error while performing signature check on %s." % (sig_filename)) + rejects.append(internal_error, "") + rejects.append("Please report the above errors to the Archive maintainers by replying to this mail.", "") + return (None, rejects) - bad = "" # Now check for obviously bad things in the processed output if keywords.has_key("KEYREVOKED"): - reject("The key used to sign %s has been revoked." % (sig_filename)) - bad = 1 + rejects.append("The key used to sign %s has been revoked." % (sig_filename)) if keywords.has_key("BADSIG"): - reject("bad signature on %s." % (sig_filename)) - bad = 1 + rejects.append("bad signature on %s." % (sig_filename)) if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"): - reject("failed to check signature on %s." % (sig_filename)) - bad = 1 + rejects.append("failed to check signature on %s." % (sig_filename)) if keywords.has_key("NO_PUBKEY"): args = keywords["NO_PUBKEY"] if len(args) >= 1: key = args[0] - reject("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename)) - bad = 1 + rejects.append("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename)) if keywords.has_key("BADARMOR"): - reject("ASCII armour of signature was corrupt in %s." % (sig_filename)) - bad = 1 + rejects.append("ASCII armour of signature was corrupt in %s." % (sig_filename)) if keywords.has_key("NODATA"): - reject("no signature found in %s." % (sig_filename)) - bad = 1 + rejects.append("no signature found in %s." % (sig_filename)) if keywords.has_key("EXPKEYSIG"): args = keywords["EXPKEYSIG"] if len(args) >= 1: key = args[0] - reject("Signature made by expired key 0x%s" % (key)) - bad = 1 + rejects.append("Signature made by expired key 0x%s" % (key)) if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"): args = keywords["KEYEXPIRED"] expiredate="" @@ -1356,53 +1338,47 @@ def check_signature (sig_filename, reject, data_filename="", keyrings=None, auto expiredate = "unknown (%s)" % (timestamp) else: expiredate = timestamp - reject("The key used to sign %s has expired on %s" % (sig_filename, expiredate)) - bad = 1 + rejects.append("The key used to sign %s has expired on %s" % (sig_filename, expiredate)) - if bad: - return None + if len(rejects) > 0: + return (None, rejects) # Next check gpgv exited with a zero return code if exit_status: - reject("gpgv failed while checking %s." % (sig_filename)) + rejects.append("gpgv failed while checking %s." % (sig_filename)) if status.strip(): - reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), "") + rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] ")) else: - reject(prefix_multi_line_string(output, " [GPG output:] "), "") - return None + rejects.append(prefix_multi_line_string(output, " [GPG output:] ")) + return (None, rejects) # Sanity check the good stuff we expect if not keywords.has_key("VALIDSIG"): - reject("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename)) - bad = 1 + rejects.append("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename)) else: args = keywords["VALIDSIG"] if len(args) < 1: - reject("internal error while checking signature on %s." % (sig_filename)) - bad = 1 + rejects.append("internal error while checking signature on %s." % (sig_filename)) else: fingerprint = args[0] if not keywords.has_key("GOODSIG"): - reject("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename)) - bad = 1 + rejects.append("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename)) if not keywords.has_key("SIG_ID"): - reject("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename)) - bad = 1 + rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename)) # Finally ensure there's not something we don't recognise - known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", + known_keywords = dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="", - NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="") + NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="",POLICY_URL="") for keyword in keywords.keys(): if not known_keywords.has_key(keyword): - reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename)) - bad = 1 + rejects.append("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename)) - if bad: - return None + if len(rejects) > 0: + return (None, rejects) else: - return fingerprint + return (fingerprint, []) ################################################################################ @@ -1411,50 +1387,41 @@ def gpg_get_key_addresses(fingerprint): addresses = key_uid_email_cache.get(fingerprint) if addresses != None: return addresses - addresses = set() + addresses = list() cmd = "gpg --no-default-keyring %s --fingerprint %s" \ % (gpg_keyring_args(), fingerprint) (result, output) = commands.getstatusoutput(cmd) if result == 0: for l in output.split('\n'): m = re_gpg_uid.match(l) - if m: - addresses.add(m.group(1)) + if not m: + continue + address = m.group(1) + if address.endswith('@debian.org'): + # prefer @debian.org addresses + # TODO: maybe not hardcode the domain + addresses.insert(0, address) + else: + addresses.append(m.group(1)) key_uid_email_cache[fingerprint] = addresses return addresses ################################################################################ -# Inspired(tm) by http://www.zopelabs.com/cookbook/1022242603 - -def wrap(paragraph, max_length, prefix=""): - line = "" - s = "" - have_started = 0 - words = paragraph.split() - - for word in words: - word_size = len(word) - if word_size > max_length: - if have_started: - s += line + '\n' + prefix - s += word + '\n' + prefix - else: - if have_started: - new_length = len(line) + word_size + 1 - if new_length > max_length: - s += line + '\n' + prefix - line = word - else: - line += ' ' + word - else: - line = word - have_started = 1 - - if have_started: - s += line - - return s +def get_logins_from_ldap(fingerprint='*'): + """retrieve login from LDAP linked to a given fingerprint""" + + LDAPDn = Cnf['Import-LDAP-Fingerprints::LDAPDn'] + LDAPServer = Cnf['Import-LDAP-Fingerprints::LDAPServer'] + l = ldap.open(LDAPServer) + l.simple_bind_s('','') + Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL, + '(keyfingerprint=%s)' % fingerprint, + ['uid', 'keyfingerprint']) + login = {} + for elem in Attrs: + login[elem[1]['keyFingerPrint'][0]] = elem[1]['uid'][0] + return login ################################################################################ @@ -1471,17 +1438,70 @@ def clean_symlink (src, dest, root): ################################################################################ -def temp_filename(directory=None, prefix="dak", suffix=""): +def temp_filename(directory=None, prefix="dak", suffix="", mode=None, group=None): """ Return a secure and unique filename by pre-creating it. - If 'directory' is non-null, it will be the directory the file is pre-created in. - If 'prefix' is non-null, the filename will be prefixed with it, default is dak. - If 'suffix' is non-null, the filename will end with it. - Returns a pair (fd, name). + @type directory: str + @param directory: If non-null it will be the directory the file is pre-created in. + + @type prefix: str + @param prefix: The filename will be prefixed with this string + + @type suffix: str + @param suffix: The filename will end with this string + + @type mode: str + @param mode: If set the file will get chmodded to those permissions + + @type group: str + @param group: If set the file will get chgrped to the specified group. + + @rtype: list + @return: Returns a pair (fd, name) """ - return tempfile.mkstemp(suffix, prefix, directory) + (tfd, tfname) = tempfile.mkstemp(suffix, prefix, directory) + if mode: + os.chmod(tfname, mode) + if group: + gid = grp.getgrnam(group).gr_gid + os.chown(tfname, -1, gid) + return (tfd, tfname) + +################################################################################ + +def temp_dirname(parent=None, prefix="dak", suffix="", mode=None, group=None): + """ + Return a secure and unique directory by pre-creating it. + + @type parent: str + @param parent: If non-null it will be the directory the directory is pre-created in. + + @type prefix: str + @param prefix: The filename will be prefixed with this string + + @type suffix: str + @param suffix: The filename will end with this string + + @type mode: str + @param mode: If set the file will get chmodded to those permissions + + @type group: str + @param group: If set the file will get chgrped to the specified group. + + @rtype: list + @return: Returns a pair (fd, name) + + """ + + tfname = tempfile.mkdtemp(suffix, prefix, parent) + if mode: + os.chmod(tfname, mode) + if group: + gid = grp.getgrnam(group).gr_gid + os.chown(tfname, -1, gid) + return tfname ################################################################################ @@ -1499,7 +1519,7 @@ def is_email_alias(email): ################################################################################ -def get_changes_files(dir): +def get_changes_files(from_dir): """ Takes a directory and lists all .changes files in it (as well as chdir'ing to the directory; this is due to broken behaviour on the part of p-u/p-a @@ -1509,21 +1529,325 @@ def get_changes_files(dir): """ try: # Much of the rest of p-u/p-a depends on being in the right place - os.chdir(dir) - changes_files = [x for x in os.listdir(dir) if x.endswith('.changes')] - except OSError, e: - fubar("Failed to read list from directory %s (%s)" % (dir, e)) + os.chdir(from_dir) + changes_files = [x for x in os.listdir(from_dir) if x.endswith('.changes')] + except OSError as e: + fubar("Failed to read list from directory %s (%s)" % (from_dir, e)) return changes_files ################################################################################ -apt_pkg.init() +Cnf = config.Config().Cnf + +################################################################################ + +def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm"): + """ + Parses the wnpp bug list available at http://qa.debian.org/data/bts/wnpp_rm + Well, actually it parsed a local copy, but let's document the source + somewhere ;) + + returns a dict associating source package name with a list of open wnpp + bugs (Yes, there might be more than one) + """ + + line = [] + try: + f = open(file) + lines = f.readlines() + except IOError as e: + print "Warning: Couldn't open %s; don't know about WNPP bugs, so won't close any." % file + lines = [] + wnpp = {} + + for line in lines: + splited_line = line.split(": ", 1) + if len(splited_line) > 1: + wnpp[splited_line[0]] = splited_line[1].split("|") + + for source in wnpp.keys(): + bugs = [] + for wnpp_bug in wnpp[source]: + bug_no = re.search("(\d)+", wnpp_bug).group() + if bug_no: + bugs.append(bug_no) + wnpp[source] = bugs + return wnpp + +################################################################################ + +def get_packages_from_ftp(root, suite, component, architecture): + """ + Returns an object containing apt_pkg-parseable data collected by + aggregating Packages.gz files gathered for each architecture. + + @type root: string + @param root: path to ftp archive root directory + + @type suite: string + @param suite: suite to extract files from + + @type component: string + @param component: component to extract files from + + @type architecture: string + @param architecture: architecture to extract files from + + @rtype: TagFile + @return: apt_pkg class containing package data + """ + filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (root, suite, component, architecture) + (fd, temp_file) = temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_file)) + if (result != 0): + fubar("Gunzip invocation failed!\n%s\n" % (output), result) + filename = "%s/dists/%s/%s/debian-installer/binary-%s/Packages.gz" % (root, suite, component, architecture) + if os.path.exists(filename): + (result, output) = commands.getstatusoutput("gunzip -c %s >> %s" % (filename, temp_file)) + if (result != 0): + fubar("Gunzip invocation failed!\n%s\n" % (output), result) + packages = open_file(temp_file) + Packages = apt_pkg.ParseTagFile(packages) + os.unlink(temp_file) + return Packages + +################################################################################ + +def deb_extract_control(fh): + """extract DEBIAN/control from a binary package""" + return apt_inst.DebFile(fh).control.extractdata("control") + +################################################################################ + +def mail_addresses_for_upload(maintainer, changed_by, fingerprint): + """mail addresses to contact for an upload -Cnf = apt_pkg.newConfiguration() -apt_pkg.ReadConfigFileISC(Cnf,default_config) + @type maintainer: str + @param maintainer: Maintainer field of the .changes file -if which_conf_file() != default_config: - apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()) + @type changed_by: str + @param changed_by: Changed-By field of the .changes file + + @type fingerprint: str + @param fingerprint: fingerprint of the key used to sign the upload + + @rtype: list of str + @return: list of RFC 2047-encoded mail addresses to contact regarding + this upload + """ + addresses = [maintainer] + if changed_by != maintainer: + addresses.append(changed_by) + + fpr_addresses = gpg_get_key_addresses(fingerprint) + if len(fpr_addresses) > 0 and fix_maintainer(changed_by)[3] not in fpr_addresses and fix_maintainer(maintainer)[3] not in fpr_addresses: + addresses.append(fpr_addresses[0]) + + encoded_addresses = [ fix_maintainer(e)[1] for e in addresses ] + return encoded_addresses + +################################################################################ + +def call_editor(text="", suffix=".txt"): + """run editor and return the result as a string + + @type text: str + @param text: initial text + + @type suffix: str + @param suffix: extension for temporary file + + @rtype: str + @return: string with the edited text + """ + editor = os.environ.get('VISUAL', os.environ.get('EDITOR', 'vi')) + tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) + try: + print >>tmp, text, + tmp.close() + subprocess.check_call([editor, tmp.name]) + return open(tmp.name, 'r').read() + finally: + os.unlink(tmp.name) ################################################################################ + +def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False): + dbsuite = get_suite(suite, session) + overridesuite = dbsuite + if dbsuite.overridesuite is not None: + overridesuite = get_suite(dbsuite.overridesuite, session) + dep_problem = 0 + p2c = {} + all_broken = {} + if arches: + all_arches = set(arches) + else: + all_arches = set([x.arch_string for x in get_suite_architectures(suite)]) + all_arches -= set(["source", "all"]) + metakey_d = get_or_set_metadatakey("Depends", session) + metakey_p = get_or_set_metadatakey("Provides", session) + params = { + 'suite_id': dbsuite.suite_id, + 'metakey_d_id': metakey_d.key_id, + 'metakey_p_id': metakey_p.key_id, + } + for architecture in all_arches | set(['all']): + deps = {} + sources = {} + virtual_packages = {} + params['arch_id'] = get_architecture(architecture, session).arch_id + + statement = ''' + SELECT b.id, b.package, s.source, c.name as component, + (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends, + (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides + FROM binaries b + JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id + JOIN source s ON b.source = s.id + JOIN files_archive_map af ON b.file = af.file_id + JOIN component c ON af.component_id = c.id + WHERE b.architecture = :arch_id''' + query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \ + from_statement(statement).params(params) + for binary_id, package, source, component, depends, provides in query: + sources[package] = source + p2c[package] = component + if depends is not None: + deps[package] = depends + # Maintain a counter for each virtual package. If a + # Provides: exists, set the counter to 0 and count all + # provides by a package not in the list for removal. + # If the counter stays 0 at the end, we know that only + # the to-be-removed packages provided this virtual + # package. + if provides is not None: + for virtual_pkg in provides.split(","): + virtual_pkg = virtual_pkg.strip() + if virtual_pkg == package: continue + if not virtual_packages.has_key(virtual_pkg): + virtual_packages[virtual_pkg] = 0 + if package not in removals: + virtual_packages[virtual_pkg] += 1 + + # If a virtual package is only provided by the to-be-removed + # packages, treat the virtual package as to-be-removed too. + for virtual_pkg in virtual_packages.keys(): + if virtual_packages[virtual_pkg] == 0: + removals.append(virtual_pkg) + + # Check binary dependencies (Depends) + for package in deps.keys(): + if package in removals: continue + parsed_dep = [] + try: + parsed_dep += apt_pkg.ParseDepends(deps[package]) + except ValueError as e: + print "Error for package %s: %s" % (package, e) + for dep in parsed_dep: + # Check for partial breakage. If a package has a ORed + # dependency, there is only a dependency problem if all + # packages in the ORed depends will be removed. + unsat = 0 + for dep_package, _, _ in dep: + if dep_package in removals: + unsat += 1 + if unsat == len(dep): + component = p2c[package] + source = sources[package] + if component != "main": + source = "%s/%s" % (source, component) + all_broken.setdefault(source, {}).setdefault(package, set()).add(architecture) + dep_problem = 1 + + if all_broken: + if cruft: + print " - broken Depends:" + else: + print "# Broken Depends:" + for source, bindict in sorted(all_broken.items()): + lines = [] + for binary, arches in sorted(bindict.items()): + if arches == all_arches or 'all' in arches: + lines.append(binary) + else: + lines.append('%s [%s]' % (binary, ' '.join(sorted(arches)))) + if cruft: + print ' %s: %s' % (source, lines[0]) + else: + print '%s: %s' % (source, lines[0]) + for line in lines[1:]: + if cruft: + print ' ' + ' ' * (len(source) + 2) + line + else: + print ' ' * (len(source) + 2) + line + if not cruft: + print + + # Check source dependencies (Build-Depends and Build-Depends-Indep) + all_broken.clear() + metakey_bd = get_or_set_metadatakey("Build-Depends", session) + metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session) + params = { + 'suite_id': dbsuite.suite_id, + 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id), + } + statement = ''' + SELECT s.id, s.source, string_agg(sm.value, ', ') as build_dep + FROM source s + JOIN source_metadata sm ON s.id = sm.src_id + WHERE s.id in + (SELECT source FROM src_associations + WHERE suite = :suite_id) + AND sm.key_id in :metakey_ids + GROUP BY s.id, s.source''' + query = session.query('id', 'source', 'build_dep').from_statement(statement). \ + params(params) + for source_id, source, build_dep in query: + if source in removals: continue + parsed_dep = [] + if build_dep is not None: + # Remove [arch] information since we want to see breakage on all arches + build_dep = re_build_dep_arch.sub("", build_dep) + try: + parsed_dep += apt_pkg.ParseDepends(build_dep) + except ValueError as e: + print "Error for source %s: %s" % (source, e) + for dep in parsed_dep: + unsat = 0 + for dep_package, _, _ in dep: + if dep_package in removals: + unsat += 1 + if unsat == len(dep): + component, = session.query(Component.component_name) \ + .join(Component.overrides) \ + .filter(Override.suite == overridesuite) \ + .filter(Override.package == re.sub('/(contrib|non-free)$', '', source)) \ + .join(Override.overridetype).filter(OverrideType.overridetype == 'dsc') \ + .first() + if component != "main": + source = "%s/%s" % (source, component) + all_broken.setdefault(source, set()).add(pp_deps(dep)) + dep_problem = 1 + + if all_broken: + if cruft: + print " - broken Build-Depends:" + else: + print "# Broken Build-Depends:" + for source, bdeps in sorted(all_broken.items()): + bdeps = sorted(bdeps) + if cruft: + print ' %s: %s' % (source, bdeps[0]) + else: + print '%s: %s' % (source, bdeps[0]) + for bdep in bdeps[1:]: + if cruft: + print ' ' + ' ' * (len(source) + 2) + bdep + else: + print ' ' * (len(source) + 2) + bdep + if not cruft: + print + + return dep_problem