X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=blobdiff_plain;f=daklib%2Futils.py;h=5e9c9b94617d3ff39722f2b7533c72a90846e3e7;hp=a37628111c93299e859725620fe3a55c7a349716;hb=ac7962e07a871d2619b475c54f6be2b3a79616ee;hpb=3d982ca701456924aa064caa16a52c10cd4a1384 diff --git a/daklib/utils.py b/daklib/utils.py index a3762811..5e9c9b94 100644 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -49,7 +49,7 @@ import daklib.config as config import daklib.daksubprocess from dbconn import DBConn, get_architecture, get_component, get_suite, \ get_override_type, Keyring, session_wrapper, \ - get_active_keyring_paths, get_primary_keyring_path, \ + get_active_keyring_paths, \ get_suite_architectures, get_or_set_metadatakey, DBSource, \ Component, Override, OverrideType from sqlalchemy import desc @@ -59,7 +59,7 @@ from textutils import fix_maintainer from regexes import re_html_escaping, html_escaping, re_single_line_field, \ re_multi_line_field, re_srchasver, re_taint_free, \ re_re_mark, re_whitespace_comment, re_issource, \ - re_is_orig_source, re_build_dep_arch, re_parse_maintainer + re_build_dep_arch, re_parse_maintainer from formats import parse_format, validate_changes_format from srcformats import get_format_from_string @@ -72,10 +72,6 @@ default_config = "/etc/dak/dak.conf" #: default dak config, defines host pro alias_cache = None #: Cache for email alias checks key_uid_email_cache = {} #: Cache for email addresses from gpg key uids -# (hashname, function, earliest_changes_version) -known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)), - ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc - # Monkeypatch commands.getstatusoutput as it may not return the correct exit # code in lenny's Python. This also affects commands.getoutput and # commands.getstatus. @@ -276,7 +272,7 @@ def parse_changes(filename, signing_rules=0, dsc_file=0, keyrings=None): missingfields.append(keyword) if len(missingfields): - raise ParseChangesError("Missing mandantory field(s) in changes file (policy 5.5): %s" % (missingfields)) + raise ParseChangesError("Missing mandatory field(s) in changes file (policy 5.5): %s" % (missingfields)) return changes @@ -311,12 +307,14 @@ def check_dsc_files(dsc_filename, dsc, dsc_files): has = defaultdict(lambda: 0) ftype_lookup = ( - (r'orig.tar.gz', ('orig_tar_gz', 'orig_tar')), - (r'diff.gz', ('debian_diff',)), - (r'tar.gz', ('native_tar_gz', 'native_tar')), + (r'orig\.tar\.(gz|bz2|xz)\.asc', ('orig_tar_sig',)), + (r'orig\.tar\.gz', ('orig_tar_gz', 'orig_tar')), + (r'diff\.gz', ('debian_diff',)), + (r'tar\.gz', ('native_tar_gz', 'native_tar')), (r'debian\.tar\.(gz|bz2|xz)', ('debian_tar',)), (r'orig\.tar\.(gz|bz2|xz)', ('orig_tar',)), (r'tar\.(gz|bz2|xz)', ('native_tar',)), + (r'orig-.+\.tar\.(gz|bz2|xz)\.asc', ('more_orig_tar_sig',)), (r'orig-.+\.tar\.(gz|bz2|xz)', ('more_orig_tar',)), ) @@ -338,10 +336,11 @@ def check_dsc_files(dsc_filename, dsc, dsc_files): # File does not match anything in lookup table; reject if not matched: - reject("%s: unexpected source file '%s'" % (dsc_filename, f)) + rejmsg.append("%s: unexpected source file '%s'" % (dsc_filename, f)) + break # Check for multiple files - for file_type in ('orig_tar', 'native_tar', 'debian_tar', 'debian_diff'): + for file_type in ('orig_tar', 'orig_tar_sig', 'native_tar', 'debian_tar', 'debian_diff'): if has[file_type] > 1: rejmsg.append("%s: lists multiple %s" % (dsc_filename, file_type)) @@ -1123,19 +1122,20 @@ def call_editor(text="", suffix=".txt"): ################################################################################ -def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False, quiet=False): +def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False, quiet=False, include_arch_all=True): dbsuite = get_suite(suite, session) overridesuite = dbsuite if dbsuite.overridesuite is not None: overridesuite = get_suite(dbsuite.overridesuite, session) dep_problem = 0 p2c = {} - all_broken = {} + all_broken = defaultdict(lambda: defaultdict(set)) if arches: all_arches = set(arches) else: - all_arches = set([x.arch_string for x in get_suite_architectures(suite)]) + all_arches = set(x.arch_string for x in get_suite_architectures(suite)) all_arches -= set(["source", "all"]) + removal_set = set(removals) metakey_d = get_or_set_metadatakey("Depends", session) metakey_p = get_or_set_metadatakey("Provides", session) params = { @@ -1143,14 +1143,18 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals 'metakey_d_id': metakey_d.key_id, 'metakey_p_id': metakey_p.key_id, } - for architecture in all_arches | set(['all']): + if include_arch_all: + rdep_architectures = all_arches | set(['all']) + else: + rdep_architectures = all_arches + for architecture in rdep_architectures: deps = {} sources = {} virtual_packages = {} params['arch_id'] = get_architecture(architecture, session).arch_id statement = ''' - SELECT b.id, b.package, s.source, c.name as component, + SELECT b.package, s.source, c.name as component, (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends, (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides FROM binaries b @@ -1159,9 +1163,9 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals JOIN files_archive_map af ON b.file = af.file_id JOIN component c ON af.component_id = c.id WHERE b.architecture = :arch_id''' - query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \ + query = session.query('package', 'source', 'component', 'depends', 'provides'). \ from_statement(statement).params(params) - for binary_id, package, source, component, depends, provides in query: + for package, source, component, depends, provides in query: sources[package] = source p2c[package] = component if depends is not None: @@ -1183,18 +1187,16 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals # If a virtual package is only provided by the to-be-removed # packages, treat the virtual package as to-be-removed too. - for virtual_pkg in virtual_packages.keys(): - if virtual_packages[virtual_pkg] == 0: - removals.append(virtual_pkg) + removal_set.update(virtual_pkg for virtual_pkg in virtual_packages if not virtual_packages[virtual_pkg]) # Check binary dependencies (Depends) - for package in deps.keys(): + for package in deps: if package in removals: continue - parsed_dep = [] try: - parsed_dep += apt_pkg.parse_depends(deps[package]) + parsed_dep = apt_pkg.parse_depends(deps[package]) except ValueError as e: print "Error for package %s: %s" % (package, e) + parsed_dep = [] for dep in parsed_dep: # Check for partial breakage. If a package has a ORed # dependency, there is only a dependency problem if all @@ -1208,7 +1210,7 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals source = sources[package] if component != "main": source = "%s/%s" % (source, component) - all_broken.setdefault(source, {}).setdefault(package, set()).add(architecture) + all_broken[source][package].add(architecture) dep_problem = 1 if all_broken and not quiet: @@ -1236,32 +1238,37 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals print # Check source dependencies (Build-Depends and Build-Depends-Indep) - all_broken.clear() + all_broken = defaultdict(set) metakey_bd = get_or_set_metadatakey("Build-Depends", session) metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session) + if include_arch_all: + metakey_ids = (metakey_bd.key_id, metakey_bdi.key_id) + else: + metakey_ids = (metakey_bd.key_id,) + params = { 'suite_id': dbsuite.suite_id, - 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id), + 'metakey_ids': metakey_ids, } statement = ''' - SELECT s.id, s.source, string_agg(sm.value, ', ') as build_dep + SELECT s.source, string_agg(sm.value, ', ') as build_dep FROM source s JOIN source_metadata sm ON s.id = sm.src_id WHERE s.id in - (SELECT source FROM src_associations + (SELECT src FROM newest_src_association WHERE suite = :suite_id) AND sm.key_id in :metakey_ids GROUP BY s.id, s.source''' - query = session.query('id', 'source', 'build_dep').from_statement(statement). \ + query = session.query('source', 'build_dep').from_statement(statement). \ params(params) - for source_id, source, build_dep in query: + for source, build_dep in query: if source in removals: continue parsed_dep = [] if build_dep is not None: # Remove [arch] information since we want to see breakage on all arches build_dep = re_build_dep_arch.sub("", build_dep) try: - parsed_dep += apt_pkg.parse_src_depends(build_dep) + parsed_dep = apt_pkg.parse_src_depends(build_dep) except ValueError as e: print "Error for source %s: %s" % (source, e) for dep in parsed_dep: @@ -1279,7 +1286,7 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals key = source if component != "main": key = "%s/%s" % (source, component) - all_broken.setdefault(key, set()).add(pp_deps(dep)) + all_broken[key].add(pp_deps(dep)) dep_problem = 1 if all_broken and not quiet: @@ -1302,3 +1309,42 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals print return dep_problem + +################################################################################ + +def parse_built_using(control): + """source packages referenced via Built-Using + + @type control: dict-like + @param control: control file to take Built-Using field from + + @rtype: list of (str, str) + @return: list of (source_name, source_version) pairs + """ + built_using = control.get('Built-Using', None) + if built_using is None: + return [] + + bu = [] + for dep in apt_pkg.parse_depends(built_using): + assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field' + source_name, source_version, comp = dep[0] + assert comp == '=', 'Built-Using must contain strict dependencies' + bu.append((source_name, source_version)) + + return bu + +################################################################################ + +def is_in_debug_section(control): + """binary package is a debug package + + @type control: dict-like + @param control: control file of binary package + + @rtype Boolean + @return: True if the binary package is a debug package + """ + section = control['Section'].split('/', 1)[-1] + auto_built_package = control.get("Auto-Built-Package") + return section == "debug" and auto_built_package == "debug-symbols"