X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fcheck_archive.py;h=926b4fb5009cf18a36ea34c8bb4f4165a7a58731;hb=6f863f95e3d1bae18aa5579e958fd8a121b06545;hp=6ca84c69823877eadf264edae87833e58a11ba4a;hpb=1c35448b880358d020e81339657e3435fdda9434;p=dak.git diff --git a/dak/check_archive.py b/dak/check_archive.py index 6ca84c69..926b4fb5 100755 --- a/dak/check_archive.py +++ b/dak/check_archive.py @@ -40,8 +40,8 @@ import apt_inst from daklib.dbconn import * from daklib import utils -from daklib.regexes import re_issource from daklib.config import Config +from daklib.dak_exceptions import InvalidDscError, ChangesUnicodeError, CantOpenError ################################################################################ @@ -68,10 +68,11 @@ The following MODEs are available: missing-overrides - check for missing overrides source-in-one-dir - ensure the source for each package is in one directory timestamps - check for future timestamps in .deb's - tar-gz-in-dsc - ensure each .dsc lists a .tar.gz file + files-in-dsc - ensure each .dsc references appropriate Files validate-indices - ensure files mentioned in Packages & Sources exist files-not-symlinks - check files in the database aren't symlinks validate-builddeps - validate build-dependencies of .dsc files in the archive + add-missing-source-checksums - add missing checksums for source packages """ sys.exit(exit_code) @@ -96,8 +97,7 @@ def process_dir (unused, dirname, filenames): if dirname.find('proposed-updates') != -1: return for name in filenames: - filename = os.path.abspath(dirname+'/'+name) - filename = filename.replace('potato-proposed-updates', 'proposed-updates') + filename = os.path.abspath(os.path.join(dirname,name)) if os.path.isfile(filename) and not os.path.islink(filename) and not db_files.has_key(filename) and not excluded.has_key(filename): waste += os.stat(filename)[stat.ST_SIZE] print "%s" % (filename) @@ -109,39 +109,61 @@ def check_files(): Prepare the dictionary of existing filenames, then walk through the archive pool/ directory to compare it. """ - global db_files - cnf = Config() + session = DBConn().session() - print "Building list of database files..." - q = DBConn().session().query(PoolFile).join(Location).order_by('path', 'location') - - print "Missing files:" - db_files.clear() - - for f in q.all(): - filename = os.path.abspath(f.location.path, f.filename) - db_files[filename] = "" - if os.access(filename, os.R_OK) == 0: - if f.last_used: - print "(last used: %s) %s" % (f.last_used, filename) - else: - print "%s" % (filename) - - - filename = os.path.join(cnf["Dir::Override"], 'override.unreferenced') - if os.path.exists(filename): - f = utils.open_file(filename) - for filename in f.readlines(): - filename = filename[:-1] - excluded[filename] = "" - - print "Existent files not in db:" - - os.path.walk(cnf["Dir::Root"] + 'pool/', process_dir, None) - - print - print "%s wasted..." % (utils.size_type(waste)) + query = """ + SELECT archive.name, suite.suite_name, f.filename + FROM binaries b + JOIN bin_associations ba ON b.id = ba.bin + JOIN suite ON ba.suite = suite.id + JOIN archive ON suite.archive_id = archive.id + JOIN files f ON b.file = f.id + WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af + WHERE af.archive_id = suite.archive_id + AND af.file_id = b.file) + ORDER BY archive.name, suite.suite_name, f.filename + """ + for row in session.execute(query): + print "MISSING-ARCHIVE-FILE {0} {1} {2}".vformat(row) + + query = """ + SELECT archive.name, suite.suite_name, f.filename + FROM source s + JOIN src_associations sa ON s.id = sa.source + JOIN suite ON sa.suite = suite.id + JOIN archive ON suite.archive_id = archive.id + JOIN dsc_files df ON s.id = df.source + JOIN files f ON df.file = f.id + WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af + WHERE af.archive_id = suite.archive_id + AND af.file_id = df.file) + ORDER BY archive.name, suite.suite_name, f.filename + """ + for row in session.execute(query): + print "MISSING-ARCHIVE-FILE {0} {1} {2}".vformat(row) + + archive_files = session.query(ArchiveFile) \ + .join(ArchiveFile.archive).join(ArchiveFile.file) \ + .order_by(Archive.archive_name, PoolFile.filename) + + expected_files = set() + for af in archive_files: + path = af.path + expected_files.add(af.path) + if not os.path.exists(path): + print "MISSING-FILE {0} {1} {2}".format(af.archive.archive_name, af.file.filename, path) + + archives = session.query(Archive).order_by(Archive.archive_name) + + for a in archives: + top = os.path.join(a.path, 'pool') + for dirpath, dirnames, filenames in os.walk(top): + for fn in filenames: + path = os.path.join(dirpath, fn) + if path in expected_files: + continue + print "UNEXPECTED-FILE {0} {1}".format(a.archive_name, path) ################################################################################ @@ -150,26 +172,24 @@ def check_dscs(): Parse every .dsc file in the archive and check for it's validity. """ - cnf = Config() - count = 0 - suite = 'unstable' - - for component in cnf.SubTree("Component").List(): - component = component.lower() - list_filename = '%s%s_%s_source.list' % (cnf["Dir::Lists"], suite, component) - list_file = utils.open_file(list_filename) - - for line in list_file.readlines(): - f = line[:-1] - try: - utils.parse_changes(f, signing_rules=1) - except InvalidDscError, line: - utils.warn("syntax error in .dsc file '%s', line %s." % (f, line)) - count += 1 - except ChangesUnicodeError: - utils.warn("found invalid changes file, not properly utf-8 encoded") - count += 1 + + for src in DBConn().session().query(DBSource).order_by(DBSource.source, DBSource.version): + f = src.poolfile.fullpath + try: + utils.parse_changes(f, signing_rules=1, dsc_file=1) + except InvalidDscError: + utils.warn("syntax error in .dsc file %s" % f) + count += 1 + except ChangesUnicodeError: + utils.warn("found invalid dsc file (%s), not properly utf-8 encoded" % f) + count += 1 + except CantOpenError: + utils.warn("missing dsc file (%s)" % f) + count += 1 + except Exception as e: + utils.warn("miscellaneous error parsing dsc file (%s): %s" % (f, str(e))) + count += 1 if count: utils.warn("Found %s invalid .dsc files." % (count)) @@ -186,8 +206,8 @@ def check_override(): print suite_name print "-" * len(suite_name) print - suite = get_suite(suite) - q = s.execute(""" + suite = get_suite(suite_name) + q = session.execute(""" SELECT DISTINCT b.package FROM binaries b, bin_associations ba WHERE b.id = ba.bin AND ba.suite = :suiteid AND NOT EXISTS (SELECT 1 FROM override o WHERE o.suite = :suiteid AND o.package = b.package)""" @@ -196,7 +216,7 @@ SELECT DISTINCT b.package FROM binaries b, bin_associations ba for j in q.fetchall(): print j[0] - q = s.execute(""" + q = session.execute(""" SELECT DISTINCT s.source FROM source s, src_associations sa WHERE s.id = sa.source AND sa.suite = :suiteid AND NOT EXISTS (SELECT 1 FROM override o WHERE o.suite = :suiteid and o.package = s.source)""" @@ -254,7 +274,7 @@ def check_checksums(): print "Checking file checksums & sizes..." for f in q: - filename = os.path.abspath(os.path.join(f.location.path, f.filename)) + filename = f.fullpath try: fi = utils.open_file(filename) @@ -320,9 +340,10 @@ def check_timestamps(): ################################################################################ -def check_missing_tar_gz_in_dsc(): +def check_files_in_dsc(): """ - Ensure each .dsc lists a .tar.gz file + Ensure each .dsc lists appropriate files in its Files field (according + to the format announced in its Format field). """ count = 0 @@ -339,23 +360,15 @@ def check_missing_tar_gz_in_dsc(): try: # NB: don't enforce .dsc syntax - dsc = utils.parse_changes(filename) + dsc = utils.parse_changes(filename, dsc_file=1) except: utils.fubar("error parsing .dsc file '%s'." % (filename)) - dsc_files = utils.build_file_list(dsc, is_a_dsc=1) - has_tar = 0 - - for f in dsc_files.keys(): - m = re_issource.match(f) - if not m: - utils.fubar("%s not recognised as source." % (f)) - ftype = m.group(3) - if ftype == "orig.tar.gz" or ftype == "tar.gz": - has_tar = 1 + reasons = utils.check_dsc_files(filename, dsc) + for r in reasons: + utils.warn(r) - if not has_tar: - utils.warn("%s has no .tar.gz in the .dsc file." % (f)) + if len(reasons) > 0: count += 1 if count: @@ -370,18 +383,18 @@ def validate_sources(suite, component): """ filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component) print "Processing %s..." % (filename) - # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... + # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance... (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) sys.exit(result) sources = utils.open_file(temp_filename) - Sources = apt_pkg.ParseTagFile(sources) - while Sources.Step(): - source = Sources.Section.Find('Package') - directory = Sources.Section.Find('Directory') - files = Sources.Section.Find('Files') + Sources = apt_pkg.TagFile(sources) + while Sources.step(): + source = Sources.section.find('Package') + directory = Sources.section.find('Directory') + files = Sources.section.find('Files') for i in files.split('\n'): (md5, size, name) = i.split() filename = "%s/%s/%s" % (Cnf["Dir::Root"], directory, name) @@ -412,16 +425,16 @@ def validate_packages(suite, component, architecture): filename = "%s/dists/%s/%s/binary-%s/Packages.gz" \ % (Cnf["Dir::Root"], suite, component, architecture) print "Processing %s..." % (filename) - # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... + # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance... (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) sys.exit(result) packages = utils.open_file(temp_filename) - Packages = apt_pkg.ParseTagFile(packages) - while Packages.Step(): - filename = "%s/%s" % (Cnf["Dir::Root"], Packages.Section.Find('Filename')) + Packages = apt_pkg.TagFile(packages) + while Packages.step(): + filename = "%s/%s" % (Cnf["Dir::Root"], Packages.section.find('Filename')) if not os.path.exists(filename): print "W: %s missing." % (filename) packages.close() @@ -434,9 +447,9 @@ def check_indices_files_exist(): Ensure files mentioned in Packages & Sources exist """ for suite in [ "stable", "testing", "unstable" ]: - for component in Cnf.ValueList("Suite::%s::Components" % (suite)): - architectures = database.get_suite_architectures(suite) - for arch in [ i.lower() for i in architectures ]: + for component in get_component_names(): + architectures = get_suite_architectures(suite) + for arch in [ i.arch_string.lower() for i in architectures ]: if arch == "source": validate_sources(suite, component) elif arch == "all": @@ -469,12 +482,12 @@ def chk_bd_process_dir (unused, dirname, filenames): if not name.endswith(".dsc"): continue filename = os.path.abspath(dirname+'/'+name) - dsc = utils.parse_changes(filename) + dsc = utils.parse_changes(filename, dsc_file=1) for field_name in [ "build-depends", "build-depends-indep" ]: field = dsc.get(field_name) if field: try: - apt_pkg.ParseSrcDepends(field) + apt_pkg.parse_src_depends(field) except: print "E: [%s] %s: %s" % (filename, field_name, field) pass @@ -483,10 +496,51 @@ def chk_bd_process_dir (unused, dirname, filenames): def check_build_depends(): """ Validate build-dependencies of .dsc files in the archive """ + cnf = Config() os.path.walk(cnf["Dir::Root"], chk_bd_process_dir, None) ################################################################################ +_add_missing_source_checksums_query = R""" +INSERT INTO source_metadata + (src_id, key_id, value) +SELECT + s.id, + :checksum_key, + E'\n' || + (SELECT STRING_AGG(' ' || tmp.checksum || ' ' || tmp.size || ' ' || tmp.basename, E'\n' ORDER BY tmp.basename) + FROM + (SELECT + CASE :checksum_type + WHEN 'Files' THEN f.md5sum + WHEN 'Checksums-Sha1' THEN f.sha1sum + WHEN 'Checksums-Sha256' THEN f.sha256sum + END AS checksum, + f.size, + SUBSTRING(f.filename FROM E'/([^/]*)\\Z') AS basename + FROM files f JOIN dsc_files ON f.id = dsc_files.file + WHERE dsc_files.source = s.id AND f.id != s.file + ) AS tmp + ) + + FROM + source s + WHERE NOT EXISTS (SELECT 1 FROM source_metadata md WHERE md.src_id=s.id AND md.key_id = :checksum_key); +""" + +def add_missing_source_checksums(): + """ Add missing source checksums to source_metadata """ + session = DBConn().session() + for checksum in ['Files', 'Checksums-Sha1', 'Checksums-Sha256']: + checksum_key = get_or_set_metadatakey(checksum, session).key_id + rows = session.execute(_add_missing_source_checksums_query, + {'checksum_key': checksum_key, 'checksum_type': checksum}).rowcount + if rows > 0: + print "Added {0} missing entries for {1}".format(rows, checksum) + session.commit() + +################################################################################ + def main (): global db_files, waste, excluded @@ -497,9 +551,9 @@ def main (): if not cnf.has_key("Check-Archive::Options::%s" % (i)): cnf["Check-Archive::Options::%s" % (i)] = "" - args = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) + args = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) - Options = cnf.SubTree("Check-Archive::Options") + Options = cnf.subtree("Check-Archive::Options") if Options["Help"]: usage() @@ -526,14 +580,16 @@ def main (): check_source_in_one_dir() elif mode == "timestamps": check_timestamps() - elif mode == "tar-gz-in-dsc": - check_missing_tar_gz_in_dsc() + elif mode == "files-in-dsc": + check_files_in_dsc() elif mode == "validate-indices": check_indices_files_exist() elif mode == "files-not-symlinks": check_files_not_symlinks() elif mode == "validate-builddeps": check_build_depends() + elif mode == "add-missing-source-checksums": + add_missing_source_checksums() else: utils.warn("unknown mode '%s'" % (mode)) usage(1)