Add database fields to override compression methods.

[dak.git] / dak / check_archive.py

diff --git a/dak/check_archive.py b/dak/check_archive.py
index a88f8ebaaacbcf99a9d705b1c233af2a4d0a93cf..a47febbd229768a952812e5537061f35cc243893 100755 (executable)
--- a/dak/check_archive.py
+++ b/dak/check_archive.py
@@ -41,6 +41,7 @@ import apt_inst
 from daklib.dbconn import *
 from daklib import utils
 from daklib.config import Config
+from daklib.dak_exceptions import InvalidDscError, ChangesUnicodeError, CantOpenError
 
 ################################################################################
 
@@ -71,6 +72,7 @@ The following MODEs are available:
   validate-indices   - ensure files mentioned in Packages & Sources exist
   files-not-symlinks - check files in the database aren't symlinks
   validate-builddeps - validate build-dependencies of .dsc files in the archive
+  add-missing-source-checksums - add missing checksums for source packages
 """
     sys.exit(exit_code)
 
@@ -148,26 +150,24 @@ def check_dscs():
     Parse every .dsc file in the archive and check for it's validity.
     """
 
-    cnf = Config()
-
     count = 0
-    suite = 'unstable'
-
-    for component in cnf.SubTree("Component").List():
-        component = component.lower()
-        list_filename = '%s%s_%s_source.list' % (cnf["Dir::Lists"], suite, component)
-        list_file = utils.open_file(list_filename)
-
-        for line in list_file.readlines():
-            f = line[:-1]
-            try:
-                utils.parse_changes(f, signing_rules=1, dsc_file=1)
-            except InvalidDscError, line:
-                utils.warn("syntax error in .dsc file '%s', line %s." % (f, line))
-                count += 1
-            except ChangesUnicodeError:
-                utils.warn("found invalid changes file, not properly utf-8 encoded")
-                count += 1
+
+    for src in DBConn().session().query(DBSource).order_by(DBSource.source, DBSource.version):
+        f = src.poolfile.fullpath
+        try:
+            utils.parse_changes(f, signing_rules=1, dsc_file=1)
+        except InvalidDscError:
+            utils.warn("syntax error in .dsc file %s" % f)
+            count += 1
+        except ChangesUnicodeError:
+            utils.warn("found invalid dsc file (%s), not properly utf-8 encoded" % f)
+            count += 1
+        except CantOpenError:
+            utils.warn("missing dsc file (%s)" % f)
+            count += 1
+        except Exception as e:
+            utils.warn("miscellaneous error parsing dsc file (%s): %s" % (f, str(e)))
+            count += 1
 
     if count:
         utils.warn("Found %s invalid .dsc files." % (count))
@@ -425,7 +425,7 @@ def check_indices_files_exist():
     Ensure files mentioned in Packages & Sources exist
     """
     for suite in [ "stable", "testing", "unstable" ]:
-        for component in Cnf.ValueList("Suite::%s::Components" % (suite)):
+        for component in get_component_names():
             architectures = get_suite_architectures(suite)
             for arch in [ i.arch_string.lower() for i in architectures ]:
                 if arch == "source":
@@ -479,6 +479,46 @@ def check_build_depends():
 
 ################################################################################
 
+_add_missing_source_checksums_query = R"""
+INSERT INTO source_metadata
+  (src_id, key_id, value)
+SELECT
+  s.id,
+  :checksum_key,
+  E'\n' ||
+    (SELECT STRING_AGG(' ' || tmp.checksum || ' ' || tmp.size || ' ' || tmp.basename, E'\n' ORDER BY tmp.basename)
+     FROM
+       (SELECT
+            CASE :checksum_type
+              WHEN 'Files' THEN f.md5sum
+              WHEN 'Checksums-Sha1' THEN f.sha1sum
+              WHEN 'Checksums-Sha256' THEN f.sha256sum
+            END AS checksum,
+            f.size,
+            SUBSTRING(f.filename FROM E'/([^/]*)\\Z') AS basename
+          FROM files f JOIN dsc_files ON f.id = dsc_files.file
+          WHERE dsc_files.source = s.id AND f.id != s.file
+       ) AS tmp
+    )
+
+  FROM
+    source s
+  WHERE NOT EXISTS (SELECT 1 FROM source_metadata md WHERE md.src_id=s.id AND md.key_id = :checksum_key);
+"""
+
+def add_missing_source_checksums():
+    """ Add missing source checksums to source_metadata """
+    session = DBConn().session()
+    for checksum in ['Files', 'Checksums-Sha1', 'Checksums-Sha256']:
+        checksum_key = get_or_set_metadatakey(checksum, session).key_id
+        rows = session.execute(_add_missing_source_checksums_query,
+            {'checksum_key': checksum_key, 'checksum_type': checksum}).rowcount
+        if rows > 0:
+            print "Added {0} missing entries for {1}".format(rows, checksum)
+    session.commit()
+
+################################################################################
+
 def main ():
     global db_files, waste, excluded
 
@@ -489,9 +529,9 @@ def main ():
         if not cnf.has_key("Check-Archive::Options::%s" % (i)):
             cnf["Check-Archive::Options::%s" % (i)] = ""
 
-    args = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+    args = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
 
-    Options = cnf.SubTree("Check-Archive::Options")
+    Options = cnf.subtree("Check-Archive::Options")
     if Options["Help"]:
         usage()
 
@@ -526,6 +566,8 @@ def main ():
         check_files_not_symlinks()
     elif mode == "validate-builddeps":
         check_build_depends()
+    elif mode == "add-missing-source-checksums":
+        add_missing_source_checksums()
     else:
         utils.warn("unknown mode '%s'" % (mode))
         usage(1)