X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fgenerate_releases.py;h=82ff6394bffa3b92765d6f403a121c3b939c10cc;hb=3d25c468b152899041da69857c3f01e9996b39d1;hp=57830390aa19c0815067adb35bd6a612e34e3c13;hpb=14ca1d5ed8b7ea783d57d76802d8cb93d7f2fabb;p=dak.git

diff --git a/dak/generate_releases.py b/dak/generate_releases.py
index 57830390..82ff6394 100755
--- a/dak/generate_releases.py
+++ b/dak/generate_releases.py
@@ -37,6 +37,7 @@ import stat
 import time
 import gzip
 import bz2
+import errno
 import apt_pkg
 import subprocess
 from tempfile import mkstemp, mkdtemp
@@ -70,7 +71,7 @@ Generate the Release files
   -h, --help                 show this help and exit
   -q, --quiet                Don't output progress
 
-SUITE can be a space seperated list, e.g.
+SUITE can be a space separated list, e.g.
    --suite=unstable testing
   """
     sys.exit(exit_code)
@@ -139,7 +140,9 @@ class ReleaseWriter(object):
                     ('Label',       'label'),
                     ('Suite',       'release_suite_output'),
                     ('Version',     'version'),
-                    ('Codename',    'codename') )
+                    ('Codename',    'codename'),
+                    ('Changelogs',  'changelog_url'),
+                  )
 
         # A "Sub" Release file has slightly different fields
         subattribs = ( ('Archive',  'suite_name'),
@@ -149,7 +152,9 @@ class ReleaseWriter(object):
 
         # Boolean stuff. If we find it true in database, write out "yes" into the release file
         boolattrs = ( ('NotAutomatic',         'notautomatic'),
-                      ('ButAutomaticUpgrades', 'butautomaticupgrades') )
+                      ('ButAutomaticUpgrades', 'butautomaticupgrades'),
+                      ('Acquire-By-Hash',      'byhash'),
+                    )
 
         cnf = Config()
 
@@ -159,6 +164,11 @@ class ReleaseWriter(object):
         out = open(outfile + ".new", "w")
 
         for key, dbfield in attribs:
+            # Hack to skip NULL Version fields as we used to do this
+            # We should probably just always ignore anything which is None
+            if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None:
+                continue
+
             out.write("%s: %s\n" % (key, getattr(suite, dbfield)))
 
         out.write("Date: %s\n" % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time()))))
@@ -217,9 +227,8 @@ class ReleaseWriter(object):
 
         os.chdir(os.path.join(suite.archive.path, "dists", suite.suite_name, suite_suffix))
 
-        hashfuncs = { 'MD5Sum' : apt_pkg.md5sum,
-                      'SHA1' : apt_pkg.sha1sum,
-                      'SHA256' : apt_pkg.sha256sum }
+        hashfuncs = dict(zip([x.upper().replace('UM', 'um') for x in suite.checksums],
+                             [getattr(apt_pkg, "%s" % (x)) for x in [x.replace("sum", "") + "sum" for x in suite.checksums]]))
 
         fileinfo = {}
 
@@ -241,11 +250,11 @@ class ReleaseWriter(object):
                 # If we find a file for which we have a compressed version and
                 # haven't yet seen the uncompressed one, store the possibility
                 # for future use
-                if entry.endswith(".gz") and entry[:-3] not in uncompnotseen.keys():
+                if entry.endswith(".gz") and filename[:-3] not in uncompnotseen:
                     uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename)
-                elif entry.endswith(".bz2") and entry[:-4] not in uncompnotseen.keys():
+                elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen:
                     uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename)
-                elif entry.endswith(".xz") and entry[:-3] not in uncompnotseen.keys():
+                elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen:
                     uncompnotseen[filename[:-3]] = (XzFile, filename)
 
                 fileinfo[filename]['len'] = len(contents)
@@ -256,12 +265,7 @@ class ReleaseWriter(object):
         for filename, comp in uncompnotseen.items():
             # If we've already seen the uncompressed file, we don't
             # need to do anything again
-            if filename in fileinfo.keys():
-                continue
-
-            # Skip uncompressed Contents files as they're huge, take ages to
-            # checksum and we checksum the compressed ones anyways
-            if os.path.basename(filename).startswith("Contents"):
+            if filename in fileinfo:
                 continue
 
             fileinfo[filename] = {}
@@ -283,6 +287,47 @@ class ReleaseWriter(object):
         out.close()
         os.rename(outfile + '.new', outfile)
 
+        if suite.byhash:
+            query = """
+                UPDATE hashfile SET unreferenced = CURRENT_TIMESTAMP
+                WHERE suite_id = :id AND unreferenced IS NULL"""
+            session.execute(query, {'id': suite.suite_id})
+
+            for filename in fileinfo:
+                if not os.path.exists(filename):
+                    # probably an uncompressed index we didn't generate
+                    continue
+
+                for h in hashfuncs:
+                    hashfile = os.path.join(os.path.dirname(filename), 'by-hash', h, fileinfo[filename][h])
+                    query = "SELECT 1 FROM hashfile WHERE path = :p AND suite_id = :id"
+                    q = session.execute(
+                            query,
+                            {'p': hashfile, 'id': suite.suite_id})
+                    if q.rowcount:
+                        session.execute('''
+                            UPDATE hashfile SET unreferenced = NULL
+                            WHERE path = :p and suite_id = :id''',
+                            {'p': hashfile, 'id': suite.suite_id})
+                    else:
+                        session.execute('''
+                            INSERT INTO hashfile (path, suite_id)
+                            VALUES (:p, :id)''',
+                            {'p': hashfile, 'id': suite.suite_id})
+
+                    try:
+                        os.makedirs(os.path.dirname(hashfile))
+                    except OSError as exc:
+                        if exc.errno != errno.EEXIST:
+                            raise
+                    try:
+                        os.link(filename, hashfile)
+                    except OSError as exc:
+                        if exc.errno != errno.EEXIST:
+                            raise
+
+                session.commit()
+
         sign_release_dir(suite, os.path.dirname(outfile))
 
         os.chdir(oldcwd)