X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fgenerate_index_diffs.py;h=e83dfa3d6f6ff5f1699c8062c3ed6b304766bb67;hb=425e44739cd77ffa01294f23e94ae7eabd5f5ec8;hp=acf6b5f36f153e69372e1fcca9bdc4b75e608df9;hpb=51aaf8ec6285ad388faf7641b39df185b12a3248;p=dak.git diff --git a/dak/generate_index_diffs.py b/dak/generate_index_diffs.py index acf6b5f3..e83dfa3d 100755 --- a/dak/generate_index_diffs.py +++ b/dak/generate_index_diffs.py @@ -1,7 +1,8 @@ #!/usr/bin/env python +""" generates partial package updates list""" + ########################################################### -# generates partial package updates list # idea and basic implementation by Anthony, some changes by Andreas # parts are stolen from 'dak generate-releases' @@ -30,10 +31,15 @@ ################################################################################ -import sys, os, tempfile +import sys +import os +import tempfile +import subprocess +import time import apt_pkg -import bz2, gzip, time +import pg from daklib import utils +from daklib import database ################################################################################ @@ -79,29 +85,16 @@ def smartlink(f, t): print "missing: %s" % (f) raise IOError, f -def smartread(filename): - """ - If filename exists, slurp the contents into a string. - if filename.gz or filename.bz2 exists instead, decompress and slurp - It returns a tuple of (filename, filecontents) - """ - actual_filename = None - contents = None - if os.path.isfile(filename): - f = open(filename, "r") - elif os.path.isfile("%s.gz" % filename): - actual_filename = "%s.gz" % filename - f = decompressors['zcat'](actual_filename) - elif os.path.isfile("%s.bz2" % filename): - actual_filename = "%s.bz2" % filename - f = decompressors['bzcat'](actual_filename) +def smartopen(file): + if os.path.isfile(file): + f = open(file, "r") + elif os.path.isfile("%s.gz" % file): + f = create_temp_file(os.popen("zcat %s.gz" % file, "r")) + elif os.path.isfile("%s.bz2" % file): + f = create_temp_file(os.popen("bzcat %s.bz2" % file, "r")) else: f = None - - if f: - contents = f.read() - - return (actual_filename, contents) + return f def pipe_file(f, t): f.seek(0) @@ -187,14 +180,17 @@ class Updates: for h in l: out.write(" %s %7d %s\n" % (hs[h][1][0], hs[h][1][1], h)) -decompressors = { 'zcat' : gzip.GzipFile, - 'bzip2' : bz2.BZ2File } - -def sizesha1_str(s): - """ - given a string, return a tuple containing its (sha1sum, length) - """ - return (apt_pkg.sha1sum(s), len( s ) ) +def create_temp_file(r): + f = tempfile.TemporaryFile() + while 1: + x = r.readline() + if not x: break + f.write(x) + r.close() + del x,r + f.flush() + f.seek(0) + return f def sizesha1(f): size = os.fstat(f.fileno())[6] @@ -234,27 +230,15 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 14): print "%s: hardlink unbroken, assuming unchanged" % (origfile) return - (oldf,contents) = smartread(oldfile) - oldsizesha1 = sizesha1_str(contents) + oldf = smartopen(oldfile) + oldsizesha1 = sizesha1(oldf) # should probably early exit if either of these checks fail # alternatively (optionally?) could just trim the patch history if upd.filesizesha1: if upd.filesizesha1 != oldsizesha1: - print "warning: old file seems to have changed! %s %s => %s %s" % (upd.filesizesha1 + oldsizesha1) - - # XXX this should be usable now - # stew: whatever this is, it won't be usable now that i removed smartopen - # - #for d in upd.history.keys(): - # df = smartopen("%s/%s" % (outdir,d)) - # act_sha1size = sizesha1(df) - # df.close() - # exp_sha1size = upd.history[d][1] - # if act_sha1size != exp_sha1size: - # print "patch file %s seems to have changed! %s %s => %s %s" % \ - # (d,) + exp_sha1size + act_sha1size + print "info: old file " + oldfile + " changed! %s %s => %s %s" % (upd.filesizesha1 + oldsizesha1) if Options.has_key("CanonicalPath"): upd.can_path=Options["CanonicalPath"] @@ -266,17 +250,20 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 14): if newsizesha1 == oldsizesha1: os.unlink(newfile) -# oldf.close() + oldf.close() print "%s: unchanged" % (origfile) else: - if not os.path.isdir(outdir): os.mkdir(outdir) - os.popen("diff --ed %s %s | gzip -c -9 > %s.gz" % - (oldf, newfile, difffile)) -# pipe_file(oldf, w) -# oldf.close() + if not os.path.isdir(outdir): + os.mkdir(outdir) + + w = os.popen("diff --ed - %s | gzip -c -9 > %s.gz" % + (newfile, difffile), "w") + pipe_file(oldf, w) + oldf.close() - (oldf,contents) = smartread(difffile) - difsizesha1 = sizesha1_str(contents) + difff = smartopen(difffile) + difsizesha1 = sizesha1(difff) + difff.close() upd.history[patchname] = (oldsizesha1, difsizesha1) upd.history_order.append(patchname) @@ -293,7 +280,7 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 14): def main(): - global Cnf, Options, Logger + global Cnf, Options, Logger, projectB os.umask(0002) @@ -324,6 +311,9 @@ def main(): if Options.has_key("RootDir"): Cnf["Dir::Root"] = Options["RootDir"] + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + database.init(Cnf, projectB) + if not suites: suites = Cnf.SubTree("Suite").List() @@ -331,13 +321,15 @@ def main(): print "Processing: " + suite SuiteBlock = Cnf.SubTree("Suite::" + suite) - if SuiteBlock.has_key("Untouchable"): + if database.get_suite_untouchable(suite): print "Skipping: " + suite + " (untouchable)" continue suite = suite.lower() - architectures = SuiteBlock.ValueList("Architectures") + architectures = database.get_suite_architectures(suite) + if architectures == None: + architectures = [] if SuiteBlock.has_key("Components"): components = SuiteBlock.ValueList("Components")