X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fgenerate_index_diffs.py;h=3391c7bbafae8714e678dc3be6249167e33bdbd8;hb=52eb03dccab0a9898bca4d36a0a96ebe377fb977;hp=acf6b5f36f153e69372e1fcca9bdc4b75e608df9;hpb=51aaf8ec6285ad388faf7641b39df185b12a3248;p=dak.git diff --git a/dak/generate_index_diffs.py b/dak/generate_index_diffs.py index acf6b5f3..3391c7bb 100755 --- a/dak/generate_index_diffs.py +++ b/dak/generate_index_diffs.py @@ -1,7 +1,8 @@ #!/usr/bin/env python +""" generates partial package updates list""" + ########################################################### -# generates partial package updates list # idea and basic implementation by Anthony, some changes by Andreas # parts are stolen from 'dak generate-releases' @@ -30,14 +31,21 @@ ################################################################################ -import sys, os, tempfile +import sys +import os +import tempfile +import time import apt_pkg -import bz2, gzip, time +import glob + from daklib import utils +from daklib.dbconn import get_suite, get_suite_architectures +#from daklib.regexes import re_includeinpdiff +import re +re_includeinpdiff = re.compile(r"(Translation-[a-zA-Z_]+\.(?:bz2|xz))") ################################################################################ -projectB = None Cnf = None Logger = None Options = None @@ -51,11 +59,13 @@ Write out ed-style diffs to Packages/Source lists -h, --help show this help and exit -c give the canonical path of the file -p name for the patch (defaults to current time) + -r use a different archive root + -d name for the hardlink farm for status + -m how many diffs to generate -n take no action """ sys.exit(exit_code) - def tryunlink(file): try: os.unlink(file) @@ -77,31 +87,18 @@ def smartlink(f, t): os.system("bzip2 -d < %s.bz2 > %s" % (f, t)) else: print "missing: %s" % (f) - raise IOError, f - -def smartread(filename): - """ - If filename exists, slurp the contents into a string. - if filename.gz or filename.bz2 exists instead, decompress and slurp - It returns a tuple of (filename, filecontents) - """ - actual_filename = None - contents = None - if os.path.isfile(filename): - f = open(filename, "r") - elif os.path.isfile("%s.gz" % filename): - actual_filename = "%s.gz" % filename - f = decompressors['zcat'](actual_filename) - elif os.path.isfile("%s.bz2" % filename): - actual_filename = "%s.bz2" % filename - f = decompressors['bzcat'](actual_filename) + raise IOError(f) + +def smartopen(file): + if os.path.isfile(file): + f = open(file, "r") + elif os.path.isfile("%s.gz" % file): + f = create_temp_file(os.popen("zcat %s.gz" % file, "r")) + elif os.path.isfile("%s.bz2" % file): + f = create_temp_file(os.popen("bzcat %s.bz2" % file, "r")) else: f = None - - if f: - contents = f.read() - - return (actual_filename, contents) + return f def pipe_file(f, t): f.seek(0) @@ -112,7 +109,7 @@ def pipe_file(f, t): t.close() class Updates: - def __init__(self, readpath = None, max = 14): + def __init__(self, readpath = None, max = 56): self.can_path = None self.history = {} self.history_order = [] @@ -187,14 +184,17 @@ class Updates: for h in l: out.write(" %s %7d %s\n" % (hs[h][1][0], hs[h][1][1], h)) -decompressors = { 'zcat' : gzip.GzipFile, - 'bzip2' : bz2.BZ2File } - -def sizesha1_str(s): - """ - given a string, return a tuple containing its (sha1sum, length) - """ - return (apt_pkg.sha1sum(s), len( s ) ) +def create_temp_file(r): + f = tempfile.TemporaryFile() + while 1: + x = r.readline() + if not x: break + f.write(x) + r.close() + del x,r + f.flush() + f.seek(0) + return f def sizesha1(f): size = os.fstat(f.fileno())[6] @@ -202,8 +202,9 @@ def sizesha1(f): sha1sum = apt_pkg.sha1sum(f) return (sha1sum, size) -def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 14): +def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 56): if Options.has_key("NoAct"): + print "Not acting on: od: %s, oldf: %s, origf: %s, md: %s" % (outdir, oldfile, origfile, maxdiffs) return patchname = Options["PatchName"] @@ -231,30 +232,18 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 14): return if oldstat[1:3] == origstat[1:3]: - print "%s: hardlink unbroken, assuming unchanged" % (origfile) + #print "%s: hardlink unbroken, assuming unchanged" % (origfile) return - (oldf,contents) = smartread(oldfile) - oldsizesha1 = sizesha1_str(contents) + oldf = smartopen(oldfile) + oldsizesha1 = sizesha1(oldf) # should probably early exit if either of these checks fail # alternatively (optionally?) could just trim the patch history - if upd.filesizesha1: - if upd.filesizesha1 != oldsizesha1: - print "warning: old file seems to have changed! %s %s => %s %s" % (upd.filesizesha1 + oldsizesha1) - - # XXX this should be usable now - # stew: whatever this is, it won't be usable now that i removed smartopen - # - #for d in upd.history.keys(): - # df = smartopen("%s/%s" % (outdir,d)) - # act_sha1size = sizesha1(df) - # df.close() - # exp_sha1size = upd.history[d][1] - # if act_sha1size != exp_sha1size: - # print "patch file %s seems to have changed! %s %s => %s %s" % \ - # (d,) + exp_sha1size + act_sha1size + #if upd.filesizesha1: + # if upd.filesizesha1 != oldsizesha1: + # print "info: old file " + oldfile + " changed! %s %s => %s %s" % (upd.filesizesha1 + oldsizesha1) if Options.has_key("CanonicalPath"): upd.can_path=Options["CanonicalPath"] @@ -266,17 +255,20 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 14): if newsizesha1 == oldsizesha1: os.unlink(newfile) -# oldf.close() - print "%s: unchanged" % (origfile) + oldf.close() + #print "%s: unchanged" % (origfile) else: - if not os.path.isdir(outdir): os.mkdir(outdir) - os.popen("diff --ed %s %s | gzip -c -9 > %s.gz" % - (oldf, newfile, difffile)) -# pipe_file(oldf, w) -# oldf.close() + if not os.path.isdir(outdir): + os.mkdir(outdir) - (oldf,contents) = smartread(difffile) - difsizesha1 = sizesha1_str(contents) + w = os.popen("diff --ed - %s | gzip --rsyncable -c -9 > %s.gz" % + (newfile, difffile), "w") + pipe_file(oldf, w) + oldf.close() + + difff = smartopen(difffile) + difsizesha1 = sizesha1(difff) + difff.close() upd.history[patchname] = (oldsizesha1, difsizesha1) upd.history_order.append(patchname) @@ -295,7 +287,7 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 14): def main(): global Cnf, Options, Logger - os.umask(0002) + os.umask(0o002) Cnf = utils.get_conf() Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"), @@ -306,11 +298,11 @@ def main(): ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"), ('n', "n-act", "Generate-Index-Diffs::Options::NoAct"), ] - suites = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) - Options = Cnf.SubTree("Generate-Index-Diffs::Options") + suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv) + Options = Cnf.subtree("Generate-Index-Diffs::Options") if Options.has_key("Help"): usage() - maxdiffs = Options.get("MaxDiffs::Default", "14") + maxdiffs = Options.get("MaxDiffs::Default", "56") maxpackages = Options.get("MaxDiffs::Packages", maxdiffs) maxcontents = Options.get("MaxDiffs::Contents", maxdiffs) maxsources = Options.get("MaxDiffs::Sources", maxdiffs) @@ -319,32 +311,34 @@ def main(): format = "%Y-%m-%d-%H%M.%S" Options["PatchName"] = time.strftime( format ) - AptCnf = apt_pkg.newConfiguration() - apt_pkg.ReadConfigFileISC(AptCnf,utils.which_apt_conf_file()) - - if Options.has_key("RootDir"): Cnf["Dir::Root"] = Options["RootDir"] + if Options.has_key("RootDir"): + Cnf["Dir::Root"] = Options["RootDir"] if not suites: - suites = Cnf.SubTree("Suite").List() + suites = Cnf.subtree("Suite").list() + + for suitename in suites: + print "Processing: " + suitename + SuiteBlock = Cnf.subtree("Suite::" + suitename) + + suiteobj = get_suite(suitename.lower()) - for suite in suites: - print "Processing: " + suite - SuiteBlock = Cnf.SubTree("Suite::" + suite) + # Use the canonical version of the suite name + suite = suiteobj.suite_name - if SuiteBlock.has_key("Untouchable"): + if suiteobj.untouchable: print "Skipping: " + suite + " (untouchable)" continue - suite = suite.lower() - - architectures = SuiteBlock.ValueList("Architectures") + architectures = get_suite_architectures(suite, skipall=True) if SuiteBlock.has_key("Components"): - components = SuiteBlock.ValueList("Components") + components = SuiteBlock.value_list("Components") else: - components = [] + print "ALERT: suite %s does not have components set in dak.conf" % (suite) + continue - suite_suffix = Cnf.Find("Dinstall::SuiteSuffix") + suite_suffix = Cnf.find("Dinstall::SuiteSuffix") if components and suite_suffix: longsuite = suite + "/" + suite_suffix else: @@ -352,32 +346,30 @@ def main(): tree = SuiteBlock.get("Tree", "dists/%s" % (longsuite)) - if AptCnf.has_key("tree::%s" % (tree)): - sections = AptCnf["tree::%s::Sections" % (tree)].split() - elif AptCnf.has_key("bindirectory::%s" % (tree)): - sections = AptCnf["bindirectory::%s::Sections" % (tree)].split() - else: - aptcnf_filename = os.path.basename(utils.which_apt_conf_file()) - print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename) - continue - - for architecture in architectures: - if architecture == "all": - continue - - if architecture != "source": - # Process Contents - file = "%s/Contents-%s" % (Cnf["Dir::Root"] + tree, - architecture) - storename = "%s/%s_contents_%s" % (Options["TempDir"], suite, architecture) - genchanges(Options, file + ".diff", storename, file, \ - Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents)) - - # use sections instead of components since dak.conf - # treats "foo/bar main" as suite "foo", suitesuffix "bar" and - # component "bar/main". suck. - - for component in sections: + # See if there are Translations which might need a new pdiff + cwd = os.getcwd() + for component in components: + #print "DEBUG: Working on %s" % (component) + workpath=os.path.join(Cnf["Dir::Root"], tree, component, "i18n") + if os.path.isdir(workpath): + os.chdir(workpath) + for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True): + for entry in filenames: + if not re_includeinpdiff.match(entry): + #print "EXCLUDING %s" % (entry) + continue + (fname, fext) = os.path.splitext(entry) + processfile=os.path.join(workpath, fname) + #print "Working: %s" % (processfile) + storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname) + #print "Storefile: %s" % (storename) + genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs) + os.chdir(cwd) + + for archobj in architectures: + architecture = archobj.arch_string + + for component in components: if architecture == "source": longarch = architecture packages = "Sources" @@ -386,6 +378,12 @@ def main(): longarch = "binary-%s"% (architecture) packages = "Packages" maxsuite = maxpackages + # Process Contents + file = "%s/%s/Contents-%s" % (Cnf["Dir::Root"] + tree, component, + architecture) + storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture) + genchanges(Options, file + ".diff", storename, file, \ + Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents)) file = "%s/%s/%s/%s" % (Cnf["Dir::Root"] + tree, component, longarch, packages)