Read files in chunks

[dak.git] / dak / generate_index_diffs.py

diff --git a/dak/generate_index_diffs.py b/dak/generate_index_diffs.py
index ba5e37d02e96fd73b2ed36b23281d511214b282c..485d3770ff6c60bb3e70cf89ba10f9110e241126 100755 (executable)
--- a/dak/generate_index_diffs.py
+++ b/dak/generate_index_diffs.py
@@ -73,7 +73,7 @@ def tryunlink(file):
         print "warning: removing of %s denied" % (file)
 
 def smartstat(file):
-    for ext in ["", ".gz", ".bz2"]:
+    for ext in ["", ".gz", ".bz2", ".xz"]:
         if os.path.isfile(file + ext):
             return (ext, os.stat(file + ext))
     return (None, None)
@@ -85,6 +85,8 @@ def smartlink(f, t):
         os.system("gzip -d < %s.gz > %s" % (f, t))
     elif os.path.isfile("%s.bz2" % (f)):
         os.system("bzip2 -d < %s.bz2 > %s" % (f, t))
+    elif os.path.isfile("%s.xz" % (f)):
+        os.system("xz -d < %s.xz > %s" % (f, t))
     else:
         print "missing: %s" % (f)
         raise IOError(f)
@@ -96,6 +98,8 @@ def smartopen(file):
         f = create_temp_file(os.popen("zcat %s.gz" % file, "r"))
     elif os.path.isfile("%s.bz2" % file):
         f = create_temp_file(os.popen("bzcat %s.bz2" % file, "r"))
+    elif os.path.isfile("%s.xz" % file):
+        f = create_temp_file(os.popen("xzcat %s.xz" % file, "r"))
     else:
         f = None
     return f
@@ -103,7 +107,7 @@ def smartopen(file):
 def pipe_file(f, t):
     f.seek(0)
     while 1:
-        l = f.read()
+        l = f.read(65536)
         if not l: break
         t.write(l)
     t.close()
@@ -239,7 +243,7 @@ class Updates:
 def create_temp_file(r):
     f = tempfile.TemporaryFile()
     while 1:
-        x = r.readline()
+        x = r.read(65536)
         if not x: break
         f.write(x)
     r.close()
@@ -337,9 +341,9 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 56):
         os.link(origfile + origext, oldfile + origext)
         os.unlink(newfile)
 
-        f = open(outdir + "/Index", "w")
-        upd.dump(f)
-        f.close()
+        with open(outdir + "/Index.new", "w") as f:
+            upd.dump(f)
+        os.rename(outdir + "/Index.new", outdir + "/Index")
 
 
 def main():
@@ -429,13 +433,14 @@ def main():
                     packages = "Sources"
                     maxsuite = maxsources
                 else:
-                    longarch = "binary-%s"% (architecture)
+                    longarch = "binary-%s" % (architecture)
                     packages = "Packages"
                     maxsuite = maxpackages
-                    # Process Contents
-                    file = "%s/%s/Contents-%s" % (tree, component, architecture)
-                    storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
-                    genchanges(Options, file + ".diff", storename, file, maxcontents)
+
+                # Process Contents
+                file = "%s/%s/Contents-%s" % (tree, component, architecture)
+                storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
+                genchanges(Options, file + ".diff", storename, file, maxcontents)
 
                 file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
                 storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)