From: Joerg Jaspert Date: Tue, 6 Jul 2010 23:57:45 +0000 (+0200) Subject: generate-packages-sources X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=c9398d303256dfedc13975b79b6b691048cfcc05;p=dak.git generate-packages-sources some more changes: - chdir before we call the generate function - dont include architecture all - use commands.getstatusoutput this breaks the try, finally: block. using os.system() it works nicely, but the output is a hell. using commands.getstatusoutput the finally: is called to early, removing the config for apt-ftparchive, breaking it. ugh, bad. - the whole mess with the config inside dbconn.py needs to go, as well as the special cases for the tree configs. should be an aptconf class, but thats a task for when this fully works Signed-off-by: Joerg Jaspert --- diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py index d91ec8fc..366d722b 100755 --- a/dak/generate_packages_sources.py +++ b/dak/generate_packages_sources.py @@ -99,15 +99,19 @@ def main (): suites=session.query(Suite).filter(Suite.untouchable == False).all() threadpool = ThreadPool() + + startdir = os.getcwd() + os.chdir(cnf["Dir::TempPath"]) + # For each given suite, each architecture, run one apt-ftparchive for s in suites: - arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session) + arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session) + Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, arch_list)]) for a in arch_list: - Logger.log(['generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string)]) - print 'generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string) threadpool.queueTask(s.generate_packages_sources, (a.arch_string)) threadpool.joinAll() + os.chdir(startdir) # this script doesn't change the database session.close() Logger.close() diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 28db5c03..b987a372 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -37,6 +37,7 @@ import os import re import psycopg2 import traceback +import commands from datetime import datetime, timedelta from errno import ENOENT from tempfile import mkstemp, mkdtemp @@ -2469,8 +2470,8 @@ Dir Default { - Packages::Compress ". bzip2 gzip"; - Sources::Compress ". bzip2 gzip"; + Packages::Compress "bzip2 gzip"; + Sources::Compress "bzip2 gzip"; Contents::Compress "gzip"; DeLinkLimit 0; MaxContentsChange 25000; @@ -2485,6 +2486,7 @@ TreeDefault """ apt_trees={} +apt_trees["di"]={} apt_trees["testing"]=""" tree "dists/testing" { @@ -2497,6 +2499,8 @@ tree "dists/testing" ExtraOverride "override.squeeze.extra.$(SECTION)"; SrcOverride "override.squeeze.$(SECTION).src"; }; +""" +apt_trees["di"]["testing"]=""" tree "dists/testing/main" { FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list"; @@ -2506,7 +2510,7 @@ tree "dists/testing/main" SrcOverride "override.squeeze.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; - Contents "$(DIST)/../Contents-udeb"; + %(contentsline)s }; tree "dists/testing/non-free" @@ -2518,7 +2522,7 @@ tree "dists/testing/non-free" SrcOverride "override.squeeze.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; - Contents "$(DIST)/../Contents-udeb-nf"; + %(contentsline)s }; """ @@ -2533,6 +2537,8 @@ tree "dists/unstable" ExtraOverride "override.sid.extra.$(SECTION)"; SrcOverride "override.sid.$(SECTION).src"; }; +""" +apt_trees["di"]["unstable"]=""" tree "dists/unstable/main" { FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list"; @@ -2542,7 +2548,7 @@ tree "dists/unstable/main" SrcOverride "override.sid.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; - Contents "$(DIST)/../Contents-udeb"; + %(contentsline)s }; tree "dists/unstable/non-free" @@ -2554,7 +2560,7 @@ tree "dists/unstable/non-free" SrcOverride "override.sid.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; - Contents "$(DIST)/../Contents-udeb-nf"; + %(contentsline)s }; """ @@ -2568,6 +2574,8 @@ tree "dists/experimental" BinOverride "override.sid.$(SECTION)"; SrcOverride "override.sid.$(SECTION).src"; }; +""" +apt_trees["di"]["experimental"]=""" tree "dists/experimental/main" { FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list"; @@ -2577,7 +2585,7 @@ tree "dists/experimental/main" SrcOverride "override.sid.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; - Contents "$(DIST)/../Contents-udeb"; + %(contentsline)s }; tree "dists/experimental/non-free" @@ -2589,7 +2597,7 @@ tree "dists/experimental/non-free" SrcOverride "override.sid.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; - Contents "$(DIST)/../Contents-udeb-nf"; + %(contentsline)s }; """ @@ -2605,6 +2613,8 @@ tree "dists/testing-proposed-updates" SrcOverride "override.squeeze.$(SECTION).src"; Contents " "; }; +""" +apt_trees["di"]["testing-proposed-updates"]=""" tree "dists/testing-proposed-updates/main" { FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; @@ -2624,12 +2634,14 @@ tree "dists/proposed-updates" FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list"; SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list"; Sections "main contrib non-free"; - Architectures "amd64"; + Architectures "%(arch)s"; BinOverride "override.lenny.$(SECTION)"; ExtraOverride "override.lenny.extra.$(SECTION)"; SrcOverride "override.lenny.$(SECTION).src"; Contents " "; }; +""" +apt_trees["di"]["proposed-updates"]=""" tree "dists/proposed-updates/main" { FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; @@ -2682,25 +2694,36 @@ class Suite(object): @param arch: Architecture name """ - tempdir = None - startdir = os.getcwd() - + cnf = Config() try: # Write apt.conf - (ac_fd, ac_name) = mkstemp() + (ac_fd, ac_name) = mkstemp(dir=cnf["Dir::TempPath"]) os.write(ac_fd, DAILY_APT_CONF) # here we want to generate the tree entries os.write(ac_fd, apt_trees[self.suite_name] % {'arch': arch}) + # this special casing needs to go away, but this whole thing may just want an + # aptconfig class anyways + if arch != 'source': + if arch == 'hurd-i386' and self.suite_name == 'experimental': + pass + else: + if arch == "amd64": + os.write(ac_fd, apt_trees["di"][self.suite_name] % + {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'}) + else: + os.write(ac_fd, apt_trees["di"][self.suite_name] % {'arch': arch, 'contentsline': ''}) os.close(ac_fd) # Run apt-ftparchive generate - os.chdir(os.path.dirname(ac_name)) # We might want to add a -q or -qq here - os.system('apt-ftparchive generate %s' % os.path.basename(ac_name)) + os.environ['GZIP'] = '--rsyncable' + os.chdir(cnf["Dir::TempPath"]) + (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name)) + sn="a-f %s,%s: " % (self.suite_name, arch) + print sn + output.replace('\n', '\n%s' % (sn)) # Clean up any left behind files finally: - os.chdir(startdir) if ac_fd: try: os.close(ac_fd)