X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fgenerate_packages_sources.py;h=9b4526fedc60085e6ab16d9fc7c67c6097fd7abe;hb=e13fd3c48245d48b0335649d294aacaee666bab5;hp=6e91aedae5470474a5130c3bfe59d002816d4682;hpb=23a6a05608604aa34382b41f70f8252c6a52a7cb;p=dak.git diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py index 6e91aeda..9b4526fe 100755 --- a/dak/generate_packages_sources.py +++ b/dak/generate_packages_sources.py @@ -6,6 +6,7 @@ @copyright: 2000, 2001, 2002, 2006 James Troup @copyright: 2009 Mark Hymers @copyright: 2010 Joerg Jaspert +@license: GNU General Public License version 2 or later """ @@ -31,7 +32,7 @@ import sys import apt_pkg from tempfile import mkstemp, mkdtemp import commands -from multiprocessing import Pool +from multiprocessing import Pool, TimeoutError from daklib import daklog from daklib.dbconn import * @@ -41,6 +42,7 @@ from daklib.config import Config Options = None #: Commandline arguments parsed into this Logger = None #: Our logging object +results = [] #: Results of the subprocesses ################################################################################ @@ -103,6 +105,60 @@ TreeDefault apt_trees={} apt_trees["di"]={} + + apt_trees["stable"]=""" +tree "dists/stable" +{ + FileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.$(SECTION)"; + ExtraOverride "override.squeeze.extra.$(SECTION)"; + SrcOverride "override.squeeze.$(SECTION).src"; +}; +""" + + apt_trees["di"]["stable"]=""" +tree "dists/stable/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/stable_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; + +tree "dists/stable/non-free" +{ + FileList "/srv/ftp-master.debian.org/database/dists/stable_non-free_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; +""" + + apt_trees["squeeze-updates"]=""" +tree "dists/squeeze-updates" +{ + FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.$(SECTION)"; + ExtraOverride "override.squeeze.extra.$(SECTION)"; + SrcOverride "override.squeeze.$(SECTION).src"; + Contents " "; +}; +""" + apt_trees["testing"]=""" tree "dists/testing" { @@ -111,19 +167,20 @@ tree "dists/testing" SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list"; Sections "main contrib non-free"; Architectures "%(arch)s"; - BinOverride "override.squeeze.$(SECTION)"; - ExtraOverride "override.squeeze.extra.$(SECTION)"; - SrcOverride "override.squeeze.$(SECTION).src"; + BinOverride "override.wheezy.$(SECTION)"; + ExtraOverride "override.wheezy.extra.$(SECTION)"; + SrcOverride "override.wheezy.$(SECTION).src"; }; """ + apt_trees["di"]["testing"]=""" tree "dists/testing/main" { FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list"; Sections "debian-installer"; Architectures "%(arch)s"; - BinOverride "override.squeeze.main.$(SECTION)"; - SrcOverride "override.squeeze.main.src"; + BinOverride "override.wheezy.main.$(SECTION)"; + SrcOverride "override.wheezy.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; %(contentsline)s @@ -134,8 +191,8 @@ tree "dists/testing/non-free" FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list"; Sections "debian-installer"; Architectures "%(arch)s"; - BinOverride "override.squeeze.main.$(SECTION)"; - SrcOverride "override.squeeze.main.src"; + BinOverride "override.wheezy.main.$(SECTION)"; + SrcOverride "override.wheezy.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; %(contentsline)s @@ -224,9 +281,9 @@ tree "dists/testing-proposed-updates" SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list"; Sections "main contrib non-free"; Architectures "%(arch)s"; - BinOverride "override.squeeze.$(SECTION)"; - ExtraOverride "override.squeeze.extra.$(SECTION)"; - SrcOverride "override.squeeze.$(SECTION).src"; + BinOverride "override.wheezy.$(SECTION)"; + ExtraOverride "override.wheezy.extra.$(SECTION)"; + SrcOverride "override.wheezy.$(SECTION).src"; Contents " "; }; """ @@ -236,8 +293,8 @@ tree "dists/testing-proposed-updates/main" FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; Sections "debian-installer"; Architectures "%(arch)s"; - BinOverride "override.squeeze.main.$(SECTION)"; - SrcOverride "override.squeeze.main.src"; + BinOverride "override.wheezy.main.$(SECTION)"; + SrcOverride "override.wheezy.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; Contents " "; @@ -251,9 +308,9 @@ tree "dists/proposed-updates" SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list"; Sections "main contrib non-free"; Architectures "%(arch)s"; - BinOverride "override.lenny.$(SECTION)"; - ExtraOverride "override.lenny.extra.$(SECTION)"; - SrcOverride "override.lenny.$(SECTION).src"; + BinOverride "override.squeeze.$(SECTION)"; + ExtraOverride "override.squeeze.extra.$(SECTION)"; + SrcOverride "override.squeeze.$(SECTION).src"; Contents " "; }; """ @@ -263,18 +320,17 @@ tree "dists/proposed-updates/main" FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; Sections "debian-installer"; Architectures "%(arch)s"; - BinOverride "override.lenny.main.$(SECTION)"; - SrcOverride "override.lenny.main.src"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; Contents " "; }; """ - cnf = Config() try: # Write apt.conf - (ac_fd, ac_name) = mkstemp(dir=tmppath) + (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch) os.write(ac_fd, DAILY_APT_CONF) # here we want to generate the tree entries os.write(ac_fd, apt_trees[suite] % {'arch': arch}) @@ -283,7 +339,7 @@ tree "dists/proposed-updates/main" if arch != 'source': if arch == 'hurd-i386' and suite == 'experimental': pass - else: + elif apt_trees["di"].has_key(suite): if arch == "amd64": os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'}) @@ -297,9 +353,10 @@ tree "dists/proposed-updates/main" # it has errormessages we like to see os.environ['GZIP'] = '--rsyncable' os.chdir(tmppath) - (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name)) + (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name)) sn="a-f %s,%s: " % (suite, arch) print sn + output.replace('\n', '\n%s' % (sn)) + return result # Clean up any left behind files finally: @@ -318,11 +375,16 @@ tree "dists/proposed-updates/main" def sname(arch): return arch.arch_string +def get_result(arg): + global results + if arg: + results.append(arg) + ######################################################################## ######################################################################## def main (): - global Options, Logger + global Options, Logger, results cnf = Config() @@ -340,7 +402,7 @@ def main (): if Options["Help"]: usage() - Logger = daklog.Logger(cnf, 'generate-packages-sources') + Logger = daklog.Logger('generate-packages-sources') session = DBConn().session() @@ -360,19 +422,26 @@ def main (): startdir = os.getcwd() os.chdir(cnf["Dir::TempPath"]) - # Setup a multiprocessing Pool. As many workers as we have CPU cores. - pool = Pool() - + broken=[] # For each given suite, each architecture, run one apt-ftparchive for s in suites: - arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session) + results=[] + # Setup a multiprocessing Pool. As many workers as we have CPU cores. + pool = Pool() + arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session) Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))]) for a in arch_list: - pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"])) + pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result) + + # No more work will be added to our pool, close it and then wait for all to finish + pool.close() + pool.join() + + if len(results) > 0: + Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)]) + print "Trouble, something with a-f broke, resultcodes: %s" % (results) + sys.exit(1) - # No more work will be added to our pool, close it and then wait for all to finish - pool.close() - pool.join() os.chdir(startdir) # this script doesn't change the database session.close()