import os
import os.path
-import stat
import sys
-from datetime import datetime
import apt_pkg
+from tempfile import mkstemp, mkdtemp
+import commands
+from multiprocessing import Pool
from daklib import daklog
from daklib.dbconn import *
from daklib.config import Config
-from daklib.threadpool import ThreadPool
################################################################################
-Options = None
-Logger = None
+Options = None #: Commandline arguments parsed into this
+Logger = None #: Our logging object
################################################################################
################################################################################
+def generate_packages_sources(arch, suite, tmppath):
+ """
+ Generate Packages/Sources files with apt-ftparchive for the given suite/arch
+
+ @type suite: string
+ @param suite: Suite name
+
+ @type arch: string
+ @param arch: Architecture name
+
+ @type tmppath: string
+ @param tmppath: The temporary path to work ing
+ """
+
+ DAILY_APT_CONF="""
+Dir
+{
+ ArchiveDir "/srv/ftp-master.debian.org/ftp/";
+ OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
+ CacheDir "/srv/ftp-master.debian.org/database/";
+};
+
+Default
+{
+ Packages::Compress "bzip2 gzip";
+ Sources::Compress "bzip2 gzip";
+ Contents::Compress "gzip";
+ DeLinkLimit 0;
+ MaxContentsChange 25000;
+ FileMode 0664;
+}
+
+TreeDefault
+{
+ Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
+};
+
+"""
+
+ apt_trees={}
+ apt_trees["di"]={}
+ apt_trees["testing"]="""
+tree "dists/testing"
+{
+ FakeDI "dists/unstable";
+ FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.wheezy.$(SECTION)";
+ ExtraOverride "override.wheezy.extra.$(SECTION)";
+ SrcOverride "override.wheezy.$(SECTION).src";
+};
+"""
+
+ apt_trees["squeeze-updates"]="""
+tree "dists/squeeze-updates"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.$(SECTION)";
+ ExtraOverride "override.squeeze.extra.$(SECTION)";
+ SrcOverride "override.squeeze.$(SECTION).src";
+ Contents " ";
+};
+"""
+
+ apt_trees["di"]["testing"]="""
+tree "dists/testing/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.wheezy.main.$(SECTION)";
+ SrcOverride "override.wheezy.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+
+tree "dists/testing/non-free"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.wheezy.main.$(SECTION)";
+ SrcOverride "override.wheezy.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+"""
+
+ apt_trees["unstable"]="""
+tree "dists/unstable"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.$(SECTION)";
+ ExtraOverride "override.sid.extra.$(SECTION)";
+ SrcOverride "override.sid.$(SECTION).src";
+};
+"""
+ apt_trees["di"]["unstable"]="""
+tree "dists/unstable/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.main.$(SECTION)";
+ SrcOverride "override.sid.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+
+tree "dists/unstable/non-free"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.main.$(SECTION)";
+ SrcOverride "override.sid.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+"""
+
+ apt_trees["experimental"]="""
+tree "dists/experimental"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.$(SECTION)";
+ SrcOverride "override.sid.$(SECTION).src";
+};
+"""
+ apt_trees["di"]["experimental"]="""
+tree "dists/experimental/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.main.$(SECTION)";
+ SrcOverride "override.sid.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+
+tree "dists/experimental/non-free"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.main.$(SECTION)";
+ SrcOverride "override.sid.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+"""
+
+ apt_trees["testing-proposed-updates"]="""
+tree "dists/testing-proposed-updates"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.wheezy.$(SECTION)";
+ ExtraOverride "override.wheezy.extra.$(SECTION)";
+ SrcOverride "override.wheezy.$(SECTION).src";
+ Contents " ";
+};
+"""
+ apt_trees["di"]["testing-proposed-updates"]="""
+tree "dists/testing-proposed-updates/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.wheezy.main.$(SECTION)";
+ SrcOverride "override.wheezy.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ Contents " ";
+};
+"""
+
+ apt_trees["proposed-updates"]="""
+tree "dists/proposed-updates"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.$(SECTION)";
+ ExtraOverride "override.squeeze.extra.$(SECTION)";
+ SrcOverride "override.squeeze.$(SECTION).src";
+ Contents " ";
+};
+"""
+ apt_trees["di"]["proposed-updates"]="""
+tree "dists/proposed-updates/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.main.$(SECTION)";
+ SrcOverride "override.squeeze.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ Contents " ";
+};
+"""
+
+ cnf = Config()
+ try:
+ # Write apt.conf
+ (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
+ os.write(ac_fd, DAILY_APT_CONF)
+ # here we want to generate the tree entries
+ os.write(ac_fd, apt_trees[suite] % {'arch': arch})
+ # this special casing needs to go away, but this whole thing may just want an
+ # aptconfig class anyways
+ if arch != 'source':
+ if arch == 'hurd-i386' and suite == 'experimental':
+ pass
+ elif apt_trees["di"].has_key(suite):
+ if arch == "amd64":
+ os.write(ac_fd, apt_trees["di"][suite] %
+ {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
+ else:
+ os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
+ os.close(ac_fd)
+
+ print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
+ # Run apt-ftparchive generate
+ # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
+ # it has errormessages we like to see
+ os.environ['GZIP'] = '--rsyncable'
+ os.chdir(tmppath)
+ (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
+ sn="a-f %s,%s: " % (suite, arch)
+ print sn + output.replace('\n', '\n%s' % (sn))
+
+ # Clean up any left behind files
+ finally:
+ if ac_fd:
+ try:
+ os.close(ac_fd)
+ except OSError:
+ pass
+
+ if ac_name:
+ try:
+ os.unlink(ac_name)
+ except OSError:
+ pass
+
+def sname(arch):
+ return arch.arch_string
+
+########################################################################
+########################################################################
+
def main ():
global Options, Logger
else:
suites=session.query(Suite).filter(Suite.untouchable == False).all()
- threadpool = ThreadPool()
+ startdir = os.getcwd()
+ os.chdir(cnf["Dir::TempPath"])
+
# For each given suite, each architecture, run one apt-ftparchive
for s in suites:
- arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session)
+ # Setup a multiprocessing Pool. As many workers as we have CPU cores.
+ pool = Pool()
+ arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
+ Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
for a in arch_list:
- Logger.log(['generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string)])
- print 'generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string)
- threadpool.queueTask(s.generate_packages_sources, (a.arch_string))
+ pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
+
+ # No more work will be added to our pool, close it and then wait for all to finish
+ pool.close()
+ pool.join()
- threadpool.joinAll()
+ os.chdir(startdir)
# this script doesn't change the database
session.close()
Logger.close()
projects / dak.git / blobdiff