Merge branch 'master' into gps

* master:
  examine-package: lock access to printed_copyrights
  show-new: multithreading support
  show-new: no longer fiddle with sys.stdout
  examine_package: make most functions functional
  examine-package: really support option -H
  examine-package: add missing newlines to output
  examine-package: make foldable_output functional()
  examine-package: make headline() functional
  call check_deb() with correct arguments
  Add config for volatile

diff --git a/dak/dak.py b/dak/dak.py
index 4ad59579aa954c4d4a0c3c6f635fd862bfc47706..bd5108fa6e2e49607b2fc59050a43d95087e88c4 100755 (executable)
--- a/dak/dak.py
+++ b/dak/dak.py
@@ -81,6 +81,8 @@ def init():
          "Generate file lists for apt-ftparchive"),
         ("generate-releases",
          "Generate Release files"),
+        ("generate-packages-sources",
+         "Generate Packages/Sources files"),
         ("contents",
          "Generate content files"),
         ("generate-index-diffs",

diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py
new file mode 100755 (executable)
index 0000000..d91ec8f
--- /dev/null
+++ b/dak/generate_packages_sources.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+
+""" Generate Packages/Sources files
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
+@copyright: 2009  Mark Hymers <mhy@debian.org>
+@copyright: 2010  Joerg Jaspert <joerg@debian.org>
+
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+################################################################################
+
+import os
+import os.path
+import stat
+import sys
+from datetime import datetime
+import apt_pkg
+
+from daklib import daklog
+from daklib.dbconn import *
+from daklib.config import Config
+from daklib.threadpool import ThreadPool
+
+################################################################################
+
+Options = None
+Logger = None
+
+################################################################################
+
+def usage (exit_code=0):
+    print """Usage: dak generate-packages-sources [OPTIONS]
+Generate the Packages/Sources files
+
+  -s, --suite=SUITE(s)       process this suite
+                             Default: All suites not marked 'untouchable'
+  -f, --force                Allow processing of untouchable suites
+                             CAREFUL: Only to be used at point release time!
+  -h, --help                 show this help and exit
+
+SUITE can be a space seperated list, e.g.
+   --suite=unstable testing
+  """
+
+    sys.exit(exit_code)
+
+################################################################################
+
+def main ():
+    global Options, Logger
+
+    cnf = Config()
+
+    for i in ["Help", "Suite", "Force"]:
+        if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
+            cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
+
+    Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
+                 ('s',"suite","Generate-Packages-Sources::Options::Suite"),
+                 ('f',"force","Generate-Packages-Sources::Options::Force")]
+
+    suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+    Options = cnf.SubTree("Generate-Packages-Sources::Options")
+
+    if Options["Help"]:
+        usage()
+
+    Logger = daklog.Logger(cnf, 'generate-packages-sources')
+
+    session = DBConn().session()
+
+    if Options["Suite"]:
+        # Something here
+        suites = []
+        for s in suite_names:
+            suite = get_suite(s.lower(), session)
+            if suite:
+                suites.append(suite)
+            else:
+                print "cannot find suite %s" % s
+                Logger.log(['cannot find suite %s' % s])
+    else:
+        suites=session.query(Suite).filter(Suite.untouchable == False).all()
+
+    threadpool = ThreadPool()
+    # For each given suite, each architecture, run one apt-ftparchive
+    for s in suites:
+        arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session)
+        for a in arch_list:
+            Logger.log(['generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string)])
+            print 'generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string)
+            threadpool.queueTask(s.generate_packages_sources, (a.arch_string))
+
+    threadpool.joinAll()
+    # this script doesn't change the database
+    session.close()
+    Logger.close()
+
+#######################################################################################
+
+if __name__ == '__main__':
+    main()

diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 559dbf17a98340169319bc06dddbf3e1d32dae0b..28db5c03b2a42bdbf965c3c435c03894bbcea49b 100755 (executable)
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -2459,6 +2459,190 @@ SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
                  ('ChangelogBase', 'changelogbase')]
 
 
+DAILY_APT_CONF="""
+Dir
+{
+   ArchiveDir "/srv/ftp-master.debian.org/ftp/";
+   OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
+   CacheDir "/srv/ftp-master.debian.org/database/";
+};
+
+Default
+{
+   Packages::Compress ". bzip2 gzip";
+   Sources::Compress ". bzip2 gzip";
+   Contents::Compress "gzip";
+   DeLinkLimit 0;
+   MaxContentsChange 25000;
+   FileMode 0664;
+}
+
+TreeDefault
+{
+   Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
+};
+
+"""
+
+apt_trees={}
+apt_trees["testing"]="""
+tree "dists/testing"
+{
+   FakeDI "dists/unstable";
+   FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.$(SECTION)";
+   ExtraOverride "override.squeeze.extra.$(SECTION)";
+   SrcOverride "override.squeeze.$(SECTION).src";
+};
+tree "dists/testing/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.main.$(SECTION)";
+   SrcOverride "override.squeeze.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents "$(DIST)/../Contents-udeb";
+};
+
+tree "dists/testing/non-free"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.main.$(SECTION)";
+   SrcOverride "override.squeeze.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents "$(DIST)/../Contents-udeb-nf";
+};
+"""
+
+apt_trees["unstable"]="""
+tree "dists/unstable"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.$(SECTION)";
+   ExtraOverride "override.sid.extra.$(SECTION)";
+   SrcOverride "override.sid.$(SECTION).src";
+};
+tree "dists/unstable/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.main.$(SECTION)";
+   SrcOverride "override.sid.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents "$(DIST)/../Contents-udeb";
+};
+
+tree "dists/unstable/non-free"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.main.$(SECTION)";
+   SrcOverride "override.sid.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents "$(DIST)/../Contents-udeb-nf";
+};
+"""
+
+apt_trees["experimental"]="""
+tree "dists/experimental"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.$(SECTION)";
+   SrcOverride "override.sid.$(SECTION).src";
+};
+tree "dists/experimental/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.main.$(SECTION)";
+   SrcOverride "override.sid.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents "$(DIST)/../Contents-udeb";
+};
+
+tree "dists/experimental/non-free"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.main.$(SECTION)";
+   SrcOverride "override.sid.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents "$(DIST)/../Contents-udeb-nf";
+};
+"""
+
+apt_trees["testing-proposed-updates"]="""
+tree "dists/testing-proposed-updates"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.$(SECTION)";
+   ExtraOverride "override.squeeze.extra.$(SECTION)";
+   SrcOverride "override.squeeze.$(SECTION).src";
+   Contents " ";
+};
+tree "dists/testing-proposed-updates/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.main.$(SECTION)";
+   SrcOverride "override.squeeze.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents " ";
+};
+"""
+
+apt_trees["proposed-updates"]="""
+tree "dists/proposed-updates"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "amd64";
+   BinOverride "override.lenny.$(SECTION)";
+   ExtraOverride "override.lenny.extra.$(SECTION)";
+   SrcOverride "override.lenny.$(SECTION).src";
+   Contents " ";
+};
+tree "dists/proposed-updates/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.lenny.main.$(SECTION)";
+   SrcOverride "override.lenny.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents " ";
+};
+"""
+
 class Suite(object):
     def __init__(self, *args, **kwargs):
         pass
@@ -2487,6 +2671,51 @@ class Suite(object):
 
         return "\n".join(ret)
 
+    def generate_packages_sources(self, arch):
+        """
+        Generate Packages/Sources files with apt-ftparchive for the given suite/arch
+
+        @type suite: string
+        @param suite: Suite name
+
+        @type arch: string
+        @param arch: Architecture name
+        """
+
+        tempdir = None
+        startdir = os.getcwd()
+
+        try:
+            # Write apt.conf
+            (ac_fd, ac_name) = mkstemp()
+            os.write(ac_fd, DAILY_APT_CONF)
+            # here we want to generate the tree entries
+            os.write(ac_fd, apt_trees[self.suite_name] % {'arch': arch})
+            os.close(ac_fd)
+
+            # Run apt-ftparchive generate
+            os.chdir(os.path.dirname(ac_name))
+            # We might want to add a -q or -qq here
+            os.system('apt-ftparchive generate %s' % os.path.basename(ac_name))
+
+        # Clean up any left behind files
+        finally:
+            os.chdir(startdir)
+            if ac_fd:
+                try:
+                    os.close(ac_fd)
+                except OSError:
+                    pass
+
+            if ac_name:
+                try:
+                    os.unlink(ac_name)
+                except OSError:
+                    pass
+
+
+
+
 __all__.append('Suite')
 
 @session_wrapper