Merge branch 'pu/xz-for-indices'

Conflicts:
	dak/generate_packages_sources2.py
	daklib/filewriter.py

diff --combined dak/generate_packages_sources2.py
index 3b58217a271df20671af2d9cf949cacbb097e64b,8b4773d8528152f4b6c32a19a38730eb4ae8eba4..7eca3c8cacc0460b858e9f2678fcc52f2a68b75c
--- 1/dak/generate_packages_sources2.py
--- 2/dak/generate_packages_sources2.py
+++ b/dak/generate_packages_sources2.py
@@@ -34,7 -34,6 +34,7 @@@ def usage()
      print """Usage: dak generate-packages-sources2 [OPTIONS]
  Generate the Packages/Sources files
  
 +  -a, --archive=ARCHIVE        process suites in ARCHIVE
    -s, --suite=SUITE            process this suite
                                 Default: All suites not marked 'untouchable'
    -f, --force                  Allow processing of untouchable suites
@@@ -67,35 -66,24 +67,35 @@@ SELEC
     WHERE s.id=sm.src_id
    )
    ||
 -  E'\nDirectory\: pool/' || SUBSTRING(f.filename FROM E'\\A(.*)/[^/]*\\Z')
 +  CASE
 +    WHEN src_associations_full.extra_source THEN E'\nExtra-Source-Only\: yes'
 +    ELSE ''
 +  END
    ||
 -  E'\nPriority\: ' || pri.priority
 +  E'\nDirectory\: pool/' || :component_name || '/' || SUBSTRING(f.filename FROM E'\\A(.*)/[^/]*\\Z')
    ||
 -  E'\nSection\: ' || sec.section
 +  E'\nPriority\: ' || COALESCE(pri.priority, 'extra')
 +  ||
 +  E'\nSection\: ' || COALESCE(sec.section, 'misc')
  
  FROM
  
  source s
 -JOIN src_associations sa ON s.id = sa.source
 +JOIN src_associations_full ON src_associations_full.suite = :suite AND s.id = src_associations_full.source
  JOIN files f ON s.file=f.id
 -JOIN override o ON o.package = s.source
 -JOIN section sec ON o.section = sec.id
 -JOIN priority pri ON o.priority = pri.id
 +JOIN files_archive_map fam
 +  ON fam.file_id = f.id
 +     AND fam.archive_id = (SELECT archive_id FROM suite WHERE id = :suite)
 +     AND fam.component_id = :component
 +LEFT JOIN override o ON o.package = s.source
 +                     AND o.suite = :overridesuite
 +                     AND o.component = :component
 +                     AND o.type = :dsc_type
 +LEFT JOIN section sec ON o.section = sec.id
 +LEFT JOIN priority pri ON o.priority = pri.id
  
  WHERE
 -  sa.suite = :suite
 -  AND o.suite = :overridesuite AND o.component = :component AND o.type = :dsc_type
 +  (src_associations_full.extra_source OR o.suite IS NOT NULL)
  
  ORDER BY
  s.source, s.version
@@@ -115,11 -103,17 +115,18 @@@ def generate_sources(suite_id, componen
  
      overridesuite_id = suite.get_overridesuite().suite_id
  
-     writer = SourcesFileWriter(archive=suite.archive.path, suite=suite.suite_name, component=component.component_name)
+     writer_args = {
++            'archive': suite.archive.path,
+             'suite': suite.suite_name,
+             'component': component.component_name
+     }
+     if suite.indices_compression is not None:
+         writer_args['compression'] = suite.indices_compression
+     writer = SourcesFileWriter(**writer_args)
      output = writer.open()
  
      # run query and write Sources
 -    r = session.execute(_sources_query, {"suite": suite_id, "component": component_id, "dsc_type": dsc_type, "overridesuite": overridesuite_id})
 +    r = session.execute(_sources_query, {"suite": suite_id, "component": component_id, "component_name": component.component_name, "dsc_type": dsc_type, "overridesuite": overridesuite_id})
      for (stanza,) in r:
          print >>output, stanza
          print >>output, ""
@@@ -153,12 -147,12 +160,12 @@@ WIT
        binaries b
        JOIN bin_associations ba ON b.id = ba.bin
        JOIN files f ON f.id = b.file
 -      JOIN location l ON l.id = f.location
 +      JOIN files_archive_map fam ON f.id = fam.file_id AND fam.archive_id = :archive_id
        JOIN source s ON b.source = s.id
      WHERE
        (b.architecture = :arch_all OR b.architecture = :arch) AND b.type = :type_name
        AND ba.suite = :suite
 -      AND l.component = :component
 +      AND fam.component_id = :component
    )
  
  SELECT
@@@ -187,7 -181,7 +194,7 @@@
    ), '')
    || E'\nSection\: ' || sec.section
    || E'\nPriority\: ' || pri.priority
 -  || E'\nFilename\: pool/' || tmp.filename
 +  || E'\nFilename\: pool/' || :component_name || '/' || tmp.filename
    || E'\nSize\: ' || tmp.size
    || E'\nMD5sum\: ' || tmp.md5sum
    || E'\nSHA1\: ' || tmp.sha1sum
@@@ -238,13 -232,18 +245,20 @@@ def generate_packages(suite_id, compone
      if include_long_description:
          metadata_skip.append("Description-md5")
  
-     writer = PackagesFileWriter(archive=suite.archive.path, suite=suite.suite_name,
-             component=component.component_name,
-             architecture=architecture.arch_string, debtype=type_name)
+     writer_args = {
++            'archive': suite.archive.path,
+             'suite': suite.suite_name,
+             'component': component.component_name,
+             'architecture': architecture.arch_string,
+             'debtype': type_name
+     }
+     if suite.indices_compression is not None:
+         writer_args['compression'] = suite.indices_compression
+     writer = PackagesFileWriter(**writer_args)
      output = writer.open()
  
 -    r = session.execute(_packages_query, {"suite": suite_id, "component": component_id,
 +    r = session.execute(_packages_query, {"archive_id": suite.archive.archive_id,
 +        "suite": suite_id, "component": component_id, 'component_name': component.component_name,
          "arch": architecture_id, "type_id": type_id, "type_name": type_name, "arch_all": arch_all_id,
          "overridesuite": overridesuite_id, "metadata_skip": metadata_skip,
          "include_long_description": 'true' if include_long_description else 'false'})
@@@ -301,7 -300,14 +315,15 @@@ def generate_translations(suite_id, com
      suite = session.query(Suite).get(suite_id)
      component = session.query(Component).get(component_id)
  
-     writer = TranslationFileWriter(archive=suite.archive.path, suite=suite.suite_name, component=component.component_name, language="en")
+     writer_args = {
++            'archive': suite.archive.path,
+             'suite': suite.suite_name,
+             'component': component.component_name,
+             'language': 'en',
+     }
+     if suite.i18n_compression is not None:
+         writer_args['compression'] = suite.i18n_compression
+     writer = TranslationFileWriter(**writer_args)
      output = writer.open()
  
      r = session.execute(_translations_query, {"suite": suite_id, "component": component_id})
@@@ -323,7 -329,6 +345,7 @@@ def main()
      cnf = Config()
  
      Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
 +                 ('a','archive','Generate-Packages-Sources::Options::Archive','HasArg'),
                   ('s',"suite","Generate-Packages-Sources::Options::Suite"),
                   ('f',"force","Generate-Packages-Sources::Options::Force"),
                   ('o','option','','ArbItem')]
@@@ -342,7 -347,7 +364,7 @@@
  
      logger = daklog.Logger('generate-packages-sources2')
  
 -    from daklib.dbconn import Component, DBConn, get_suite, Suite
 +    from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
      session = DBConn().session()
      session.execute("SELECT add_missing_description_md5()")
      session.commit()
@@@ -357,13 -362,11 +379,13 @@@
                  print "I: Cannot find suite %s" % s
                  logger.log(['Cannot find suite %s' % s])
      else:
 -        suites = session.query(Suite).filter(Suite.untouchable == False).all()
 +        query = session.query(Suite).filter(Suite.untouchable == False)
 +        if 'Archive' in Options:
 +            query = query.join(Suite.archive).filter(Archive.archive_name==Options['Archive'])
 +        suites = query.all()
  
      force = Options.has_key("Force") and Options["Force"]
  
 -    component_ids = [ c.component_id for c in session.query(Component).all() ]
  
      def parse_results(message):
          # Split out into (code, msg)
@@@ -376,10 -379,9 +398,10 @@@
              logger.log(['E: ', msg])
  
      for s in suites:
 +        component_ids = [ c.component_id for c in s.components ]
          if s.untouchable and not force:
 -            import utils
 -            utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name)
 +            import daklib.utils
 +            daklib.utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name)
          for c in component_ids:
              pool.apply_async(generate_sources, [s.suite_id, c], callback=parse_results)
              if not s.include_long_description:

diff --combined daklib/filewriter.py
index 148dd171e8ce7e5105d31566f952021367f144a7,3b816ee9b5f53261151f7090d9f110774294274e..2f080e681b8cc050a0f73f1c317c72e921329db4
mode 100644,100755..100644
--- 1/daklib/filewriter.py
--- 2/daklib/filewriter.py
+++ b/daklib/filewriter.py
@@@ -27,7 -27,7 +27,7 @@@ Helper code for file writing with optio
  
  from daklib.config import Config
  
 -from subprocess import check_call
 +from daklib.daksubprocess import check_call
  
  import os, os.path
  
@@@ -47,7 -47,10 +47,8 @@@ class BaseFileWriter(object)
          self.uncompressed = 'none' in compression
          self.gzip = 'gzip' in compression
          self.bzip2 = 'bzip2' in compression
 -        root_dir = Config()['Dir::Root']
 -        relative_dir = template % keywords
 -        self.path = os.path.join(root_dir, relative_dir)
+         self.xz = 'xz' in compression
 +        self.path = template % keywords
  
      def open(self):
          '''
@@@ -79,6 -82,9 +80,9 @@@
          if self.bzip2:
              check_call('bzip2 -9 <%s.new >%s.bz2.new' % (self.path, self.path), shell = True)
              self.rename('%s.bz2' % self.path)
+         if self.xz:
+             check_call('xz -c <{0}.new >{0}.xz.new'.format(self.path), shell=True)
+             self.rename('{0}.xz'.format(self.path))
          if self.uncompressed:
              self.rename(self.path)
          else:
@@@ -96,9 -102,9 +100,9 @@@ class BinaryContentsFileWriter(BaseFile
          }
          flags.update(keywords)
          if flags['debtype'] == 'deb':
 -            template = "dists/%(suite)s/%(component)s/Contents-%(architecture)s"
 +            template = "%(archive)s/dists/%(suite)s/%(component)s/Contents-%(architecture)s"
          else: # udeb
 -            template = "dists/%(suite)s/%(component)s/Contents-udeb-%(architecture)s"
 +            template = "%(archive)s/dists/%(suite)s/%(component)s/Contents-udeb-%(architecture)s"
          BaseFileWriter.__init__(self, template, **flags)
  
  class SourceContentsFileWriter(BaseFileWriter):
@@@ -111,7 -117,7 +115,7 @@@
              'compression': ['gzip'],
          }
          flags.update(keywords)
 -        template = "dists/%(suite)s/%(component)s/Contents-source"
 +        template = "%(archive)s/dists/%(suite)s/%(component)s/Contents-source"
          BaseFileWriter.__init__(self, template, **flags)
  
  class PackagesFileWriter(BaseFileWriter):
@@@ -125,9 -131,9 +129,9 @@@
          }
          flags.update(keywords)
          if flags['debtype'] == 'deb':
 -            template = "dists/%(suite)s/%(component)s/binary-%(architecture)s/Packages"
 +            template = "%(archive)s/dists/%(suite)s/%(component)s/binary-%(architecture)s/Packages"
          else: # udeb
 -            template = "dists/%(suite)s/%(component)s/debian-installer/binary-%(architecture)s/Packages"
 +            template = "%(archive)s/dists/%(suite)s/%(component)s/debian-installer/binary-%(architecture)s/Packages"
          BaseFileWriter.__init__(self, template, **flags)
  
  class SourcesFileWriter(BaseFileWriter):
@@@ -140,7 -146,7 +144,7 @@@
              'compression': ['gzip', 'bzip2'],
          }
          flags.update(keywords)
 -        template = "dists/%(suite)s/%(component)s/source/Sources"
 +        template = "%(archive)s/dists/%(suite)s/%(component)s/source/Sources"
          BaseFileWriter.__init__(self, template, **flags)
  
  class TranslationFileWriter(BaseFileWriter):
@@@ -154,5 -160,5 +158,5 @@@
              'language':     'en',
          }
          flags.update(keywords)
 -        template = "dists/%(suite)s/%(component)s/i18n/Translation-%(language)s"
 +        template = "%(archive)s/dists/%(suite)s/%(component)s/i18n/Translation-%(language)s"
          super(TranslationFileWriter, self).__init__(template, **flags)