Split binary contents into components.

Signed-off-by: Torsten Werner <twerner@debian.org>

diff --git a/dak/contents.py b/dak/contents.py
index ee904b2a2467e4f0d100e03e5dca3396e3587b14..c2d04eb697dd683c176ebe6e1f4581887b43893a 100755 (executable)
--- a/dak/contents.py
+++ b/dak/contents.py
@@ -70,6 +70,9 @@ OPTIONS for generate
      -s, --suite={stable,testing,unstable,...}
         only operate on specified suite names
 
+     -c, --component={main,contrib,non-free}
+        only operate on specified components
+
      -f, --force
         write Contents files for suites marked as untouchable, too
 
@@ -81,9 +84,9 @@ OPTIONS for scan-source and scan-binary
 
 ################################################################################
 
-def write_all(cnf, suite_names = [], force = None):
+def write_all(cnf, suite_names = [], component_names = [], force = None):
     Logger = daklog.Logger(cnf.Cnf, 'contents generate')
-    ContentsWriter.write_all(Logger, suite_names, force)
+    ContentsWriter.write_all(Logger, suite_names, component_names, force)
     Logger.close()
 
 ################################################################################
@@ -112,12 +115,14 @@ def main():
     cnf = Config()
     cnf['Contents::Options::Help'] = ''
     cnf['Contents::Options::Suite'] = ''
+    cnf['Contents::Options::Component'] = ''
     cnf['Contents::Options::Limit'] = ''
     cnf['Contents::Options::Force'] = ''
-    arguments = [('h', "help",  'Contents::Options::Help'),
-                 ('s', "suite", 'Contents::Options::Suite', "HasArg"),
-                 ('l', "limit", 'Contents::Options::Limit', "HasArg"),
-                 ('f', "force", 'Contents::Options::Force'),
+    arguments = [('h', "help",      'Contents::Options::Help'),
+                 ('s', "suite",     'Contents::Options::Suite',     "HasArg"),
+                 ('c', "component", 'Contents::Options::Component', "HasArg"),
+                 ('l', "limit",     'Contents::Options::Limit',     "HasArg"),
+                 ('f', "force",     'Contents::Options::Force'),
                 ]
     args = apt_pkg.ParseCommandLine(cnf.Cnf, arguments, sys.argv)
     options = cnf.SubTree('Contents::Options')
@@ -137,12 +142,13 @@ def main():
         binary_scan_all(cnf, limit)
         return
 
-    suite_names = utils.split_args(options['Suite'])
+    suite_names     = utils.split_args(options['Suite'])
+    component_names = utils.split_args(options['Component'])
 
     force = bool(options['Force'])
 
     if args[0] == 'generate':
-        write_all(cnf, suite_names, force)
+        write_all(cnf, suite_names, component_names, force)
         return
 
     usage()

diff --git a/dak/generate_index_diffs.py b/dak/generate_index_diffs.py
index 7e4b0058e335186a26d609ee5e5cb18ad0d6165d..3a69f3a57e7b1b4c2a9f1e82b36455de1f6f763f 100755 (executable)
--- a/dak/generate_index_diffs.py
+++ b/dak/generate_index_diffs.py
@@ -352,14 +352,6 @@ def main():
         for archobj in architectures:
             architecture = archobj.arch_string
 
-            if architecture != "source":
-                # Process Contents
-                file = "%s/Contents-%s" % (Cnf["Dir::Root"] + tree,
-                        architecture)
-                storename = "%s/%s_contents_%s" % (Options["TempDir"], suite, architecture)
-                genchanges(Options, file + ".diff", storename, file, \
-                  Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents))
-
             # use sections instead of components since dak.conf
             # treats "foo/bar main" as suite "foo", suitesuffix "bar" and
             # component "bar/main". suck.
@@ -373,6 +365,12 @@ def main():
                     longarch = "binary-%s"% (architecture)
                     packages = "Packages"
                     maxsuite = maxpackages
+                    # Process Contents
+                    file = "%s/%s/Contents-%s" % (Cnf["Dir::Root"] + tree, component,
+                            longarch)
+                    storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
+                    genchanges(Options, file + ".diff", storename, file, \
+                      Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents))
 
                 file = "%s/%s/%s/%s" % (Cnf["Dir::Root"] + tree,
                            component, longarch, packages)

diff --git a/daklib/contents.py b/daklib/contents.py
index a906b54d20e2cb8f75a83c525449a73bd601be68..f5f27488c02149f8493859dd790d42c427670912 100755 (executable)
--- a/daklib/contents.py
+++ b/daklib/contents.py
@@ -41,7 +41,7 @@ class BinaryContentsWriter(object):
     '''
     BinaryContentsWriter writes the Contents-$arch.gz files.
     '''
-    def __init__(self, suite, architecture, overridetype, component = None):
+    def __init__(self, suite, architecture, overridetype, component):
         self.suite = suite
         self.architecture = architecture
         self.overridetype = overridetype
@@ -58,15 +58,14 @@ class BinaryContentsWriter(object):
         params = {
             'suite':         self.suite.suite_id,
             'overridesuite': overridesuite.suite_id,
+            'component':     self.component.component_id,
             'arch_all':      get_architecture('all', self.session).arch_id,
             'arch':          self.architecture.arch_id,
             'type_id':       self.overridetype.overridetype_id,
             'type':          self.overridetype.overridetype,
         }
 
-        if self.component is not None:
-            params['component'] = self.component.component_id
-            sql = '''
+        sql = '''
 create temp table newest_binaries (
     id integer primary key,
     package text);
@@ -88,34 +87,6 @@ unique_override as
         where o.suite = :overridesuite and o.type = :type_id and o.section = s.id and
         o.component = :component)
 
-select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package) as pkglist
-    from newest_binaries b, bin_contents bc, unique_override o
-    where b.id = bc.binary_id and o.package = b.package
-    group by bc.file'''
-
-        else:
-            sql = '''
-create temp table newest_binaries (
-    id integer primary key,
-    package text);
-
-create index newest_binaries_by_package on newest_binaries (package);
-
-insert into newest_binaries (id, package)
-    select distinct on (package) id, package from binaries
-        where type = :type and
-            (architecture = :arch_all or architecture = :arch) and
-            id in (select bin from bin_associations where suite = :suite)
-        order by package, version desc;
-
-with
-
-unique_override as
-    (select distinct on (o.package, s.section) o.package, s.section
-        from override o, section s
-        where o.suite = :overridesuite and o.type = :type_id and o.section = s.id
-        order by o.package, s.section, o.modified desc)
-
 select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package) as pkglist
     from newest_binaries b, bin_contents bc, unique_override o
     where b.id = bc.binary_id and o.package = b.package
@@ -151,10 +122,9 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package
         '''
         values = {
             'suite':        self.suite.suite_name,
+            'debtype':      self.overridetype.overridetype,
             'architecture': self.architecture.arch_string,
         }
-        if self.component is not None:
-            values['component'] = self.component.component_name
         return BinaryContentsFileWriter(**values)
 
     def get_header(self):
@@ -264,7 +234,7 @@ select sc.file, string_agg(s.source, ',' order by s.source) as pkglist
         writer.close()
 
 
-def binary_helper(suite_id, arch_id, overridetype_id, component_id = None):
+def binary_helper(suite_id, arch_id, overridetype_id, component_id):
     '''
     This function is called in a new subprocess and multiprocessing wants a top
     level function.
@@ -273,12 +243,9 @@ def binary_helper(suite_id, arch_id, overridetype_id, component_id = None):
     suite = Suite.get(suite_id, session)
     architecture = Architecture.get(arch_id, session)
     overridetype = OverrideType.get(overridetype_id, session)
-    log_message = [suite.suite_name, architecture.arch_string, overridetype.overridetype]
-    if component_id is None:
-        component = None
-    else:
-        component = Component.get(component_id, session)
-        log_message.append(component.component_name)
+    component = Component.get(component_id, session)
+    log_message = [suite.suite_name, architecture.arch_string, \
+        overridetype.overridetype, component.component_name]
     contents_writer = BinaryContentsWriter(suite, architecture, overridetype, component)
     contents_writer.write_file()
     return log_message
@@ -309,7 +276,7 @@ class ContentsWriter(object):
         class_.logger.log(result)
 
     @classmethod
-    def write_all(class_, logger, suite_names = [], force = False):
+    def write_all(class_, logger, suite_names = [], component_names = [], force = False):
         '''
         Writes all Contents files for suites in list suite_names which defaults
         to all 'touchable' suites if not specified explicitely. Untouchable
@@ -320,33 +287,29 @@ class ContentsWriter(object):
         suite_query = session.query(Suite)
         if len(suite_names) > 0:
             suite_query = suite_query.filter(Suite.suite_name.in_(suite_names))
+        component_query = session.query(Component)
+        if len(component_names) > 0:
+            component_query = component_query.filter(Component.component_name.in_(component_names))
         if not force:
             suite_query = suite_query.filter_by(untouchable = False)
         deb_id = get_override_type('deb', session).overridetype_id
         udeb_id = get_override_type('udeb', session).overridetype_id
-        main_id = get_component('main', session).component_id
-        contrib_id = get_component('contrib', session).component_id
-        non_free_id = get_component('non-free', session).component_id
         pool = Pool()
         for suite in suite_query:
             suite_id = suite.suite_id
-            # handle source packages
-            pool.apply_async(source_helper, (suite_id, main_id),
-                callback = class_.log_result)
-            pool.apply_async(source_helper, (suite_id, contrib_id),
-                callback = class_.log_result)
-            pool.apply_async(source_helper, (suite_id, non_free_id),
-                callback = class_.log_result)
-            for architecture in suite.get_architectures(skipsrc = True, skipall = True):
-                arch_id = architecture.arch_id
-                # handle 'deb' packages
-                pool.apply_async(binary_helper, (suite_id, arch_id, deb_id), \
-                    callback = class_.log_result)
-                # handle 'udeb' packages for 'main' and 'non-free'
-                pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, main_id), \
-                    callback = class_.log_result)
-                pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, non_free_id), \
+            for component in component_query:
+                component_id = component.component_id
+                # handle source packages
+                pool.apply_async(source_helper, (suite_id, component_id),
                     callback = class_.log_result)
+                for architecture in suite.get_architectures(skipsrc = True, skipall = True):
+                    arch_id = architecture.arch_id
+                    # handle 'deb' packages
+                    pool.apply_async(binary_helper, (suite_id, arch_id, deb_id, component_id), \
+                        callback = class_.log_result)
+                    # handle 'udeb' packages
+                    pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, component_id), \
+                        callback = class_.log_result)
         pool.close()
         pool.join()
         session.close()

diff --git a/daklib/filewriter.py b/daklib/filewriter.py
index 714531afe4b1a4163f120b81743dd29f91cfa097..a3c16ea828c9f7b10b6881fec336fa9b294542a1 100755 (executable)
--- a/daklib/filewriter.py
+++ b/daklib/filewriter.py
@@ -93,10 +93,10 @@ class BinaryContentsFileWriter(BaseFileWriter):
             'bzip2':        False
         }
         flags.update(keywords)
-        if 'component' in flags:
+        if flags['debtype'] == 'deb':
             template = "dists/%(suite)s/%(component)s/Contents-%(architecture)s"
-        else:
-            template = "dists/%(suite)s/Contents-%(architecture)s"
+        else: # udeb
+            template = "dists/%(suite)s/%(component)s/Contents-udeb-%(architecture)s"
         BaseFileWriter.__init__(self, template, **flags)
 
 class SourceContentsFileWriter(BaseFileWriter):