From: Mike O'Connor <stew@vireo.org>
Date: Sat, 21 Feb 2009 14:29:47 +0000 (-0500)
Subject: Merge branch 'master' into content_generation, make changes based on Joerg's review
X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=commitdiff_plain;h=fdf3c42445b4f11f4cd71634dd2b57cb7d7a4f36

Merge branch 'master' into content_generation, make changes based on Joerg's review

Conflicts:

	dak/dakdb/update4.py
	dak/init_db.py
	dak/process_accepted.py
	dak/update_db.py
	daklib/database.py

Signed-off-by: Mike O'Connor <stew@vireo.org>
---

fdf3c42445b4f11f4cd71634dd2b57cb7d7a4f36
diff --cc dak/contents.py
index 519ebba8,00000000..dbd94f92
mode 100644,000000..100644
--- a/dak/contents.py
+++ b/dak/contents.py
@@@ -1,367 -1,0 +1,399 @@@
 +#!/usr/bin/env python
 +"""
 +Create all the contents files
 +
 +@contact: Debian FTPMaster <ftpmaster@debian.org>
 +@copyright: 2008, 2009 Michael Casadevall <mcasadevall@debian.org>
 +@copyright: 2009 Mike O'Connor <stew@debian.org>
 +@license: GNU General Public License version 2 or later
 +"""
 +
 +################################################################################
 +
 +# This program is free software; you can redistribute it and/or modify
 +# it under the terms of the GNU General Public License as published by
 +# the Free Software Foundation; either version 2 of the License, or
 +# (at your option) any later version.
 +
 +# This program is distributed in the hope that it will be useful,
 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 +# GNU General Public License for more details.
 +
 +# You should have received a copy of the GNU General Public License
 +# along with this program; if not, write to the Free Software
 +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 +
 +################################################################################
 +
 +# <Ganneff> there is the idea to slowly replace contents files
 +# <Ganneff> with a new generation of such files.
 +# <Ganneff> having more info.
 +
 +# <Ganneff> of course that wont help for now where we need to generate them :)
 +
 +################################################################################
 +
 +import sys
 +import os
 +import tempfile
 +import logging
 +import math
 +import gzip
 +import apt_pkg
 +from daklib import utils
 +from daklib.config import Config
 +from daklib.dbconn import DBConn
 +################################################################################
 +
 +def usage (exit_code=0):
 +    print """Usage: dak contents [options] command [arguments]
 +
 +COMMANDS
 +    generate
 +        generate Contents-$arch.gz files
 +
 +    bootstrap
 +        scan the debs in the existing pool and load contents in the the database
 +
 +    cruft
 +        remove files/paths which are no longer referenced by a binary
 +
 +OPTIONS
 +     -h, --help
 +        show this help and exit
 +
 +     -v, --verbose
 +        show verbose information messages
 +
 +     -q, --quiet
 +        supress all output but errors
 +
 +     -s, --suite={stable,testing,unstable,...}
 +        only operate on a signle suite
 +
 +     -a, --arch={i386,amd64}
 +        only operate on a signle architecture
 +"""
 +    sys.exit(exit_code)
 +
 +################################################################################
 +
 +# where in dak.conf all of our configuration will be stowed
 +
 +options_prefix = "Contents"
 +options_prefix = "%s::Opitons" % options_prefix
 +
 +log = logging.getLogger()
 +
 +################################################################################
 +
++# we unfortunately still have broken stuff in headers
 +latin1_q = """SET CLIENT_ENCODING TO 'LATIN1'"""
 +
++# get all the arches delivered for a given suite
++# this should probably exist somehere common
 +arches_q = """PREPARE arches_q as
 +              SELECT s.architecture, a.arch_string
 +              FROM suite_architectures s
 +              JOIN architecture a ON (s.architecture=a.id)
 +                  WHERE suite = $1"""
 +
++# find me the .deb for a given binary id
 +debs_q = """PREPARE debs_q as
 +              SELECT b.id, f.filename FROM bin_assoc_by_arch baa
 +              JOIN binaries b ON baa.bin=b.id
 +              JOIN files f ON b.file=f.id
 +              WHERE suite = $1
 +                  AND arch = $2"""
 +
++# ask if we already have contents associated with this binary
 +olddeb_q = """PREPARE olddeb_q as
 +              SELECT 1 FROM content_associations
 +              WHERE binary_pkg = $1
 +              LIMIT 1"""
 +
++# find me all of the contents for a given .deb
 +contents_q = """PREPARE contents_q as
 +              SELECT (p.path||'/'||n.file) AS fn,
 +                      comma_separated_list(s.section||'/'||b.package)
 +              FROM content_associations c
 +              JOIN content_file_paths p ON (c.filepath=p.id)
 +              JOIN content_file_names n ON (c.filename=n.id)
 +              JOIN binaries b ON (b.id=c.binary_pkg)
 +              JOIN bin_associations ba ON (b.id=ba.bin)
 +              JOIN override o ON (o.package=b.package)
 +              JOIN section s ON (s.id=o.section)
 +              WHERE (b.architecture = $1 OR b.architecture = $2)
 +                  AND ba.suite = $3
 +                  AND o.suite = $4
 +                  AND b.type = 'deb'
 +                  AND o.type = '7'
 +              GROUP BY fn
 +              ORDER BY fn"""
 +
++# find me all of the contents for a given .udeb
 +udeb_contents_q = """PREPARE udeb_contents_q as
 +              SELECT (p.path||'/'||n.file) as fn,
 +                      comma_separated_list(s.section||'/'||b.package)
 +              FROM content_associations c
 +              JOIN content_file_paths p ON (c.filepath=p.id)
 +              JOIN content_file_names n ON (c.filename=n.id)
 +              JOIN binaries b ON (b.id=c.binary_pkg)
 +              JOIN bin_associations ba ON (b.id=ba.bin)
 +              JOIN override o ON (o.package=b.package)
 +              JOIN section s ON (s.id=o.section)
 +              WHERE s.id = $1
 +                  AND ba.suite = $2
 +                  AND o.suite = $3
 +                  AND b.type = 'udeb'
 +                  AND o.type = '8'
 +              GROUP BY fn
 +              ORDER BY fn"""
 +
++# clear out all of the temporarily stored content associations
++# this should be run only after p-a has run.  after a p-a
++# run we should have either accepted or rejected every package
++# so there should no longer be anything in the queue
++remove_temp_contents_cruft_q = """DELETE FROM temp_content_associations"""
++
++# delete any filenames we are storing which have no binary associated with them
++remove_filename_cruft_q = """DELETE FROM content_file_names
++                             WHERE id IN (SELECT cfn.id FROM content_file_names cfn
++                                          LEFT JOIN content_associations ca
++                                            ON ca.filename=cfn.id
++                                          WHERE ca.id IS NULL)""" );
++
++# delete any paths we are storing which have no binary associated with them
++remove_filepath_cruft_q = """DELETE FROM content_file_paths
++                             WHERE id IN (SELECT cfn.id FROM content_file_paths cfn
++                                          LEFT JOIN content_associations ca
++                                             ON ca.filepath=cfn.id
++                                          WHERE ca.id IS NULL)"""
 +class Contents(object):
 +    """
 +    Class capable of generating Contents-$arch.gz files
 +
 +    Usage GenerateContents().generateContents( ["main","contrib","non-free"] )
 +    """
 +
 +    def __init__(self):
 +        self.header = None
 +
 +    def _getHeader(self):
-         # Internal method to return the header for Contents.gz files
++        """
++        Internal method to return the header for Contents.gz files
++
++        This is boilerplate which explains the contents of the file and how
++        it can be used.
++        """
 +        if self.header == None:
 +            if Config().has_key("Contents::Header"):
 +                try:
 +                    h = open(os.path.join( Config()["Dir::Templates"],
 +                                           Config()["Contents::Header"] ), "r")
 +                    self.header = h.read()
 +                    print( "header: %s" % self.header )
 +                    h.close()
 +                except:
-                     log.error( "error openeing header file: %d\n%s" % (Config()["Contents::Header"],
-                                                                        traceback.format_exc() ))
++                    log.error( "error opening header file: %d\n%s" % (Config()["Contents::Header"],
++                                                                      traceback.format_exc() ))
 +                    self.header = False
 +            else:
 +                print( "no header" )
 +                self.header = False
 +
 +        return self.header
 +
 +    # goal column for section column
 +    _goal_column = 54
 +
 +    def _write_content_file(self, cursor, filename):
-         # Internal method for writing all the results to a given file
++        """
++        Internal method for writing all the results to a given file.
++        The cursor should have a result set generated from a query already.
++        """
 +        f = gzip.open(Config()["Dir::Root"] + filename, "w")
 +        try:
 +            header = self._getHeader()
 +
 +            if header:
 +                f.write(header)
 +
 +            while True:
 +                contents = cursor.fetchone()
 +                if not contents:
 +                    return
 +
 +                num_tabs = max(1,
 +                               int( math.ceil( (self._goal_column - len(contents[0])) / 8) ) )
 +                f.write(contents[0] + ( '\t' * num_tabs ) + contents[-1] + "\n")
 +
 +        finally:
 +            f.close()
 +
 +    def cruft(self):
 +        """
-         remove files/paths from the DB which are no longer referenced by binaries
++        remove files/paths from the DB which are no longer referenced
++        by binaries and clean the temporary table
 +        """
 +        cursor = DBConn().cursor();
 +        cursor.execute( "BEGIN WORK" )
-         cursor.execute( """DELETE FROM content_file_names
-                            WHERE id IN (SELECT cfn.id FROM content_file_names cfn
-                                         LEFT JOIN content_associations ca
-                                             ON ca.filename=cfn.id
-                                         WHERE ca.id IS NULL)""" );
-         cursor.execute( """DELETE FROM content_file_paths
-                            WHERE id IN (SELECT cfn.id FROM content_file_paths cfn
-                                         LEFT JOIN content_associations ca
-                                             ON ca.filepath=cfn.id
-                                         WHERE ca.id IS NULL)""" );
++        cursor.execute( remove_temp_contents_cruft_q )
++        cursor.execute( remove_filename_cruft_q )
++        cursor.execute( remove_filepath_cruft_q )
 +        cursor.execute( "COMMIT" )
 +
 +
 +    def bootstrap(self):
 +        """
 +        scan the existing debs in the pool to populate the contents database tables
 +        """
 +        pooldir = Config()[ 'Dir::Pool' ]
 +
 +        cursor = DBConn().cursor();
 +        cursor.execute( latin1_q )
 +        cursor.execute( debs_q )
 +        cursor.execute( olddeb_q )
 +        cursor.execute( arches_q )
 +
 +        suites = self._suites()
 +        for suite in [i.lower() for i in suites]:
 +            suite_id = DBConn().get_suite_id(suite)
 +
 +            arch_list = self._arches(cursor, suite_id)
 +            arch_all_id = DBConn().get_architecture_id("all")
 +            for arch_id in arch_list:
 +                cursor.execute( "EXECUTE debs_q(%d, %d)" % ( suite_id, arch_id[0] ) )
 +
 +                debs = cursor.fetchall()
 +                count = 0
 +                for deb in debs:
 +                    count += 1
 +                    cursor.execute( "EXECUTE olddeb_q(%d)" % (deb[0] ) )
 +                    old = cursor.fetchone()
 +                    if old:
 +                        log.debug( "already imported: %s" % deb[1] )
 +                    else:
 +                        debfile = os.path.join( pooldir, deb[1] )
 +                        if os.path.exists( debfile ):
 +                            contents = utils.generate_contents_information( debfile )
 +                            DBConn().insert_content_paths(deb[0], contents)
 +                            log.info( "imported (%d/%d): %s" % (count,len(debs),deb[1] ) )
 +                        else:
 +                            log.error( "missing .deb: %s" % deb[1] )
 +
 +    def generate(self):
 +        """
 +        Generate Contents-$arch.gz files for every aviailable arch in each given suite.
 +        """
 +        cursor = DBConn().cursor();
 +
 +        cursor.execute( arches_q )
 +        cursor.execute( contents_q )
 +        cursor.execute( udeb_contents_q )
 +
 +        suites = self._suites()
 +
 +        # Get our suites, and the architectures
 +        for suite in [i.lower() for i in suites]:
 +            suite_id = DBConn().get_suite_id(suite)
 +            arch_list = self._arches(cursor, suite_id)
 +
 +            arch_all_id = DBConn().get_architecture_id("all")
 +
 +            for arch_id in arch_list:
 +                cursor.execute( "EXECUTE contents_q(%d,%d,%d,%d)" % (arch_id[0], arch_all_id, suite_id, suite_id))
 +                self._write_content_file(cursor, "dists/%s/Contents-%s.gz" % (suite, arch_id[1]))
 +
 +            # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free)
 +            # This is HORRIBLY debian specific :-/
 +            # First off, udeb
 +            section_id = DBConn().get_section_id('debian-installer') # all udebs should be here)
 +            if section_id != -1:
 +                cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (section_id, suite_id, suite_id))
 +                self._write_content_file(cursor, "dists/%s/Contents-udeb.gz" % suite)
 +
 +            # Once more, with non-free
 +            section_id = DBConn().get_section_id('non-free/debian-installer') # all udebs should be here)
 +
 +            if section_id != -1:
 +                cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (section_id, suite_id, suite_id))
 +                self._write_content_file(cursor, "dists/%s/Contents-udeb-nf.gz" % suite)
 +
 +
 +################################################################################
 +
 +    def _suites(self):
-         # return a list of suites to operate on
++        """
++        return a list of suites to operate on
++        """
 +        if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
 +            suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
 +        else:
 +            suites = Config().SubTree("Suite").List()
 +
 +        return suites
 +
 +    def _arches(self, cursor, suite):
-         # return a list of archs to operate on
++        """
++        return a list of archs to operate on
++        """
 +        arch_list = [ ]
 +        if Config().has_key( "%s::%s" %(options_prefix,"Arch")):
 +            archs = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Arch")])
 +            for arch_name in archs:
 +                arch_list.append((DBConn().get_architecture_id(arch_name), arch_name))
 +        else:
 +            cursor.execute("EXECUTE arches_q(%d)" % (suite))
 +            while True:
 +                r = cursor.fetchone()
 +                if not r:
 +                    break
 +
 +                if r[1] != "source" and r[1] != "all":
 +                    arch_list.append((r[0], r[1]))
 +
 +        return arch_list
 +
 +################################################################################
 +
 +def main():
 +    cnf = Config()
 +
 +    arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
 +                 ('s',"suite", "%s::%s" % (options_prefix,"Suite"),"HasArg"),
 +                 ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
 +                 ('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
 +                 ('a',"arch", "%s::%s" % (options_prefix,"Arch"),"HasArg"),
 +                ]
 +
 +    commands = {'generate' : Contents.generate,
 +                'bootstrap' : Contents.bootstrap,
 +                'cruft' : Contents.cruft,
 +                }
 +
 +    level=logging.INFO
 +    if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
 +        level=logging.ERROR
 +
 +    elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
 +        level=logging.DEBUG
 +
 +
-     logging.basicConfig( level=logging.DEBUG,
++    logging.basicConfig( level=level,
 +                         format='%(asctime)s %(levelname)s %(message)s',
 +                         stream = sys.stderr )
 +
 +    args = apt_pkg.ParseCommandLine(cnf.Cnf, arguments,sys.argv)
 +
 +    if (len(args) < 1) or not commands.has_key(args[0]):
 +        usage()
 +
 +    if cnf.has_key("%s::%s" % (options_prefix,"Help")):
 +        usage()
 +
 +    commands[args[0]](Contents())
 +
 +if __name__ == '__main__':
 +    main()
diff --cc dak/dak.py
index 15c3e1ad,981a31a9..14cddd56
--- a/dak/dak.py
+++ b/dak/dak.py
@@@ -33,8 -33,8 +33,10 @@@ G{importgraph
  
  ################################################################################
  
--import sys, imp
--import daklib.utils, daklib.extensions
++import sys
++import imp
++import daklib.utils
++import daklib.extensions
  
  ################################################################################
  
@@@ -112,8 -112,6 +114,8 @@@ def init()
           "Generate package <-> file mapping"),
          ("generate-releases",
           "Generate Release files"),
 +        ("contents",
-          "Generate contest files"),
++         "Generate content files"),
          ("generate-index-diffs",
           "Generate .diff/Index files"),
          ("clean-suites",
diff --cc dak/dakdb/update4.py
index f707a311,8c55d09b..1a9d9c3a
mode 100644,100755..100644
--- a/dak/dakdb/update4.py
+++ b/dak/dakdb/update4.py
@@@ -1,12 -1,12 +1,10 @@@
  #!/usr/bin/env python
- # coding=utf8
--
  """
- Debian Archive Kit Database Update Script
- Copyright © 2008  Michael Casadevall <mcasadevall@debian.org>
- Copyright © 2008  Roger Leigh <rleigh@debian.org>
+ Database Update Script - Get suite_architectures table use sane values
  
- Debian Archive Kit Database Update Script 2
+ @contact: Debian FTP Master <ftpmaster@debian.org>
+ @copyright: 2009  Joerg Jaspert <joerg@debian.org>
+ @license: GNU General Public License version 2 or later
 -
  """
  
  # This program is free software; you can redistribute it and/or modify
diff --cc dak/init_db.py
index bbf682af,e1f1bced..af0a03b2
--- a/dak/init_db.py
+++ b/dak/init_db.py
@@@ -49,172 -52,138 +49,167 @@@ def sql_get (config, key)
  
  ################################################################################
  
 -def do_archive():
 -    """Initalize the archive table."""
 -
 -    projectB.query("BEGIN WORK")
 -    projectB.query("DELETE FROM archive")
 -    for name in Cnf.SubTree("Archive").List():
 -        archive_config = Cnf.SubTree("Archive::%s" % (name))
 -        origin_server = sql_get(archive_config, "OriginServer")
 -        description = sql_get(archive_config, "Description")
 -        projectB.query("INSERT INTO archive (name, origin_server, description) "
 -                       "VALUES ('%s', %s, %s)"
 -                       % (name, origin_server, description))
 -    projectB.query("COMMIT WORK")
 -
 -def do_architecture():
 -    """Initalize the architecture table."""
 -
 -    projectB.query("BEGIN WORK")
 -    projectB.query("DELETE FROM architecture")
 -    for arch in Cnf.SubTree("Architectures").List():
 -        description = Cnf["Architectures::%s" % (arch)]
 -        projectB.query("INSERT INTO architecture (arch_string, description) "
 -                       "VALUES ('%s', '%s')" % (arch, description))
 -    projectB.query("COMMIT WORK")
 -
 -def do_component():
 -    """Initalize the component table."""
 -
 -    projectB.query("BEGIN WORK")
 -    projectB.query("DELETE FROM component")
 -    for name in Cnf.SubTree("Component").List():
 -        component_config = Cnf.SubTree("Component::%s" % (name))
 -        description = sql_get(component_config, "Description")
 -        if component_config.get("MeetsDFSG").lower() == "true":
 -            meets_dfsg = "true"
 -        else:
 -            meets_dfsg = "false"
 -        projectB.query("INSERT INTO component (name, description, meets_dfsg) "
 -                       "VALUES ('%s', %s, %s)"
 -                       % (name, description, meets_dfsg))
 -    projectB.query("COMMIT WORK")
 -
 -def do_location():
 -    """Initalize the location table."""
 -
 -    projectB.query("BEGIN WORK")
 -    projectB.query("DELETE FROM location")
 -    for location in Cnf.SubTree("Location").List():
 -        location_config = Cnf.SubTree("Location::%s" % (location))
 -        archive_id = database.get_archive_id(location_config["Archive"])
 -        if archive_id == -1:
 -            utils.fubar("Archive '%s' for location '%s' not found."
 -                               % (location_config["Archive"], location))
 -        location_type = location_config.get("type")
 -        if location_type == "pool":
 -            for component in Cnf.SubTree("Component").List():
 -                component_id = database.get_component_id(component)
 -                projectB.query("INSERT INTO location (path, component, "
 -                               "archive, type) VALUES ('%s', %d, %d, '%s')"
 -                               % (location, component_id, archive_id,
 -                                  location_type))
 -        else:
 -            utils.fubar("E: type '%s' not recognised in location %s."
 -                               % (location_type, location))
 -    projectB.query("COMMIT WORK")
 -
 -def do_suite():
 -    """Initalize the suite table."""
 -
 -    projectB.query("BEGIN WORK")
 -    projectB.query("DELETE FROM suite")
 -    for suite in Cnf.SubTree("Suite").List():
 -        suite_config = Cnf.SubTree("Suite::%s" %(suite))
 -        version = sql_get(suite_config, "Version")
 -        origin = sql_get(suite_config, "Origin")
 -        description = sql_get(suite_config, "Description")
 -        projectB.query("INSERT INTO suite (suite_name, version, origin, "
 -                       "description) VALUES ('%s', %s, %s, %s)"
 -                       % (suite.lower(), version, origin, description))
 -        for architecture in database.get_suite_architectures(suite):
 -            architecture_id = database.get_architecture_id (architecture)
 -            if architecture_id < 0:
 -                utils.fubar("architecture '%s' not found in architecture"
 -                                   " table for suite %s."
 -                                   % (architecture, suite))
 -            projectB.query("INSERT INTO suite_architectures (suite, "
 -                           "architecture) VALUES (currval('suite_id_seq'), %d)"
 -                           % (architecture_id))
 -    projectB.query("COMMIT WORK")
 -
 -def do_override_type():
 -    """Initalize the override_type table."""
 -
 -    projectB.query("BEGIN WORK")
 -    projectB.query("DELETE FROM override_type")
 -    for override_type in Cnf.ValueList("OverrideType"):
 -        projectB.query("INSERT INTO override_type (type) VALUES ('%s')"
 -                       % (override_type))
 -    projectB.query("COMMIT WORK")
 -
 -def do_priority():
 -    """Initialize the priority table."""
 -
 -    projectB.query("BEGIN WORK")
 -    projectB.query("DELETE FROM priority")
 -    for priority in Cnf.SubTree("Priority").List():
 -        projectB.query("INSERT INTO priority (priority, level) VALUES "
 -                       "('%s', %s)"
 -                       % (priority, Cnf["Priority::%s" % (priority)]))
 -    projectB.query("COMMIT WORK")
 -
 -def do_section():
 -    """Initalize the section table."""
 -    projectB.query("BEGIN WORK")
 -    projectB.query("DELETE FROM section")
 -    for component in Cnf.SubTree("Component").List():
 -        if Cnf["Control-Overrides::ComponentPosition"] == "prefix":
 -            suffix = ""
 -            if component != "main":
 -                prefix = component + '/'
 -            else:
 -                prefix = ""
 -        else:
 -            prefix = ""
 -            if component != "main":
 -                suffix = '/' + component
 +class InitDB(object):
 +    def __init__(self, Cnf, projectB):
 +        self.Cnf = Cnf
 +        self.projectB = projectB
 +
 +    def do_archive(self):
-         """Initalize the archive table."""
++        """initalize the archive table."""
 +
 +        c = self.projectB.cursor()
 +        c.execute("DELETE FROM archive")
 +        archive_add = "INSERT INTO archive (name, origin_server, description) VALUES (%s, %s, %s)"
 +        for name in self.Cnf.SubTree("Archive").List():
 +            archive_config = self.Cnf.SubTree("Archive::%s" % (name))
 +            origin_server = sql_get(archive_config, "OriginServer")
 +            description = sql_get(archive_config, "Description")
 +            c.execute(archive_add, [name, origin_server, description])
 +        self.projectB.commit()
 +
 +    def do_architecture(self):
 +        """Initalize the architecture table."""
 +
 +        c = self.projectB.cursor()
 +        c.execute("DELETE FROM architecture")
 +        arch_add = "INSERT INTO architecture (arch_string, description) VALUES (%s, %s)"
 +        for arch in self.Cnf.SubTree("Architectures").List():
 +            description = self.Cnf["Architectures::%s" % (arch)]
 +            c.execute(arch_add, [arch, description])
 +        self.projectB.commit()
 +
 +    def do_component(self):
 +        """Initalize the component table."""
 +
 +        c = self.projectB.cursor()
 +        c.execute("DELETE FROM component")
 +
 +        comp_add = "INSERT INTO component (name, description, meets_dfsg) " + \
 +                   "VALUES (%s, %s, %s)"
 +
 +        for name in self.Cnf.SubTree("Component").List():
 +            component_config = self.Cnf.SubTree("Component::%s" % (name))
 +            description = sql_get(component_config, "Description")
 +            meets_dfsg = (component_config.get("MeetsDFSG").lower() == "true")
 +            c.execute(comp_add, [name, description, meets_dfsg])
 +
 +        self.projectB.commit()
 +
 +    def do_location(self):
 +        """Initalize the location table."""
 +
 +        c = self.projectB.cursor()
 +        c.execute("DELETE FROM location")
 +
-         loc_add_mixed = "INSERT INTO location (path, archive, type) " + \
-                         "VALUES (%s, %s, %s)"
- 
 +        loc_add = "INSERT INTO location (path, component, archive, type) " + \
 +                  "VALUES (%s, %s, %s, %s)"
 +
 +        for location in self.Cnf.SubTree("Location").List():
 +            location_config = self.Cnf.SubTree("Location::%s" % (location))
 +            archive_id = self.projectB.get_archive_id(location_config["Archive"])
 +            if archive_id == -1:
 +                utils.fubar("Archive '%s' for location '%s' not found."
 +                                   % (location_config["Archive"], location))
 +            location_type = location_config.get("type")
-             if location_type == "legacy-mixed":
-                 c.execute(loc_add_mixed, [location, archive_id, location_config["type"]])
-             elif location_type == "legacy" or location_type == "pool":
++            if location_type == "pool":
 +                for component in self.Cnf.SubTree("Component").List():
 +                    component_id = self.projectB.get_component_id(component)
 +                    c.execute(loc_add, [location, component_id, archive_id, location_type])
              else:
 +                utils.fubar("E: type '%s' not recognised in location %s."
 +                                   % (location_type, location))
 +
 +        self.projectB.commit()
 +
 +    def do_suite(self):
 +        """Initalize the suite table."""
 +
 +        c = self.projectB.cursor()
 +        c.execute("DELETE FROM suite")
 +
 +        suite_add = "INSERT INTO suite (suite_name, version, origin, description) " + \
 +                    "VALUES (%s, %s, %s, %s)"
 +
 +        sa_add = "INSERT INTO suite_architectures (suite, architecture) " + \
 +                 "VALUES (currval('suite_id_seq'), %s)"
 +
 +        for suite in self.Cnf.SubTree("Suite").List():
 +            suite_config = self.Cnf.SubTree("Suite::%s" %(suite))
 +            version = sql_get(suite_config, "Version")
 +            origin = sql_get(suite_config, "Origin")
 +            description = sql_get(suite_config, "Description")
 +            c.execute(suite_add, [suite.lower(), version, origin, description])
 +            for architecture in self.Cnf.ValueList("Suite::%s::Architectures" % (suite)):
 +                architecture_id = self.projectB.get_architecture_id (architecture)
 +                if architecture_id < 0:
 +                    utils.fubar("architecture '%s' not found in architecture"
-                                        " table for suite %s."
-                                    % (architecture, suite))
++                                " table for suite %s."
++                                % (architecture, suite))
 +                c.execute(sa_add, [architecture_id])
 +
 +        self.projectB.commit()
 +
 +    def do_override_type(self):
 +        """Initalize the override_type table."""
 +
 +        c = self.projectB.cursor()
 +        c.execute("DELETE FROM override_type")
 +
 +        over_add = "INSERT INTO override_type (type) VALUES (%s)"
 +
 +        for override_type in self.Cnf.ValueList("OverrideType"):
 +            c.execute(over_add, [override_type])
 +
 +        self.projectB.commit()
 +
 +    def do_priority(self):
 +        """Initialize the priority table."""
 +
 +        c = self.projectB.cursor()
 +        c.execute("DELETE FROM priority")
 +
 +        prio_add = "INSERT INTO priority (priority, level) VALUES (%s, %s)"
 +
 +        for priority in self.Cnf.SubTree("Priority").List():
 +            c.execute(prio_add, [priority, self.Cnf["Priority::%s" % (priority)]])
 +
 +        self.projectB.commit()
 +
 +    def do_section(self):
 +        """Initalize the section table."""
 +
 +        c = self.projectB.cursor()
 +        c.execute("DELETE FROM section")
 +
 +        sect_add = "INSERT INTO section (section) VALUES (%s)"
 +
 +        for component in self.Cnf.SubTree("Component").List():
 +            if self.Cnf["Control-Overrides::ComponentPosition"] == "prefix":
                  suffix = ""
 -        for section in Cnf.ValueList("Section"):
 -            projectB.query("INSERT INTO section (section) VALUES "
 -                           "('%s%s%s')" % (prefix, section, suffix))
 -    projectB.query("COMMIT WORK")
 +                if component != "main":
 +                    prefix = component + '/'
 +                else:
 +                    prefix = ""
 +            else:
 +                prefix = ""
 +                if component != "main":
 +                    suffix = '/' + component
 +                else:
 +                    suffix = ""
 +            for section in self.Cnf.ValueList("Section"):
 +                c.execute(sect_add, [prefix + section + suffix])
 +
 +        self.projectB.commit()
 +
 +    def do_all(self):
 +        self.do_archive()
 +        self.do_architecture()
 +        self.do_component()
 +        self.do_location()
 +        self.do_suite()
 +        self.do_override_type()
 +        self.do_priority()
 +        self.do_section()
  
  ################################################################################
  
diff --cc dak/process_accepted.py
index e597a8e2,dce0cdbe..cc3f7eab
--- a/dak/process_accepted.py
+++ b/dak/process_accepted.py
@@@ -29,8 -29,8 +29,13 @@@
  
  ###############################################################################
  
--import errno, fcntl, os, sys, time, re
 -import apt_pkg
++import errno
++import fcntl
++import os
++import sys
++import time
++import re
 +import apt_pkg, commands
  from daklib import database
  from daklib import logging
  from daklib import queue
@@@ -374,33 -372,7 +379,10 @@@ def install ()
                  suite_id = database.get_suite_id(suite)
                  projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id))
  
- 
 +            if not database.copy_temporary_contents(package, version, files[newfile]):
 +                reject("Missing contents for package")
 +
-     # If the .orig.tar.gz is in a legacy directory we need to poolify
-     # it, so that apt-get source (and anything else that goes by the
-     # "Directory:" field in the Sources.gz file) works.
-     orig_tar_id = Upload.pkg.orig_tar_id
-     orig_tar_location = Upload.pkg.orig_tar_location
-     legacy_source_untouchable = Upload.pkg.legacy_source_untouchable
-     if orig_tar_id and orig_tar_location == "legacy":
-         q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id))
-         qd = q.dictresult()
-         for qid in qd:
-             # Is this an old upload superseded by a newer -sa upload?  (See check_dsc() for details)
-             if legacy_source_untouchable.has_key(qid["files_id"]):
-                 continue
-             # First move the files to the new location
-             legacy_filename = qid["path"] + qid["filename"]
-             pool_location = utils.poolify (changes["source"], files[newfile]["component"])
-             pool_filename = pool_location + os.path.basename(qid["filename"])
-             destination = Cnf["Dir::Pool"] + pool_location
-             utils.move(legacy_filename, destination)
-             # Then Update the DB's files table
-             q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"]))
- 
-     # If this is a sourceful diff only upload that is moving non-legacy
+     # If this is a sourceful diff only upload that is moving
      # cross-component we need to copy the .orig.tar.gz into the new
      # component too for the same reasons as above.
      #
diff --cc dak/process_unchecked.py
index 5097b24a,2e60435d..07e4592b
--- a/dak/process_unchecked.py
+++ b/dak/process_unchecked.py
@@@ -28,11 -28,9 +28,22 @@@
  
  ################################################################################
  
- import commands, errno, fcntl, os, re, shutil, stat, sys, time, tempfile, traceback, tarfile
 -import commands, errno, fcntl, os, re, shutil, stat, sys, time, tempfile, traceback
++import commands
++import errno
++import fcntl
++import os
++import re
++import shutil
++import stat
++import sys
++import time
++import tempfile
++import traceback
++import tarfile
  import apt_inst, apt_pkg
 -from daklib import database
 +from debian_bundle import deb822
 +from daklib.dbconn import DBConn
 +from daklib.binary import Binary
  from daklib import logging
  from daklib import queue
  from daklib import utils
diff --cc dak/rm.py
index 005653e5,903e138e..6844738f
--- a/dak/rm.py
+++ b/dak/rm.py
@@@ -39,8 -39,8 +39,13 @@@
  
  ################################################################################
  
--import commands, os, pg, re, sys
--import apt_pkg, apt_inst
++import commands
++import os
++import pg
++import re
++import sys
++import apt_pkg
++import apt_inst
  from daklib import database
  from daklib import utils
  from daklib.dak_exceptions import *
diff --cc daklib/database.py
index 5818733e,3fbd2a50..94e0a7cc
--- a/daklib/database.py
+++ b/daklib/database.py
@@@ -47,6 -48,6 +48,7 @@@ location_id_cache = {}        #: cache 
  maintainer_id_cache = {}      #: cache for maintainers
  keyring_id_cache = {}         #: cache for keyrings
  source_id_cache = {}          #: cache for sources
++
  files_id_cache = {}           #: cache for files
  maintainer_cache = {}         #: cache for maintainer names
  fingerprint_id_cache = {}     #: cache for fingerprints
@@@ -412,50 -410,33 +414,76 @@@ def get_suite_version(source, suite)
  
      return version
  
 +def get_latest_binary_version_id(binary, section, suite, arch):
 +    global suite_bin_version_cache
 +    cache_key = "%s_%s_%s_%s" % (binary, section, suite, arch)
 +    cache_key_all = "%s_%s_%s_%s" % (binary, section, suite, get_architecture_id("all"))
 +
 +    # Check for the cache hit for its arch, then arch all
 +    if suite_bin_version_cache.has_key(cache_key):
 +        return suite_bin_version_cache[cache_key]
 +    if suite_bin_version_cache.has_key(cache_key_all):
 +        return suite_bin_version_cache[cache_key_all]
 +    if cache_preloaded == True:
 +        return # package does not exist
 +
 +    q = projectB.query("SELECT DISTINCT b.id FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN override o ON (o.package=b.package) WHERE b.package = '%s' AND b.architecture = '%d' AND ba.suite = '%d' AND o.section = '%d'" % (binary, int(arch), int(suite), int(section)))
 +
 +    if not q.getresult():
 +        return False
 +
 +    highest_bid = q.getresult()[0][0]
 +
 +    suite_bin_version_cache[cache_key] = highest_bid
 +    return highest_bid
 +
 +def preload_binary_id_cache():
 +    global suite_bin_version_cache, cache_preloaded
 +
 +    # Get suite info
 +    q = projectB.query("SELECT id FROM suite")
 +    suites = q.getresult()
 +
 +    # Get arch mappings
 +    q = projectB.query("SELECT id FROM architecture")
 +    arches = q.getresult()
 +
 +    for suite in suites:
 +        for arch in arches:
 +            q = projectB.query("SELECT DISTINCT b.id, b.package, o.section FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN override o ON (o.package=b.package) WHERE b.architecture = '%d' AND ba.suite = '%d'" % (int(arch[0]), int(suite[0])))
 +
 +            for bi in q.getresult():
 +                cache_key = "%s_%s_%s_%s" % (bi[1], bi[2], suite[0], arch[0])
 +                suite_bin_version_cache[cache_key] = int(bi[0])
 +
 +    cache_preloaded = True
 +
+ def get_suite_architectures(suite):
+     """
+     Returns list of architectures for C{suite}.
+ 
+     @type suite: string, int
+     @param suite: the suite name or the suite_id
+ 
+     @rtype: list
+     @return: the list of architectures for I{suite}
+     """
+ 
+     suite_id = None
+     if type(suite) == str:
+         suite_id = get_suite_id(suite)
+     elif type(suite) == int:
+         suite_id = suite
+     else:
+         return None
+ 
+     sql = """ SELECT a.arch_string FROM suite_architectures sa
+               JOIN architecture a ON (a.id = sa.architecture)
+               WHERE suite='%s' """ % (suite_id)
+ 
+     q = projectB.query(sql)
+     return map(lambda x: x[0], q.getresult())
+ 
 -
  ################################################################################
  
  def get_or_set_maintainer_id (maintainer):
@@@ -777,46 -758,3 +805,47 @@@ def get_suites(pkgname, src=False)
  
      q = projectB.query(sql)
      return map(lambda x: x[0], q.getresult())
 +
 +
 +################################################################################
 +
 +def copy_temporary_contents(package, version, deb):
 +    """
 +    copy the previously stored contents from the temp table to the permanant one
 +
 +    during process-unchecked, the deb should have been scanned and the
 +    contents stored in temp_content_associations
 +    """
 +
 +    # first see if contents exist:
 +
 +    exists = projectB.query("""SELECT 1 FROM temp_content_associations
 +                               WHERE package='%s' LIMIT 1""" % package ).getresult()
 +
 +    if not exists:
 +        # This should NOT happen.  We should have added contents
 +        # during process-unchecked.  if it did, log an error, and send
 +        # an email.
 +        subst = {
 +            "__PACKAGE__": package,
 +            "__VERSION__": version,
++            "__TO_ADDRESS__": Cnf["Dinstall::MyAdminAddress"]
 +            "__DAK_ADDRESS__": Cnf["Dinstall::MyEmailAddress"]
 +            }
 +
-         message = utils.TemplateSubst(Subst, Cnf["Dir::Templates"]+"/bts-categorize")
++        message = utils.TemplateSubst(Subst, Cnf["Dir::Templates"]+"/missing-contents")
 +        utils.send_mail( message )
 +
 +        exists = DBConn().insert_content_path(package, version, deb)
 +
 +    if exists:
 +        sql = """INSERT INTO content_associations(binary_pkg,filepath,filename)
 +                 SELECT currval('binaries_id_seq'), filepath, filename FROM temp_content_associations
 +                 WHERE package='%s'
 +                     AND version='%s'""" % (package, version)
 +        projectB.query(sql)
 +        projectB.query("""DELETE from temp_content_associations
 +                          WHERE package='%s'
 +                            AND version='%s'""" % (package, version))
 +
 +    return exists
diff --cc templates/missing-contents
index a4477fa7,00000000..90f16dc5
mode 100644,000000..100644
--- a/templates/missing-contents
+++ b/templates/missing-contents
@@@ -1,14 -1,0 +1,15 @@@
 +From: __DAK_ADDRESS__
++To: __TO_ADDRESS__
 +X-Debian: DAK
 +X-Debian-Package: __PACKAGE__
 +MIME-Version: 1.0
 +Content-Type: text/plain; charset="utf-8"
 +Content-Transfer-Encoding: 8bit
 +Subject: Missing contents for __PACKAGE__ in accepted queue
 +
 +While processing the accepted queue, I didn't have contents in the
 +database for __PACKAGE__ version __VERSION__.  These contents should
 +have been put into the database by process-unchecked when the package
 +first arrived.
 +
 +This is probably stew's fault.