Add option to forbid source and/or binary uploads to a suite

[dak.git] / daklib / archive.py

diff --git a/daklib/archive.py b/daklib/archive.py
index 914bc0f33c0c43851ad0d9734d10a492eec9342c..6577d3591f479e960a0ae23b213e1f5c75ccfef3 100644 (file)
--- a/daklib/archive.py
+++ b/daklib/archive.py
@@ -19,20 +19,22 @@
 This module provides classes to manipulate the archive.
 """
 
 This module provides classes to manipulate the archive.
 """
 

-from .dbconn import *
+from daklib.dbconn import *

 import daklib.checks as checks
 from daklib.config import Config
 import daklib.upload as upload
 import daklib.utils as utils
 import daklib.checks as checks
 from daklib.config import Config
 import daklib.upload as upload
 import daklib.utils as utils

-from .fstransactions import FilesystemTransaction
-from .regexes import re_changelog_versions, re_bin_only_nmu
+from daklib.fstransactions import FilesystemTransaction
+from daklib.regexes import re_changelog_versions, re_bin_only_nmu
+import daklib.daksubprocess

 
 import apt_pkg
 from datetime import datetime
 import os
 import shutil
 
 import apt_pkg
 from datetime import datetime
 import os
 import shutil

-import subprocess

 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.orm.exc import NoResultFound

+from sqlalchemy.orm import object_session
+import sqlalchemy.exc

 import tempfile
 import traceback
 
 import tempfile
 import traceback
 


@@ -49,23 +51,31 @@ class ArchiveTransaction(object):
         self.fs = FilesystemTransaction()
         self.session = DBConn().session()
 
         self.fs = FilesystemTransaction()
         self.session = DBConn().session()
 

-    def get_file(self, hashed_file, source_name):
-        """Look for file `hashed_file` in database
+    def get_file(self, hashed_file, source_name, check_hashes=True):
+        """Look for file C{hashed_file} in database

 
 

-        Args:
-           hashed_file (daklib.upload.HashedFile): file to look for in the database
+        @type  hashed_file: L{daklib.upload.HashedFile}
+        @param hashed_file: file to look for in the database

 
 

-        Raises:
-           KeyError: file was not found in the database
-           HashMismatchException: hash mismatch
+        @type  source_name: str
+        @param source_name: source package name

 
 

-        Returns:
-           `daklib.dbconn.PoolFile` object for the database
+        @type  check_hashes: bool
+        @param check_hashes: check size and hashes match
+
+        @raise KeyError: file was not found in the database
+        @raise HashMismatchException: hash mismatch
+
+        @rtype:  L{daklib.dbconn.PoolFile}
+        @return: database entry for the file

         """
         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
         try:
             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
         """
         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
         try:
             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()

-            if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
+            if check_hashes and (poolfile.filesize != hashed_file.size
+                                 or poolfile.md5sum != hashed_file.md5sum
+                                 or poolfile.sha1sum != hashed_file.sha1sum
+                                 or poolfile.sha256sum != hashed_file.sha256sum):

                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
             return poolfile
         except NoResultFound:
                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
             return poolfile
         except NoResultFound:


@@ -76,8 +86,8 @@ class ArchiveTransaction(object):
 
         Will not give an error when the file is already present.
 
 
         Will not give an error when the file is already present.
 

-        Returns:
-           `daklib.dbconn.PoolFile` object for the new file
+        @rtype:  L{daklib.dbconn.PoolFile}
+        @return: database object for the new file

         """
         session = self.session
 
         """
         session = self.session
 


@@ -100,7 +110,7 @@ class ArchiveTransaction(object):
             session.flush()
 
             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
             session.flush()
 
             path = os.path.join(archive.path, 'pool', component.component_name, poolname)

-            hashed_file_path = os.path.join(directory, hashed_file.filename)
+            hashed_file_path = os.path.join(directory, hashed_file.input_filename)

             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 
         return poolfile
             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 
         return poolfile


@@ -108,23 +118,34 @@ class ArchiveTransaction(object):
     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
         """Install a binary package
 
     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
         """Install a binary package
 

-        Args:
-           directory (str): directory the binary package is located in
-           binary (daklib.upload.Binary): binary package to install
-           suite (daklib.dbconn.Suite): target suite
-           component (daklib.dbconn.Component): target component
-
-        Kwargs:
-           allow_tainted (bool): allow to copy additional files from tainted archives
-           fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
-           source_suites (list of daklib.dbconn.Suite or True): suites to copy
-              the source from if they are not in `suite` or True to allow
-              copying from any suite.
-              This can also be a SQLAlchemy (sub)query object.
-           extra_source_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
-
-        Returns:
-           `daklib.dbconn.DBBinary` object for the new package
+        @type  directory: str
+        @param directory: directory the binary package is located in
+
+        @type  binary: L{daklib.upload.Binary}
+        @param binary: binary package to install
+
+        @type  suite: L{daklib.dbconn.Suite}
+        @param suite: target suite
+
+        @type  component: L{daklib.dbconn.Component}
+        @param component: target component
+
+        @type  allow_tainted: bool
+        @param allow_tainted: allow to copy additional files from tainted archives
+
+        @type  fingerprint: L{daklib.dbconn.Fingerprint}
+        @param fingerprint: optional fingerprint
+
+        @type  source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
+        @param source_suites: suites to copy the source from if they are not
+                              in C{suite} or C{True} to allow copying from any
+                              suite.
+
+        @type  extra_source_archives: list of L{daklib.dbconn.Archive}
+        @param extra_source_archives: extra archives to copy Built-Using sources from
+
+        @rtype:  L{daklib.dbconn.DBBinary}
+        @return: databse object for the new package

         """
         session = self.session
         control = binary.control
         """
         session = self.session
         control = binary.control


@@ -155,6 +176,10 @@ class ArchiveTransaction(object):
             maintainer=maintainer,
             poolfile=db_file,
             binarytype=binary.type,
             maintainer=maintainer,
             poolfile=db_file,
             binarytype=binary.type,

+            )
+        # Other attributes that are ignored for purposes of equality with
+        # an existing source
+        rest2 = dict(

             fingerprint=fingerprint,
             )
 
             fingerprint=fingerprint,
             )
 


@@ -167,6 +192,8 @@ class ArchiveTransaction(object):
             db_binary = DBBinary(**unique)
             for key, value in rest.iteritems():
                 setattr(db_binary, key, value)
             db_binary = DBBinary(**unique)
             for key, value in rest.iteritems():
                 setattr(db_binary, key, value)

+            for key, value in rest2.iteritems():
+                setattr(db_binary, key, value)

             session.add(db_binary)
             session.flush()
             import_metadata_into_db(db_binary, session)
             session.add(db_binary)
             session.flush()
             import_metadata_into_db(db_binary, session)


@@ -185,14 +212,18 @@ class ArchiveTransaction(object):
 
         This is intended to be used to check that Built-Using sources exist.
 
 
         This is intended to be used to check that Built-Using sources exist.
 

-        Args:
-           filename (str): filename to use in error messages
-           source (daklib.dbconn.DBSource): source to look for
-           archive (daklib.dbconn.Archive): archive to look in
+        @type  filename: str
+        @param filename: filename to use in error messages
+
+        @type  source: L{daklib.dbconn.DBSource}
+        @param source: source to look for
+
+        @type  archive: L{daklib.dbconn.Archive}
+        @param archive: archive to look in

 
 

-        Kwargs:
-           extra_archives (list of daklib.dbconn.Archive): list of archives to copy
-               the source package from if it is not yet present in `archive`
+        @type  extra_archives: list of L{daklib.dbconn.Archive}
+        @param extra_archives: list of archives to copy the source package from
+                               if it is not yet present in C{archive}

         """
         session = self.session
         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
         """
         session = self.session
         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()


@@ -213,44 +244,21 @@ class ArchiveTransaction(object):
             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 
     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 
     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):

-        """Add Built-Using sources to `db_binary.extra_sources`
+        """Add Built-Using sources to C{db_binary.extra_sources}

         """
         session = self.session
         """
         session = self.session

-        built_using = control.get('Built-Using', None)
-
-        if built_using is not None:
-            for dep in apt_pkg.parse_depends(built_using):
-                assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
-                bu_source_name, bu_source_version, comp = dep[0]
-                assert comp == '=', 'Built-Using must contain strict dependencies'
-
-                bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
-                if bu_source is None:
-                    raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))

 
 

-                self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
+        for bu_source_name, bu_source_version in daklib.utils.parse_built_using(control):
+            bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
+            if bu_source is None:
+                raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))

 
 

-                db_binary.extra_sources.append(bu_source)
+            self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)

 
 

-    def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
-        """Install a source package
-
-        Args:
-           directory (str): directory the source package is located in
-           source (daklib.upload.Source): source package to install
-           suite (daklib.dbconn.Suite): target suite
-           component (daklib.dbconn.Component): target component
-           changed_by (daklib.dbconn.Maintainer): person who prepared this version of the package
+            db_binary.extra_sources.append(bu_source)

 
 

-        Kwargs:
-           allow_tainted (bool): allow to copy additional files from tainted archives
-           fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
-
-        Returns:
-           `daklib.dbconn.DBSource` object for the new source
-        """
+    def install_source_to_archive(self, directory, source, archive, component, changed_by, allow_tainted=False, fingerprint=None):

         session = self.session
         session = self.session

-        archive = suite.archive

         control = source.dsc
         maintainer = get_or_set_maintainer(control['Maintainer'], session)
         source_name = control['Source']
         control = source.dsc
         maintainer = get_or_set_maintainer(control['Maintainer'], session)
         source_name = control['Source']


@@ -266,12 +274,15 @@ class ArchiveTransaction(object):
             )
         rest = dict(
             maintainer=maintainer,
             )
         rest = dict(
             maintainer=maintainer,

-            changedby=changed_by,
-            #install_date=datetime.now().date(),

             poolfile=db_file_dsc,
             poolfile=db_file_dsc,

-            fingerprint=fingerprint,

             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
             )
             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
             )

+        # Other attributes that are ignored for purposes of equality with
+        # an existing source
+        rest2 = dict(
+            changedby=changed_by,
+            fingerprint=fingerprint,
+            )

 
         created = False
         try:
 
         created = False
         try:


@@ -284,8 +295,8 @@ class ArchiveTransaction(object):
             db_source = DBSource(**unique)
             for key, value in rest.iteritems():
                 setattr(db_source, key, value)
             db_source = DBSource(**unique)
             for key, value in rest.iteritems():
                 setattr(db_source, key, value)

-            # XXX: set as default in postgres?
-            db_source.install_date = datetime.now().date()
+            for key, value in rest2.iteritems():
+                setattr(db_source, key, value)

             session.add(db_source)
             session.flush()
 
             session.add(db_source)
             session.flush()
 


@@ -296,18 +307,15 @@ class ArchiveTransaction(object):
             session.add(db_dsc_file)
             session.flush()
 
             session.add(db_dsc_file)
             session.flush()
 

-        if suite in db_source.suites:
-            return db_source
-
-        db_source.suites.append(suite)
-

         if not created:
         if not created:

+            for f in db_source.srcfiles:
+                self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)

             return db_source
 
         ### Now add remaining files and copy them to the archive.
 
         for hashed_file in source.files.itervalues():
             return db_source
 
         ### Now add remaining files and copy them to the archive.
 
         for hashed_file in source.files.itervalues():

-            hashed_file_path = os.path.join(directory, hashed_file.filename)
+            hashed_file_path = os.path.join(directory, hashed_file.input_filename)

             if os.path.exists(hashed_file_path):
                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
                 session.add(db_file)
             if os.path.exists(hashed_file_path):
                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
                 session.add(db_file)


@@ -328,38 +336,74 @@ class ArchiveTransaction(object):
         # Uploaders are the maintainer and co-maintainers from the Uploaders field
         db_source.uploaders.append(maintainer)
         if 'Uploaders' in control:
         # Uploaders are the maintainer and co-maintainers from the Uploaders field
         db_source.uploaders.append(maintainer)
         if 'Uploaders' in control:

-            def split_uploaders(field):
-                import re
-                for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
-                    yield u.strip()
-
+            from daklib.textutils import split_uploaders

             for u in split_uploaders(control['Uploaders']):
                 db_source.uploaders.append(get_or_set_maintainer(u, session))
         session.flush()
 
         return db_source
 
             for u in split_uploaders(control['Uploaders']):
                 db_source.uploaders.append(get_or_set_maintainer(u, session))
         session.flush()
 
         return db_source
 

+    def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
+        """Install a source package
+
+        @type  directory: str
+        @param directory: directory the source package is located in
+
+        @type  source: L{daklib.upload.Source}
+        @param source: source package to install
+
+        @type  suite: L{daklib.dbconn.Suite}
+        @param suite: target suite
+
+        @type  component: L{daklib.dbconn.Component}
+        @param component: target component
+
+        @type  changed_by: L{daklib.dbconn.Maintainer}
+        @param changed_by: person who prepared this version of the package
+
+        @type  allow_tainted: bool
+        @param allow_tainted: allow to copy additional files from tainted archives
+
+        @type  fingerprint: L{daklib.dbconn.Fingerprint}
+        @param fingerprint: optional fingerprint
+
+        @rtype:  L{daklib.dbconn.DBSource}
+        @return: database object for the new source
+        """
+        db_source = self.install_source_to_archive(directory, source, suite.archive, component, changed_by, allow_tainted, fingerprint)
+
+        if suite in db_source.suites:
+            return db_source
+        db_source.suites.append(suite)
+        self.session.flush()
+
+        return db_source
+

     def _copy_file(self, db_file, archive, component, allow_tainted=False):
         """Copy a file to the given archive and component
 
     def _copy_file(self, db_file, archive, component, allow_tainted=False):
         """Copy a file to the given archive and component
 

-        Args:
-           db_file (daklib.dbconn.PoolFile): file to copy
-           archive (daklib.dbconn.Archive): target archive
-           component (daklib.dbconn.Component): target component
+        @type  db_file: L{daklib.dbconn.PoolFile}
+        @param db_file: file to copy

 
 

-        Kwargs:
-           allow_tainted (bool): allow to copy from tainted archives (such as NEW)
+        @type  archive: L{daklib.dbconn.Archive}
+        @param archive: target archive
+
+        @type  component: L{daklib.dbconn.Archive}
+        @param component: target component
+
+        @type  allow_tainted: bool
+        @param allow_tainted: allow to copy from tainted archives (such as NEW)

         """
         session = self.session
 
         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
         """
         session = self.session
 
         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:

-            query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
+            query = session.query(ArchiveFile).filter_by(file=db_file)

             if not allow_tainted:
                 query = query.join(Archive).filter(Archive.tainted == False)
 
             source_af = query.first()
             if source_af is None:
             if not allow_tainted:
                 query = query.join(Archive).filter(Archive.tainted == False)
 
             source_af = query.first()
             if source_af is None:

-                raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
+                raise ArchiveException('cp: Could not find {0} in any archive.'.format(db_file.filename))

             target_af = ArchiveFile(archive, component, db_file)
             session.add(target_af)
             session.flush()
             target_af = ArchiveFile(archive, component, db_file)
             session.add(target_af)
             session.flush()


@@ -368,22 +412,34 @@ class ArchiveTransaction(object):
     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
         """Copy a binary package to the given suite and component
 
     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
         """Copy a binary package to the given suite and component
 

-        Args:
-           db_binary (daklib.dbconn.DBBinary): binary to copy
-           suite (daklib.dbconn.Suite): target suite
-           component (daklib.dbconn.Component): target component
+        @type  db_binary: L{daklib.dbconn.DBBinary}
+        @param db_binary: binary to copy
+
+        @type  suite: L{daklib.dbconn.Suite}
+        @param suite: target suite
+
+        @type  component: L{daklib.dbconn.Component}
+        @param component: target component
+
+        @type  allow_tainted: bool
+        @param allow_tainted: allow to copy from tainted archives (such as NEW)

 
 

-        Kwargs:
-           allow_tainted (bool): allow to copy from tainted archives (such as NEW)
-           extra_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
+        @type  extra_archives: list of L{daklib.dbconn.Archive}
+        @param extra_archives: extra archives to copy Built-Using sources from

         """
         session = self.session
         archive = suite.archive
         if archive.tainted:
             allow_tainted = True
 
         """
         session = self.session
         archive = suite.archive
         if archive.tainted:
             allow_tainted = True
 

-        # make sure built-using packages are present in target archive

         filename = db_binary.poolfile.filename
         filename = db_binary.poolfile.filename

+
+        # make sure source is present in target archive
+        db_source = db_binary.source
+        if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
+            raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
+
+        # make sure built-using packages are present in target archive

         for db_source in db_binary.extra_sources:
             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 
         for db_source in db_binary.extra_sources:
             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 


@@ -397,13 +453,17 @@ class ArchiveTransaction(object):
     def copy_source(self, db_source, suite, component, allow_tainted=False):
         """Copy a source package to the given suite and component
 
     def copy_source(self, db_source, suite, component, allow_tainted=False):
         """Copy a source package to the given suite and component
 

-        Args:
-           db_source (daklib.dbconn.DBSource): source to copy
-           suite (daklib.dbconn.Suite): target suite
-           component (daklib.dbconn.Component): target component
+        @type  db_source: L{daklib.dbconn.DBSource}
+        @param db_source: source to copy
+
+        @type  suite: L{daklib.dbconn.Suite}
+        @param suite: target suite

 
 

-        Kwargs:
-           allow_tainted (bool): allow to copy from tainted archives (such as NEW)
+        @type  component: L{daklib.dbconn.Component}
+        @param component: target component
+
+        @type  allow_tainted: bool
+        @param allow_tainted: allow to copy from tainted archives (such as NEW)

         """
         archive = suite.archive
         if archive.tainted:
         """
         archive = suite.archive
         if archive.tainted:


@@ -417,10 +477,14 @@ class ArchiveTransaction(object):
     def remove_file(self, db_file, archive, component):
         """Remove a file from a given archive and component
 
     def remove_file(self, db_file, archive, component):
         """Remove a file from a given archive and component
 

-        Args:
-           db_file (daklib.dbconn.PoolFile): file to remove
-           archive (daklib.dbconn.Archive): archive to remove the file from
-           component (daklib.dbconn.Component): component to remove the file from
+        @type  db_file: L{daklib.dbconn.PoolFile}
+        @param db_file: file to remove
+
+        @type  archive: L{daklib.dbconn.Archive}
+        @param archive: archive to remove the file from
+
+        @type  component: L{daklib.dbconn.Component}
+        @param component: component to remove the file from

         """
         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
         self.fs.unlink(af.path)
         """
         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
         self.fs.unlink(af.path)


@@ -429,9 +493,11 @@ class ArchiveTransaction(object):
     def remove_binary(self, binary, suite):
         """Remove a binary from a given suite and component
 
     def remove_binary(self, binary, suite):
         """Remove a binary from a given suite and component
 

-        Args:
-           binary (daklib.dbconn.DBBinary): binary to remove
-           suite (daklib.dbconn.Suite): suite to remove the package from
+        @type  binary: L{daklib.dbconn.DBBinary}
+        @param binary: binary to remove
+
+        @type  suite: L{daklib.dbconn.Suite}
+        @param suite: suite to remove the package from

         """
         binary.suites.remove(suite)
         self.session.flush()
         """
         binary.suites.remove(suite)
         self.session.flush()


@@ -439,13 +505,14 @@ class ArchiveTransaction(object):
     def remove_source(self, source, suite):
         """Remove a source from a given suite and component
 
     def remove_source(self, source, suite):
         """Remove a source from a given suite and component
 

-        Raises:
-           ArchiveException: source package is still referenced by other
-                             binaries in the suite
+        @type  source: L{daklib.dbconn.DBSource}
+        @param source: source to remove
+
+        @type  suite: L{daklib.dbconn.Suite}
+        @param suite: suite to remove the package from

 
 

-        Args:
-           binary (daklib.dbconn.DBSource): source to remove
-           suite (daklib.dbconn.Suite): suite to remove the package from
+        @raise ArchiveException: source package is still referenced by other
+                                 binaries in the suite

         """
         session = self.session
 
         """
         session = self.session
 


@@ -471,6 +538,9 @@ class ArchiveTransaction(object):
         self.session.rollback()
         self.fs.rollback()
 
         self.session.rollback()
         self.fs.rollback()
 

+    def flush(self):
+        self.session.flush()
+

     def __enter__(self):
         return self
 
     def __enter__(self):
         return self
 


@@ -481,55 +551,118 @@ class ArchiveTransaction(object):
             self.rollback()
         return None
 
             self.rollback()
         return None
 

+def source_component_from_package_list(package_list, suite):
+    """Get component for a source package
+
+    This function will look at the Package-List field to determine the
+    component the source package belongs to. This is the first component
+    the source package provides binaries for (first with respect to the
+    ordering of components).
+
+    It the source package has no Package-List field, None is returned.
+
+    @type  package_list: L{daklib.packagelist.PackageList}
+    @param package_list: package list of the source to get the override for
+
+    @type  suite: L{daklib.dbconn.Suite}
+    @param suite: suite to consider for binaries produced
+
+    @rtype:  L{daklib.dbconn.Component} or C{None}
+    @return: component for the given source or C{None}
+    """
+    if package_list.fallback:
+        return None
+    session = object_session(suite)
+    packages = package_list.packages_for_suite(suite)
+    components = set(p.component for p in packages)
+    query = session.query(Component).order_by(Component.ordering) \
+            .filter(Component.component_name.in_(components))
+    return query.first()
+

 class ArchiveUpload(object):
     """handle an upload
 
 class ArchiveUpload(object):
     """handle an upload
 

-    This class can be used in a with-statement:
+    This class can be used in a with-statement::

 
        with ArchiveUpload(...) as upload:
           ...
 
     Doing so will automatically run any required cleanup and also rollback the
     transaction if it was not committed.
 
        with ArchiveUpload(...) as upload:
           ...
 
     Doing so will automatically run any required cleanup and also rollback the
     transaction if it was not committed.

-
-    Attributes:
-       changes (daklib.upload.Changes): upload to process
-       directory (str): directory with temporary copy of files. set by `prepare`
-       fingerprint (daklib.dbconn.Fingerprint): fingerprint used to sign the upload
-       new (bool): upload is NEW. set by `check`
-       reject_reasons (list of str): reasons why the upload cannot be accepted
-       session: database session
-       transaction (daklib.archive.ArchiveTransaction): transaction used to handle the upload
-       warnings (list of str): warnings (NOT USED YET)

     """
     def __init__(self, directory, changes, keyrings):
         self.transaction = ArchiveTransaction()
     """
     def __init__(self, directory, changes, keyrings):
         self.transaction = ArchiveTransaction()

+        """transaction used to handle the upload
+        @type: L{daklib.archive.ArchiveTransaction}
+        """
+

         self.session = self.transaction.session
         self.session = self.transaction.session

+        """database session"""

 
         self.original_directory = directory
         self.original_changes = changes
 
         self.original_directory = directory
         self.original_changes = changes

+

         self.changes = None
         self.changes = None

+        """upload to process
+        @type: L{daklib.upload.Changes}
+        """
+

         self.directory = None
         self.directory = None

+        """directory with temporary copy of files. set by C{prepare}
+        @type: str
+        """
+

         self.keyrings = keyrings
 
         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
         self.keyrings = keyrings
 
         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()

+        """fingerprint of the key used to sign the upload
+        @type: L{daklib.dbconn.Fingerprint}
+        """

 
         self.reject_reasons = []
 
         self.reject_reasons = []

+        """reasons why the upload cannot by accepted
+        @type: list of str
+        """
+

         self.warnings = []
         self.warnings = []

+        """warnings
+        @note: Not used yet.
+        @type: list of str
+        """
+

         self.final_suites = None
         self.final_suites = None

+

         self.new = False
         self.new = False

+        """upload is NEW. set by C{check}
+        @type: bool
+        """
+
+        self._checked = False
+        """checks passes. set by C{check}
+        @type: bool
+        """

 
         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
         self._new = self._new_queue.suite
 
 
         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
         self._new = self._new_queue.suite
 

+    def warn(self, message):
+        """add a warning message
+
+        Adds a warning message that can later be seen in C{self.warnings}
+
+        @type  message: string
+        @param message: warning message
+        """
+        self.warnings.append(message)
+

     def prepare(self):
         """prepare upload for further processing
 
         This copies the files involved to a temporary directory.  If you use
         this method directly, you have to remove the directory given by the
     def prepare(self):
         """prepare upload for further processing
 
         This copies the files involved to a temporary directory.  If you use
         this method directly, you have to remove the directory given by the

-        `directory` attribute later on your own.
+        C{directory} attribute later on your own.

 
 

-        Instead of using the method directly, you can also use a with-statement:
+        Instead of using the method directly, you can also use a with-statement::

 
            with ArchiveUpload(...) as upload:
               ...
 
            with ArchiveUpload(...) as upload:
               ...


@@ -542,39 +675,54 @@ class ArchiveUpload(object):
         cnf = Config()
         session = self.transaction.session
 
         cnf = Config()
         session = self.transaction.session
 

-        self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
+        group = cnf.get('Dinstall::UnprivGroup') or None
+        self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
+                                            mode=0o2750, group=group)

         with FilesystemTransaction() as fs:
             src = os.path.join(self.original_directory, self.original_changes.filename)
             dst = os.path.join(self.directory, self.original_changes.filename)
         with FilesystemTransaction() as fs:
             src = os.path.join(self.original_directory, self.original_changes.filename)
             dst = os.path.join(self.directory, self.original_changes.filename)

-            fs.copy(src, dst)
+            fs.copy(src, dst, mode=0o640)

 
             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 
             for f in self.changes.files.itervalues():
                 src = os.path.join(self.original_directory, f.filename)
                 dst = os.path.join(self.directory, f.filename)
 
             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 
             for f in self.changes.files.itervalues():
                 src = os.path.join(self.original_directory, f.filename)
                 dst = os.path.join(self.directory, f.filename)

-                fs.copy(src, dst)
+                if not os.path.exists(src):
+                    continue
+                fs.copy(src, dst, mode=0o640)
+
+            source = None
+            try:
+                source = self.changes.source
+            except Exception:
+                # Do not raise an exception here if the .dsc is invalid.
+                pass

 
 

-            source = self.changes.source

             if source is not None:
                 for f in source.files.itervalues():
                     src = os.path.join(self.original_directory, f.filename)
                     dst = os.path.join(self.directory, f.filename)
             if source is not None:
                 for f in source.files.itervalues():
                     src = os.path.join(self.original_directory, f.filename)
                     dst = os.path.join(self.directory, f.filename)

-                    if f.filename not in self.changes.files:
-                        db_file = self.transaction.get_file(f, source.dsc['Source'])
-                        db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
-                        fs.copy(db_archive_file.path, dst, symlink=True)
+                    if not os.path.exists(dst):
+                        try:
+                            db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
+                            db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
+                            fs.copy(db_archive_file.path, dst, mode=0o640)
+                        except KeyError:
+                            # Ignore if get_file could not find it. Upload will
+                            # probably be rejected later.
+                            pass

 
     def unpacked_source(self):
         """Path to unpacked source
 
         Get path to the unpacked source. This method does unpack the source
 
     def unpacked_source(self):
         """Path to unpacked source
 
         Get path to the unpacked source. This method does unpack the source

-        into a temporary directory under `self.directory` if it has not
+        into a temporary directory under C{self.directory} if it has not

         been done so already.
 
         been done so already.
 

-        Returns:
-           String giving the path to the unpacked source directory
-           or None if no source was included in the upload.
+        @rtype:  str or C{None}
+        @return: string giving the path to the unpacked source directory
+                 or C{None} if no source was included in the upload.

         """
         assert self.directory is not None
 
         """
         assert self.directory is not None
 


@@ -585,7 +733,8 @@ class ArchiveUpload(object):
 
         sourcedir = os.path.join(self.directory, 'source')
         if not os.path.exists(sourcedir):
 
         sourcedir = os.path.join(self.directory, 'source')
         if not os.path.exists(sourcedir):

-            subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
+            devnull = open('/dev/null', 'w')
+            daklib.daksubprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)

         if not os.path.isdir(sourcedir):
             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
         return sourcedir
         if not os.path.isdir(sourcedir):
             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
         return sourcedir


@@ -599,7 +748,7 @@ class ArchiveUpload(object):
                 if src == suite_name:
                     suite_name = dst
                     if rtype != "silent-map":
                 if src == suite_name:
                     suite_name = dst
                     if rtype != "silent-map":

-                        self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
+                        self.warnings.append('Mapping {0} to {1}.'.format(src, dst))

             elif rtype == "ignore":
                 ignored = fields[1]
                 if suite_name == ignored:
             elif rtype == "ignore":
                 ignored = fields[1]
                 if suite_name == ignored:


@@ -608,15 +757,15 @@ class ArchiveUpload(object):
             elif rtype == "reject":
                 rejected = fields[1]
                 if suite_name == rejected:
             elif rtype == "reject":
                 rejected = fields[1]
                 if suite_name == rejected:

-                    self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
+                    raise checks.Reject('Uploads to {0} are not accepted.'.format(rejected))

             ## XXX: propup-version and map-unreleased not yet implemented
         return suite_name
 
     def _mapped_suites(self):
         """Get target suites after mappings
 
             ## XXX: propup-version and map-unreleased not yet implemented
         return suite_name
 
     def _mapped_suites(self):
         """Get target suites after mappings
 

-        Returns:
-           list of daklib.dbconn.Suite giving the mapped target suites of this upload
+        @rtype:  list of L{daklib.dbconn.Suite}
+        @return: list giving the mapped target suites of this upload

         """
         session = self.session
 
         """
         session = self.session
 


@@ -629,51 +778,54 @@ class ArchiveUpload(object):
         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
         return suites
 
         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
         return suites
 

-    def _mapped_component(self, component_name):
-        """get component after mappings
-
-        Evaluate component mappings from ComponentMappings in dak.conf for the
-        given component name.
-
-        NOTE: ansgar wants to get rid of this. It's currently only used for
-        the security archive
-
-        Args:
-           component_name (str): component name
+    def _check_new_binary_overrides(self, suite, overridesuite):
+        new = False
+        source = self.changes.source

 
 

-        Returns:
-           `daklib.dbconn.Component` object
-        """
-        cnf = Config()
-        for m in cnf.value_list("ComponentMappings"):
-            (src, dst) = m.split()
-            if component_name == src:
-                component_name = dst
-        component = self.session.query(Component).filter_by(component_name=component_name).one()
-        return component
-
-    def _check_new(self, suite):
+        if source is not None and not source.package_list.fallback:
+            packages = source.package_list.packages_for_suite(suite)
+            binaries = [ entry for entry in packages ]
+            for b in binaries:
+                override = self._binary_override(overridesuite, b)
+                if override is None:
+                    self.warnings.append('binary:{0} is NEW.'.format(b.name))
+                    new = True
+        else:
+            binaries = self.changes.binaries
+            for b in binaries:
+                if utils.is_in_debug_section(b.control) and suite.debug_suite is not None:
+                    continue
+                override = self._binary_override(overridesuite, b)
+                if override is None:
+                    self.warnings.append('binary:{0} is NEW.'.format(b.name))
+                    new = True
+
+        return new
+
+    def _check_new(self, suite, overridesuite):

         """Check if upload is NEW
 
         An upload is NEW if it has binary or source packages that do not have
         """Check if upload is NEW
 
         An upload is NEW if it has binary or source packages that do not have

-        an override in `suite` OR if it references files ONLY in a tainted
-        archive (eg. when it references files in NEW).
+        an override in C{overridesuite} OR if it references files ONLY in a
+        tainted archive (eg. when it references files in NEW).

 
 

-        Returns:
-           True if the upload is NEW, False otherwise
+        Debug packages (*-dbgsym in Section: debug) are not considered as NEW
+        if C{suite} has a seperate debug suite.
+
+        @rtype:  bool
+        @return: C{True} if the upload is NEW, C{False} otherwise

         """
         session = self.session
         """
         session = self.session

+        new = False

 
         # Check for missing overrides
 
         # Check for missing overrides

-        for b in self.changes.binaries:
-            override = self._binary_override(suite, b)
-            if override is None:
-                return True
-
+        if self._check_new_binary_overrides(suite, overridesuite):
+            new = True

         if self.changes.source is not None:
         if self.changes.source is not None:

-            override = self._source_override(suite, self.changes.source)
+            override = self._source_override(overridesuite, self.changes.source)

             if override is None:
             if override is None:

-                return True
+                self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
+                new = True

 
         # Check if we reference a file only in a tainted archive
         files = self.changes.files.values()
 
         # Check if we reference a file only in a tainted archive
         files = self.changes.files.values()


@@ -687,7 +839,10 @@ class ArchiveUpload(object):
             in_untainted_archive = (query_untainted.first() is not None)
 
             if in_archive and not in_untainted_archive:
             in_untainted_archive = (query_untainted.first() is not None)
 
             if in_archive and not in_untainted_archive:

-                return True
+                self.warnings.append('{0} is only available in NEW.'.format(f.filename))
+                new = True
+
+        return new

 
     def _final_suites(self):
         session = self.session
 
     def _final_suites(self):
         session = self.session


@@ -699,7 +854,7 @@ class ArchiveUpload(object):
             overridesuite = suite
             if suite.overridesuite is not None:
                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
             overridesuite = suite
             if suite.overridesuite is not None:
                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()

-            if self._check_new(overridesuite):
+            if self._check_new(suite, overridesuite):

                 self.new = True
             final_suites.add(suite)
 
                 self.new = True
             final_suites.add(suite)
 


@@ -708,18 +863,24 @@ class ArchiveUpload(object):
     def _binary_override(self, suite, binary):
         """Get override entry for a binary
 
     def _binary_override(self, suite, binary):
         """Get override entry for a binary
 

-        Args:
-           suite (daklib.dbconn.Suite)
-           binary (daklib.upload.Binary)
+        @type  suite: L{daklib.dbconn.Suite}
+        @param suite: suite to get override for
+
+        @type  binary: L{daklib.upload.Binary} or L{daklib.packagelist.PackageListEntry}
+        @param binary: binary to get override for

 
 

-        Returns:
-           daklib.dbconn.Override or None
+        @rtype:  L{daklib.dbconn.Override} or C{None}
+        @return: override for the given binary or C{None}

         """
         if suite.overridesuite is not None:
             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 
         """
         if suite.overridesuite is not None:
             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 

-        query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
-                .join(Component).filter(Component.component_name == binary.component) \
+        mapped_component = get_mapped_component(binary.component)
+        if mapped_component is None:
+            return None
+
+        query = self.session.query(Override).filter_by(suite=suite, package=binary.name) \
+                .join(Component).filter(Component.component_name == mapped_component.component_name) \

                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 
         try:
                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 
         try:


@@ -730,20 +891,25 @@ class ArchiveUpload(object):
     def _source_override(self, suite, source):
         """Get override entry for a source
 
     def _source_override(self, suite, source):
         """Get override entry for a source
 

-        Args:
-           suite (daklib.dbconn.Suite)
-           source (daklib.upload.Source)
+        @type  suite: L{daklib.dbconn.Suite}
+        @param suite: suite to get override for

 
 

-        Returns:
-           daklib.dbconn.Override or None
+        @type  source: L{daklib.upload.Source}
+        @param source: source to get override for
+
+        @rtype:  L{daklib.dbconn.Override} or C{None}
+        @return: override for the given source or C{None}

         """
         if suite.overridesuite is not None:
             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 
         """
         if suite.overridesuite is not None:
             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 

-        # XXX: component for source?

         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 
         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 

+        component = source_component_from_package_list(source.package_list, suite)
+        if component is not None:
+            query = query.filter(Override.component == component)
+

         try:
             return query.one()
         except NoResultFound:
         try:
             return query.one()
         except NoResultFound:


@@ -753,59 +919,69 @@ class ArchiveUpload(object):
         """get component for a binary
 
         By default this will only look at overrides to get the right component;
         """get component for a binary
 
         By default this will only look at overrides to get the right component;

-        if `only_overrides` is False this method will also look at the Section field.
+        if C{only_overrides} is C{False} this method will also look at the
+        Section field.
+
+        @type  suite: L{daklib.dbconn.Suite}

 
 

-        Args:
-           suite (daklib.dbconn.Suite)
-           binary (daklib.upload.Binary)
+        @type  binary: L{daklib.upload.Binary}

 
 

-        Kwargs:
-           only_overrides (bool): only use overrides to get the right component.
-               defaults to True.
+        @type  only_overrides: bool
+        @param only_overrides: only use overrides to get the right component

 
 

-        Returns:
-           `daklib.dbconn.Component` object or None
+        @rtype: L{daklib.dbconn.Component} or C{None}

         """
         override = self._binary_override(suite, binary)
         if override is not None:
             return override.component
         if only_overrides:
             return None
         """
         override = self._binary_override(suite, binary)
         if override is not None:
             return override.component
         if only_overrides:
             return None

-        return self._mapped_component(binary.component)
+        return get_mapped_component(binary.component, self.session)

 
     def check(self, force=False):
         """run checks against the upload
 
 
     def check(self, force=False):
         """run checks against the upload
 

-        Args:
-           force (bool): ignore failing forcable checks
+        @type  force: bool
+        @param force: ignore failing forcable checks

 
 

-        Returns:
-           True if all checks passed, False otherwise
+        @rtype:  bool
+        @return: C{True} if all checks passed, C{False} otherwise

         """
         # XXX: needs to be better structured.
         assert self.changes.valid_signature
 
         try:
         """
         # XXX: needs to be better structured.
         assert self.changes.valid_signature
 
         try:

+            # Validate signatures and hashes before we do any real work:

             for chk in (
             for chk in (

-                    checks.SignatureCheck,
+                    checks.SignatureAndHashesCheck,
+                    checks.SignatureTimestampCheck,

                     checks.ChangesCheck,
                     checks.ChangesCheck,

-                    checks.HashesCheck,
+                    checks.ExternalHashesCheck,

                     checks.SourceCheck,
                     checks.BinaryCheck,
                     checks.BinaryTimestampCheck,
                     checks.SourceCheck,
                     checks.BinaryCheck,
                     checks.BinaryTimestampCheck,

-                    checks.ACLCheck,

                     checks.SingleDistributionCheck,
                     checks.SingleDistributionCheck,

-                    checks.NoSourceOnlyCheck,
-                    checks.LintianCheck,

                     ):
                 chk().check(self)
 
             final_suites = self._final_suites()
             if len(final_suites) == 0:
                     ):
                 chk().check(self)
 
             final_suites = self._final_suites()
             if len(final_suites) == 0:

-                self.reject_reasons.append('Ended with no suite to install to.')
+                self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')

                 return False
 
                 return False
 

+            self.final_suites = final_suites
+
+            for chk in (
+                    checks.SuiteCheck,
+                    checks.TransitionCheck,
+                    checks.ACLCheck,
+                    checks.NoSourceOnlyCheck,
+                    checks.LintianCheck,
+                    ):
+                chk().check(self)
+

             for chk in (
             for chk in (

+                    checks.ACLCheck,

                     checks.SourceFormatCheck,
                     checks.SuiteArchitectureCheck,
                     checks.VersionCheck,
                     checks.SourceFormatCheck,
                     checks.SuiteArchitectureCheck,
                     checks.VersionCheck,


@@ -816,7 +992,7 @@ class ArchiveUpload(object):
             if len(self.reject_reasons) != 0:
                 return False
 
             if len(self.reject_reasons) != 0:
                 return False
 

-            self.final_suites = final_suites
+            self._checked = True

             return True
         except checks.Reject as e:
             self.reject_reasons.append(unicode(e))
             return True
         except checks.Reject as e:
             self.reject_reasons.append(unicode(e))


@@ -827,23 +1003,24 @@ class ArchiveUpload(object):
     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
         """Install upload to the given suite
 
     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
         """Install upload to the given suite
 

-        Args:
-           suite (daklib.dbconn.Suite): suite to install the package into.
-              This is the real suite, ie. after any redirection to NEW or a policy queue
-           source_component_func: function to get the `daklib.dbconn.Component`
-              for a `daklib.upload.Source` object
-           binary_component_func: function to get the `daklib.dbconn.Component`
-              for a `daklib.upload.Binary` object
-
-        Kwargs:
-           source_suites: see `daklib.archive.ArchiveTransaction.install_binary`
-           extra_source_archives: see `daklib.archive.ArchiveTransaction.install_binary`
-
-        Returns:
-           tuple with two elements. The first is a `daklib.dbconn.DBSource`
-           object for the install source or None if no source was included.
-           The second is a list of `daklib.dbconn.DBBinary` objects for the
-           installed binary packages.
+        @type  suite: L{daklib.dbconn.Suite}
+        @param suite: suite to install the package into. This is the real suite,
+                      ie. after any redirection to NEW or a policy queue
+
+        @param source_component_func: function to get the L{daklib.dbconn.Component}
+                                      for a L{daklib.upload.Source} object
+
+        @param binary_component_func: function to get the L{daklib.dbconn.Component}
+                                      for a L{daklib.upload.Binary} object
+
+        @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
+
+        @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
+
+        @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
+                 object for the install source or C{None} if no source was
+                 included. The second is a list of L{daklib.dbconn.DBBinary}
+                 objects for the installed binary packages.

         """
         # XXX: move this function to ArchiveTransaction?
 
         """
         # XXX: move this function to ArchiveTransaction?
 


@@ -851,25 +1028,44 @@ class ArchiveUpload(object):
         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 
         if source_suites is None:
         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 
         if source_suites is None:

-            source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
+            source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()

 
         source = self.changes.source
         if source is not None:
             component = source_component_func(source)
 
         source = self.changes.source
         if source is not None:
             component = source_component_func(source)

-            db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
+            db_source = self.transaction.install_source(
+                self.directory,
+                source,
+                suite,
+                component,
+                changed_by,
+                fingerprint=self.fingerprint
+            )

         else:
             db_source = None
 
         db_binaries = []
         for binary in self.changes.binaries:
         else:
             db_source = None
 
         db_binaries = []
         for binary in self.changes.binaries:

+            copy_to_suite = suite
+            if utils.is_in_debug_section(binary.control) and suite.debug_suite is not None:
+                copy_to_suite = suite.debug_suite
+

             component = binary_component_func(binary)
             component = binary_component_func(binary)

-            db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
+            db_binary = self.transaction.install_binary(
+                self.directory,
+                binary,
+                copy_to_suite,
+                component,
+                fingerprint=self.fingerprint,
+                source_suites=source_suites,
+                extra_source_archives=extra_source_archives
+            )

             db_binaries.append(db_binary)
 
         if suite.copychanges:
             src = os.path.join(self.directory, self.changes.filename)
             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
             db_binaries.append(db_binary)
 
         if suite.copychanges:
             src = os.path.join(self.directory, self.changes.filename)
             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)

-            self.transaction.fs.copy(src, dst)
+            self.transaction.fs.copy(src, dst, mode=suite.archive.mode)

 
         return (db_source, db_binaries)
 
 
         return (db_source, db_binaries)
 


@@ -901,8 +1097,11 @@ class ArchiveUpload(object):
         db_changes.changelog_id = changelog_id
         db_changes.closes = self.changes.closed_bugs
 
         db_changes.changelog_id = changelog_id
         db_changes.closes = self.changes.closed_bugs
 

-        self.transaction.session.add(db_changes)
-        self.transaction.session.flush()
+        try:
+            self.transaction.session.add(db_changes)
+            self.transaction.session.flush()
+        except sqlalchemy.exc.IntegrityError:
+            raise ArchiveException('{0} is already known.'.format(self.changes.filename))

 
         return db_changes
 
 
         return db_changes
 


@@ -917,7 +1116,7 @@ class ArchiveUpload(object):
         self.transaction.session.flush()
 
         dst = os.path.join(policy_queue.path, self.changes.filename)
         self.transaction.session.flush()
 
         dst = os.path.join(policy_queue.path, self.changes.filename)

-        self.transaction.fs.copy(self.changes.path, dst)
+        self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)

 
         return u
 
 
         return u
 


@@ -926,12 +1125,13 @@ class ArchiveUpload(object):
 
         Try to handle byhand packages automatically.
 
 
         Try to handle byhand packages automatically.
 

-        Returns:
-           list of `daklib.upload.hashed_file` for the remaining byhand packages
+        @rtype:  list of L{daklib.upload.HashedFile}
+        @return: list of remaining byhand files

         """
         assert len(self.reject_reasons) == 0
         assert self.changes.valid_signature
         assert self.final_suites is not None
         """
         assert len(self.reject_reasons) == 0
         assert self.changes.valid_signature
         assert self.final_suites is not None

+        assert self._checked

 
         byhand = self.changes.byhand_files
         if len(byhand) == 0:
 
         byhand = self.changes.byhand_files
         if len(byhand) == 0:


@@ -947,26 +1147,41 @@ class ArchiveUpload(object):
 
         remaining = []
         for f in byhand:
 
         remaining = []
         for f in byhand:

-            parts = f.filename.split('_', 2)
-            if len(parts) != 3:
-                print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
-                remaining.append(f)
-                continue
-
-            package, version, archext = parts
-            arch, ext = archext.split('.', 1)
-
-            rule = automatic_byhand_packages.get(package)
-            if rule is None:
+            if '_' in f.filename:
+                parts = f.filename.split('_', 2)
+                if len(parts) != 3:
+                    print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
+                    remaining.append(f)
+                    continue
+
+                package, version, archext = parts
+                arch, ext = archext.split('.', 1)
+            else:
+                parts = f.filename.split('.')
+                if len(parts) < 2:
+                    print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
+                    remaining.append(f)
+                    continue
+
+                package = parts[0]
+                version = '0'
+                arch = 'all'
+                ext = parts[-1]
+
+            try:
+                rule = automatic_byhand_packages.subtree(package)
+            except KeyError:

                 remaining.append(f)
                 continue
 
                 remaining.append(f)
                 continue
 

-            if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
+            if rule['Source'] != self.changes.source_name \
+                    or rule['Section'] != f.section \
+                    or ('Extension' in rule and rule['Extension'] != ext):

                 remaining.append(f)
                 continue
 
             script = rule['Script']
                 remaining.append(f)
                 continue
 
             script = rule['Script']

-            retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
+            retcode = daklib.daksubprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename), suite.suite_name], shell=False)

             if retcode != 0:
                 print "W: error processing {0}.".format(f.filename)
                 remaining.append(f)
             if retcode != 0:
                 print "W: error processing {0}.".format(f.filename)
                 remaining.append(f)


@@ -974,10 +1189,11 @@ class ArchiveUpload(object):
         return len(remaining) == 0
 
     def _install_byhand(self, policy_queue_upload, hashed_file):
         return len(remaining) == 0
 
     def _install_byhand(self, policy_queue_upload, hashed_file):

-        """
-        Args:
-           policy_queue_upload (daklib.dbconn.PolicyQueueUpload): XXX
-           hashed_file (daklib.upload.HashedFile): XXX
+        """install byhand file
+
+        @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
+
+        @type  hashed_file: L{daklib.upload.HashedFile}

         """
         fs = self.transaction.fs
         session = self.transaction.session
         """
         fs = self.transaction.fs
         session = self.transaction.session


@@ -991,7 +1207,7 @@ class ArchiveUpload(object):
 
         src = os.path.join(self.directory, hashed_file.filename)
         dst = os.path.join(policy_queue.path, hashed_file.filename)
 
         src = os.path.join(self.directory, hashed_file.filename)
         dst = os.path.join(policy_queue.path, hashed_file.filename)

-        fs.copy(src, dst)
+        fs.copy(src, dst, mode=policy_queue.change_perms)

 
         return byhand_file
 
 
         return byhand_file
 


@@ -1021,21 +1237,27 @@ class ArchiveUpload(object):
         for binary in self.changes.binaries:
             control = binary.control
             source_package, source_version = binary.source
         for binary in self.changes.binaries:
             control = binary.control
             source_package, source_version = binary.source

-            line = " ".join([control['Package'], control['Version'], source_package, source_version])
+            line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])

             print >>debinfo, line
         debinfo.close()
 
             print >>debinfo, line
         debinfo.close()
 

+    def _policy_queue(self, suite):
+        if suite.policy_queue is not None:
+            return suite.policy_queue
+        return None
+

     def install(self):
         """install upload
 
     def install(self):
         """install upload
 

-        Install upload to a suite or policy queue.  This method does *not*
+        Install upload to a suite or policy queue.  This method does B{not}

         handle uploads to NEW.
 
         handle uploads to NEW.
 

-        You need to have called the `check` method before calling this method.
+        You need to have called the C{check} method before calling this method.

         """
         assert len(self.reject_reasons) == 0
         assert self.changes.valid_signature
         assert self.final_suites is not None
         """
         assert len(self.reject_reasons) == 0
         assert self.changes.valid_signature
         assert self.final_suites is not None

+        assert self._checked

         assert not self.new
 
         db_changes = self._install_changes()
         assert not self.new
 
         db_changes = self._install_changes()


@@ -1045,32 +1267,43 @@ class ArchiveUpload(object):
             if suite.overridesuite is not None:
                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 
             if suite.overridesuite is not None:
                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 

+            policy_queue = self._policy_queue(suite)
+

             redirected_suite = suite
             redirected_suite = suite

-            if suite.policy_queue is not None:
-                redirected_suite = suite.policy_queue.suite
+            if policy_queue is not None:
+                redirected_suite = policy_queue.suite
+
+            # source can be in the suite we install to or any suite we enhance
+            source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
+            for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
+                    .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
+                    .filter(VersionCheck.check == 'Enhances'):
+                source_suite_ids.add(enhanced_suite_id)
+
+            source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()

 
             source_component_func = lambda source: self._source_override(overridesuite, source).component
 
             source_component_func = lambda source: self._source_override(overridesuite, source).component

-            binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
+            binary_component_func = lambda binary: self._binary_component(overridesuite, binary, only_overrides=False)

 
 

-            (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
+            (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])

 
 

-            if suite.policy_queue is not None:
-                self._install_policy(suite.policy_queue, suite, db_changes, db_source, db_binaries)
+            if policy_queue is not None:
+                self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)

 
             # copy to build queues
 
             # copy to build queues

-            if suite.policy_queue is None or suite.policy_queue.send_to_build_queues:
+            if policy_queue is None or policy_queue.send_to_build_queues:

                 for build_queue in suite.copy_queues:
                 for build_queue in suite.copy_queues:

-                    self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
+                    self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])

 
         self._do_bts_versiontracking()
 
     def install_to_new(self):
         """install upload to NEW
 
 
         self._do_bts_versiontracking()
 
     def install_to_new(self):
         """install upload to NEW
 

-        Install upload to NEW.  This method does *not* handle regular uploads
+        Install upload to NEW.  This method does B{not} handle regular uploads

         to suites or policy queues.
 
         to suites or policy queues.
 

-        You need to have called the `check` method before calling this method.
+        You need to have called the C{check} method before calling this method.

         """
         # Uploads to NEW are special as we don't have overrides.
         assert len(self.reject_reasons) == 0
         """
         # Uploads to NEW are special as we don't have overrides.
         assert len(self.reject_reasons) == 0


@@ -1081,16 +1314,22 @@ class ArchiveUpload(object):
         binaries = self.changes.binaries
         byhand = self.changes.byhand_files
 
         binaries = self.changes.binaries
         byhand = self.changes.byhand_files
 

-        new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
-        if len(byhand) > 0:
-            new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
-        new_suite = new_queue.suite
-

         # we need a suite to guess components
         suites = list(self.final_suites)
         assert len(suites) == 1, "NEW uploads must be to a single suite"
         suite = suites[0]
 
         # we need a suite to guess components
         suites = list(self.final_suites)
         assert len(suites) == 1, "NEW uploads must be to a single suite"
         suite = suites[0]
 

+        # decide which NEW queue to use
+        if suite.new_queue is None:
+            new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
+        else:
+            new_queue = suite.new_queue
+        if len(byhand) > 0:
+            # There is only one global BYHAND queue
+            new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
+        new_suite = new_queue.suite
+
+

         def binary_component_func(binary):
             return self._binary_component(suite, binary, only_overrides=False)
 
         def binary_component_func(binary):
             return self._binary_component(suite, binary, only_overrides=False)
 


@@ -1101,13 +1340,15 @@ class ArchiveUpload(object):
             component = binary_component_func(binary)
             binary_component_names.add(component.component_name)
         source_component_name = None
             component = binary_component_func(binary)
             binary_component_names.add(component.component_name)
         source_component_name = None

-        for guess in ('main', 'contrib', 'non-free'):
+        for c in self.session.query(Component).order_by(Component.component_id):
+            guess = c.component_name

             if guess in binary_component_names:
                 source_component_name = guess
                 break
         if source_component_name is None:
             if guess in binary_component_names:
                 source_component_name = guess
                 break
         if source_component_name is None:

-            raise Exception('Could not guess source component.')
-        source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
+            source_component = self.session.query(Component).order_by(Component.component_id).first()
+        else:
+            source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()

         source_component_func = lambda source: source_component
 
         db_changes = self._install_changes()
         source_component_func = lambda source: source_component
 
         db_changes = self._install_changes()