Adjust filename passed to get_poolfile_like_name

[dak.git] / daklib / queue.py

diff --git a/daklib/queue.py b/daklib/queue.py
index bb62aaeee273c506502fef112218ac61dfb3321a..58a8783442f70a87281dc96ffabdadae6d4e05ae 100755 (executable)
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -6,7 +6,7 @@ Queue utility functions for dak
 
 @contact: Debian FTP Master <ftpmaster@debian.org>
 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
 
 @contact: Debian FTP Master <ftpmaster@debian.org>
 @copyright: 2001 - 2006 James Troup <james@nocrew.org>

-@copyright: 2009  Joerg Jaspert <joerg@debian.org>
+@copyright: 2009, 2010  Joerg Jaspert <joerg@debian.org>

 @license: GNU General Public License version 2 or later
 """
 
 @license: GNU General Public License version 2 or later
 """
 


@@ -39,6 +39,7 @@ import shutil
 import textwrap
 from types import *
 from sqlalchemy.sql.expression import desc
 import textwrap
 from types import *
 from sqlalchemy.sql.expression import desc

+from sqlalchemy.orm.exc import NoResultFound

 
 import yaml
 
 
 import yaml
 


@@ -47,11 +48,13 @@ from changes import *
 from regexes import *
 from config import Config
 from holding import Holding
 from regexes import *
 from config import Config
 from holding import Holding

+from urgencylog import UrgencyLog

 from dbconn import *
 from summarystats import SummaryStats
 from utils import parse_changes, check_dsc_files
 from textutils import fix_maintainer
 from binary import Binary
 from dbconn import *
 from summarystats import SummaryStats
 from utils import parse_changes, check_dsc_files
 from textutils import fix_maintainer
 from binary import Binary

+from lintian import parse_lintian_output, generate_reject_messages

 
 ###############################################################################
 
 
 ###############################################################################
 


@@ -112,8 +115,8 @@ def determine_new(changes, files, warn=1):
     # Build up a list of potentially new things
     for name, f in files.items():
         # Skip byhand elements
     # Build up a list of potentially new things
     for name, f in files.items():
         # Skip byhand elements

-        if f["type"] == "byhand":
-            continue
+#        if f["type"] == "byhand":
+#            continue

         pkg = f["package"]
         priority = f["priority"]
         section = f["section"]
         pkg = f["package"]
         priority = f["priority"]
         section = f["section"]


@@ -145,6 +148,22 @@ def determine_new(changes, files, warn=1):
         if f.has_key("othercomponents"):
             new[pkg]["othercomponents"] = f["othercomponents"]
 
         if f.has_key("othercomponents"):
             new[pkg]["othercomponents"] = f["othercomponents"]
 

+    # Fix up the list of target suites
+    cnf = Config()
+    for suite in changes["suite"].keys():
+        override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
+        if override:
+            (olderr, newerr) = (get_suite(suite, session) == None,
+                                get_suite(override, session) == None)
+            if olderr or newerr:
+                (oinv, newinv) = ("", "")
+                if olderr: oinv = "invalid "
+                if newerr: ninv = "invalid "
+                print "warning: overriding %ssuite %s to %ssuite %s" % (
+                        oinv, suite, ninv, override)
+            del changes["suite"][suite]
+            changes["suite"][override] = 1
+

     for suite in changes["suite"].keys():
         for pkg in new.keys():
             ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
     for suite in changes["suite"].keys():
         for pkg in new.keys():
             ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)


@@ -212,17 +231,6 @@ def check_valid(new):
 
 ###############################################################################
 
 
 ###############################################################################
 

-def check_status(files):
-    new = byhand = 0
-    for f in files.keys():
-        if files[f]["type"] == "byhand":
-            byhand = 1
-        elif files[f].has_key("new"):
-            new = 1
-    return (new, byhand)
-
-###############################################################################
-

 # Used by Upload.check_timestamps
 class TarTime(object):
     def __init__(self, future_cutoff, past_cutoff):
 # Used by Upload.check_timestamps
 class TarTime(object):
     def __init__(self, future_cutoff, past_cutoff):


@@ -269,6 +277,8 @@ class Upload(object):
         self.warnings = []
         self.notes = []
 
         self.warnings = []
         self.notes = []
 

+        self.later_check_files = []
+

         self.pkg.reset()
 
     def package_info(self):
         self.pkg.reset()
 
     def package_info(self):


@@ -286,6 +296,7 @@ class Upload(object):
         for title, messages in msgs:
             if messages:
                 msg += '\n\n%s:\n%s' % (title, '\n'.join(messages))
         for title, messages in msgs:
             if messages:
                 msg += '\n\n%s:\n%s' % (title, '\n'.join(messages))

+        msg += '\n\n'

 
         return msg
 
 
         return msg
 


@@ -339,8 +350,14 @@ class Upload(object):
     ###########################################################################
     def load_changes(self, filename):
         """
     ###########################################################################
     def load_changes(self, filename):
         """

+        Load a changes file and setup a dictionary around it. Also checks for mandantory
+        fields  within.
+
+        @type filename: string
+        @param filename: Changes filename, full path.
+

         @rtype: boolean
         @rtype: boolean

-        @rvalue: whether the changes file was valid or not.  We may want to
+        @return: whether the changes file was valid or not.  We may want to

                  reject even if this is True (see what gets put in self.rejects).
                  This is simply to prevent us even trying things later which will
                  fail because we couldn't properly parse the file.
                  reject even if this is True (see what gets put in self.rejects).
                  This is simply to prevent us even trying things later which will
                  fail because we couldn't properly parse the file.


@@ -423,7 +440,7 @@ class Upload(object):
             self.pkg.changes["changedbyemail"] = ""
 
             self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
             self.pkg.changes["changedbyemail"] = ""
 
             self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \

-                   % (filename, changes["changed-by"], msg))
+                   % (filename, self.pkg.changes["changed-by"], msg))

 
         # Ensure all the values in Closes: are numbers
         if self.pkg.changes.has_key("closes"):
 
         # Ensure all the values in Closes: are numbers
         if self.pkg.changes.has_key("closes"):


@@ -435,12 +452,6 @@ class Upload(object):
         self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
         self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
 
         self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
         self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
 

-        # Check there isn't already a changes file of the same name in one
-        # of the queue directories.
-        base_filename = os.path.basename(filename)
-        if get_knownchange(base_filename):
-            self.rejects.append("%s: a file with this name already exists." % (base_filename))
-

         # Check the .changes is non-empty
         if not self.pkg.files:
             self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
         # Check the .changes is non-empty
         if not self.pkg.files:
             self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))


@@ -663,7 +674,7 @@ class Upload(object):
                     entry["new"] = 1
                 else:
                     dsc_file_exists = False
                     entry["new"] = 1
                 else:
                     dsc_file_exists = False

-                    for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
+                    for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:

                         if cnf.has_key("Dir::Queue::%s" % (myq)):
                             if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
                                 dsc_file_exists = True
                         if cnf.has_key("Dir::Queue::%s" % (myq)):
                             if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
                                 dsc_file_exists = True


@@ -723,7 +734,6 @@ class Upload(object):
     def per_suite_file_checks(self, f, suite, session):
         cnf = Config()
         entry = self.pkg.files[f]
     def per_suite_file_checks(self, f, suite, session):
         cnf = Config()
         entry = self.pkg.files[f]

-        archive = utils.where_am_i()

 
         # Skip byhand
         if entry.has_key("byhand"):
 
         # Skip byhand
         if entry.has_key("byhand"):


@@ -767,9 +777,9 @@ class Upload(object):
 
         # Determine the location
         location = cnf["Dir::Pool"]
 
         # Determine the location
         location = cnf["Dir::Pool"]

-        l = get_location(location, entry["component"], archive, session)
+        l = get_location(location, entry["component"], session=session)

         if l is None:
         if l is None:

-            self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (entry["component"], archive))
+            self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %)" % entry["component"])

             entry["location id"] = -1
         else:
             entry["location id"] = l.location_id
             entry["location id"] = -1
         else:
             entry["location id"] = l.location_id


@@ -807,33 +817,35 @@ class Upload(object):
             for f in file_keys:
                 ret = holding.copy_to_holding(f)
                 if ret is not None:
             for f in file_keys:
                 ret = holding.copy_to_holding(f)
                 if ret is not None:

-                    # XXX: Should we bail out here or try and continue?
-                    self.rejects.append(ret)
+                    self.warnings.append('Could not copy %s to holding; will attempt to find in DB later' % f)

 
             os.chdir(cwd)
 
 
             os.chdir(cwd)
 

-        # Check there isn't already a .changes file of the same name in
-        # the proposed-updates "CopyChanges" storage directories.
+        # check we already know the changes file

         # [NB: this check must be done post-suite mapping]
         base_filename = os.path.basename(self.pkg.changes_file)
 
         # [NB: this check must be done post-suite mapping]
         base_filename = os.path.basename(self.pkg.changes_file)
 

-        for suite in self.pkg.changes["distribution"].keys():
-            copychanges = "Suite::%s::CopyChanges" % (suite)
-            if cnf.has_key(copychanges) and \
-                   os.path.exists(os.path.join(cnf[copychanges], base_filename)):
-                self.rejects.append("%s: a file with this name already exists in %s" \
-                           % (base_filename, cnf[copychanges]))
+        session = DBConn().session()
+
+        try:
+            dbc = session.query(DBChange).filter_by(changesname=base_filename).one()
+            # if in the pool or in a queue other than unchecked, reject
+            if (dbc.in_queue is None) \
+                   or (dbc.in_queue is not None
+                       and dbc.in_queue.queue_name not in ["unchecked", "newstage"]):
+                self.rejects.append("%s file already known to dak" % base_filename)
+        except NoResultFound, e:
+            # not known, good
+            pass

 
         has_binaries = False
         has_source = False
 
 
         has_binaries = False
         has_source = False
 

-        session = DBConn().session()
-

         for f, entry in self.pkg.files.items():
             # Ensure the file does not already exist in one of the accepted directories
         for f, entry in self.pkg.files.items():
             # Ensure the file does not already exist in one of the accepted directories

-            for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
+            for d in [ "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:

                 if not cnf.has_key("Dir::Queue::%s" % (d)): continue
                 if not cnf.has_key("Dir::Queue::%s" % (d)): continue

-                if os.path.exists(cnf["Dir::Queue::%s" % (d) ] + '/' + f):
+                if os.path.exists(os.path.join(cnf["Dir::Queue::%s" % (d) ], f)):

                     self.rejects.append("%s file already exists in the %s directory." % (f, d))
 
             if not re_taint_free.match(f):
                     self.rejects.append("%s file already exists in the %s directory." % (f, d))
 
             if not re_taint_free.match(f):


@@ -847,7 +859,9 @@ class Upload(object):
                     if os.path.exists(f):
                         self.rejects.append("Can't read `%s'. [permission denied]" % (f))
                     else:
                     if os.path.exists(f):
                         self.rejects.append("Can't read `%s'. [permission denied]" % (f))
                     else:

-                        self.rejects.append("Can't read `%s'. [file not found]" % (f))
+                        # Don't directly reject, mark to check later to deal with orig's
+                        # we can find in the pool
+                        self.later_check_files.append(f)

                 entry["type"] = "unreadable"
                 continue
 
                 entry["type"] = "unreadable"
                 continue
 


@@ -992,6 +1006,10 @@ class Upload(object):
         self.check_dsc_against_db(dsc_filename, session)
         session.close()
 
         self.check_dsc_against_db(dsc_filename, session)
         session.close()
 

+        # Finally, check if we're missing any files
+        for f in self.later_check_files:
+            self.rejects.append("Could not find file %s references in changes" % f)
+

         return True
 
     ###########################################################################
         return True
 
     ###########################################################################


@@ -1073,9 +1091,7 @@ class Upload(object):
     def check_source(self):
         # Bail out if:
         #    a) there's no source
     def check_source(self):
         # Bail out if:
         #    a) there's no source

-        # or c) the orig files are MIA
-        if not self.pkg.changes["architecture"].has_key("source") \
-           or len(self.pkg.orig_files) == 0:
+        if not self.pkg.changes["architecture"].has_key("source"):

             return
 
         tmpdir = utils.temp_dirname()
             return
 
         tmpdir = utils.temp_dirname()


@@ -1209,7 +1225,7 @@ class Upload(object):
             found = False
 
             # Look in the pool
             found = False
 
             # Look in the pool

-            for poolfile in get_poolfile_like_name('/%s' % filename, session_):
+            for poolfile in get_poolfile_like_name('%s' % filename, session_):

                 poolfile_path = os.path.join(
                     poolfile.location.path, poolfile.filename
                 )
                 poolfile_path = os.path.join(
                     poolfile.location.path, poolfile.filename
                 )


@@ -1225,7 +1241,7 @@ class Upload(object):
                 continue
 
             # Look in some other queues for the file
                 continue
 
             # Look in some other queues for the file

-            queues = ('Accepted', 'New', 'Byhand', 'ProposedUpdates',
+            queues = ('New', 'Byhand', 'ProposedUpdates',

                 'OldProposedUpdates', 'Embargoed', 'Unembargoed')
 
             for queue in queues:
                 'OldProposedUpdates', 'Embargoed', 'Unembargoed')
 
             for queue in queues:


@@ -1248,6 +1264,11 @@ class Upload(object):
     ###########################################################################
 
     def check_lintian(self):
     ###########################################################################
 
     def check_lintian(self):

+        """
+        Extends self.rejects by checking the output of lintian against tags
+        specified in Dinstall::LintianTags.
+        """
+

         cnf = Config()
 
         # Don't reject binary uploads
         cnf = Config()
 
         # Don't reject binary uploads


@@ -1255,24 +1276,22 @@ class Upload(object):
             return
 
         # Only check some distributions
             return
 
         # Only check some distributions

-        valid_dist = False

         for dist in ('unstable', 'experimental'):
             if dist in self.pkg.changes['distribution']:
         for dist in ('unstable', 'experimental'):
             if dist in self.pkg.changes['distribution']:

-                valid_dist = True

                 break
                 break

-
-        if not valid_dist:
+        else:

             return
 
             return
 

+        # If we do not have a tagfile, don't do anything

         tagfile = cnf.get("Dinstall::LintianTags")
         if tagfile is None:
         tagfile = cnf.get("Dinstall::LintianTags")
         if tagfile is None:

-            # We don't have a tagfile, so just don't do anything.

             return
 
         # Parse the yaml file
         sourcefile = file(tagfile, 'r')
         sourcecontent = sourcefile.read()
         sourcefile.close()
             return
 
         # Parse the yaml file
         sourcefile = file(tagfile, 'r')
         sourcecontent = sourcefile.read()
         sourcefile.close()

+

         try:
             lintiantags = yaml.load(sourcecontent)['lintian']
         except yaml.YAMLError, msg:
         try:
             lintiantags = yaml.load(sourcecontent)['lintian']
         except yaml.YAMLError, msg:


@@ -1282,78 +1301,42 @@ class Upload(object):
         # Try and find all orig mentioned in the .dsc
         symlinked = self.ensure_orig()
 
         # Try and find all orig mentioned in the .dsc
         symlinked = self.ensure_orig()
 

-        # Now setup the input file for lintian. lintian wants "one tag per line" only,
-        # so put it together like it. We put all types of tags in one file and then sort
-        # through lintians output later to see if its a fatal tag we detected, or not.
-        # So we only run lintian once on all tags, even if we might reject on some, but not
-        # reject on others.
-        # Additionally build up a set of tags
-        tags = set()
-        (fd, temp_filename) = utils.temp_filename()
+        # Setup the input file for lintian
+        fd, temp_filename = utils.temp_filename()

         temptagfile = os.fdopen(fd, 'w')
         temptagfile = os.fdopen(fd, 'w')

-        for tagtype in lintiantags:
-            for tag in lintiantags[tagtype]:
-                temptagfile.write("%s\n" % tag)
-                tags.add(tag)
+        for tags in lintiantags.values():
+            temptagfile.writelines(['%s\n' % x for x in tags])

         temptagfile.close()
 
         temptagfile.close()
 

-        # So now we should look at running lintian at the .changes file, capturing output
-        # to then parse it.
-        command = "lintian --show-overrides --tags-from-file %s %s" % (temp_filename, self.pkg.changes_file)
-        (result, output) = commands.getstatusoutput(command)
+        try:
+            cmd = "lintian --show-overrides --tags-from-file %s %s" % \
+                (temp_filename, self.pkg.changes_file)

 
 

-        # We are done with lintian, remove our tempfile and any symlinks we created
-        os.unlink(temp_filename)
-        for symlink in symlinked:
-            os.unlink(symlink)
+            result, output = commands.getstatusoutput(cmd)
+        finally:
+            # Remove our tempfile and any symlinks we created
+            os.unlink(temp_filename)

 
 

-        if (result == 2):
-            utils.warn("lintian failed for %s [return code: %s]." % (self.pkg.changes_file, result))
-            utils.warn(utils.prefix_multi_line_string(output, " [possible output:] "))
+            for symlink in symlinked:
+                os.unlink(symlink)

 
 

-        if len(output) == 0:
-            return
+        if result == 2:
+            utils.warn("lintian failed for %s [return code: %s]." % \
+                (self.pkg.changes_file, result))
+            utils.warn(utils.prefix_multi_line_string(output, \
+                " [possible output:] "))

 
         def log(*txt):
             if self.logger:
 
         def log(*txt):
             if self.logger:

-                self.logger.log([self.pkg.changes_file, "check_lintian"] + list(txt))
-
-        # We have output of lintian, this package isn't clean. Lets parse it and see if we
-        # are having a victim for a reject.
-        # W: tzdata: binary-without-manpage usr/sbin/tzconfig
-        for line in output.split('\n'):
-            m = re_parse_lintian.match(line)
-            if m is None:
-                continue
-
-            etype = m.group(1)
-            epackage = m.group(2)
-            etag = m.group(3)
-            etext = m.group(4)
-
-            # So lets check if we know the tag at all.
-            if etag not in tags:
-                continue
+                self.logger.log(
+                    [self.pkg.changes_file, "check_lintian"] + list(txt)
+                )

 
 

-            if etype == 'O':
-                # We know it and it is overriden. Check that override is allowed.
-                if etag in lintiantags['warning']:
-                    # The tag is overriden, and it is allowed to be overriden.
-                    # Don't add a reject message.
-                    pass
-                elif etag in lintiantags['error']:
-                    # The tag is overriden - but is not allowed to be
-                    self.rejects.append("%s: Overriden tag %s found, but this tag may not be overwritten." % (epackage, etag))
-                    log("ftpmaster does not allow tag to be overridable", etag)
-            else:
-                # Tag is known, it is not overriden, direct reject.
-                self.rejects.append("%s: Found lintian output: '%s %s', automatically rejected package." % (epackage, etag, etext))
-                # Now tell if they *might* override it.
-                if etag in lintiantags['warning']:
-                    log("auto rejecting", "overridable", etag)
-                    self.rejects.append("%s: If you have a good reason, you may override this lintian tag." % (epackage))
-                else:
-                    log("auto rejecting", "not overridable", etag)
+        # Generate messages
+        parsed_tags = parse_lintian_output(output)
+        self.rejects.extend(
+            generate_reject_messages(parsed_tags, lintiantags, log=log)
+        )

 
     ###########################################################################
     def check_urgency(self):
 
     ###########################################################################
     def check_urgency(self):


@@ -1478,16 +1461,15 @@ class Upload(object):
             self.check_dm_upload(fpr, session)
         else:
             # Check source-based permissions for other types
             self.check_dm_upload(fpr, session)
         else:
             # Check source-based permissions for other types

-            if self.pkg.changes["architecture"].has_key("source"):
-                if fpr.source_acl.access_level is None:
-                    rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
-                    rej += '\nPlease contact ftpmaster if you think this is incorrect'
-                    self.rejects.append(rej)
-                    return
-            else:
-                # If not a DM, we allow full upload rights
-                uid_email = "%s@debian.org" % (fpr.uid.uid)
-                self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)
+            if self.pkg.changes["architecture"].has_key("source") and \
+                fpr.source_acl.access_level is None:
+                rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
+                rej += '\nPlease contact ftpmaster if you think this is incorrect'
+                self.rejects.append(rej)
+                return
+            # If not a DM, we allow full upload rights
+            uid_email = "%s@debian.org" % (fpr.uid.uid)
+            self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)

 
 
         # Check binary upload permissions
 
 
         # Check binary upload permissions


@@ -1819,8 +1801,8 @@ distribution."""
         return summary
 
     ###########################################################################
         return summary
 
     ###########################################################################

-
-    def accept (self, summary, short_summary, session):
+    @session_wrapper
+    def accept (self, summary, short_summary, session=None):

         """
         Accept an upload.
 
         """
         Accept an upload.
 


@@ -1841,27 +1823,32 @@ distribution."""
         stats = SummaryStats()
 
         print "Installing."
         stats = SummaryStats()
 
         print "Installing."

-        Logger.log(["installing changes", u.pkg.changes_file])
+        self.logger.log(["installing changes", self.pkg.changes_file])
+
+        poolfiles = []

 
         # Add the .dsc file to the DB first
 
         # Add the .dsc file to the DB first

-        for newfile, entry in u.pkg.files.items():
+        for newfile, entry in self.pkg.files.items():

             if entry["type"] == "dsc":
             if entry["type"] == "dsc":

-                dsc_component, dsc_location_id = add_dsc_to_db(u, newfile, session)
+                source, dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session)
+                for j in pfs:
+                    poolfiles.append(j)

 
         # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
 
         # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)

-        for newfile, entry in u.pkg.files.items():
+        for newfile, entry in self.pkg.files.items():

             if entry["type"] == "deb":
             if entry["type"] == "deb":

-                add_deb_to_db(u, newfile, session)
+                poolfiles.append(add_deb_to_db(self, newfile, session))

 
         # If this is a sourceful diff only upload that is moving
         # cross-component we need to copy the .orig files into the new
         # component too for the same reasons as above.
 
         # If this is a sourceful diff only upload that is moving
         # cross-component we need to copy the .orig files into the new
         # component too for the same reasons as above.

-        if u.pkg.changes["architecture"].has_key("source"):
-            for orig_file in u.pkg.orig_files.keys():
-                if not u.pkg.orig_files[orig_file].has_key("id"):
+        # XXX: mhy: I think this should be in add_dsc_to_db
+        if self.pkg.changes["architecture"].has_key("source"):
+            for orig_file in self.pkg.orig_files.keys():
+                if not self.pkg.orig_files[orig_file].has_key("id"):

                     continue # Skip if it's not in the pool
                     continue # Skip if it's not in the pool

-                orig_file_id = u.pkg.orig_files[orig_file]["id"]
-                if u.pkg.orig_files[orig_file]["location"] == dsc_location_id:
+                orig_file_id = self.pkg.orig_files[orig_file]["id"]
+                if self.pkg.orig_files[orig_file]["location"] == dsc_location_id:

                     continue # Skip if the location didn't change
 
                 # Do the move
                     continue # Skip if the location didn't change
 
                 # Do the move


@@ -1870,37 +1857,63 @@ distribution."""
                 old_dat = {'size': oldf.filesize,   'md5sum': oldf.md5sum,
                            'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
 
                 old_dat = {'size': oldf.filesize,   'md5sum': oldf.md5sum,
                            'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
 

-                new_filename = os.path.join(utils.poolify(u.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
+                new_filename = os.path.join(utils.poolify(self.pkg.changes["source"], dsc_component), os.path.basename(old_filename))

 
                 # TODO: Care about size/md5sum collisions etc
 
                 # TODO: Care about size/md5sum collisions etc

-                (found, newf) = check_poolfile(new_filename, file_size, file_md5sum, dsc_location_id, session)
+                (found, newf) = check_poolfile(new_filename, old_dat['size'], old_dat['md5sum'], dsc_location_id, session)

 
 

+                # TODO: Uhm, what happens if newf isn't None - something has gone badly and we should cope

                 if newf is None:
                     utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
                     newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
 
                 if newf is None:
                     utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
                     newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
 

-                    # TODO: Check that there's only 1 here
-                    source = get_sources_from_name(u.pkg.changes["source"], u.pkg.changes["version"])[0]
-                    dscf = get_dscfiles(source_id=source.source_id, poolfile_id=orig_file_id, session=session)[0]
-                    dscf.poolfile_id = newf.file_id
-                    session.add(dscf)

                     session.flush()
 
                     session.flush()
 

+                    # Don't reference the old file from this changes
+                    for p in poolfiles:
+                        if p.file_id == oldf.file_id:
+                            poolfiles.remove(p)
+
+                    poolfiles.append(newf)
+
+                    # Fix up the DSC references
+                    toremove = []
+
+                    for df in source.srcfiles:
+                        if df.poolfile.file_id == oldf.file_id:
+                            # Add a new DSC entry and mark the old one for deletion
+                            # Don't do it in the loop so we don't change the thing we're iterating over
+                            newdscf = DSCFile()
+                            newdscf.source_id = source.source_id
+                            newdscf.poolfile_id = newf.file_id
+                            session.add(newdscf)
+
+                            toremove.append(df)
+
+                    for df in toremove:
+                        session.delete(df)
+
+                    # Flush our changes
+                    session.flush()
+
+                    # Make sure that our source object is up-to-date
+                    session.expire(source)
+

         # Install the files into the pool
         # Install the files into the pool

-        for newfile, entry in u.pkg.files.items():
+        for newfile, entry in self.pkg.files.items():

             destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
             utils.move(newfile, destination)
             destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
             utils.move(newfile, destination)

-            Logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
-            summarystats.accept_bytes += float(entry["size"])
+            self.logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
+            stats.accept_bytes += float(entry["size"])

 
         # Copy the .changes file across for suite which need it.
         copy_changes = {}
 
         # Copy the .changes file across for suite which need it.
         copy_changes = {}

-        for suite_name in u.pkg.changes["distribution"].keys():
+        for suite_name in self.pkg.changes["distribution"].keys():

             if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)):
                 copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = ""
 
         for dest in copy_changes.keys():
             if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)):
                 copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = ""
 
         for dest in copy_changes.keys():

-            utils.copy(u.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
+            utils.copy(self.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))

 
         # We're done - commit the database changes
         session.commit()
 
         # We're done - commit the database changes
         session.commit()


@@ -1908,32 +1921,23 @@ distribution."""
         # the last commit
 
         # Move the .changes into the 'done' directory
         # the last commit
 
         # Move the .changes into the 'done' directory

-        utils.move(u.pkg.changes_file,
-                   os.path.join(cnf["Dir::Queue::Done"], os.path.basename(u.pkg.changes_file)))
-
-        if u.pkg.changes["architecture"].has_key("source") and log_urgency:
-            UrgencyLog().log(u.pkg.dsc["source"], u.pkg.dsc["version"], u.pkg.changes["urgency"])
-
-        # Send accept mail, announce to lists, close bugs and check for
-        # override disparities
-        if not cnf["Dinstall::Options::No-Mail"]:
-            self.update_subst()
-            self.Subst["__SUITE__"] = ""
-            self.Subst["__SUMMARY__"] = summary
-            mail_message = utils.TemplateSubst(self.Subst, accepttemplate)
-            utils.send_mail(mail_message)
-            self.announce(short_summary, 1)
+        utils.move(self.pkg.changes_file,
+                   os.path.join(cnf["Dir::Queue::Done"], os.path.basename(self.pkg.changes_file)))

 
 

-        ## Helper stuff for DebBugs Version Tracking
-        if cnf.Find("Dir::Queue::BTSVersionTrack"):
-            # ??? once queue/* is cleared on *.d.o and/or reprocessed
-            # the conditionalization on dsc["bts changelog"] should be
-            # dropped.
+        if self.pkg.changes["architecture"].has_key("source") and cnf.get("Dir::UrgencyLog"):
+            UrgencyLog().log(self.pkg.dsc["source"], self.pkg.dsc["version"], self.pkg.changes["urgency"])

 
 

-            # Write out the version history from the changelog
-            if self.pkg.changes["architecture"].has_key("source") and \
-               self.pkg.dsc.has_key("bts changelog"):
+        self.update_subst()
+        self.Subst["__SUITE__"] = ""
+        self.Subst["__SUMMARY__"] = summary
+        mail_message = utils.TemplateSubst(self.Subst,
+                                           os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted'))
+        utils.send_mail(mail_message)
+        self.announce(short_summary, 1)

 
 

+        ## Helper stuff for DebBugs Version Tracking
+        if cnf.Find("Dir::Queue::BTSVersionTrack"):
+            if self.pkg.changes["architecture"].has_key("source"):

                 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
                 version_history = os.fdopen(fd, 'w')
                 version_history.write(self.pkg.dsc["bts changelog"])
                 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
                 version_history = os.fdopen(fd, 'w')
                 version_history.write(self.pkg.dsc["bts changelog"])


@@ -1958,16 +1962,19 @@ distribution."""
             os.rename(temp_filename, filename)
             os.chmod(filename, 0644)
 
             os.rename(temp_filename, filename)
             os.chmod(filename, 0644)
 

-        # auto-build queue
-#        res = get_or_set_queue('buildd', session).autobuild_upload(self.pkg, session)
-#        if res:
-#            utils.fubar(res)
-#            now_date = datetime.now()
+        session.commit()
+
+        # Set up our copy queues (e.g. buildd queues)
+        for suite_name in self.pkg.changes["distribution"].keys():
+            suite = get_suite(suite_name, session)
+            for q in suite.copy_queues:
+                for f in poolfiles:
+                    q.add_file_from_pool(f)

 
         session.commit()
 
         # Finally...
 
         session.commit()
 
         # Finally...

-        summarystats.accept_count += 1
+        stats.accept_count += 1

 
     def check_override(self):
         """
 
     def check_override(self):
         """


@@ -1981,11 +1988,8 @@ distribution."""
 
         cnf = Config()
 
 
         cnf = Config()
 

-        # Abandon the check if:
-        #  a) override disparity checks have been disabled
-        #  b) we're not sending mail
-        if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
-           cnf["Dinstall::Options::No-Mail"]:
+        # Abandon the check if override disparity checks have been disabled
+        if not cnf.FindB("Dinstall::OverrideDisparityCheck"):

             return
 
         summary = self.pkg.check_override()
             return
 
         summary = self.pkg.check_override()


@@ -2017,22 +2021,22 @@ distribution."""
             os.unlink(os.path.join(from_dir, f))
             if os.path.exists(os.path.join(h.holding_dir, f)):
                 os.unlink(os.path.join(h.holding_dir, f))
             os.unlink(os.path.join(from_dir, f))
             if os.path.exists(os.path.join(h.holding_dir, f)):
                 os.unlink(os.path.join(h.holding_dir, f))

-                          
+

         os.unlink(os.path.join(from_dir, self.pkg.changes_file))
         if os.path.exists(os.path.join(h.holding_dir, self.pkg.changes_file)):
             os.unlink(os.path.join(h.holding_dir, self.pkg.changes_file))
 
     ###########################################################################
 
         os.unlink(os.path.join(from_dir, self.pkg.changes_file))
         if os.path.exists(os.path.join(h.holding_dir, self.pkg.changes_file)):
             os.unlink(os.path.join(h.holding_dir, self.pkg.changes_file))
 
     ###########################################################################
 

-    def move_to_dir (self, dest, perms=0660, changesperms=0664):
+    def move_to_queue (self, queue):

         """
         """

-        Move files to dest with certain perms/changesperms
+        Move files to a destination queue using the permissions in the table

         """
         h = Holding()
         utils.move(os.path.join(h.holding_dir, self.pkg.changes_file),
         """
         h = Holding()
         utils.move(os.path.join(h.holding_dir, self.pkg.changes_file),

-                   dest, perms=changesperms)
+                   queue.path, perms=int(queue.change_perms, 8))

         for f in self.pkg.files.keys():
         for f in self.pkg.files.keys():

-            utils.move(os.path.join(h.holding_dir, f), dest, perms=perms)
+            utils.move(os.path.join(h.holding_dir, f), queue.path, perms=int(queue.perms, 8))

 
     ###########################################################################
 
 
     ###########################################################################
 


@@ -2043,8 +2047,8 @@ distribution."""
         directory it will be moved to the morgue to make way for
         the new file.
 
         directory it will be moved to the morgue to make way for
         the new file.
 

-        @type files: dict
-        @param files: file dictionary
+        @type reject_files: dict
+        @param reject_files: file dictionary

 
         """
 
 
         """
 


@@ -2060,17 +2064,17 @@ distribution."""
             try:
                 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
             except OSError, e:
             try:
                 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
             except OSError, e:

-                # File exists?  Let's try and move it to the morgue
+                # File exists?  Let's find a new name by adding a number

                 if e.errno == errno.EEXIST:
                 if e.errno == errno.EEXIST:

-                    morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)

                     try:
                     try:

-                        morgue_file = utils.find_next_free(morgue_file)
+                        dest_file = utils.find_next_free(dest_file, 255)

                     except NoFreeFilenameError:
                         # Something's either gone badly Pete Tong, or
                         # someone is trying to exploit us.
                     except NoFreeFilenameError:
                         # Something's either gone badly Pete Tong, or
                         # someone is trying to exploit us.

-                        utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
+                        utils.warn("**WARNING** failed to find a free filename for %s in %s." % (file_entry, cnf["Dir::Queue::Reject"]))

                         return
                         return

-                    utils.move(dest_file, morgue_file, perms=0660)
+
+                    # Make sure we really got it

                     try:
                         dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
                     except OSError, e:
                     try:
                         dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
                     except OSError, e:


@@ -2085,7 +2089,7 @@ distribution."""
             os.close(dest_fd)
 
     ###########################################################################
             os.close(dest_fd)
 
     ###########################################################################

-    def do_reject (self, manual=0, reject_message="", note=""):
+    def do_reject (self, manual=0, reject_message="", notes=""):

         """
         Reject an upload. If called without a reject message or C{manual} is
         true, spawn an editor so the user can write one.
         """
         Reject an upload. If called without a reject message or C{manual} is
         true, spawn an editor so the user can write one.


@@ -2104,9 +2108,10 @@ distribution."""
         if manual and not reject_message:
             (fd, temp_filename) = utils.temp_filename()
             temp_file = os.fdopen(fd, 'w')
         if manual and not reject_message:
             (fd, temp_filename) = utils.temp_filename()
             temp_file = os.fdopen(fd, 'w')

-            if len(note) > 0:
-                for line in note:
-                    temp_file.write(line)
+            if len(notes) > 0:
+                for note in notes:
+                    temp_file.write("\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
+                                    % (note.author, note.version, note.notedate, note.comment))

             temp_file.close()
             editor = os.environ.get("EDITOR","vi")
             answer = 'E'
             temp_file.close()
             editor = os.environ.get("EDITOR","vi")
             answer = 'E'


@@ -2162,6 +2167,7 @@ distribution."""
             user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
             self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
             self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
             user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
             self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
             self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message

+            self.Subst["__REJECT_MESSAGE__"] = ""

             self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
             # Write the rejection email out as the <foo>.reason file
             self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
             # Write the rejection email out as the <foo>.reason file


@@ -2173,9 +2179,8 @@ distribution."""
 
         os.close(reason_fd)
 
 
         os.close(reason_fd)
 

-        # Send the rejection mail if appropriate
-        if not cnf["Dinstall::Options::No-Mail"]:
-            utils.send_mail(reject_mail_message)
+        # Send the rejection mail
+        utils.send_mail(reject_mail_message)

 
         if self.logger:
             self.logger.log(["rejected", self.pkg.changes_file])
 
         if self.logger:
             self.logger.log(["rejected", self.pkg.changes_file])


@@ -2330,7 +2335,7 @@ distribution."""
                             cansave = 1
 
                     if not cansave:
                             cansave = 1
 
                     if not cansave:

-                        self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
+                        self.rejects.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))

 
     ################################################################################
     def check_binary_against_db(self, filename, session):
 
     ################################################################################
     def check_binary_against_db(self, filename, session):


@@ -2353,8 +2358,6 @@ distribution."""
     ################################################################################
 
     def check_source_against_db(self, filename, session):
     ################################################################################
 
     def check_source_against_db(self, filename, session):

-        """
-        """

         source = self.pkg.dsc.get("source")
         version = self.pkg.dsc.get("version")
 
         source = self.pkg.dsc.get("source")
         version = self.pkg.dsc.get("version")
 


@@ -2423,11 +2426,19 @@ distribution."""
                                 # This would fix the stupidity of changing something we often iterate over
                                 # whilst we're doing it
                                 del self.pkg.files[dsc_name]
                                 # This would fix the stupidity of changing something we often iterate over
                                 # whilst we're doing it
                                 del self.pkg.files[dsc_name]

+                                dsc_entry["files id"] = i.file_id

                                 if not orig_files.has_key(dsc_name):
                                     orig_files[dsc_name] = {}
                                 orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
                                 match = 1
 
                                 if not orig_files.has_key(dsc_name):
                                     orig_files[dsc_name] = {}
                                 orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
                                 match = 1
 

+                                # Don't bitch that we couldn't find this file later
+                                try:
+                                    self.later_check_files.remove(dsc_name)
+                                except ValueError:
+                                    pass
+
+

                     if not match:
                         self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
 
                     if not match:
                         self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
 


@@ -2477,7 +2488,7 @@ distribution."""
                 else:
                     # TODO: Record the queues and info in the DB so we don't hardcode all this crap
                     # Not there? Check the queue directories...
                 else:
                     # TODO: Record the queues and info in the DB so we don't hardcode all this crap
                     # Not there? Check the queue directories...

-                    for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
+                    for directory in [ "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:

                         if not Cnf.has_key("Dir::Queue::%s" % (directory)):
                             continue
                         in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
                         if not Cnf.has_key("Dir::Queue::%s" % (directory)):
                             continue
                         in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)


@@ -2526,7 +2537,7 @@ distribution."""
                     source_epochless_version = re_no_epoch.sub('', source_version)
                     dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
                     found = False
                     source_epochless_version = re_no_epoch.sub('', source_version)
                     dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
                     found = False

-                    for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
+                    for q in ["Embargoed", "Unembargoed", "Newstage"]:

                         if cnf.has_key("Dir::Queue::%s" % (q)):
                             if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
                                 found = True
                         if cnf.has_key("Dir::Queue::%s" % (q)):
                             if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
                                 found = True