Merge remote-tracking branch 'ansgar/pu/multiarchive-2' into merge

* ansgar/pu/multiarchive-2: (51 commits)
  dak/show_new.py: remove obsolete import
  config/debian/*: update for multiarchive changes
  dak/contents.py, daklib/contents.py: add archive option
  dak/export.py, daklib/policy.py: ignore existing files
  Revert "config/*/common: do not touch incoming"
  add export-suite subcommand to export a suite to a flat directory
  dak/clean_suites.py: allow limiting file removal to specific archives
  dak/process_upload.py: ignore non-existing files when rejecting
  dak/process_upload.py: add missing import
  daklib/archive.py: try to get all source files from pool, even when listed in changes
  daklib/archive.py: also ignore files from .changes that do not exist
  daklib/archive.py: ignore missing source files when copying to temporary directory
  dak/dakdb/update75.py: fix adding architectures for new and byhand
  dak/clean_suites.py: implement per-archive stay of execution
  dak/dakdb/update77.py: move stayofexecution to the database
  dak/control_suite.py: prefetch files table when looking up a binary or source package
  dak/generate_releases.py: add archive option
  dak/generate_packages_sources2.py: add archive option
  indicate who rejected a package
  config/*/dinstall.functions: include Built-Using sources
  ...

Signed-off-by: Joerg Jaspert <joerg@debian.org>

diff --combined config/debian/dinstall.functions
index ae534ba60e3a60a66d8531456bacfb20ff70ea15,d6870a7c4ef78b2a6459e3269c544b2c2b26feb0..e9f8bfd3af6c0b784d140379f9567d71bae3817c
--- 1/config/debian/dinstall.functions
--- 2/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@@ -152,8 -152,8 +152,8 @@@ function mpfm() 
  
  function packages() {
      log "Generating Packages and Sources files"
-     dak generate-packages-sources2
-     dak contents generate
+     dak generate-packages-sources2 -a ftp-master
+     dak contents generate -a ftp-master
  }
  
  function pdiff() {
@@@ -171,13 -171,14 +171,14 @@@ function release() 
          done
      )
      log "Generating Release files"
-     dak generate-releases
+     dak generate-releases -a ftp-master
  }
  
  function dakcleanup() {
      log "Cleanup old packages/files"
      dak clean-suites -m 10000
-     dak clean-queues
+     # XXX: reactivate once clean-queues is fixed
+     #dak clean-queues
  }
  
  function buildd_dir() {
@@@ -193,14 -194,31 +194,14 @@@ function mklslar() 
      FILENAME=ls-lR
  
      log "Removing any core files ..."
 -    find -type f -name core -print0 | xargs -0r rm -v
 -
 -    log "Checking permissions on files in the FTP tree ..."
 -    find -type f \( \! -perm -444 -o -perm +002 \) -ls
 -    find -type d \( \! -perm -555 -o -perm +002 \) -ls
 +    find -type f -name core -print -delete
  
      log "Checking symlinks ..."
      symlinks -rd .
  
      log "Creating recursive directory listing ... "
 -    rm -f .${FILENAME}.new
 -    TZ=UTC ls -lR > .${FILENAME}.new
 -
 -    if [ -r ${FILENAME}.gz ] ; then
 -        mv -f ${FILENAME}.gz ${FILENAME}.old.gz
 -        mv -f .${FILENAME}.new ${FILENAME}
 -        rm -f ${FILENAME}.patch.gz
 -        zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
 -        rm -f ${FILENAME}.old.gz
 -    else
 -        mv -f .${FILENAME}.new ${FILENAME}
 -    fi
 -
 -    gzip -9cfN ${FILENAME} >${FILENAME}.gz
 -    rm -f ${FILENAME}
 +    rm -f ${FILENAME}.gz
 +    TZ=UTC ls -lR | gzip -9c --rsyncable > ${FILENAME}.gz
  }
  
  function mkmaintainers() {
@@@ -223,15 -241,26 +224,27 @@@ function copyoverrides() 
  }
  
  function mkfilesindices() {
 +    set +o pipefail
      umask 002
      cd $base/ftp/indices/files/components
  
      ARCHLIST=$(tempfile)
  
      log "Querying postgres"
-     local query='SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)'
-     psql -At -c "$query" | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
+     local query="
+       SELECT CONCAT('./pool/', c.name, '/', f.filename) AS path, a.arch_string AS arch_string
+       FROM files f
+       JOIN files_archive_map af ON f.id = af.file_id
+       JOIN component c ON af.component_id = c.id
+       JOIN archive ON af.archive_id = archive.id
+       LEFT OUTER JOIN
+         (binaries b
+          JOIN architecture a ON b.architecture = a.id)
+         ON f.id = b.file
+       WHERE archive.name = 'ftp-master'
+       ORDER BY path, arch_string
+     "
+     psql -At -c "$query" >$ARCHLIST
  
      includedirs () {
          perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
@@@ -264,11 -293,42 +277,42 @@@
      log "Generating suite lists"
  
      suite_list () {
+       local suite_id="$(printf %d $1)"
        local query
-       query="$(printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)' $1 $1)"
+       query="
+           SELECT DISTINCT CONCAT('./pool/', c.name, '/', f.filename)
+           FROM
+             (SELECT sa.source AS source
+                FROM src_associations sa
+               WHERE sa.suite = $suite_id
+              UNION
+              SELECT esr.src_id
+                FROM extra_src_references esr
+                JOIN bin_associations ba ON esr.bin_id = ba.bin
+                WHERE ba.suite = $suite_id
+              UNION
+              SELECT b.source AS source
+                FROM bin_associations ba
+                JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s
+             JOIN dsc_files df ON s.source = df.source
+             JOIN files f ON df.file = f.id
+             JOIN files_archive_map af ON f.id = af.file_id
+             JOIN component c ON af.component_id = c.id
+             JOIN archive ON af.archive_id = archive.id
+             WHERE archive.name = 'ftp-master'
+         "
        psql -F' ' -A -t -c "$query"
  
-       query="$(printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d' $1)"
+       query="
+           SELECT CONCAT('./pool/', c.name, '/', f.filename)
+           FROM bin_associations ba
+           JOIN binaries b ON ba.bin = b.id
+           JOIN files f ON b.file = f.id
+           JOIN files_archive_map af ON f.id = af.file_id
+           JOIN component c ON af.component_id = c.id
+           JOIN archive ON af.archive_id = archive.id
+           WHERE ba.suite = $suite_id AND archive.name = 'ftp-master'
+         "
        psql -F' ' -A -t -c "$query"
      }
  
@@@ -283,7 -343,7 +327,7 @@@
                      [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
                  done
              )
-             suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
+             suite_list $id
          ) | sort -u | gzip -9 > suite-${suite}.list.gz
      done
  
@@@ -313,7 -373,6 +357,7 @@@
  
      rm -f $ARCHLIST
      log "Done!"
 +    set -o pipefail
  }
  
  function mkchecksums() {