Merge commit 'pkern/master'

* commit 'pkern/master':
  dak/process_new.py: fix up binary diversion into various queues
  dak/process_new.py: handle uploads to (oldstable)-proposed-updates differently

diff --git a/ChangeLog b/ChangeLog
index 6af8f757551753895794fc7fb750f920e6d08d9b..bed4e2df77abf78c4c2b41d549f1bd5e40a4d47d 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2008-08-08  Joerg Jaspert  <joerg@debian.org>
+
+       * config/debian/cron.dinstall: Create a hardlinked tree of the
+       ftp/ in mirror/ so we have more atomic mirror updates for the
+       buildds
+
+       * config/debian/cron.unchecked: Added signing of buildd incoming
+
 2008-08-07  Philipp Kern  <pkern@debian.org>
 
        * dak/process_new.py (do_accept): handle uploads to (oldstable-)

diff --git a/config/debian/apt.conf.buildd b/config/debian/apt.conf.buildd
index 2ad4483c2306be4e66161feec9f6d25f678c28e8..65a8363e74ec7f4046d57a1c89edfff3c9a36617 100644 (file)
--- a/config/debian/apt.conf.buildd
+++ b/config/debian/apt.conf.buildd
@@ -7,8 +7,8 @@ Dir
 
 Default
 {
-   Packages::Compress "bzip2 gzip";
-   Sources::Compress "bzip2 gzip";
+   Packages::Compress ". bzip2 gzip";
+   Sources::Compress ". bzip2 gzip";
    DeLinkLimit 0;
    FileMode 0664;
 }

diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index b8b25d7a3d0911b072bb57991ad596825ac03117..6704fe4e0fe82393b5b0c895abaecbe7d9b242a0 100755 (executable)
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -24,6 +24,7 @@ TS=0
 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
 LOCKCU="$lockdir/daily.lock"
 LOCKAC="$lockdir/unchecked.lock"
+BRITNEYLOCK="$lockdir/britney.lock"
 lockac=0
 
 cleanup() {
@@ -37,6 +38,11 @@ cleanup() {
 lockfile -l 3600 $LOCKCU
 trap cleanup 0
 
+# This file is simply used to indicate to britney whether or not
+# the Packages file updates completed sucessfully.  It's not a lock
+# from our point of view
+touch ${BRITNEYLOCK}
+
 rm -f "$NOTICE"
 cat > "$NOTICE" <<EOF
 Packages are currently being installed and indices rebuilt.
@@ -172,6 +178,14 @@ cd $scriptsdir
 ./mkfilesindices
 ./mkchecksums
 
+ts
+
+# (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
+echo "Regenerating mirror/ hardlink fun"
+cd ${mirrordir}
+rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
+
+
 ts
 
 echo "Trigger daily wanna-build run"
@@ -240,6 +254,9 @@ R --slave --vanilla < $base/misc/ftpstats.R
 
 ts
 
+# Remove the britney lock
+rm -f ${BRITNEYLOCK}
+
 # Clean up apt-ftparchive's databases
 echo "Clean up apt-ftparchive's databases"
 cd $configdir

diff --git a/config/debian/cron.unchecked b/config/debian/cron.unchecked
index 92f40cc90db0a722063d36c78e9ea491f9e3ba46..406636ef65b0e0504e0c92bfdcae143c4420208c 100755 (executable)
--- a/config/debian/cron.unchecked
+++ b/config/debian/cron.unchecked
@@ -11,47 +11,62 @@ NOTICE="$lockdir/daily.lock"
 
 if [ -e $NOTICE ]; then exit 0; fi
 
+STAMP=$(date "+%Y%m%d%H%M")
+
 cleanup() {
-  rm -f "$LOCKFILE"
-  if [ ! -z "$LOCKDAILY" ]; then
-         rm -f "$NOTICE"
-  fi
+    rm -f "$LOCKFILE"
+    if [ ! -z "$LOCKDAILY" ]; then
+       rm -f "$NOTICE"
+    fi
 }
 
 # only run one cron.unchecked
 if lockfile -r3 $LOCKFILE; then
-       trap cleanup 0
-       cd $unchecked
-
-       changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs)
-       report=$queuedir/REPORT
-       timestamp=$(date "+%Y-%m-%d %H:%M")
-
-       if [ ! -z "$changes" ]; then
-               echo "$timestamp": "$changes"  >> $report
-               dak process-unchecked -a $changes >> $report
-               echo "--" >> $report
-
-               # sync with debbugs
-               rsync -aq --remove-source-files  $queuedir/bts_version_track/ bugs-sync:/org/bugs.debian.org/versions/queue/ftp-master/
-
-               if lockfile -r3 $NOTICE; then
-                       LOCKDAILY="YES"
-                       psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE queue = 0 AND suite = 5 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
-                       cd $overridedir
-                       dak make-overrides &>/dev/null
-                       rm -f override.sid.all3 override.sid.all3.src
-                       for i in main contrib non-free main.debian-installer; do
-                               cat override.sid.$i >> override.sid.all3
-                               if [ "$i" != "main.debian-installer" ]; then
-                                       cat override.sid.$i.src >> override.sid.all3.src
-                               fi
-                       done
-                       cd $configdir
-                       apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate apt.conf.buildd
-                       . $configdir/cron.buildd
+    trap cleanup 0
+    cd $unchecked
+
+    changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs)
+    report=$queuedir/REPORT
+    timestamp=$(date "+%Y-%m-%d %H:%M")
+
+    if [ ! -z "$changes" ]; then
+       echo "$timestamp": "$changes"  >> $report
+       dak process-unchecked -a $changes >> $report
+       echo "--" >> $report
+
+       # sync with debbugs
+       rsync -aq --remove-source-files  $queuedir/bts_version_track/ bugs-sync:/org/bugs.debian.org/versions/queue/ftp-master/
+
+       if lockfile -r3 $NOTICE; then
+           LOCKDAILY="YES"
+           psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE queue = 0 AND suite = 5 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
+           cd $overridedir
+           dak make-overrides &>/dev/null
+           rm -f override.sid.all3 override.sid.all3.src
+           for i in main contrib non-free main.debian-installer; do
+               cat override.sid.$i >> override.sid.all3
+               if [ "$i" != "main.debian-installer" ]; then
+                   cat override.sid.$i.src >> override.sid.all3.src
                fi
-       else
-               echo "$timestamp": Nothing to do >> $report
+           done
+           cd $configdir
+           apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate apt.conf.buildd
+
+           cd  ${incoming}
+           rm -f buildd/Release*
+           apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="Debian" -o APT::FTPArchive::Release::Label="Debian" -o APT::FTPArchive::Release::Description="buildd incoming" -o APT::FTPArchive::Release::Architectures="${archs}" release buildd > Release
+           gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o Release.gpg Release 
+               mv Release* buildd/.
+
+           cd ${incoming}
+           mkdir -p tree/${STAMP}
+           cp -al ${incoming}/buildd/. tree/${STAMP}/
+           ln -sfT tree/${STAMP} ${incoming}/builddweb
+           find ./tree -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
+
+           . $configdir/cron.buildd
        fi
+    else
+               echo "$timestamp": Nothing to do >> $report
+    fi
 fi

diff --git a/config/debian/vars b/config/debian/vars
index 3f993fadda259afc24bcac9214a311848fb11b33..ef4b670c75ee5dd1f0e1ced12e11cb7320d65051 100644 (file)
--- a/config/debian/vars
+++ b/config/debian/vars
@@ -19,6 +19,7 @@ logdir=$base/log/cron/
 queuedir=$base/queue/
 unchecked=$queuedir/unchecked/
 accepted=$queuedir/accepted/
+mirrordir=$base/mirror/
 incoming=$base/incoming
 
 ftpgroup=debadmin