X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fcron.dinstall;h=6704fe4e0fe82393b5b0c895abaecbe7d9b242a0;hb=6e18b0d9c051739fa15d5de4e44def11998653fc;hp=d8a69d62de4478af48045c1ad2c4b0fa368993ec;hpb=d7bdf461c9c5cdc1d5502917f5b306237f65ed83;p=dak.git

diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index d8a69d62..6704fe4e 100755
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -13,12 +13,18 @@ NOW=`date "+%Y.%m.%d-%H:%M:%S"`
 LOGFILE="$logdir/dinstall_${NOW}.log"
 exec > "$LOGFILE" 2>&1
 
-echo Archive maintenance started at $(date +%X)
+ts() {
+	TS=$(($TS+1));
+	echo Archive maintenance timestamp $TS: $(date +%H:%M:%S)
+}
+
+echo Archive maintenance started at $(date +%H:%M:%S)
 TS=0
 
 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
 LOCKCU="$lockdir/daily.lock"
 LOCKAC="$lockdir/unchecked.lock"
+BRITNEYLOCK="$lockdir/britney.lock"
 lockac=0
 
 cleanup() {
@@ -32,18 +38,24 @@ cleanup() {
 lockfile -l 3600 $LOCKCU
 trap cleanup 0
 
+# This file is simply used to indicate to britney whether or not
+# the Packages file updates completed sucessfully.  It's not a lock
+# from our point of view
+touch ${BRITNEYLOCK}
+
 rm -f "$NOTICE"
 cat > "$NOTICE" <<EOF
 Packages are currently being installed and indices rebuilt.
-Maintenance is automatic, starting at 13:52 US Central time, and
-ending at about 15:30.  This file is then removed.
+Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
+ending about an hour later.  This file is then removed.
 
 You should not mirror the archive during this period.
 EOF
 
 # Push merkels qa user, so the qa pages can show "dinstall is running" information
 echo "Telling merkels QA user that we start dinstall"
-ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
+ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
+ts
 
 ################################################################################
 
@@ -52,7 +64,7 @@ pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
 
 ################################################################################
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
 cd $configdir
 $scriptsdir/update-bugdoctxt
@@ -61,23 +73,23 @@ $scriptsdir/update-mailingliststxt
 
 ################################################################################
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 echo "Doing automated p-u-new processing"
 cd $queuedir/p-u-new
 date -u -R >> REPORT
-dak process-new -a -C COMMENTS >> REPORT
+dak process-new -a -C COMMENTS >> REPORT || true
 echo >> REPORT
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 echo "Doing automated o-p-u-new processing"
 cd $queuedir/o-p-u-new
 date -u -R >> REPORT
-dak process-new -a -C COMMENTS >> REPORT
+dak process-new -a -C COMMENTS >> REPORT || true
 echo >> REPORT
 
 ################################################################################
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 lockfile $LOCKAC
 lockac=1
 echo "Processing queue/accepted"
@@ -88,7 +100,7 @@ dak process-accepted -pa *.changes | tee REPORT | \
 chgrp debadmin REPORT
 chmod 664 REPORT
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 echo "Checking for cruft in overrides"
 dak check-overrides
 rm -f $LOCKAC
@@ -97,57 +109,57 @@ lockac=0
 echo "Fixing symlinks in $ftpdir"
 symlinks -d -r $ftpdir
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 echo "Generating suite file lists for apt-ftparchive"
 dak make-suite-file-list
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 echo "Updating fingerprints"
 # Update fingerprints
-dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
+dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 # Generate override files
 echo "Writing overrides into text files"
 cd $overridedir
 dak make-overrides
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 # FIXME
 rm -f override.sid.all3
 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 # Generate Packages and Sources files
 echo "Generating Packages and Sources files"
 cd $configdir
 apt-ftparchive generate apt.conf
-# Generate *.diff/ incremental updates
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
+# Generate *.diff/ incremental updates
 echo "Generating pdiff files"
 dak generate-index-diffs
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 # Generate Release files
 echo "Generating Release files"
 dak generate-releases
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 # Clean out old packages
 echo "Cleanup old packages/files"
 dak clean-suites
 dak clean-queues
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 # Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
 # update this before wanna-build is updated.
@@ -156,7 +168,7 @@ psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND
 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
 apt-ftparchive generate apt.conf.buildd
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 echo "Running various scripts from $scriptsdir"
 cd $scriptsdir
@@ -165,16 +177,23 @@ cd $scriptsdir
 ./mklslar
 ./mkfilesindices
 ./mkchecksums
-#
-rm -f $NOTICE
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
+
+# (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
+echo "Regenerating mirror/ hardlink fun"
+cd ${mirrordir}
+rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
+
+
+ts
 
 echo "Trigger daily wanna-build run"
-ssh buildd@buildd /org/wanna-build/trigger.daily
+ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 buildd@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
 
+rm -f $NOTICE
 rm -f $LOCKCU
-echo Archive maintenance finished at $(date +%X)
+echo Archive maintenance finished at $(date +%H:%M:%S)
 
 ################################################################################
 
@@ -185,14 +204,14 @@ pg_dump projectb > $POSTDUMP
 
 ################################################################################
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 echo "Expiring old database dumps..."
 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
 
 ################################################################################
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 # Send a report on NEW/BYHAND packages
 echo "Nagging ftpteam about NEW/BYHAND packages"
@@ -203,7 +222,7 @@ dak cruft-report > $webdir/cruft-report-daily.txt
 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 echo "Updating DM html page"
 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
@@ -212,11 +231,11 @@ $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
 
 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
 echo "Trigger merkels projectb sync"
-ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
+ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
 
 ################################################################################
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 ulimit -m 90000 -d 90000 -s 10000 -v 200000
 
@@ -225,7 +244,7 @@ run-parts --report $base/scripts/distmnt
 
 echo "Daily cron scripts successful."
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 # Stats pr0n
 echo "Updating stats data"
@@ -233,26 +252,29 @@ cd $configdir
 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
 R --slave --vanilla < $base/misc/ftpstats.R
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
+
+# Remove the britney lock
+rm -f ${BRITNEYLOCK}
 
 # Clean up apt-ftparchive's databases
 echo "Clean up apt-ftparchive's databases"
 cd $configdir
 apt-ftparchive -q clean apt.conf
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
-# Compress psql backups older than a week, but no more than 20 of them
+# Compress psql backups
 echo "Compress old psql backups"
 (cd $base/backup/
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | 
-   sort | head -n20 | while read dumpname; do
-     echo "Compressing $dumpname"
-     bzip2 -9 "$dumpname"
-   done
+	find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 | 
+	while read dumpname; do
+		echo "Compressing $dumpname"
+		bzip2 -9 "$dumpname"
+	done
 )
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
 
 echo "Finally, all is done, sending mail and compressing logfile"
 exec > /dev/null 2>&1