+2008-05-03 Joerg Jaspert <joerg@debian.org>
+
+ * config/debian/cron.daily: Rename to
+ * config/debian/cron.dinstall: this, as its not really something run
+ daily for some time now. And while dinstall is an OLD name, its
+ recognized pretty well within Debian
+ Also add some more timing information
+
2008-05-03 Thomas Viehmann <tv@beamnet.de>
* dak/process_unchecked.py (check_hashes): Reject on error while
+++ /dev/null
-#! /bin/sh
-#
-# Executed daily via cron, out of dak's crontab.
-
-set -e
-export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-################################################################################
-
-# Start logging
-NOW=`date "+%Y.%m.%d-%H:%M:%S"`
-LOGFILE="$logdir/daily_${NOW}.log"
-exec > "$LOGFILE" 2>&1
-
-echo Archive maintenance started at $(date +%X)
-TS=0
-
-NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
-LOCKCU="$lockdir/daily.lock"
-LOCKAC="$lockdir/unchecked.lock"
-lockac=0
-
-cleanup() {
- rm -f "$NOTICE"
- rm -f "$LOCKCU"
- if [ "$lockac" -eq "1" ]; then
- rm -f "$LOCKAC"
- fi
- echo "Cleanup"
- rm -f "$LOGFILE"
-}
-lockfile -l 3600 $LOCKCU
-trap cleanup 0
-
-rm -f "$NOTICE"
-cat > "$NOTICE" <<EOF
-Packages are currently being installed and indices rebuilt.
-Maintenance is automatic, starting at 13:52 US Central time, and
-ending at about 15:30. This file is then removed.
-
-You should not mirror the archive during this period.
-EOF
-
-# Push merkels qa user, so the qa pages can show "dinstall is running" information
-echo "Telling merkels QA user that we start dinstall"
-ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
-
-################################################################################
-
-echo "Creating pre-daily-cron-job backup of projectb database..."
-pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
-cd $configdir
-$scriptsdir/update-bugdoctxt
-$scriptsdir/update-mirrorlists
-$scriptsdir/update-mailingliststxt
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-echo "Doing automated p-u-new processing"
-cd $queuedir/p-u-new
-date -u -R >> REPORT
-dak process-new -a -C COMMENTS >> REPORT
-echo >> REPORT
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-echo "Doing automated o-p-u-new processing"
-cd $queuedir/o-p-u-new
-date -u -R >> REPORT
-dak process-new -a -C COMMENTS >> REPORT
-echo >> REPORT
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-lockfile $LOCKAC
-lockac=1
-echo "Processing queue/accepted"
-cd $accepted
-rm -f REPORT
-dak process-accepted -pa *.changes | tee REPORT | \
- mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
-chgrp debadmin REPORT
-chmod 664 REPORT
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-echo "Checking for cruft in overrides"
-dak check-overrides
-rm -f $LOCKAC
-lockac=0
-
-echo "Fixing symlinks in $ftpdir"
-symlinks -d -r $ftpdir
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-echo "Generating suite file lists for apt-ftparchive"
-dak make-suite-file-list
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-echo "Updating fingerprints"
-# Update fingerprints
-dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
-
-# Generate override files
-echo "Writing overrides into text files"
-cd $overridedir
-dak make-overrides
-
-# FIXME
-rm -f override.sid.all3
-for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Generate Packages and Sources files
-echo "Generating Packages and Sources files"
-cd $configdir
-apt-ftparchive generate apt.conf
-# Generate *.diff/ incremental updates
-echo "Generating pdiff files"
-dak generate-index-diffs
-# Generate Release files
-echo "Generating Release files"
-dak generate-releases
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Clean out old packages
-echo "Cleanup old packages/files"
-dak clean-suites
-dak clean-queues
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
-# update this before wanna-build is updated.
-echo "Regenerating wanna-build/buildd information"
-psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
-symlinks -d /srv/incoming.debian.org/buildd > /dev/null
-apt-ftparchive generate apt.conf.buildd
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-echo "Running various scripts from $scriptsdir"
-cd $scriptsdir
-./mkmaintainers
-./copyoverrides
-./mklslar
-./mkfilesindices
-./mkchecksums
-#
-rm -f $NOTICE
-echo "Trigger daily wanna-build run"
-ssh buildd@buildd /org/wanna-build/trigger.daily
-
-rm -f $LOCKCU
-echo Archive maintenance finished at $(date +%X)
-
-################################################################################
-
-echo "Creating post-daily-cron-job backup of projectb database..."
-POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
-pg_dump projectb > $POSTDUMP
-(cd $base/backup; ln -sf $POSTDUMP current)
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-echo "Expiring old database dumps..."
-(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Send a report on NEW/BYHAND packages
-echo "Nagging ftpteam about NEW/BYHAND packages"
-dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
-# and one on crufty packages
-echo "Sending information about crufty packages"
-dak cruft-report > $webdir/cruft-report-daily.txt
-dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
-cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
-
-echo "Updating DM html page"
-$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
-
-################################################################################
-
-# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
-echo "Trigger merkels projectb sync"
-ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-ulimit -m 90000 -d 90000 -s 10000 -v 200000
-
-echo "Using run-parts to run scripts in $base/scripts/distmnt"
-run-parts --report $base/scripts/distmnt
-
-echo "Daily cron scripts successful."
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Stats pr0n
-echo "Updating stats data"
-cd $configdir
-$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
-R --slave --vanilla < $base/misc/ftpstats.R
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Clean up apt-ftparchive's databases
-echo "Clean up apt-ftparchive's databases"
-cd $configdir
-apt-ftparchive -q clean apt.conf
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Compress psql backups older than a week, but no more than 20 of them
-echo "Compress old psql backups"
-(cd $base/backup/
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 |
- sort | head -n20 | while read dumpname; do
- echo "Compressing $dumpname"
- bzip2 -9 "$dumpname"
- done
-)
-
-echo "Finally, all is done, sending mail and compressing logfile"
-exec > /dev/null 2>&1
-
-cat "$LOGFILE" | mail -s "Log for cron.daily run of $(date +%Y.%m.%d)" cron@ftp-master.debian.org
-bzip2 -9 "$LOGFILE"
-
-################################################################################
--- /dev/null
+#! /bin/sh
+#
+# Executed daily via cron, out of dak's crontab.
+
+set -e
+export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
+. $SCRIPTVARS
+
+################################################################################
+
+# Start logging
+NOW=`date "+%Y.%m.%d-%H:%M:%S"`
+LOGFILE="$logdir/dinstall_${NOW}.log"
+exec > "$LOGFILE" 2>&1
+
+echo Archive maintenance started at $(date +%X)
+TS=0
+
+NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
+LOCKCU="$lockdir/daily.lock"
+LOCKAC="$lockdir/unchecked.lock"
+lockac=0
+
+cleanup() {
+ rm -f "$NOTICE"
+ rm -f "$LOCKCU"
+ if [ "$lockac" -eq "1" ]; then
+ rm -f "$LOCKAC"
+ fi
+ echo "Cleanup"
+ rm -f "$LOGFILE"
+}
+lockfile -l 3600 $LOCKCU
+trap cleanup 0
+
+rm -f "$NOTICE"
+cat > "$NOTICE" <<EOF
+Packages are currently being installed and indices rebuilt.
+Maintenance is automatic, starting at 13:52 US Central time, and
+ending at about 15:30. This file is then removed.
+
+You should not mirror the archive during this period.
+EOF
+
+# Push merkels qa user, so the qa pages can show "dinstall is running" information
+echo "Telling merkels QA user that we start dinstall"
+ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
+
+################################################################################
+
+echo "Creating pre-daily-cron-job backup of projectb database..."
+pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
+
+################################################################################
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
+cd $configdir
+$scriptsdir/update-bugdoctxt
+$scriptsdir/update-mirrorlists
+$scriptsdir/update-mailingliststxt
+
+################################################################################
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+echo "Doing automated p-u-new processing"
+cd $queuedir/p-u-new
+date -u -R >> REPORT
+dak process-new -a -C COMMENTS >> REPORT
+echo >> REPORT
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+echo "Doing automated o-p-u-new processing"
+cd $queuedir/o-p-u-new
+date -u -R >> REPORT
+dak process-new -a -C COMMENTS >> REPORT
+echo >> REPORT
+
+################################################################################
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+lockfile $LOCKAC
+lockac=1
+echo "Processing queue/accepted"
+cd $accepted
+rm -f REPORT
+dak process-accepted -pa *.changes | tee REPORT | \
+ mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
+chgrp debadmin REPORT
+chmod 664 REPORT
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+echo "Checking for cruft in overrides"
+dak check-overrides
+rm -f $LOCKAC
+lockac=0
+
+echo "Fixing symlinks in $ftpdir"
+symlinks -d -r $ftpdir
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Generating suite file lists for apt-ftparchive"
+dak make-suite-file-list
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Updating fingerprints"
+# Update fingerprints
+dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Generate override files
+echo "Writing overrides into text files"
+cd $overridedir
+dak make-overrides
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# FIXME
+rm -f override.sid.all3
+for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Generate Packages and Sources files
+echo "Generating Packages and Sources files"
+cd $configdir
+apt-ftparchive generate apt.conf
+# Generate *.diff/ incremental updates
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Generating pdiff files"
+dak generate-index-diffs
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Generate Release files
+echo "Generating Release files"
+dak generate-releases
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Clean out old packages
+echo "Cleanup old packages/files"
+dak clean-suites
+dak clean-queues
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
+# update this before wanna-build is updated.
+echo "Regenerating wanna-build/buildd information"
+psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
+symlinks -d /srv/incoming.debian.org/buildd > /dev/null
+apt-ftparchive generate apt.conf.buildd
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Running various scripts from $scriptsdir"
+cd $scriptsdir
+./mkmaintainers
+./copyoverrides
+./mklslar
+./mkfilesindices
+./mkchecksums
+#
+rm -f $NOTICE
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Trigger daily wanna-build run"
+ssh buildd@buildd /org/wanna-build/trigger.daily
+
+rm -f $LOCKCU
+echo Archive maintenance finished at $(date +%X)
+
+################################################################################
+
+echo "Creating post-daily-cron-job backup of projectb database..."
+POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
+pg_dump projectb > $POSTDUMP
+(cd $base/backup; ln -sf $POSTDUMP current)
+
+################################################################################
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Expiring old database dumps..."
+(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
+
+################################################################################
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Send a report on NEW/BYHAND packages
+echo "Nagging ftpteam about NEW/BYHAND packages"
+dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
+# and one on crufty packages
+echo "Sending information about crufty packages"
+dak cruft-report > $webdir/cruft-report-daily.txt
+dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
+cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Updating DM html page"
+$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
+
+################################################################################
+
+# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
+echo "Trigger merkels projectb sync"
+ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
+
+################################################################################
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+ulimit -m 90000 -d 90000 -s 10000 -v 200000
+
+echo "Using run-parts to run scripts in $base/scripts/distmnt"
+run-parts --report $base/scripts/distmnt
+
+echo "Daily cron scripts successful."
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Stats pr0n
+echo "Updating stats data"
+cd $configdir
+$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
+R --slave --vanilla < $base/misc/ftpstats.R
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Clean up apt-ftparchive's databases
+echo "Clean up apt-ftparchive's databases"
+cd $configdir
+apt-ftparchive -q clean apt.conf
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Compress psql backups older than a week, but no more than 20 of them
+echo "Compress old psql backups"
+(cd $base/backup/
+ find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 |
+ sort | head -n20 | while read dumpname; do
+ echo "Compressing $dumpname"
+ bzip2 -9 "$dumpname"
+ done
+)
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Finally, all is done, sending mail and compressing logfile"
+exec > /dev/null 2>&1
+
+cat "$LOGFILE" | mail -s "Log for dinstall run of $(date +%Y.%m.%d' at '%H:%M)" cron@ftp-master.debian.org
+bzip2 -9 "$LOGFILE"
+
+################################################################################
projects / dak.git / commitdiff