From: Joerg Jaspert Date: Sat, 3 May 2008 11:35:35 +0000 (+0200) Subject: Rename daily cronscript to dinstall, as its not run daily for a long time now X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=259e0951de653dafa87f3f6b6d1d92a92e6800c9;p=dak.git Rename daily cronscript to dinstall, as its not run daily for a long time now --- diff --git a/ChangeLog b/ChangeLog index fc6e9f88..0407faa1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +2008-05-03 Joerg Jaspert + + * config/debian/cron.daily: Rename to + * config/debian/cron.dinstall: this, as its not really something run + daily for some time now. And while dinstall is an OLD name, its + recognized pretty well within Debian + Also add some more timing information + 2008-05-03 Thomas Viehmann * dak/process_unchecked.py (check_hashes): Reject on error while diff --git a/config/debian/cron.daily b/config/debian/cron.daily deleted file mode 100755 index 09d6a6b3..00000000 --- a/config/debian/cron.daily +++ /dev/null @@ -1,247 +0,0 @@ -#! /bin/sh -# -# Executed daily via cron, out of dak's crontab. - -set -e -export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars -. $SCRIPTVARS - -################################################################################ - -# Start logging -NOW=`date "+%Y.%m.%d-%H:%M:%S"` -LOGFILE="$logdir/daily_${NOW}.log" -exec > "$LOGFILE" 2>&1 - -echo Archive maintenance started at $(date +%X) -TS=0 - -NOTICE="$ftpdir/Archive_Maintenance_In_Progress" -LOCKCU="$lockdir/daily.lock" -LOCKAC="$lockdir/unchecked.lock" -lockac=0 - -cleanup() { - rm -f "$NOTICE" - rm -f "$LOCKCU" - if [ "$lockac" -eq "1" ]; then - rm -f "$LOCKAC" - fi - echo "Cleanup" - rm -f "$LOGFILE" -} -lockfile -l 3600 $LOCKCU -trap cleanup 0 - -rm -f "$NOTICE" -cat > "$NOTICE" < $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) - -################################################################################ - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -echo "Updating Bugs docu, Mirror list and mailing-lists.txt" -cd $configdir -$scriptsdir/update-bugdoctxt -$scriptsdir/update-mirrorlists -$scriptsdir/update-mailingliststxt - -################################################################################ - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -echo "Doing automated p-u-new processing" -cd $queuedir/p-u-new -date -u -R >> REPORT -dak process-new -a -C COMMENTS >> REPORT -echo >> REPORT - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -echo "Doing automated o-p-u-new processing" -cd $queuedir/o-p-u-new -date -u -R >> REPORT -dak process-new -a -C COMMENTS >> REPORT -echo >> REPORT - -################################################################################ - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -lockfile $LOCKAC -lockac=1 -echo "Processing queue/accepted" -cd $accepted -rm -f REPORT -dak process-accepted -pa *.changes | tee REPORT | \ - mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org -chgrp debadmin REPORT -chmod 664 REPORT - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -echo "Checking for cruft in overrides" -dak check-overrides -rm -f $LOCKAC -lockac=0 - -echo "Fixing symlinks in $ftpdir" -symlinks -d -r $ftpdir - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -echo "Generating suite file lists for apt-ftparchive" -dak make-suite-file-list - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -echo "Updating fingerprints" -# Update fingerprints -dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg - -# Generate override files -echo "Writing overrides into text files" -cd $overridedir -dak make-overrides - -# FIXME -rm -f override.sid.all3 -for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -# Generate Packages and Sources files -echo "Generating Packages and Sources files" -cd $configdir -apt-ftparchive generate apt.conf -# Generate *.diff/ incremental updates -echo "Generating pdiff files" -dak generate-index-diffs -# Generate Release files -echo "Generating Release files" -dak generate-releases - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -# Clean out old packages -echo "Cleanup old packages/files" -dak clean-suites -dak clean-queues - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -# Needs to be rebuilt, as files have moved. Due to unaccepts, we need to -# update this before wanna-build is updated. -echo "Regenerating wanna-build/buildd information" -psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list -symlinks -d /srv/incoming.debian.org/buildd > /dev/null -apt-ftparchive generate apt.conf.buildd - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -echo "Running various scripts from $scriptsdir" -cd $scriptsdir -./mkmaintainers -./copyoverrides -./mklslar -./mkfilesindices -./mkchecksums -# -rm -f $NOTICE -echo "Trigger daily wanna-build run" -ssh buildd@buildd /org/wanna-build/trigger.daily - -rm -f $LOCKCU -echo Archive maintenance finished at $(date +%X) - -################################################################################ - -echo "Creating post-daily-cron-job backup of projectb database..." -POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) -pg_dump projectb > $POSTDUMP -(cd $base/backup; ln -sf $POSTDUMP current) - -################################################################################ - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -echo "Expiring old database dumps..." -(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*") - -################################################################################ - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -# Send a report on NEW/BYHAND packages -echo "Nagging ftpteam about NEW/BYHAND packages" -dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org -# and one on crufty packages -echo "Sending information about crufty packages" -dak cruft-report > $webdir/cruft-report-daily.txt -dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt -cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org - -echo "Updating DM html page" -$scriptsdir/dm-monitor >$webdir/dm-uploaders.html - -################################################################################ - -# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached -echo "Trigger merkels projectb sync" -ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 - -################################################################################ - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -ulimit -m 90000 -d 90000 -s 10000 -v 200000 - -echo "Using run-parts to run scripts in $base/scripts/distmnt" -run-parts --report $base/scripts/distmnt - -echo "Daily cron scripts successful." - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -# Stats pr0n -echo "Updating stats data" -cd $configdir -$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data -R --slave --vanilla < $base/misc/ftpstats.R - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -# Clean up apt-ftparchive's databases -echo "Clean up apt-ftparchive's databases" -cd $configdir -apt-ftparchive -q clean apt.conf - -TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - -# Compress psql backups older than a week, but no more than 20 of them -echo "Compress old psql backups" -(cd $base/backup/ - find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | - sort | head -n20 | while read dumpname; do - echo "Compressing $dumpname" - bzip2 -9 "$dumpname" - done -) - -echo "Finally, all is done, sending mail and compressing logfile" -exec > /dev/null 2>&1 - -cat "$LOGFILE" | mail -s "Log for cron.daily run of $(date +%Y.%m.%d)" cron@ftp-master.debian.org -bzip2 -9 "$LOGFILE" - -################################################################################ diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall new file mode 100755 index 00000000..4e3befb7 --- /dev/null +++ b/config/debian/cron.dinstall @@ -0,0 +1,264 @@ +#! /bin/sh +# +# Executed daily via cron, out of dak's crontab. + +set -e +export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars +. $SCRIPTVARS + +################################################################################ + +# Start logging +NOW=`date "+%Y.%m.%d-%H:%M:%S"` +LOGFILE="$logdir/dinstall_${NOW}.log" +exec > "$LOGFILE" 2>&1 + +echo Archive maintenance started at $(date +%X) +TS=0 + +NOTICE="$ftpdir/Archive_Maintenance_In_Progress" +LOCKCU="$lockdir/daily.lock" +LOCKAC="$lockdir/unchecked.lock" +lockac=0 + +cleanup() { + rm -f "$NOTICE" + rm -f "$LOCKCU" + if [ "$lockac" -eq "1" ]; then + rm -f "$LOCKAC" + fi + echo "Cleanup" + rm -f "$LOGFILE" +} +lockfile -l 3600 $LOCKCU +trap cleanup 0 + +rm -f "$NOTICE" +cat > "$NOTICE" < $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) + +################################################################################ + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Updating Bugs docu, Mirror list and mailing-lists.txt" +cd $configdir +$scriptsdir/update-bugdoctxt +$scriptsdir/update-mirrorlists +$scriptsdir/update-mailingliststxt + +################################################################################ + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Doing automated p-u-new processing" +cd $queuedir/p-u-new +date -u -R >> REPORT +dak process-new -a -C COMMENTS >> REPORT +echo >> REPORT + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Doing automated o-p-u-new processing" +cd $queuedir/o-p-u-new +date -u -R >> REPORT +dak process-new -a -C COMMENTS >> REPORT +echo >> REPORT + +################################################################################ + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +lockfile $LOCKAC +lockac=1 +echo "Processing queue/accepted" +cd $accepted +rm -f REPORT +dak process-accepted -pa *.changes | tee REPORT | \ + mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org +chgrp debadmin REPORT +chmod 664 REPORT + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Checking for cruft in overrides" +dak check-overrides +rm -f $LOCKAC +lockac=0 + +echo "Fixing symlinks in $ftpdir" +symlinks -d -r $ftpdir + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +echo "Generating suite file lists for apt-ftparchive" +dak make-suite-file-list + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +echo "Updating fingerprints" +# Update fingerprints +dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Generate override files +echo "Writing overrides into text files" +cd $overridedir +dak make-overrides + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# FIXME +rm -f override.sid.all3 +for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Generate Packages and Sources files +echo "Generating Packages and Sources files" +cd $configdir +apt-ftparchive generate apt.conf +# Generate *.diff/ incremental updates + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +echo "Generating pdiff files" +dak generate-index-diffs + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Generate Release files +echo "Generating Release files" +dak generate-releases + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Clean out old packages +echo "Cleanup old packages/files" +dak clean-suites +dak clean-queues + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Needs to be rebuilt, as files have moved. Due to unaccepts, we need to +# update this before wanna-build is updated. +echo "Regenerating wanna-build/buildd information" +psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list +symlinks -d /srv/incoming.debian.org/buildd > /dev/null +apt-ftparchive generate apt.conf.buildd + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +echo "Running various scripts from $scriptsdir" +cd $scriptsdir +./mkmaintainers +./copyoverrides +./mklslar +./mkfilesindices +./mkchecksums +# +rm -f $NOTICE + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +echo "Trigger daily wanna-build run" +ssh buildd@buildd /org/wanna-build/trigger.daily + +rm -f $LOCKCU +echo Archive maintenance finished at $(date +%X) + +################################################################################ + +echo "Creating post-daily-cron-job backup of projectb database..." +POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) +pg_dump projectb > $POSTDUMP +(cd $base/backup; ln -sf $POSTDUMP current) + +################################################################################ + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +echo "Expiring old database dumps..." +(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*") + +################################################################################ + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Send a report on NEW/BYHAND packages +echo "Nagging ftpteam about NEW/BYHAND packages" +dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org +# and one on crufty packages +echo "Sending information about crufty packages" +dak cruft-report > $webdir/cruft-report-daily.txt +dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt +cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +echo "Updating DM html page" +$scriptsdir/dm-monitor >$webdir/dm-uploaders.html + +################################################################################ + +# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached +echo "Trigger merkels projectb sync" +ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 + +################################################################################ + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +ulimit -m 90000 -d 90000 -s 10000 -v 200000 + +echo "Using run-parts to run scripts in $base/scripts/distmnt" +run-parts --report $base/scripts/distmnt + +echo "Daily cron scripts successful." + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Stats pr0n +echo "Updating stats data" +cd $configdir +$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data +R --slave --vanilla < $base/misc/ftpstats.R + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Clean up apt-ftparchive's databases +echo "Clean up apt-ftparchive's databases" +cd $configdir +apt-ftparchive -q clean apt.conf + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Compress psql backups older than a week, but no more than 20 of them +echo "Compress old psql backups" +(cd $base/backup/ + find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | + sort | head -n20 | while read dumpname; do + echo "Compressing $dumpname" + bzip2 -9 "$dumpname" + done +) + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +echo "Finally, all is done, sending mail and compressing logfile" +exec > /dev/null 2>&1 + +cat "$LOGFILE" | mail -s "Log for dinstall run of $(date +%Y.%m.%d' at '%H:%M)" cron@ftp-master.debian.org +bzip2 -9 "$LOGFILE" + +################################################################################