X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fcron.daily;h=09d6a6b36b00df088a80bcaf2f9790bd86d10ed6;hb=f8d3d222b8464d1f8e96d0b4f2d8538ca239f1a7;hp=5d85823b471a1d3bc783b1bcd501731f22915f69;hpb=da564af9a40d05d078e5ec7e49c0c778ae08ea0e;p=dak.git diff --git a/config/debian/cron.daily b/config/debian/cron.daily index 5d85823b..09d6a6b3 100755 --- a/config/debian/cron.daily +++ b/config/debian/cron.daily @@ -3,26 +3,37 @@ # Executed daily via cron, out of dak's crontab. set -e -export SCRIPTVARS=/org/ftp.debian.org/dak/config/debian/vars +export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars . $SCRIPTVARS ################################################################################ +# Start logging +NOW=`date "+%Y.%m.%d-%H:%M:%S"` +LOGFILE="$logdir/daily_${NOW}.log" +exec > "$LOGFILE" 2>&1 + echo Archive maintenance started at $(date +%X) TS=0 NOTICE="$ftpdir/Archive_Maintenance_In_Progress" LOCKCU="$lockdir/daily.lock" LOCKAC="$lockdir/unchecked.lock" +lockac=0 cleanup() { rm -f "$NOTICE" rm -f "$LOCKCU" + if [ "$lockac" -eq "1" ]; then + rm -f "$LOCKAC" + fi + echo "Cleanup" + rm -f "$LOGFILE" } +lockfile -l 3600 $LOCKCU trap cleanup 0 rm -f "$NOTICE" -lockfile -l 3600 $LOCKCU cat > "$NOTICE" < /org/ftp.debian.org/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) +pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) ################################################################################ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -update-bugdoctxt -update-mirrorlists -update-mailingliststxt +echo "Updating Bugs docu, Mirror list and mailing-lists.txt" +cd $configdir +$scriptsdir/update-bugdoctxt +$scriptsdir/update-mirrorlists +$scriptsdir/update-mailingliststxt + +################################################################################ + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Doing automated p-u-new processing" +cd $queuedir/p-u-new +date -u -R >> REPORT +dak process-new -a -C COMMENTS >> REPORT +echo >> REPORT + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Doing automated o-p-u-new processing" +cd $queuedir/o-p-u-new +date -u -R >> REPORT +dak process-new -a -C COMMENTS >> REPORT +echo >> REPORT ################################################################################ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) lockfile $LOCKAC +lockac=1 +echo "Processing queue/accepted" cd $accepted rm -f REPORT -dak process-unchecked -pa *.changes | tee REPORT | \ +dak process-accepted -pa *.changes | tee REPORT | \ mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org chgrp debadmin REPORT chmod 664 REPORT TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - +echo "Checking for cruft in overrides" dak check-overrides rm -f $LOCKAC +lockac=0 +echo "Fixing symlinks in $ftpdir" symlinks -d -r $ftpdir TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Generating suite file lists for apt-ftparchive" dak make-suite-file-list TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Updating fingerprints" # Update fingerprints -# [JT - disabled, dak import-ldap-fingerprints currently can ask questions] -#dak import-ldap-fingerprints +dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg # Generate override files +echo "Writing overrides into text files" cd $overridedir dak make-overrides -# Update task overrides for testing and unstable -# [JT 2004-02-04 disabled; copying in by hand for now] -#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sarge.extra.main -#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sid.extra.main - # FIXME -rm -f override.potato.all3 override.sid.all3 -for i in main contrib non-free; do cat override.potato.$i >> override.potato.all3; done +rm -f override.sid.all3 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Generate Packages and Sources files +echo "Generating Packages and Sources files" cd $configdir apt-ftparchive generate apt.conf # Generate *.diff/ incremental updates +echo "Generating pdiff files" dak generate-index-diffs # Generate Release files +echo "Generating Release files" dak generate-releases TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Clean out old packages +echo "Cleanup old packages/files" dak clean-suites dak clean-queues @@ -105,20 +142,23 @@ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to # update this before wanna-build is updated. +echo "Regenerating wanna-build/buildd information" psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list +symlinks -d /srv/incoming.debian.org/buildd > /dev/null apt-ftparchive generate apt.conf.buildd TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -mkmaintainers -copyoverrides -mklslar -mkfilesindices -mkchecksums +echo "Running various scripts from $scriptsdir" +cd $scriptsdir +./mkmaintainers +./copyoverrides +./mklslar +./mkfilesindices +./mkchecksums # -# Fetch bugs information before unchecked processing is allowed again. -/org/ftp.debian.org/testing/britney bugs rm -f $NOTICE +echo "Trigger daily wanna-build run" ssh buildd@buildd /org/wanna-build/trigger.daily rm -f $LOCKCU @@ -127,46 +167,81 @@ echo Archive maintenance finished at $(date +%X) ################################################################################ echo "Creating post-daily-cron-job backup of projectb database..." -POSTDUMP=/org/ftp.debian.org/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) +POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) pg_dump projectb > $POSTDUMP -(cd /org/ftp.debian.org/backup; ln -sf $POSTDUMP current) +(cd $base/backup; ln -sf $POSTDUMP current) ################################################################################ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -# Vacuum the database -echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$" +echo "Expiring old database dumps..." +(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*") ################################################################################ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Send a report on NEW/BYHAND packages +echo "Nagging ftpteam about NEW/BYHAND packages" dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org # and one on crufty packages -dak cruft-report | tee $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org +echo "Sending information about crufty packages" +dak cruft-report > $webdir/cruft-report-daily.txt +dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt +cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org -################################################################################ +echo "Updating DM html page" +$scriptsdir/dm-monitor >$webdir/dm-uploaders.html -# Run mirror-split +################################################################################ -#time dak mirror-split +# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached +echo "Trigger merkels projectb sync" +ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 ################################################################################ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -ulimit -m 90000 -d 90000 -s 10000 -v 90000 +ulimit -m 90000 -d 90000 -s 10000 -v 200000 -run-parts --report /org/ftp.debian.org/scripts/distmnt +echo "Using run-parts to run scripts in $base/scripts/distmnt" +run-parts --report $base/scripts/distmnt -echo Daily cron scripts successful. +echo "Daily cron scripts successful." TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Stats pr0n -update-ftpstats $base/log/* > $base/misc/ftpstats.data +echo "Updating stats data" +cd $configdir +$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data R --slave --vanilla < $base/misc/ftpstats.R TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Clean up apt-ftparchive's databases +echo "Clean up apt-ftparchive's databases" +cd $configdir +apt-ftparchive -q clean apt.conf + +TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) + +# Compress psql backups older than a week, but no more than 20 of them +echo "Compress old psql backups" +(cd $base/backup/ + find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | + sort | head -n20 | while read dumpname; do + echo "Compressing $dumpname" + bzip2 -9 "$dumpname" + done +) + +echo "Finally, all is done, sending mail and compressing logfile" +exec > /dev/null 2>&1 + +cat "$LOGFILE" | mail -s "Log for cron.daily run of $(date +%Y.%m.%d)" cron@ftp-master.debian.org +bzip2 -9 "$LOGFILE" + +################################################################################