X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fcron.daily;h=09d6a6b36b00df088a80bcaf2f9790bd86d10ed6;hb=f89e353cb89593444b3e841cadc7175a3f4e9081;hp=77832a0546375c63cbe60aa812b6ea3bd7a6e66a;hpb=59fd5aa2a8be3b76dbc968429c457f096adfa472;p=dak.git

diff --git a/config/debian/cron.daily b/config/debian/cron.daily
index 77832a05..09d6a6b3 100755
--- a/config/debian/cron.daily
+++ b/config/debian/cron.daily
@@ -1,27 +1,39 @@
 #! /bin/sh
 #
-# Executed daily via cron, out of katie's crontab.
+# Executed daily via cron, out of dak's crontab.
 
 set -e
-export SCRIPTVARS=/org/ftp.debian.org/katie/vars
+export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
 . $SCRIPTVARS
 
 ################################################################################
 
+# Start logging
+NOW=`date "+%Y.%m.%d-%H:%M:%S"`
+LOGFILE="$logdir/daily_${NOW}.log"
+exec > "$LOGFILE" 2>&1
+
 echo Archive maintenance started at $(date +%X)
+TS=0
 
 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
 LOCKCU="$lockdir/daily.lock"
 LOCKAC="$lockdir/unchecked.lock"
+lockac=0
 
 cleanup() {
   rm -f "$NOTICE"
   rm -f "$LOCKCU"
+  if [ "$lockac" -eq "1" ]; then
+    rm -f "$LOCKAC"
+  fi
+  echo "Cleanup"
+  rm -f "$LOGFILE"
 }
+lockfile -l 3600 $LOCKCU
 trap cleanup 0
 
 rm -f "$NOTICE"
-lockfile -l 3600 $LOCKCU
 cat > "$NOTICE" <<EOF
 Packages are currently being installed and indices rebuilt.
 Maintenance is automatic, starting at 13:52 US Central time, and
@@ -30,80 +42,124 @@ ending at about 15:30.  This file is then removed.
 You should not mirror the archive during this period.
 EOF
 
+# Push merkels qa user, so the qa pages can show "dinstall is running" information
+echo "Telling merkels QA user that we start dinstall"
+ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
+
 ################################################################################
 
 echo "Creating pre-daily-cron-job backup of projectb database..."
-pg_dump projectb > /org/ftp.debian.org/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
+pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
 
 ################################################################################
 
-update-bugdoctxt
-update-mirrorlists
-update-mailingliststxt
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
+cd $configdir
+$scriptsdir/update-bugdoctxt
+$scriptsdir/update-mirrorlists
+$scriptsdir/update-mailingliststxt
+
+################################################################################
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+echo "Doing automated p-u-new processing"
+cd $queuedir/p-u-new
+date -u -R >> REPORT
+dak process-new -a -C COMMENTS >> REPORT
+echo >> REPORT
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+echo "Doing automated o-p-u-new processing"
+cd $queuedir/o-p-u-new
+date -u -R >> REPORT
+dak process-new -a -C COMMENTS >> REPORT
+echo >> REPORT
 
 ################################################################################
 
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
 lockfile $LOCKAC
+lockac=1
+echo "Processing queue/accepted"
 cd $accepted
 rm -f REPORT
-kelly -pa *.changes | tee REPORT | \
+dak process-accepted -pa *.changes | tee REPORT | \
      mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
 chgrp debadmin REPORT
 chmod 664 REPORT
 
-cd $masterdir
-cindy
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+echo "Checking for cruft in overrides"
+dak check-overrides
 rm -f $LOCKAC
+lockac=0
 
+echo "Fixing symlinks in $ftpdir"
 symlinks -d -r $ftpdir
 
-cd $masterdir
-jenna
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
 
+echo "Generating suite file lists for apt-ftparchive"
+dak make-suite-file-list
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Updating fingerprints"
 # Update fingerprints
-# [JT - disabled, emilie currently can ask questions]
-#emilie
+dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
 
 # Generate override files
+echo "Writing overrides into text files"
 cd $overridedir
-denise
-
-# Update task overrides for testing and unstable
-# [JT 2004-02-04 disabled; copying in by hand for now]
-#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sarge.extra.main
-#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sid.extra.main
+dak make-overrides
 
 # FIXME
-rm -f override.potato.all3 override.sid.all3
-for i in main contrib non-free; do cat override.potato.$i >> override.potato.all3; done
+rm -f override.sid.all3
 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
 
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
 # Generate Packages and Sources files
-cd $masterdir
+echo "Generating Packages and Sources files"
+cd $configdir
 apt-ftparchive generate apt.conf
 # Generate *.diff/ incremental updates
-tiffani
+echo "Generating pdiff files"
+dak generate-index-diffs
 # Generate Release files
-ziyi
+echo "Generating Release files"
+dak generate-releases
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
 
 # Clean out old packages
-rhona
-shania
+echo "Cleanup old packages/files"
+dak clean-suites
+dak clean-queues
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
 
 # Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
 # update this before wanna-build is updated.
+echo "Regenerating wanna-build/buildd information"
 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
+symlinks -d /srv/incoming.debian.org/buildd > /dev/null
 apt-ftparchive generate apt.conf.buildd
 
-mkmaintainers
-copyoverrides
-mklslar
-mkchecksums
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Running various scripts from $scriptsdir"
+cd $scriptsdir
+./mkmaintainers
+./copyoverrides
+./mklslar
+./mkfilesindices
+./mkchecksums
 #
-# Fetch bugs information before unchecked processing is allowed again.
-/org/ftp.debian.org/testing/britney bugs
 rm -f $NOTICE
-sudo -u archvsync /home/archvsync/pushmerkel
+echo "Trigger daily wanna-build run"
+ssh buildd@buildd /org/wanna-build/trigger.daily
 
 rm -f $LOCKCU
 echo Archive maintenance finished at $(date +%X)
@@ -111,37 +167,81 @@ echo Archive maintenance finished at $(date +%X)
 ################################################################################
 
 echo "Creating post-daily-cron-job backup of projectb database..."
-POSTDUMP=/org/ftp.debian.org/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
+POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
 pg_dump projectb > $POSTDUMP
-(cd /org/ftp.debian.org/backup; ln -sf $POSTDUMP current)
+(cd $base/backup; ln -sf $POSTDUMP current)
 
 ################################################################################
 
-# Vacuum the database
-echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE:  Skipping.*only table owner can VACUUM it$"
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+echo "Expiring old database dumps..."
+(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
 
 ################################################################################
 
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
 # Send a report on NEW/BYHAND packages
-helena | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
+echo "Nagging ftpteam about NEW/BYHAND packages"
+dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
 # and one on crufty packages
-rene | tee $webdir/rene-daily.txt | mail -e -s "rene run for $(date +%D)" ftpmaster@ftp-master.debian.org
+echo "Sending information about crufty packages"
+dak cruft-report > $webdir/cruft-report-daily.txt
+dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
+cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
 
-################################################################################
+echo "Updating DM html page"
+$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
 
-# Run billie
+################################################################################
 
-#time billie
+# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
+echo "Trigger merkels projectb sync"
+ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
 
 ################################################################################
 
-ulimit -m 90000 -d 90000 -s 10000 -v 90000
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
 
-run-parts --report /org/ftp.debian.org/scripts/distmnt
+ulimit -m 90000 -d 90000 -s 10000 -v 200000
 
-echo Daily cron scripts successful.
-# Stats pr0n
+echo "Using run-parts to run scripts in $base/scripts/distmnt"
+run-parts --report $base/scripts/distmnt
 
-cd $masterdir
-update-ftpstats $base/log/* > $base/misc/ftpstats.data
+echo "Daily cron scripts successful."
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Stats pr0n
+echo "Updating stats data"
+cd $configdir
+$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
 R --slave --vanilla < $base/misc/ftpstats.R
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Clean up apt-ftparchive's databases
+echo "Clean up apt-ftparchive's databases"
+cd $configdir
+apt-ftparchive -q clean apt.conf
+
+TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+
+# Compress psql backups older than a week, but no more than 20 of them
+echo "Compress old psql backups"
+(cd $base/backup/
+ find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | 
+   sort | head -n20 | while read dumpname; do
+     echo "Compressing $dumpname"
+     bzip2 -9 "$dumpname"
+   done
+)
+
+echo "Finally, all is done, sending mail and compressing logfile"
+exec > /dev/null 2>&1
+
+cat "$LOGFILE" | mail -s "Log for cron.daily run of $(date +%Y.%m.%d)" cron@ftp-master.debian.org
+bzip2 -9 "$LOGFILE"
+
+################################################################################