X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fcron.daily;h=90c990317a3b09c6bcb90f42024f335ea21cc364;hb=2a0aae5ffea0cca4b9581c2d3afcc2752bbd04c9;hp=5d85823b471a1d3bc783b1bcd501731f22915f69;hpb=342a24f0e31c3b6637cd9926235c514f531ad604;p=dak.git

diff --git a/config/debian/cron.daily b/config/debian/cron.daily
index 5d85823b..90c99031 100755
--- a/config/debian/cron.daily
+++ b/config/debian/cron.daily
@@ -1,172 +1,66 @@
-#! /bin/sh
+#! /bin/bash
 #
-# Executed daily via cron, out of dak's crontab.
+# Run daily via cron, out of dak's crontab.
 
 set -e
-export SCRIPTVARS=/org/ftp.debian.org/dak/config/debian/vars
+set -o pipefail
+set -u
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
 . $SCRIPTVARS
 
-################################################################################
-
-echo Archive maintenance started at $(date +%X)
-TS=0
-
-NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
-LOCKCU="$lockdir/daily.lock"
-LOCKAC="$lockdir/unchecked.lock"
-
-cleanup() {
-  rm -f "$NOTICE"
-  rm -f "$LOCKCU"
-}
-trap cleanup 0
-
-rm -f "$NOTICE"
-lockfile -l 3600 $LOCKCU
-cat > "$NOTICE" <<EOF
-Packages are currently being installed and indices rebuilt.
-Maintenance is automatic, starting at 13:52 US Central time, and
-ending at about 15:30.  This file is then removed.
-
-You should not mirror the archive during this period.
-EOF
-
-################################################################################
-
-echo "Creating pre-daily-cron-job backup of projectb database..."
-pg_dump projectb > /org/ftp.debian.org/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
+# common functions are "outsourced"
+. "${configdir}/common"
 
 ################################################################################
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-update-bugdoctxt
-update-mirrorlists
-update-mailingliststxt
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-lockfile $LOCKAC
-cd $accepted
-rm -f REPORT
-dak process-unchecked -pa *.changes | tee REPORT | \
-     mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
-chgrp debadmin REPORT
-chmod 664 REPORT
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-dak check-overrides
-rm -f $LOCKAC
-
-symlinks -d -r $ftpdir
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-dak make-suite-file-list
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Update fingerprints
-# [JT - disabled, dak import-ldap-fingerprints currently can ask questions]
-#dak import-ldap-fingerprints
-
-# Generate override files
-cd $overridedir
-dak make-overrides
-
-# Update task overrides for testing and unstable
-# [JT 2004-02-04 disabled; copying in by hand for now]
-#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sarge.extra.main
-#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sid.extra.main
-
-# FIXME
-rm -f override.potato.all3 override.sid.all3
-for i in main contrib non-free; do cat override.potato.$i >> override.potato.all3; done
-for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
+TMPFILE=$( mktemp -p ${TMPDIR} )
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Generate Packages and Sources files
-cd $configdir
-apt-ftparchive generate apt.conf
-# Generate *.diff/ incremental updates
-dak generate-index-diffs
-# Generate Release files
-dak generate-releases
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Clean out old packages
-dak clean-suites
-dak clean-queues
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
-# update this before wanna-build is updated.
-psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
-apt-ftparchive generate apt.conf.buildd
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-mkmaintainers
-copyoverrides
-mklslar
-mkfilesindices
-mkchecksums
-#
-# Fetch bugs information before unchecked processing is allowed again.
-/org/ftp.debian.org/testing/britney bugs
-rm -f $NOTICE
-ssh buildd@buildd /org/wanna-build/trigger.daily
-
-rm -f $LOCKCU
-echo Archive maintenance finished at $(date +%X)
-
-################################################################################
+function cleanup {
+    ERRVAL=$?
+    rm -f ${TMPFILE}
+    exit ${ERRVAL}
+}
+trap cleanup SIGHUP SIGINT SIGPIPE SIGTERM EXIT ERR
 
-echo "Creating post-daily-cron-job backup of projectb database..."
-POSTDUMP=/org/ftp.debian.org/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
-pg_dump projectb > $POSTDUMP
-(cd /org/ftp.debian.org/backup; ln -sf $POSTDUMP current)
+# log to dinstall's logfile instead of sending email
+PROGRAM="cron.daily"
+LOGFILE="$logdir/dinstall.log"
+exec >> "$LOGFILE" 2>&1
 
-################################################################################
+# get the latest list of wnpp bugs and their source packages
+wget -q -O${TMPFILE} http://qa.debian.org/data/bts/wnpp_rm
+chmod go+r ${TMPFILE}
+mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+# Push files over to security
+# The key over there should have the following set for the ssh key:
+#  command="/usr/bin/xzcat | /usr/bin/psql -f - -1 obscurity"
+pg_dump -a -F p -t files | \
+            sed -e "s,^COPY files (,DELETE FROM external_files; COPY external_files (," | \
+            xz -3 | \
+            ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \
+                -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync
 
-# Vacuum the database
-echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE:  Skipping.*only table owner can VACUUM it$"
+# Update wanna-build dump
+echo "Update wanna-build database dump"
+$base/dak/scripts/nfu/get-w-b-db
 
-################################################################################
+reports
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+clean_debbugs
 
-# Send a report on NEW/BYHAND packages
-dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
-# and one on crufty packages
-dak cruft-report | tee $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
+# Generate list of override disparities
+dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz
 
-################################################################################
+# Generate stats about the new queue
+dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null
 
-# Run mirror-split
+# Generate the contributor data
+# FIXME: In a day or three, when this worked from cron without
+# failure, redirect its output to dev/null. Alternatively until then
+# enrico added a --quiet and we use that.
+dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --post
 
-#time dak mirror-split
+${scriptsdir}/link_morgue.sh
 
 ################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-ulimit -m 90000 -d 90000 -s 10000 -v 90000
-
-run-parts --report /org/ftp.debian.org/scripts/distmnt
-
-echo Daily cron scripts successful.
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Stats pr0n
-update-ftpstats $base/log/* > $base/misc/ftpstats.data
-R --slave --vanilla < $base/misc/ftpstats.R
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)