X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fcron.daily;h=0b822bc5148d06b0bc90ee1739ef5f399090866b;hb=5fd1e87ecc948980add7e9b41da71f69a3be9844;hp=09d6a6b36b00df088a80bcaf2f9790bd86d10ed6;hpb=d9822f04453a1b62ca0aa66e2efeea35f654778f;p=dak.git

diff --git a/config/debian/cron.daily b/config/debian/cron.daily
index 09d6a6b3..0b822bc5 100755
--- a/config/debian/cron.daily
+++ b/config/debian/cron.daily
@@ -1,247 +1,86 @@
-#! /bin/sh
+#! /bin/bash
 #
-# Executed daily via cron, out of dak's crontab.
+# Run daily via cron, out of dak's crontab.
 
 set -e
-export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
+set -o pipefail
+set -u
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
 . $SCRIPTVARS
 
-################################################################################
-
-# Start logging
-NOW=`date "+%Y.%m.%d-%H:%M:%S"`
-LOGFILE="$logdir/daily_${NOW}.log"
-exec > "$LOGFILE" 2>&1
-
-echo Archive maintenance started at $(date +%X)
-TS=0
-
-NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
-LOCKCU="$lockdir/daily.lock"
-LOCKAC="$lockdir/unchecked.lock"
-lockac=0
-
-cleanup() {
-  rm -f "$NOTICE"
-  rm -f "$LOCKCU"
-  if [ "$lockac" -eq "1" ]; then
-    rm -f "$LOCKAC"
-  fi
-  echo "Cleanup"
-  rm -f "$LOGFILE"
-}
-lockfile -l 3600 $LOCKCU
-trap cleanup 0
-
-rm -f "$NOTICE"
-cat > "$NOTICE" <<EOF
-Packages are currently being installed and indices rebuilt.
-Maintenance is automatic, starting at 13:52 US Central time, and
-ending at about 15:30.  This file is then removed.
-
-You should not mirror the archive during this period.
-EOF
-
-# Push merkels qa user, so the qa pages can show "dinstall is running" information
-echo "Telling merkels QA user that we start dinstall"
-ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
-
-################################################################################
-
-echo "Creating pre-daily-cron-job backup of projectb database..."
-pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
-cd $configdir
-$scriptsdir/update-bugdoctxt
-$scriptsdir/update-mirrorlists
-$scriptsdir/update-mailingliststxt
+# common functions are "outsourced"
+. "${configdir}/common"
 
 ################################################################################
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-echo "Doing automated p-u-new processing"
-cd $queuedir/p-u-new
-date -u -R >> REPORT
-dak process-new -a -C COMMENTS >> REPORT
-echo >> REPORT
+TMPFILE=$( mktemp -p ${TMPDIR} )
+TMPCNTB=$( mktemp -p ${TMPDIR} )
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-echo "Doing automated o-p-u-new processing"
-cd $queuedir/o-p-u-new
-date -u -R >> REPORT
-dak process-new -a -C COMMENTS >> REPORT
-echo >> REPORT
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-lockfile $LOCKAC
-lockac=1
-echo "Processing queue/accepted"
-cd $accepted
-rm -f REPORT
-dak process-accepted -pa *.changes | tee REPORT | \
-     mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
-chgrp debadmin REPORT
-chmod 664 REPORT
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-echo "Checking for cruft in overrides"
-dak check-overrides
-rm -f $LOCKAC
-lockac=0
-
-echo "Fixing symlinks in $ftpdir"
-symlinks -d -r $ftpdir
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-echo "Generating suite file lists for apt-ftparchive"
-dak make-suite-file-list
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-echo "Updating fingerprints"
-# Update fingerprints
-dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
-
-# Generate override files
-echo "Writing overrides into text files"
-cd $overridedir
-dak make-overrides
-
-# FIXME
-rm -f override.sid.all3
-for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Generate Packages and Sources files
-echo "Generating Packages and Sources files"
-cd $configdir
-apt-ftparchive generate apt.conf
-# Generate *.diff/ incremental updates
-echo "Generating pdiff files"
-dak generate-index-diffs
-# Generate Release files
-echo "Generating Release files"
-dak generate-releases
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Clean out old packages
-echo "Cleanup old packages/files"
-dak clean-suites
-dak clean-queues
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
-# update this before wanna-build is updated.
-echo "Regenerating wanna-build/buildd information"
-psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
-symlinks -d /srv/incoming.debian.org/buildd > /dev/null
-apt-ftparchive generate apt.conf.buildd
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-echo "Running various scripts from $scriptsdir"
-cd $scriptsdir
-./mkmaintainers
-./copyoverrides
-./mklslar
-./mkfilesindices
-./mkchecksums
-#
-rm -f $NOTICE
-echo "Trigger daily wanna-build run"
-ssh buildd@buildd /org/wanna-build/trigger.daily
+function cleanup {
+    ERRVAL=$?
+    rm -f ${TMPFILE} ${TMPCNTB}
+    exit ${ERRVAL}
+}
+trap cleanup SIGHUP SIGINT SIGPIPE SIGTERM EXIT ERR
 
-rm -f $LOCKCU
-echo Archive maintenance finished at $(date +%X)
+# log to dinstall's logfile instead of sending email
+PROGRAM="cron.daily"
+# Start logging
+NOW=`date "+%Y.%m.%d-%H:%M:%S"`
+LOGFILE="$logdir/daily_${NOW}.log"
+exec >> "$LOGFILE" 2>&1
 
-################################################################################
+# get the latest list of wnpp bugs and their source packages
+wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
+chmod go+r ${TMPFILE}
+mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
 
-echo "Creating post-daily-cron-job backup of projectb database..."
-POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
-pg_dump projectb > $POSTDUMP
-(cd $base/backup; ln -sf $POSTDUMP current)
+# Push files over to security
+# The key over there should have the following set for the ssh key:
+#  command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity"
+psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \
+            xz -3 | \
+            ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \
+                -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync
 
-################################################################################
+# Update wanna-build dump
+log "Update wanna-build database dump"
+$base/dak/scripts/nfu/get-w-b-db
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+reports
 
-echo "Expiring old database dumps..."
-(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
+clean_debbugs
 
-################################################################################
+# Generate list of override disparities
+dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+# Generate stats about the new queue
+dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null
 
-# Send a report on NEW/BYHAND packages
-echo "Nagging ftpteam about NEW/BYHAND packages"
-dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
-# and one on crufty packages
-echo "Sending information about crufty packages"
-dak cruft-report > $webdir/cruft-report-daily.txt
-dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
-cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
+# Generate the contributor data
+# FIXME: In a day or three, when this worked from cron without
+# failure, redirect its output to dev/null. Alternatively until then
+# enrico added a --quiet and we use that.
+log "Submitting data to contributors"
+REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
 
-echo "Updating DM html page"
-$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
+# Post with curl as a workaround for #801506
+# See https://wiki.debian.org/ServicesSSL#curl
+dir=/etc/ssl/ca-debian
+test -d $dir && capath="--capath $dir"
+curl -s $capath https://contributors.debian.org/contributors/post \
+    -F source=ftp.debian.org \
+    -F auth_token="$(cat ${base}/s3kr1t/contributor.auth)" \
+    -F data=@${TMPCNTB} > ${TMPCNTB}.result
+cat ${TMPCNTB}.result
+rm -f ${TMPCNTB}.result
 
-################################################################################
 
-# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
-echo "Trigger merkels projectb sync"
-ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
+${scriptsdir}/link_morgue.sh
 
 ################################################################################
 
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-ulimit -m 90000 -d 90000 -s 10000 -v 200000
-
-echo "Using run-parts to run scripts in $base/scripts/distmnt"
-run-parts --report $base/scripts/distmnt
-
-echo "Daily cron scripts successful."
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Stats pr0n
-echo "Updating stats data"
-cd $configdir
-$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
-R --slave --vanilla < $base/misc/ftpstats.R
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Clean up apt-ftparchive's databases
-echo "Clean up apt-ftparchive's databases"
-cd $configdir
-apt-ftparchive -q clean apt.conf
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Compress psql backups older than a week, but no more than 20 of them
-echo "Compress old psql backups"
-(cd $base/backup/
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | 
-   sort | head -n20 | while read dumpname; do
-     echo "Compressing $dumpname"
-     bzip2 -9 "$dumpname"
-   done
-)
-
-echo "Finally, all is done, sending mail and compressing logfile"
+log "Finally, all is done, compressing logfile"
 exec > /dev/null 2>&1
 
-cat "$LOGFILE" | mail -s "Log for cron.daily run of $(date +%Y.%m.%d)" cron@ftp-master.debian.org
 bzip2 -9 "$LOGFILE"
-
-################################################################################