-#! /bin/sh
+#! /bin/bash
#
-# Executed daily via cron, out of dak's crontab.
+# Run daily via cron, out of dak's crontab.
set -e
-export SCRIPTVARS=/org/ftp.debian.org/dak/config/debian/vars
+set -o pipefail
+set -u
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS
-################################################################################
-
-echo Archive maintenance started at $(date +%X)
-TS=0
-
-NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
-LOCKCU="$lockdir/daily.lock"
-LOCKAC="$lockdir/unchecked.lock"
-
-cleanup() {
- rm -f "$NOTICE"
- rm -f "$LOCKCU"
-}
-trap cleanup 0
-
-rm -f "$NOTICE"
-lockfile -l 3600 $LOCKCU
-cat > "$NOTICE" <<EOF
-Packages are currently being installed and indices rebuilt.
-Maintenance is automatic, starting at 13:52 US Central time, and
-ending at about 15:30. This file is then removed.
-
-You should not mirror the archive during this period.
-EOF
-
-################################################################################
-
-echo "Creating pre-daily-cron-job backup of projectb database..."
-pg_dump projectb > /org/ftp.debian.org/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
-
-################################################################################
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-update-bugdoctxt
-update-mirrorlists
-update-mailingliststxt
+# common functions are "outsourced"
+. "${configdir}/common"
################################################################################
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-lockfile $LOCKAC
-cd $accepted
-rm -f REPORT
-dak process-unchecked -pa *.changes | tee REPORT | \
- mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
-chgrp debadmin REPORT
-chmod 664 REPORT
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-dak check-overrides
-rm -f $LOCKAC
-
-symlinks -d -r $ftpdir
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-dak make-suite-file-list
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Update fingerprints
-# [JT - disabled, dak import-ldap-fingerprints currently can ask questions]
-#dak import-ldap-fingerprints
-
-# Generate override files
-cd $overridedir
-dak make-overrides
-
-# Update task overrides for testing and unstable
-# [JT 2004-02-04 disabled; copying in by hand for now]
-#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sarge.extra.main
-#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sid.extra.main
+TMPFILE=$( mktemp -p ${TMPDIR} )
-# FIXME
-rm -f override.potato.all3 override.sid.all3
-for i in main contrib non-free; do cat override.potato.$i >> override.potato.all3; done
-for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Generate Packages and Sources files
-cd $configdir
-apt-ftparchive generate apt.conf
-# Generate *.diff/ incremental updates
-dak generate-index-diffs
-# Generate Release files
-dak generate-releases
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Clean out old packages
-dak clean-suites
-dak clean-queues
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
-# update this before wanna-build is updated.
-psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
-apt-ftparchive generate apt.conf.buildd
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-mkmaintainers
-copyoverrides
-mklslar
-mkfilesindices
-mkchecksums
-#
-# Fetch bugs information before unchecked processing is allowed again.
-/org/ftp.debian.org/testing/britney bugs
-rm -f $NOTICE
-ssh buildd@buildd /org/wanna-build/trigger.daily
-
-rm -f $LOCKCU
-echo Archive maintenance finished at $(date +%X)
-
-################################################################################
+function cleanup {
+ ERRVAL=$?
+ rm -f ${TMPFILE}
+ exit ${ERRVAL}
+}
+trap cleanup SIGHUP SIGINT SIGPIPE SIGTERM EXIT ERR
-echo "Creating post-daily-cron-job backup of projectb database..."
-POSTDUMP=/org/ftp.debian.org/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
-pg_dump projectb > $POSTDUMP
-(cd /org/ftp.debian.org/backup; ln -sf $POSTDUMP current)
+# log to dinstall's logfile instead of sending email
+PROGRAM="cron.daily"
+# Start logging
+NOW=`date "+%Y.%m.%d-%H:%M:%S"`
+LOGFILE="$logdir/daily_${NOW}.log"
+exec >> "$LOGFILE" 2>&1
-################################################################################
+# get the latest list of wnpp bugs and their source packages
+wget -q -O${TMPFILE} https://qa.debian.org/data/bts/wnpp_rm
+chmod go+r ${TMPFILE}
+mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+# Push files over to security
+# The key over there should have the following set for the ssh key:
+# command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity"
+psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \
+ xz -3 | \
+ ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \
+ -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync
-# Vacuum the database
-echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$"
+# Update wanna-build dump
+echo "Update wanna-build database dump"
+$base/dak/scripts/nfu/get-w-b-db
-################################################################################
+reports
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+clean_debbugs
-# Send a report on NEW/BYHAND packages
-dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
-# and one on crufty packages
-dak cruft-report | tee $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
+# Generate list of override disparities
+dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz
-################################################################################
+# Generate stats about the new queue
+dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null
-# Run mirror-split
+# Generate the contributor data
+# FIXME: In a day or three, when this worked from cron without
+# failure, redirect its output to dev/null. Alternatively until then
+# enrico added a --quiet and we use that.
+REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --post
-#time dak mirror-split
+${scriptsdir}/link_morgue.sh
################################################################################
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-ulimit -m 90000 -d 90000 -s 10000 -v 90000
-
-run-parts --report /org/ftp.debian.org/scripts/distmnt
-
-echo Daily cron scripts successful.
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Stats pr0n
-update-ftpstats $base/log/* > $base/misc/ftpstats.data
-R --slave --vanilla < $base/misc/ftpstats.R
+echo "Finally, all is done, compressing logfile"
+exec > /dev/null 2>&1
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+bzip2 -9 "$LOGFILE"
projects / dak.git / blobdiff