LOGFILE="$logdir/dinstall_${NOW}.log"
exec > "$LOGFILE" 2>&1
-echo Archive maintenance started at $(date +%X)
+ts() {
+ TS=$(($TS+1));
+ echo Archive maintenance timestamp $TS: $(date +%H:%M:%S)
+}
+
+echo Archive maintenance started at $(date +%H:%M:%S)
TS=0
NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
LOCKCU="$lockdir/daily.lock"
LOCKAC="$lockdir/unchecked.lock"
+BRITNEYLOCK="$lockdir/britney.lock"
lockac=0
cleanup() {
rm -f "$LOCKAC"
fi
echo "Cleanup"
- rm -f "$LOGFILE"
}
lockfile -l 3600 $LOCKCU
trap cleanup 0
+# This file is simply used to indicate to britney whether or not
+# the Packages file updates completed sucessfully. It's not a lock
+# from our point of view
+touch ${BRITNEYLOCK}
+
rm -f "$NOTICE"
cat > "$NOTICE" <<EOF
Packages are currently being installed and indices rebuilt.
-Maintenance is automatic, starting at 13:52 US Central time, and
-ending at about 15:30. This file is then removed.
+Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
+ending about an hour later. This file is then removed.
You should not mirror the archive during this period.
EOF
# Push merkels qa user, so the qa pages can show "dinstall is running" information
echo "Telling merkels QA user that we start dinstall"
-ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
+ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
+ts
################################################################################
################################################################################
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
cd $configdir
$scriptsdir/update-bugdoctxt
################################################################################
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Doing automated p-u-new processing"
cd $queuedir/p-u-new
date -u -R >> REPORT
-dak process-new -a -C COMMENTS >> REPORT
+dak process-new -a -C COMMENTS >> REPORT || true
echo >> REPORT
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Doing automated o-p-u-new processing"
cd $queuedir/o-p-u-new
date -u -R >> REPORT
-dak process-new -a -C COMMENTS >> REPORT
+dak process-new -a -C COMMENTS >> REPORT || true
echo >> REPORT
################################################################################
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
lockfile $LOCKAC
lockac=1
echo "Processing queue/accepted"
chgrp debadmin REPORT
chmod 664 REPORT
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Checking for cruft in overrides"
dak check-overrides
rm -f $LOCKAC
echo "Fixing symlinks in $ftpdir"
symlinks -d -r $ftpdir
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Generating suite file lists for apt-ftparchive"
dak make-suite-file-list
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Updating fingerprints"
# Update fingerprints
-dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
+dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
# Generate override files
echo "Writing overrides into text files"
cd $overridedir
dak make-overrides
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
# FIXME
rm -f override.sid.all3
for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
# Generate Packages and Sources files
echo "Generating Packages and Sources files"
cd $configdir
apt-ftparchive generate apt.conf
-# Generate *.diff/ incremental updates
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
+# Generate *.diff/ incremental updates
echo "Generating pdiff files"
dak generate-index-diffs
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
# Generate Release files
echo "Generating Release files"
dak generate-releases
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
# Clean out old packages
echo "Cleanup old packages/files"
dak clean-suites
dak clean-queues
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
# Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
# update this before wanna-build is updated.
symlinks -d /srv/incoming.debian.org/buildd > /dev/null
apt-ftparchive generate apt.conf.buildd
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Running various scripts from $scriptsdir"
cd $scriptsdir
./mklslar
./mkfilesindices
./mkchecksums
-#
-rm -f $NOTICE
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
+
+# (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
+echo "Regenerating mirror/ hardlink fun"
+cd ${mirrordir}
+rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
+
+
+ts
echo "Trigger daily wanna-build run"
-ssh buildd@buildd /org/wanna-build/trigger.daily
+ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 buildd@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
+rm -f $NOTICE
rm -f $LOCKCU
-echo Archive maintenance finished at $(date +%X)
+echo Archive maintenance finished at $(date +%H:%M:%S)
################################################################################
################################################################################
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Expiring old database dumps..."
(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
################################################################################
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
# Send a report on NEW/BYHAND packages
echo "Nagging ftpteam about NEW/BYHAND packages"
dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Updating DM html page"
$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
echo "Trigger merkels projectb sync"
-ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
+ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
################################################################################
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
ulimit -m 90000 -d 90000 -s 10000 -v 200000
echo "Daily cron scripts successful."
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
# Stats pr0n
echo "Updating stats data"
$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
R --slave --vanilla < $base/misc/ftpstats.R
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
+
+# Remove the britney lock
+rm -f ${BRITNEYLOCK}
# Clean up apt-ftparchive's databases
echo "Clean up apt-ftparchive's databases"
cd $configdir
apt-ftparchive -q clean apt.conf
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
-# Compress psql backups older than a week, but no more than 20 of them
+# Compress psql backups
echo "Compress old psql backups"
(cd $base/backup/
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 |
- sort | head -n20 | while read dumpname; do
- echo "Compressing $dumpname"
- bzip2 -9 "$dumpname"
- done
+ find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 |
+ while read dumpname; do
+ echo "Compressing $dumpname"
+ bzip2 -9 "$dumpname"
+ done
)
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts
echo "Finally, all is done, sending mail and compressing logfile"
exec > /dev/null 2>&1
projects / dak.git / blobdiff