X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fcron.dinstall;h=c730a116c28120978e8bc040f79ec574150de43b;hb=610b08292f816e81e7dea5446bbdd22effb434d7;hp=6ade212b32d9ec07e0c7e20b77dced82d2af4c27;hpb=115cce2bd25c16edee4e3dce96b8ac5deeb23846;p=dak.git diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall index 6ade212b..a9e9bd18 100755 --- a/config/debian/cron.dinstall +++ b/config/debian/cron.dinstall @@ -1,355 +1,538 @@ -#! /bin/sh -# -# Executed daily via cron, out of dak's crontab. +#!/bin/bash +# No way I try to deal with a crippled sh just for POSIX foo. +# Copyright (C) 2009 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +# Homer: Are you saying you're never going to eat any animal again? What +# about bacon? +# Lisa: No. +# Homer: Ham? +# Lisa: No. +# Homer: Pork chops? +# Lisa: Dad, those all come from the same animal. +# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal. + +# exit on errors set -e -export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars +# make sure to only use defined variables +set -u +# ERR traps should be inherited from functions too. (And command +# substitutions and subshells and whatnot, but for us the functions is +# the important part here) +set -E + +# import the general variable set. +export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars . $SCRIPTVARS -################################################################################ +######################################################################## +# Functions # +######################################################################## +# common functions are "outsourced" +. "${configdir}/common" + +# source the dinstall functions +. "${configdir}/dinstall.functions" -# Start logging -NOW=`date "+%Y.%m.%d-%H:%M:%S"` -LOGFILE="$logdir/dinstall_${NOW}.log" -exec > "$LOGFILE" 2>&1 +######################################################################## +######################################################################## -ts() { - TS=$(($TS+1)); - echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)" +# Function to save which stage we are in, so we can restart an interrupted +# dinstall. Or even run actions in parallel, if we dare to, by simply +# backgrounding the call to this function. But that should only really be +# done for things we don't care much about. +# +# This should be called with the first argument being an array, with the +# members +# - FUNC - the function name to call +# - ARGS - Possible arguments to hand to the function. Can be the empty string +# - TIME - The timestamp name. Can be the empty string +# - ERR - if this is the string false, then the call will be surrounded by +# set +e ... set -e calls, so errors in the function do not exit +# dinstall. Can be the empty string, meaning true. +# +# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES +# ADDED FOR DINSTALL FEATURES! +function stage() { + ARGS='GO[@]' + local "${!ARGS}" + + error=${ERR:-"true"} + + STAGEFILE="${stagedir}/${FUNC}" + if [ -f "${STAGEFILE}" ]; then + stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}") + unixtime=$(date +%s) + difference=$(( $unixtime - $stamptime )) + if [ ${difference} -ge 14400 ]; then + log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check." + else + log "Did already run ${FUNC}, not calling again..." + fi + return + fi + + debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}" + + # Make sure we are always at the same place. If a function wants to be elsewhere, + # it has to cd first! + cd ${configdir} + + # Now redirect the output into $STAGEFILE.log. In case it errors out somewhere our + # errorhandler trap can then mail the contents of $STAGEFILE.log only, instead of a whole + # dinstall logfile. Short error mails ftw! + exec >> "${STAGEFILE}.log" 2>&1 + + if [ -f "${LOCK_STOP}" ]; then + log "${LOCK_STOP} exists, exiting immediately" + exit 42 + fi + + if [ "${error}" = "false" ]; then + set +e + fi + ${FUNC} ${ARGS} + + # No matter what happened in the function, we make sure we have set -e default state back + set -e + + # Make sure we are always at the same place. + cd ${configdir} + + # We always use the same umask. If a function wants to do different, fine, but we reset. + umask 022 + + touch "${STAGEFILE}" + + if [ -n "${TIME}" ]; then + ts "${TIME}" + fi + + # And the output goes back to the normal logfile + exec >> "$LOGFILE" 2>&1 + + # Now we should make sure that we have a usable dinstall.log, so append the $STAGEFILE.log + # to it. + cat "${STAGEFILE}.log" >> "${LOGFILE}" + rm -f "${STAGEFILE}.log" + + if [ -f "${LOCK_STOP}" ]; then + log "${LOCK_STOP} exists, exiting immediately" + exit 42 + fi } -TS=-1 +######################################################################## + +# We need logs. +LOGFILE="$logdir/dinstall.log" + +exec >> "$LOGFILE" 2>&1 + +# And now source our default config +. "${configdir}/dinstall.variables" + +# Make sure we start out with a sane umask setting +umask 022 + +# And use one locale, no matter what the caller has set +export LANG=C +export LC_ALL=C + +touch "${DINSTALLSTART}" ts "startup" +DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")" +state "Startup" -NOTICE="$ftpdir/Archive_Maintenance_In_Progress" -LOCKCU="$lockdir/daily.lock" -LOCKAC="$lockdir/unchecked.lock" -BRITNEYLOCK="$lockdir/britney.lock" -lockac=0 - -cleanup() { - rm -f "$NOTICE" - rm -f "$LOCKCU" - if [ "$lockac" -eq "1" ]; then - rm -f "$LOCKAC" - fi - echo "Cleanup" -} -lockfile -l 3600 $LOCKCU -trap cleanup 0 - -# This file is simply used to indicate to britney whether or not -# the Packages file updates completed sucessfully. It's not a lock -# from our point of view -touch ${BRITNEYLOCK} - -rm -f "$NOTICE" -cat > "$NOTICE" < $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) -ts "pg_dump1" - -################################################################################ - -echo "Updating Bugs docu, Mirror list and mailing-lists.txt" -cd $configdir -$scriptsdir/update-bugdoctxt -$scriptsdir/update-mirrorlists -$scriptsdir/update-mailingliststxt -$scriptsdir/update-pseudopackages.sh -ts "External Updates" - -################################################################################ - -echo "Doing automated p-u-new processing" -cd $queuedir/p-u-new -date -u -R >> REPORT -dak process-new -a -C COMMENTS >> REPORT || true -echo >> REPORT -ts "p-u-new" - -echo "Doing automated o-p-u-new processing" -cd $queuedir/o-p-u-new -date -u -R >> REPORT -dak process-new -a -C COMMENTS >> REPORT || true -echo >> REPORT -ts "o-p-u-new" - -################################################################################ - -# Temporary until lenny is there -dak control-suite -s lenny-r0 /srv/ftp.debian.org/misc/lenny-r0.txt - -echo "Synchronizing i18n package descriptions" -# First sync their newest data -cd ${scriptdir}/i18nsync -rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true - -# Now check if we still know about the packages for which they created the files -# is the timestamp signed by us? -if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then - # now read it. As its signed by us we are sure the content is what we expect, no need - # to do more here. And we only test -d a directory on it anyway. - TSTAMP=$(cat timestamp) - # do we have the dir still? - if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then - # Lets check! - if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then - # Yay, worked, lets copy around - for dir in lenny sid; do - if [ -d dists/${dir}/ ]; then - cd dists/${dir}/main/i18n - rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/. - fi - cd ${scriptdir}/i18nsync - done - else - echo "ARRRR, bad guys, wrong files, ARRR" - echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org - fi - else - echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR" - echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org - fi -else - echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR." - echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org -fi -ts "i18n 1" - -################################################################################ - -lockfile $LOCKAC -lockac=1 -echo "Processing queue/accepted" -cd $accepted -rm -f REPORT -dak process-accepted -pa *.changes | tee REPORT | \ - mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org -chgrp debadmin REPORT -chmod 664 REPORT -ts "accepted" - -echo "Checking for cruft in overrides" -dak check-overrides -rm -f $LOCKAC -lockac=0 - -echo "Fixing symlinks in $ftpdir" -symlinks -d -r $ftpdir -ts "cruft" - -echo "Generating suite file lists for apt-ftparchive" -dak make-suite-file-list -ts "make-suite-file-list" - -echo "Updating fingerprints" -# Update fingerprints -dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true -ts "import-keyring" - -# Generate override files -echo "Writing overrides into text files" -cd $overridedir -dak make-overrides - -# FIXME -rm -f override.sid.all3 -for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done -ts "overrides" - -# Generate package / file mapping -echo "Generating package / file mapping" -dak make-pkg-file-mapping -ts "pkg-file-mapping" - -# Generate Packages and Sources files -echo "Generating Packages and Sources files" -cd $configdir -apt-ftparchive generate apt.conf -ts "apt-ftparchive" - -# Generate *.diff/ incremental updates -echo "Generating pdiff files" -dak generate-index-diffs -ts "pdiff" - -# Generate Release files -echo "Generating Release files" -dak generate-releases -ts "release files" - -# Clean out old packages -echo "Cleanup old packages/files" -dak clean-suites -dak clean-queues -ts "cleanup" - -# Needs to be rebuilt, as files have moved. Due to unaccepts, we need to -# update this before wanna-build is updated. -echo "Regenerating wanna-build/buildd information" -psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list -symlinks -d /srv/incoming.debian.org/buildd > /dev/null -apt-ftparchive generate apt.conf.buildd -ts "buildd" - -echo "Running various scripts from $scriptsdir" -cd $scriptsdir -./mkmaintainers -./copyoverrides -./mklslar -./mkfilesindices -./mkchecksums -ts "scripts" - -# (Re)generate the hardlinked mirror directory for "public" buildd / mirror access -echo "Regenerating mirror/ hardlink fun" -cd ${mirrordir} -rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. . -ts "mirror hardlinks" - -echo "Trigger daily wanna-build run" -ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org -ts "w-b" - -rm -f $NOTICE -rm -f $LOCKCU +lockfile -l 3600 "${LOCK_DAILY}" +trap onerror ERR +trap cleanup EXIT TERM HUP INT QUIT + +touch "${LOCK_BRITNEY}" + +GO=( + FUNC="savetimestamp" + TIME="" + ARGS="" + ERR="false" +) +stage $GO + +GO=( + FUNC="merkel1" + TIME="init" + ARGS="" + ERR="false" +) +stage $GO & + +GO=( + FUNC="pgdump_pre" + TIME="pg_dump1" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="updates" + TIME="External Updates" + ARGS="" + ERR="false" +) +stage $GO + +GO=( + FUNC="i18n1" + TIME="i18n 1" + ARGS="" + ERR="false" +) +stage $GO + +lockfile "$LOCK_ACCEPTED" +lockfile "$LOCK_NEW" + +GO=( + FUNC="punew" + TIME="p-u-new" + ARGS="proposedupdates" + ERR="false" +) +stage $GO + +#GO=( +# FUNC="opunew" +# TIME="o-p-u-new" +# ARGS="oldproposedupdates" +# ERR="false" +#) +#stage $GO + +GO=( + FUNC="newstage" + TIME="newstage" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="cruft" + TIME="cruft" + ARGS="" + ERR="" +) +stage $GO + +state "indices" + +GO=( + FUNC="dominate" + TIME="dominate" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="filelist" + TIME="generate-filelist" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="fingerprints" + TIME="import-keyring" + ARGS="" + ERR="false" +) +stage $GO + +GO=( + FUNC="overrides" + TIME="overrides" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mpfm" + TIME="pkg-file-mapping" + ARGS="" + ERR="false" +) +stage $GO + +state "packages/contents" +GO=( + FUNC="packages" + TIME="apt-ftparchive" + ARGS="" + ERR="" +) +# Careful: When we ever go and remove this monster-long thing, we have to check the backgrounded +# functions before it. We no longer have a 1.5hour sync point then. +stage $GO + +state "dists/" +GO=( + FUNC="pdiff" + TIME="pdiff" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="release" + TIME="release files" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="dakcleanup" + TIME="cleanup" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="buildd_dir" + TIME="buildd_dir" + ARGS="" + ERR="" +) +stage $GO + +state "scripts" +GO=( + FUNC="mkmaintainers" + TIME="mkmaintainers" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mkuploaders" + TIME="mkuploaders" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="copyoverrides" + TIME="copyoverrides" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mklslar" + TIME="mklslar" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mkfilesindices" + TIME="mkfilesindices" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mkchecksums" + TIME="mkchecksums" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mirror" + TIME="mirror hardlinks" + ARGS="" + ERR="" +) +stage $GO + +rm -f "$LOCK_ACCEPTED" +rm -f "$LOCK_NEW" +rm -f "${LOCK_DAILY}" ts "locked part finished" +state "postlock" + +GO=( + FUNC="changelogs" + TIME="changelogs" + ARGS="" + ERR="false" +) +stage $GO & -################################################################################ +GO=( + FUNC="pgdump_post" + TIME="pg_dump2" + ARGS="" + ERR="" +) +stage $GO & -echo "Creating post-daily-cron-job backup of projectb database..." -POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) -pg_dump projectb > $POSTDUMP -(cd $base/backup; ln -sf $POSTDUMP current) -ts "pg_dump2" - -################################################################################ - - -echo "Expiring old database dumps..." -(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*") -ts "expire_dumps" - -################################################################################ +GO=( + FUNC="expire" + TIME="expire_dumps" + ARGS="" + ERR="" +) +stage $GO & +GO=( + FUNC="transitionsclean" + TIME="transitionsclean" + ARGS="" + ERR="" +) +stage $GO & -# Send a report on NEW/BYHAND packages -echo "Nagging ftpteam about NEW/BYHAND packages" -dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org -# and one on crufty packages -echo "Sending information about crufty packages" -dak cruft-report > $webdir/cruft-report-daily.txt -dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt -cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org -ts "reports" +GO=( + FUNC="reports" + TIME="reports" + ARGS="" + ERR="" +) +stage $GO & -echo "Updating DM html page" -$scriptsdir/dm-monitor >$webdir/dm-uploaders.html +GO=( + FUNC="dm" + TIME="" + ARGS="" + ERR="" +) +stage $GO & -echo "Categorizing uncategorized bugs filed against ftp.debian.org" -dak bts-categorize +GO=( + FUNC="bts" + TIME="" + ARGS="" + ERR="false" +) +stage $GO & -################################################################################ +GO=( + FUNC="merkel2" + TIME="merkel projectb push" + ARGS="" + ERR="false" +) +stage $GO & -# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached -echo "Trigger merkels projectb sync" -ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true -ts "merkel projectb push" +GO=( + FUNC="mirrorpush" + TIME="mirrorpush" + ARGS="" + ERR="false" +) +stage $GO & -################################################################################ +GO=( + FUNC="i18n2" + TIME="i18n 2" + ARGS="" + ERR="false" +) +stage $GO & +GO=( + FUNC="stats" + TIME="stats" + ARGS="" + ERR="false" +) +stage $GO & -ulimit -m 90000 -d 90000 -s 10000 -v 200000 +GO=( + FUNC="testingsourcelist" + TIME="" + ARGS="" + ERR="false" +) +stage $GO & -echo "Using run-parts to run scripts in $base/scripts/distmnt" -run-parts --report $base/scripts/distmnt -ts "run-parts" +rm -f "${LOCK_BRITNEY}" -echo "Exporting package data foo for i18n project" -STAMP=$(date "+%Y%m%d%H%M") -mkdir -p ${scriptdir}/i18n/${STAMP} -cd ${scriptdir}/i18n/${STAMP} -dak control-suite -l stable > etch -dak control-suite -l testing > lenny -dak control-suite -l unstable > sid -echo "${STAMP}" > timestamp -gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp -rm -f md5sum -md5sum * > md5sum -cd ${webdir}/ -ln -sfT ${scriptdir}/i18n/${STAMP} i18n +GO=( + FUNC="pgdakdev" + TIME="dak-dev db" + ARGS="" + ERR="false" +) +stage $GO & -cd ${scriptdir} -find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf -ts "i18n 2" +GO=( + FUNC="merkel3" + TIME="merkel ddaccessible sync" + ARGS="" + ERR="false" +) +stage $GO & -echo "Daily cron scripts successful." +GO=( + FUNC="compress" + TIME="compress" + ARGS="" + ERR="" +) +stage $GO & -# Stats pr0n -echo "Updating stats data" -cd $configdir -$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data -R --slave --vanilla < $base/misc/ftpstats.R -ts "stats" +GO=( + FUNC="aptftpcleanup" + TIME="apt-ftparchive cleanup" + ARGS="" + ERR="false" +) +stage $GO -# Remove the britney lock -rm -f ${BRITNEYLOCK} +log "Daily cron scripts successful, all done" -# Clean up apt-ftparchive's databases -echo "Clean up apt-ftparchive's databases" -cd $configdir -apt-ftparchive -q clean apt.conf -ts "apt-ftparchive cleanup" +exec > "$logdir/afterdinstall.log" 2>&1 -# Compress psql backups -echo "Compress old psql backups" -(cd $base/backup/ - find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 | - while read dumpname; do - echo "Compressing $dumpname" - bzip2 -9 "$dumpname" - done +GO=( + FUNC="renamelogfile" + TIME="" + ARGS="" + ERR="false" ) -ts "compress" +stage $GO +state "all done" -echo "Removing old dinstall logfiles" -(cd $logdir - find -maxdepth 1 -mindepth 1 -type f -name 'dinstall_*' -mtime +60 | - while read dumpname; do - echo "Removing $dumpname" - rm -f "$dumpname" - done - find -maxdepth 1 -mindepth 1 -type f -name 'weekly_*' -mtime +60 | - while read dumpname; do - echo "Removing $dumpname" - rm -f "$dumpname" - done -) -ts "logremove" - -echo "Finally, all is done, sending mail and compressing logfile" -exec > /dev/null 2>&1 - -$masterdir/tools/logs.py "$LOGFILE" - -cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -bzip2 -9 "$LOGFILE" - -################################################################################ +# Now, at the very (successful) end of dinstall, make sure we remove +# our stage files, so the next dinstall run will do it all again. +rm -f ${stagedir}/* +touch "${DINSTALLEND}"