X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;ds=inline;f=config%2Fbackports%2Fcron.dinstall;fp=config%2Fbackports%2Fcron.dinstall;h=0841f8380aab8c51136c709805d952556c30bb23;hb=99ba94686d0dc810dad10a1f5531d23bd886cf5a;hp=0000000000000000000000000000000000000000;hpb=fa35358d461b03ff5833385ca2970cecce454d0e;p=dak.git diff --git a/config/backports/cron.dinstall b/config/backports/cron.dinstall new file mode 100755 index 00000000..0841f838 --- /dev/null +++ b/config/backports/cron.dinstall @@ -0,0 +1,530 @@ +#!/bin/bash +# No way I try to deal with a crippled sh just for POSIX foo. + +# Copyright (C) 2009 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +# Homer: Are you saying you're never going to eat any animal again? What +# about bacon? +# Lisa: No. +# Homer: Ham? +# Lisa: No. +# Homer: Pork chops? +# Lisa: Dad, those all come from the same animal. +# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal. + +# exit on errors +set -e +# make sure to only use defined variables +set -u +# ERR traps should be inherited from functions too. (And command +# substitutions and subshells and whatnot, but for us the functions is +# the important part here) +set -E + +# import the general variable set. +export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars +. $SCRIPTVARS + +######################################################################## +# Functions # +######################################################################## +# common functions are "outsourced" +. "${configdir}/common" + +# source the dinstall functions +. "${configdir}/dinstall.functions" + +######################################################################## +######################################################################## + +# Function to save which stage we are in, so we can restart an interrupted +# dinstall. Or even run actions in parallel, if we dare to, by simply +# backgrounding the call to this function. But that should only really be +# done for things we don't care much about. +# +# This should be called with the first argument being an array, with the +# members +# - FUNC - the function name to call +# - ARGS - Possible arguments to hand to the function. Can be the empty string +# - TIME - The timestamp name. Can be the empty string +# - ERR - if this is the string false, then the call will be surrounded by +# set +e ... set -e calls, so errors in the function do not exit +# dinstall. Can be the empty string, meaning true. +# +# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES +# ADDED FOR DINSTALL FEATURES! +function stage() { + ARGS='GO[@]' + local "${!ARGS}" + + error=${ERR:-"true"} + + STAGEFILE="${stagedir}/${FUNC}" + if [ -f "${STAGEFILE}" ]; then + stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}") + unixtime=$(date +%s) + difference=$(( $unixtime - $stamptime )) + if [ ${difference} -ge 14400 ]; then + log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check." + else + log "Did already run ${FUNC}, not calling again..." + fi + return + fi + + debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}" + + # Make sure we are always at the same place. If a function wants to be elsewhere, + # it has to cd first! + cd ${configdir} + + # Now redirect the output into $STAGEFILE.log. In case it errors out somewhere our + # errorhandler trap can then mail the contents of $STAGEFILE.log only, instead of a whole + # dinstall logfile. Short error mails ftw! + exec >> "${STAGEFILE}.log" 2>&1 + + if [ -f "${LOCK_STOP}" ]; then + log "${LOCK_STOP} exists, exiting immediately" + exit 42 + fi + + if [ "${error}" = "false" ]; then + set +e + fi + ${FUNC} ${ARGS} + + # No matter what happened in the function, we make sure we have set -e default state back + set -e + + # Make sure we are always at the same place. + cd ${configdir} + + # We always use the same umask. If a function wants to do different, fine, but we reset. + umask 022 + + touch "${STAGEFILE}" + + if [ -n "${TIME}" ]; then + ts "${TIME}" + fi + + # And the output goes back to the normal logfile + exec >> "$LOGFILE" 2>&1 + + # Now we should make sure that we have a usable dinstall.log, so append the $STAGEFILE.log + # to it. + cat "${STAGEFILE}.log" >> "${LOGFILE}" + rm -f "${STAGEFILE}.log" + + if [ -f "${LOCK_STOP}" ]; then + log "${LOCK_STOP} exists, exiting immediately" + exit 42 + fi +} + +######################################################################## + +# We need logs. +LOGFILE="$logdir/dinstall.log" + +exec >> "$LOGFILE" 2>&1 + +# And now source our default config +. "${configdir}/dinstall.variables" + +# Make sure we start out with a sane umask setting +umask 022 + +# And use one locale, no matter what the caller has set +export LANG=C +export LC_ALL=C + +touch "${DINSTALLSTART}" +ts "startup" +DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")" +state "Startup" + +lockfile -l 3600 "${LOCK_DAILY}" +trap onerror ERR +trap cleanup EXIT TERM HUP INT QUIT + +touch "${LOCK_BRITNEY}" + +GO=( + FUNC="savetimestamp" + TIME="" + ARGS="" + ERR="false" +) +stage $GO + +GO=( + FUNC="merkel1" + TIME="init" + ARGS="" + ERR="false" +) +stage $GO & + +GO=( + FUNC="pgdump_pre" + TIME="pg_dump1" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="updates" + TIME="External Updates" + ARGS="" + ERR="false" +) +stage $GO + +GO=( + FUNC="i18n1" + TIME="i18n 1" + ARGS="" + ERR="false" +) +stage $GO + +lockfile "$LOCK_ACCEPTED" +lockfile "$LOCK_NEW" + +GO=( + FUNC="punew" + TIME="p-u-new" + ARGS="proposedupdates" + ERR="false" +) +stage $GO + +#GO=( +# FUNC="opunew" +# TIME="o-p-u-new" +# ARGS="oldproposedupdates" +# ERR="false" +#) +#stage $GO + +GO=( + FUNC="newstage" + TIME="newstage" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="cruft" + TIME="cruft" + ARGS="" + ERR="" +) +stage $GO + +state "indices" + +GO=( + FUNC="dominate" + TIME="dominate" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="filelist" + TIME="generate-filelist" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="fingerprints" + TIME="import-keyring" + ARGS="" + ERR="false" +) +stage $GO + +GO=( + FUNC="overrides" + TIME="overrides" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mpfm" + TIME="pkg-file-mapping" + ARGS="" + ERR="false" +) +stage $GO + +state "packages/contents" +GO=( + FUNC="packages" + TIME="apt-ftparchive" + ARGS="" + ERR="" +) +# Careful: When we ever go and remove this monster-long thing, we have to check the backgrounded +# functions before it. We no longer have a 1.5hour sync point then. +stage $GO + +state "dists/" +GO=( + FUNC="pdiff" + TIME="pdiff" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="release" + TIME="release files" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="dakcleanup" + TIME="cleanup" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="buildd_dir" + TIME="buildd_dir" + ARGS="" + ERR="" +) +stage $GO + +state "scripts" +GO=( + FUNC="mkmaintainers" + TIME="mkmaintainers" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mkuploaders" + TIME="mkuploaders" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="copyoverrides" + TIME="copyoverrides" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mklslar" + TIME="mklslar" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mkfilesindices" + TIME="mkfilesindices" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mkchecksums" + TIME="mkchecksums" + ARGS="" + ERR="" +) +stage $GO + +GO=( + FUNC="mirror" + TIME="mirror hardlinks" + ARGS="" + ERR="" +) +stage $GO + +rm -f "$LOCK_ACCEPTED" +rm -f "$LOCK_NEW" +rm -f "${LOCK_DAILY}" + +ts "locked part finished" +state "postlock" + +GO=( + FUNC="pgdump_post" + TIME="pg_dump2" + ARGS="" + ERR="" +) +stage $GO & + +GO=( + FUNC="expire" + TIME="expire_dumps" + ARGS="" + ERR="" +) +stage $GO & + +GO=( + FUNC="transitionsclean" + TIME="transitionsclean" + ARGS="" + ERR="" +) +stage $GO & + +GO=( + FUNC="reports" + TIME="reports" + ARGS="" + ERR="" +) +stage $GO & + +GO=( + FUNC="dm" + TIME="" + ARGS="" + ERR="" +) +stage $GO & + +GO=( + FUNC="bts" + TIME="" + ARGS="" + ERR="false" +) +stage $GO & + +GO=( + FUNC="merkel2" + TIME="merkel projectb push" + ARGS="" + ERR="false" +) +stage $GO & + +GO=( + FUNC="mirrorpush" + TIME="mirrorpush" + ARGS="" + ERR="false" +) +stage $GO & + +GO=( + FUNC="i18n2" + TIME="i18n 2" + ARGS="" + ERR="false" +) +stage $GO & + +GO=( + FUNC="stats" + TIME="stats" + ARGS="" + ERR="false" +) +stage $GO & + +GO=( + FUNC="testingsourcelist" + TIME="" + ARGS="" + ERR="false" +) +stage $GO & + +rm -f "${LOCK_BRITNEY}" + +GO=( + FUNC="pgdakdev" + TIME="dak-dev db" + ARGS="" + ERR="false" +) +stage $GO & + +GO=( + FUNC="merkel3" + TIME="merkel ddaccessible sync" + ARGS="" + ERR="false" +) +stage $GO & + +GO=( + FUNC="compress" + TIME="compress" + ARGS="" + ERR="" +) +stage $GO & + +GO=( + FUNC="aptftpcleanup" + TIME="apt-ftparchive cleanup" + ARGS="" + ERR="false" +) +stage $GO + +log "Daily cron scripts successful, all done" + +exec > "$logdir/afterdinstall.log" 2>&1 + +GO=( + FUNC="renamelogfile" + TIME="" + ARGS="" + ERR="false" +) +stage $GO +state "all done" + + +# Now, at the very (successful) end of dinstall, make sure we remove +# our stage files, so the next dinstall run will do it all again. +rm -f ${stagedir}/* +touch "${DINSTALLEND}"