Refactor functions getSource() and getBinaries().

[dak.git] / config / debian / cron.dinstall

#!/bin/bash
# No way I try to deal with a crippled sh just for POSIX foo.

# Copyright (C) 2009 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# Homer: Are you saying you're never going to eat any animal again? What
#        about bacon?
# Lisa: No.
# Homer: Ham?
# Lisa: No.
# Homer: Pork chops?
# Lisa: Dad, those all come from the same animal.
# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.

# exit on errors
set -e
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the functions is
# the important part here)
set -E

# import the general variable set.
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS

########################################################################
# Functions                                                            #
########################################################################
# common functions are "outsourced"
. "${configdir}/common"

# source the dinstall functions
. "${configdir}/dinstall.functions"

########################################################################
########################################################################

# Function to save which stage we are in, so we can restart an interrupted
# dinstall. Or even run actions in parallel, if we dare to, by simply
# backgrounding the call to this function. But that should only really be
# done for things we don't care much about.
#
# This should be called with the first argument being an array, with the
# members
#  - FUNC - the function name to call
#  - ARGS - Possible arguments to hand to the function. Can be the empty string
#  - TIME - The timestamp name. Can be the empty string
#  - ERR  - if this is the string false, then the call will be surrounded by
#           set +e ... set -e calls, so errors in the function do not exit
#           dinstall. Can be the empty string, meaning true.
#
# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
# ADDED FOR DINSTALL FEATURES!
function stage() {
    ARGS='GO[@]'
    local "${!ARGS}"

    error=${ERR:-"true"}

    STAGEFILE="${stagedir}/${FUNC}"
    if [ -f "${STAGEFILE}" ]; then
        stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
        unixtime=$(date +%s)
        difference=$(( $unixtime - $stamptime ))
        if [ ${difference} -ge 14400 ]; then
            log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
        else
            log "Did already run ${FUNC}, not calling again..."
        fi
        return
    fi

    debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"

    # Make sure we are always at the same place. If a function wants to be elsewhere,
    # it has to cd first!
    cd ${configdir}

    # Now redirect the output into $STAGEFILE.log. In case it errors out somewhere our
    # errorhandler trap can then mail the contents of $STAGEFILE.log only, instead of a whole
    # dinstall logfile. Short error mails ftw!
    exec >> "${STAGEFILE}.log" 2>&1

    if [ -f "${LOCK_STOP}" ]; then
        log "${LOCK_STOP} exists, exiting immediately"
        exit 42
    fi

    if [ "${error}" = "false" ]; then
        set +e
    fi
    ${FUNC} ${ARGS}

    # No matter what happened in the function, we make sure we have set -e default state back
    set -e

    # Make sure we are always at the same place.
    cd ${configdir}

    # We always use the same umask. If a function wants to do different, fine, but we reset.
    umask 022

    touch "${STAGEFILE}"

    if [ -n "${TIME}" ]; then
        ts "${TIME}"
    fi

    # And the output goes back to the normal logfile
    exec >> "$LOGFILE" 2>&1

    # Now we should make sure that we have a usable dinstall.log, so append the $STAGEFILE.log
    # to it.
    cat "${STAGEFILE}.log" >> "${LOGFILE}"
    rm -f "${STAGEFILE}.log"

    if [ -f "${LOCK_STOP}" ]; then
        log "${LOCK_STOP} exists, exiting immediately"
        exit 42
    fi
}

########################################################################

# We need logs.
LOGFILE="$logdir/dinstall.log"

exec >> "$LOGFILE" 2>&1

# And now source our default config
. "${configdir}/dinstall.variables"

# Make sure we start out with a sane umask setting
umask 022

# And use one locale, no matter what the caller has set
export LANG=C
export LC_ALL=C

touch "${DINSTALLSTART}"
ts "startup"
DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
state "Startup"

lockfile -l 3600 "${LOCK_DAILY}"
trap onerror ERR
trap remove_daily_lock EXIT TERM HUP INT QUIT

touch "${LOCK_BRITNEY}"

GO=(
    FUNC="savetimestamp"
    TIME=""
    ARGS=""
    ERR="false"
)
stage $GO

GO=(
    FUNC="qa1"
    TIME="init"
    ARGS=""
    ERR="false"
)
stage $GO &

GO=(
    FUNC="pg_timestamp"
    TIME="pg_dump1"
    ARGS="predinstall"
    ERR=""
)
stage $GO

GO=(
    FUNC="updates"
    TIME="External Updates"
    ARGS=""
    ERR="false"
)
stage $GO

GO=(
    FUNC="i18n1"
    TIME="i18n 1"
    ARGS=""
    ERR="false"
)
stage $GO

lockfile "$LOCK_ACCEPTED"
lockfile "$LOCK_NEW"
trap remove_all_locks EXIT TERM HUP INT QUIT

GO=(
    FUNC="punew"
    TIME="p-u-new"
    ARGS="proposedupdates"
    ERR="false"
)
stage $GO

GO=(
    FUNC="opunew"
    TIME="o-p-u-new"
    ARGS="oldproposedupdates"
    ERR="false"
)
stage $GO

GO=(
    FUNC="newstage"
    TIME="newstage"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="cruft"
    TIME="cruft"
    ARGS=""
    ERR=""
)
stage $GO

state "indices"

GO=(
    FUNC="dominate"
    TIME="dominate"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="filelist"
    TIME="generate-filelist"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="fingerprints"
    TIME="import-keyring"
    ARGS=""
    ERR="false"
)
stage $GO

GO=(
    FUNC="overrides"
    TIME="overrides"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="mpfm"
    TIME="pkg-file-mapping"
    ARGS=""
    ERR="false"
)
stage $GO

state "packages/contents"
GO=(
    FUNC="packages"
    TIME="apt-ftparchive"
    ARGS=""
    ERR=""
)
# Careful: When we ever go and remove this monster-long thing, we have to check the backgrounded
# functions before it. We no longer have a 1.5hour sync point then.
stage $GO

state "dists/"
GO=(
    FUNC="pdiff"
    TIME="pdiff"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="release"
    TIME="release files"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="dakcleanup"
    TIME="cleanup"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="buildd_dir"
    TIME="buildd_dir"
    ARGS=""
    ERR=""
)
stage $GO

state "scripts"
GO=(
    FUNC="mkmaintainers"
    TIME="mkmaintainers"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="mkuploaders"
    TIME="mkuploaders"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="copyoverrides"
    TIME="copyoverrides"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="mklslar"
    TIME="mklslar"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="mkfilesindices"
    TIME="mkfilesindices"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="mkchecksums"
    TIME="mkchecksums"
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="mirror"
    TIME="mirror hardlinks"
    ARGS=""
    ERR=""
)
stage $GO

remove_all_locks
trap - EXIT TERM HUP INT QUIT

ts "locked part finished"
state "postlock"

GO=(
    FUNC="changelogs"
    TIME="changelogs"
    ARGS=""
    ERR="false"
)
stage $GO &

GO=(
    FUNC="pg_timestamp"
    TIME="pg_dump2"
    ARGS="postdinstall"
    ERR=""
)
stage $GO

GO=(
    FUNC="expire"
    TIME="expire_dumps"
    ARGS=""
    ERR=""
)
stage $GO &

GO=(
    FUNC="transitionsclean"
    TIME="transitionsclean"
    ARGS=""
    ERR=""
)
stage $GO &

GO=(
    FUNC="dm"
    TIME=""
    ARGS=""
    ERR=""
)
stage $GO &

GO=(
    FUNC="bts"
    TIME=""
    ARGS=""
    ERR="false"
)
stage $GO &

GO=(
    FUNC="mirrorpush"
    TIME="mirrorpush"
    ARGS=""
    ERR="false"
)
stage $GO &

GO=(
    FUNC="i18n2"
    TIME="i18n 2"
    ARGS=""
    ERR="false"
)
stage $GO &

GO=(
    FUNC="stats"
    TIME="stats"
    ARGS=""
    ERR="false"
)
stage $GO &

GO=(
    FUNC="testingsourcelist"
    TIME=""
    ARGS=""
    ERR="false"
)
stage $GO &

rm -f "${LOCK_BRITNEY}"

GO=(
    FUNC="ddaccess"
    TIME="ddaccessible sync"
    ARGS=""
    ERR="false"
)
stage $GO &

GO=(
    FUNC="cleantransactions"
    TIME=""
    ARGS=""
    ERR=""
)
stage $GO

GO=(
    FUNC="aptftpcleanup"
    TIME="apt-ftparchive cleanup"
    ARGS=""
    ERR="false"
)
stage $GO

# we need to wait for the background processes before the end of dinstall
wait

log "Daily cron scripts successful, all done"

exec > "$logdir/afterdinstall.log" 2>&1

GO=(
    FUNC="renamelogfile"
    TIME=""
    ARGS=""
    ERR="false"
)
stage $GO
state "all done"


# Now, at the very (successful) end of dinstall, make sure we remove
# our stage files, so the next dinstall run will do it all again.
rm -f ${stagedir}/*
touch "${DINSTALLEND}"