X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fcron.dinstall;h=001ab117f2f0cc89ffe0dfc70b78de0bacf4eda6;hb=43e3cfd0724d425cdfe6528c76a5677ac95e6177;hp=e31ea80ed9aeaa004a25daf99fa4685461ad33be;hpb=2de2a91cbde91f3ecef5475829f7620edd94a14b;p=dak.git

diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index e31ea80e..001ab117 100755
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -1,7 +1,7 @@
 #!/bin/bash
 # No way I try to deal with a crippled sh just for POSIX foo.
 
-# Copyright (C) 2009-2012 Joerg Jaspert <joerg@debian.org>
+# Copyright (C) 2009-2015 Joerg Jaspert <joerg@debian.org>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as
@@ -71,15 +71,17 @@ function stage() {
     ARGS='GO[@]'
     local "${!ARGS}"
 
-    error=${ERR:-"true"}
+    local error=${ERR:-"true"}
 
     ARGS=${ARGS:-""}
-    STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
+
+    log "########## DINSTALL BEGIN: ${FUNC} ${ARGS} ##########"
+    local STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
     STAGEFILE=${STAGEFILE// /_}
     if [ -f "${STAGEFILE}" ]; then
-        stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
-        unixtime=$(date +%s)
-        difference=$(( $unixtime - $stamptime ))
+        local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
+        local unixtime=$(date +%s)
+        local difference=$(( $unixtime - $stamptime ))
         if [ ${difference} -ge 14400 ]; then
             log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
         else
@@ -90,13 +92,14 @@ function stage() {
 
     debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"
 
-    # Make sure we are always at the same place. If a function wants to be elsewhere,
-    # it has to cd first!
+    # Make sure we are always at the same place. If a function wants
+    # to be elsewhere, it has to cd first!
     cd ${configdir}
 
-    # Now redirect the output into $STAGEFILE.log. In case it errors out somewhere our
-    # errorhandler trap can then mail the contents of $STAGEFILE.log only, instead of a whole
-    # dinstall logfile. Short error mails ftw!
+    # Now redirect the output into $STAGEFILE.log. In case it errors
+    # out somewhere our errorhandler trap can then mail the contents
+    # of $STAGEFILE.log only, instead of a whole dinstall logfile.
+    # Short error mails ftw!
     exec >> "${STAGEFILE}.log" 2>&1
 
     if [ -f "${LOCK_STOP}" ]; then
@@ -104,18 +107,21 @@ function stage() {
         exit 42
     fi
 
+    # Do we care about trouble in the function we call?
     if [ "${error}" = "false" ]; then
         set +e
     fi
     ${FUNC} ${ARGS}
 
-    # No matter what happened in the function, we make sure we have set -e default state back
+    # No matter what happened in the function, we make sure we have
+    # set -e default state back
     set -e
 
     # Make sure we are always at the same place.
     cd ${configdir}
 
-    # We always use the same umask. If a function wants to do different, fine, but we reset.
+    # We always use the same umask. If a function wants to do
+    # different, fine, but we reset.
     umask 022
 
     touch "${STAGEFILE}"
@@ -127,11 +133,13 @@ function stage() {
     # And the output goes back to the normal logfile
     exec >> "$LOGFILE" 2>&1
 
-    # Now we should make sure that we have a usable dinstall.log, so append the $STAGEFILE.log
-    # to it.
+    # Now we should make sure that we have a usable dinstall.log, so
+    # append the $STAGEFILE.log to it.
     cat "${STAGEFILE}.log" >> "${LOGFILE}"
     rm -f "${STAGEFILE}.log"
 
+    log "########## DINSTALL END: ${FUNC} ##########"
+
     if [ -f "${LOCK_STOP}" ]; then
         log "${LOCK_STOP} exists, exiting immediately"
         exit 42
@@ -166,317 +174,117 @@ trap remove_daily_lock EXIT TERM HUP INT QUIT
 
 touch "${LOCK_BRITNEY}"
 
-GO=(
-    FUNC="savetimestamp"
-    TIME=""
-    ARGS=""
-    ERR="false"
-)
-stage $GO
-
-GO=(
-    FUNC="qa1"
-    TIME="init"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="pg_timestamp"
-    TIME="pg_dump1"
-    ARGS="predinstall"
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="updates"
-    TIME="External Updates"
-    ARGS=""
-    ERR="false"
-)
-stage $GO
-
-GO=(
-    FUNC="i18n1"
-    TIME="i18n 1"
-    ARGS=""
-    ERR="false"
-)
-stage $GO
-
-lockfile "$LOCK_ACCEPTED"
-trap remove_all_locks EXIT TERM HUP INT QUIT
-
-GO=(
-    FUNC="punew"
-    TIME="p-u-new"
-    ARGS="stable-new"
-    ERR="false"
-)
-stage $GO
-
-GO=(
-    FUNC="opunew"
-    TIME="o-p-u-new"
-    ARGS="oldstable-new"
-    ERR="false"
-)
-stage $GO
-
-GO=(
-    FUNC="cruft"
-    TIME="cruft"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-state "indices"
-
-GO=(
-    FUNC="dominate"
-    TIME="dominate"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="filelist"
-    TIME="generate-filelist"
-    ARGS=""
-    ERR=""
-)
-#stage $GO
-
-GO=(
-    FUNC="fingerprints"
-    TIME="import-keyring"
-    ARGS=""
-    ERR="false"
-)
-stage $GO
-
-GO=(
-    FUNC="overrides"
-    TIME="overrides"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mpfm"
-    TIME="pkg-file-mapping"
-    ARGS=""
-    ERR="false"
-)
-stage $GO
-
-state "packages/contents"
-GO=(
-    FUNC="packages"
-    TIME="apt-ftparchive"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-state "dists/"
-GO=(
-    FUNC="pdiff"
-    TIME="pdiff"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="gitpdiff"
-    TIME="gitpdiff"
-    ARGS=""
-    ERR=""
-)
-#stage $GO
-
-GO=(
-    FUNC="release"
-    TIME="release files"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="dakcleanup"
-    TIME="cleanup"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="buildd_dir"
-    TIME="buildd_dir"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-state "scripts"
-GO=(
-    FUNC="mkmaintainers"
-    TIME="mkmaintainers"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="copyoverrides"
-    TIME="copyoverrides"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mklslar"
-    TIME="mklslar"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mkfilesindices"
-    TIME="mkfilesindices"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mkchecksums"
-    TIME="mkchecksums"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mirror"
-    TIME="mirror hardlinks"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="ddaccess"
-    TIME="ddaccessible sync"
-    ARGS=""
-    ERR="false"
-)
-stage $GO
-
-remove_all_locks
-trap - EXIT TERM HUP INT QUIT
-
-ts "locked part finished"
-state "postlock"
-
-GO=(
-    FUNC="changelogs"
-    TIME="changelogs"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="pg_timestamp"
-    TIME="pg_dump2"
-    ARGS="postdinstall"
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="expire"
-    TIME="expire_dumps"
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-GO=(
-    FUNC="transitionsclean"
-    TIME="transitionsclean"
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-GO=(
-    FUNC="dm"
-    TIME=""
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-GO=(
-    FUNC="bts"
-    TIME=""
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="mirrorpush"
-    TIME="mirrorpush"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="i18n2"
-    TIME="i18n 2"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="stats"
-    TIME="stats"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="testingsourcelist"
-    TIME=""
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-rm -f "${LOCK_BRITNEY}"
-
-GO=(
-    FUNC="cleantransactions"
-    TIME=""
-    ARGS=""
-    ERR=""
-)
-stage $GO
+# This loop simply wants to be fed by a list of values (see below)
+# made out of 5 columns.
+# The first four are the array values for the stage function, the
+# fifth tells us if we should background the stage call.
+#
+#  - FUNC - the function name to call
+#  - ARGS - Possible arguments to hand to the function. Can be the empty string
+#  - TIME - The timestamp name. Can be the empty string
+#  - ERR  - if this is the string false, then the call will be surrounded by
+#           set +e ... set -e calls, so errors in the function do not exit
+#           dinstall. Can be the empty string, meaning true.
+#  - BG   - Background the function stage?
+#
+# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \
+#
+# NOTE 1: There are two special values for the first column (FUNC).
+#         STATE   - do not call stage function, call the state
+#                   function to update the public statefile "where is dinstall"
+#         NOSTAGE - do not call stage function, call the command directly.
+#
+# Note 2: If you want to hand an empty value to the stage function,
+#         use the word "none" in the list below.
+while read FUNC ARGS TIME ERR BACKGROUND; do
+    debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND"
+
+    # Empty values in the value list are the string "none" (or the
+    # while read loop won't work). Here we ensure that variables that
+    # can be empty, are empty if the string none is set for them.
+    for var in ARGS TIME; do
+        if [[ ${!var} == none ]]; then
+            typeset ${var}=''
+        fi
+    done
+
+    # ERR/BACKGROUND are boolean, check that they are.
+    for var in ERR BACKGROUND; do
+        if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then
+            error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}"
+        fi
+    done
+
+    case ${FUNC} in
+        STATE)
+            state ${ARGS}
+        ;;
+        NOSTAGE)
+            ${ARGS}
+        ;;
+        *)
+            GO=(
+                FUNC=${FUNC}
+                TIME=${TIME}
+                ARGS=${ARGS}
+                ERR=${ERR}
+            )
+            if [[ ${BACKGROUND} == true ]]; then
+                stage $GO &
+            else
+                stage $GO
+            fi
+        ;;
+    esac
+done < <(cat - <<EOF
+savetimestamp          none                       none                       false   false
+qa1                    none                       init                       true    true
+pg_timestamp           predinstall                pg_dump1                   false   false
+updates                none                       External\ Updates          false   false
+i18n1                  none                       i18n\ 1                    false   false
+dep11                  none                       dep11\ 1                   false   false
+NOSTAGE                lockaccepted               none                       false   false
+punew                  stable-new                 p-u-new                    false   false
+opunew                 oldstable-new              o-p-u-new                  false   false
+backports_policy       none                       backports-policy           false   false
+cruft                  none                       cruft                      false   false
+STATE                  indices                    none                       false   false
+dominate               none                       dominate                   false   false
+autocruft              none                       autocruft                  false   false
+fingerprints           none                       import-keyring             false   false
+overrides              none                       overrides                  false   false
+mpfm                   none                       pkg-file-mapping           false   false
+STATE                  packages/contents          none                       false   false
+packages               none                       apt-ftparchive             false   false
+STATE                  dists/                     none                       false   false
+pdiff                  none                       pdiff                      false   false
+release                none                       release\ files             false   false
+dakcleanup             none                       cleanup                    false   false
+STATE                  scripts                    none                       false   false
+mkmaintainers          none                       mkmaintainers              false   false
+copyoverrides          none                       copyoverrides              false   false
+mklslar                none                       mklslar                    false   false
+mkfilesindices         none                       mkfilesindices             false   false
+mkchecksums            none                       mkchecksums                false   false
+mirror                 none                       mirror\ hardlinks          false   false
+NOSTAGE                remove_locks               none                       false   false
+STATE                  postlock                   none                       false   false
+ddaccess               none                       ddaccessible\ sync         true    true
+changelogs             none                       changelogs                 false   true
+pg_timestamp           postdinstall               pg_dump2                   false   false
+expire                 none                       expire_dumps               false   true
+transitionsclean       none                       transitionsclean           false   true
+dm                     none                       none                       false   true
+bts                    none                       none                       false   true
+mirrorpush             none                       mirrorpush                 false   true
+mirrorpush-backports   none                       mirrorpush-backports       false   true
+i18n2                  none                       i18n\ 2                    false   true
+stats                  none                       stats                      false   true
+testingsourcelist      none                       none                       false   true
+NOSTAGE                rm\ -f\ "\${LOCK_BRITNEY}" none                       false   false
+cleantransactions      none                       none                       false   false
+EOF
+        )
 
 # we need to wait for the background processes before the end of dinstall
 wait
@@ -485,15 +293,19 @@ log "Daily cron scripts successful, all done"
 
 exec > "$logdir/afterdinstall.log" 2>&1
 
-GO=(
-    FUNC="renamelogfile"
-    TIME=""
-    ARGS=""
-    ERR="false"
-)
-stage $GO
-state "all done"
+if [ -f "${dbdir}/dinstallstart" ]; then
+    NOW=$(cat "${dbdir}/dinstallstart")
+    mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
+    logstats "$logdir/dinstall_${NOW}.log"
+    bzip2 -9 "$logdir/dinstall_${NOW}.log"
+else
+    error "Problem, I don't know when dinstall started, unable to do log statistics."
+    NOW=`date "+%Y.%m.%d-%H:%M:%S"`
+    mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
+    bzip2 -9 "$logdir/dinstall_${NOW}.log"
+fi
 
+state "all done"
 
 # Now, at the very (successful) end of dinstall, make sure we remove
 # our stage files, so the next dinstall run will do it all again.