X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fcron.dinstall;h=4c3613d507d17ed7c70630202540d3b48a611786;hb=11c5e4d559120ac40c31b27c9422c79dfa5e0960;hp=747db2f1333b2c7bec2d9c6313930e1b184fa87b;hpb=4c7eee9642e82b6286f807ad92a93e7ef30288e6;p=dak.git

diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index 747db2f1..4c3613d5 100755
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -1,7 +1,7 @@
 #!/bin/bash
 # No way I try to deal with a crippled sh just for POSIX foo.
 
-# Copyright (C) 2009 Joerg Jaspert <joerg@debian.org>
+# Copyright (C) 2009-2012 Joerg Jaspert <joerg@debian.org>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as
@@ -27,6 +27,7 @@
 
 # exit on errors
 set -e
+set -o pipefail
 # make sure to only use defined variables
 set -u
 # ERR traps should be inherited from functions too. (And command
@@ -35,7 +36,7 @@ set -u
 set -E
 
 # import the general variable set.
-export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
 . $SCRIPTVARS
 
 ########################################################################
@@ -70,13 +71,17 @@ function stage() {
     ARGS='GO[@]'
     local "${!ARGS}"
 
-    error=${ERR:-"true"}
+    local error=${ERR:-"true"}
 
-    STAGEFILE="${stagedir}/${FUNC}"
+    ARGS=${ARGS:-""}
+
+    log "########## DINSTALL BEGIN: ${FUNC} ${ARGS} ##########"
+    local STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
+    STAGEFILE=${STAGEFILE// /_}
     if [ -f "${STAGEFILE}" ]; then
-        stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
-        unixtime=$(date +%s)
-        difference=$(( $unixtime - $stamptime ))
+        local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
+        local unixtime=$(date +%s)
+        local difference=$(( $unixtime - $stamptime ))
         if [ ${difference} -ge 14400 ]; then
             log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
         else
@@ -129,6 +134,8 @@ function stage() {
     cat "${STAGEFILE}.log" >> "${LOGFILE}"
     rm -f "${STAGEFILE}.log"
 
+    log "########## DINSTALL END: ${FUNC} ##########"
+
     if [ -f "${LOCK_STOP}" ]; then
         log "${LOCK_STOP} exists, exiting immediately"
         exit 42
@@ -154,358 +161,119 @@ export LC_ALL=C
 
 touch "${DINSTALLSTART}"
 ts "startup"
+DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
+state "Startup"
 
 lockfile -l 3600 "${LOCK_DAILY}"
 trap onerror ERR
-trap cleanup EXIT TERM HUP INT QUIT
+trap remove_daily_lock EXIT TERM HUP INT QUIT
 
 touch "${LOCK_BRITNEY}"
 
-GO=(
-    FUNC="savetimestamp"
-    TIME=""
-    ARGS=""
-    ERR="false"
-)
-stage $GO
-
-GO=(
-    FUNC="merkel1"
-    TIME="init"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="pgdump_pre"
-    TIME="pg_dump1"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="updates"
-    TIME="External Updates"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="punew"
-    TIME="p-u-new"
-    ARGS="p-u-new"
-    ERR=""
-)
-### TODO: policy-new
-#stage $GO
-
-GO=(
-    FUNC="opunew"
-    TIME="o-p-u-new"
-    ARGS="o-p-u-new"
-    ERR=""
-)
-### TODO: policy-new
-#stage $GO
-
-GO=(
-    FUNC="i18n1"
-    TIME="i18n 1"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-lockfile "$LOCK_ACCEPTED"
-lockfile "$LOCK_NEW"
-
-GO=(
-    FUNC="process_unchecked"
-    TIME="unchecked"
-    ARGS=""
-    ERR=""
-)
-# disabled until p-u is faster than now. it runs often enough, so wont hurt to save
-# the time here.
-#stage $GO
-
-GO=(
-    FUNC="cruft"
-    TIME="cruft"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-rm -f "$LOCK_ACCEPTED"
-rm -f "$LOCK_NEW"
-
-GO=(
-    FUNC="dominate"
-    TIME="dominate"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="filelist"
-    TIME="generate-filelist"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="fingerprints"
-    TIME="import-keyring"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="overrides"
-    TIME="overrides"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mpfm"
-    TIME="pkg-file-mapping"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="packages"
-    TIME="apt-ftparchive"
-    ARGS=""
-    ERR=""
-)
-# Careful: When we ever go and remove this monster-long thing, we have to check the backgrounded
-# functions before it. We no longer have a 1.5hour sync point then.
-stage $GO
-
-GO=(
-    FUNC="pdiff"
-    TIME="pdiff"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="release"
-    TIME="release files"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="dakcleanup"
-    TIME="cleanup"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="buildd_dir"
-    TIME="buildd_dir"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mkmaintainers"
-    TIME="mkmaintainers"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="copyoverrides"
-    TIME="copyoverrides"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mklslar"
-    TIME="mklslar"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mkfilesindices"
-    TIME="mkfilesindices"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mkchecksums"
-    TIME="mkchecksums"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="mirror"
-    TIME="mirror hardlinks"
-    ARGS=""
-    ERR=""
-)
-stage $GO
-
-GO=(
-    FUNC="wb"
-    TIME="w-b"
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-rm -f "${LOCK_DAILY}"
-
-ts "locked part finished"
-
-GO=(
-    FUNC="pgdump_post"
-    TIME="pg_dump2"
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-GO=(
-    FUNC="expire"
-    TIME="expire_dumps"
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-GO=(
-    FUNC="transitionsclean"
-    TIME="transitionsclean"
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-GO=(
-    FUNC="reports"
-    TIME="reports"
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-GO=(
-    FUNC="dm"
-    TIME=""
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-GO=(
-    FUNC="bts"
-    TIME=""
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="merkel2"
-    TIME="merkel projectb push"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="mirrorpush"
-    TIME="mirrorpush"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="i18n2"
-    TIME="i18n 2"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="stats"
-    TIME="stats"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="testingsourcelist"
-    TIME=""
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-rm -f "${LOCK_BRITNEY}"
-
-GO=(
-    FUNC="pgdakdev"
-    TIME="dak-dev db"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="merkel3"
-    TIME="merkel ddaccessible sync"
-    ARGS=""
-    ERR="false"
-)
-stage $GO &
-
-GO=(
-    FUNC="compress"
-    TIME="compress"
-    ARGS=""
-    ERR=""
-)
-stage $GO &
-
-GO=(
-    FUNC="aptftpcleanup"
-    TIME="apt-ftparchive cleanup"
-    ARGS=""
-    ERR="false"
-)
-stage $GO
+# This loop simply wants to be fed by a list of values (see below)
+# which consists of 5 values currently.
+# The first four are the array values for the stage function, the
+# fifths tells us if we should background the stage call.
+#
+#  - FUNC - the function name to call
+#  - ARGS - Possible arguments to hand to the function. Can be the empty string
+#  - TIME - The timestamp name. Can be the empty string
+#  - ERR  - if this is the string false, then the call will be surrounded by
+#           set +e ... set -e calls, so errors in the function do not exit
+#           dinstall. Can be the empty string, meaning true.
+#  - BG   - Background the function stage?
+#
+# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \
+#
+# ATTENTION: There are two special values for the first column (FUNC).
+#            STATE   - do not call stage function, call the state
+#                      function to update the public statefile "where is dinstall"
+#            NOSTAGE - do not call stage function, call the command directly.
+while read FUNC ARGS TIME ERR BACKGROUND; do
+    debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND"
+
+    # Empty values in the value list are the string "none" (or the
+    # while read loop won't work). Here we ensure that variables that
+    # can be empty, are empty if the string none is set for them.
+    for var in ARGS TIME; do
+        if [[ ${!var} == none ]]; then
+            typeset ${var}=''
+        fi
+    done
+
+    case ${FUNC} in
+        STATE)
+            state ${ARGS}
+        ;;
+        NOSTAGE)
+            ${ARGS}
+        ;;
+        *)
+            GO=(
+                FUNC=${FUNC}
+                TIME=${TIME}
+                ARGS=${ARGS}
+                ERR=${ERR}
+            )
+            if [[ ${BACKGROUND} == true ]]; then
+                stage $GO &
+            else
+                stage $GO
+            fi
+        ;;
+    esac
+done < <(cat - <<EOF
+savetimestamp          none                       none                       false   false
+qa1                    none                       init                       true    true
+pg_timestamp           predinstall                pg_dump1                   false   false
+updates                none                       External\ Updates          false   false
+i18n1                  none                       i18n\ 1                    false   false
+dep11                  none                       dep11\ 1                   false   false
+NOSTAGE                lockaccepted               none                       false   false
+punew                  stable-new                 p-u-new                    false   false
+opunew                 oldstable-new              o-p-u-new                  false   false
+backports_policy       none                       backports-policy           false   false
+cruft                  none                       cruft                      false   false
+STATE                  indices                    none                       false   false
+dominate               none                       dominate                   false   false
+autocruft              none                       autocruft                  false   false
+fingerprints           none                       import-keyring             false   false
+overrides              none                       overrides                  false   false
+mpfm                   none                       pkg-file-mapping           false   false
+STATE                  packages/contents          none                       false   false
+packages               none                       apt-ftparchive             false   false
+STATE                  dists/                     none                       false   false
+pdiff                  none                       pdiff                      false   false
+release                none                       release\ files             false   false
+dakcleanup             none                       cleanup                    false   false
+STATE                  scripts                    none                       false   false
+mkmaintainers          none                       mkmaintainers              false   false
+copyoverrides          none                       copyoverrides              false   false
+mklslar                none                       mklslar                    false   false
+mkfilesindices         none                       mkfilesindices             false   false
+mkchecksums            none                       mkchecksums                false   false
+mirror                 none                       mirror\ hardlinks          false   false
+ddaccess               none                       ddaccessible\ sync         false   false
+NOSTAGE                remove_locks               none                       false   false
+STATE                  postlock                   none                       false   false
+changelogs             none                       changelogs                 false   true
+pg_timestamp           postdinstall               pg_dump2                   false   false
+expire                 none                       expire_dumps               false   true
+transitionsclean       none                       transitionsclean           false   true
+dm                     none                       none                       false   true
+bts                    none                       none                       false   true
+mirrorpush             none                       mirrorpush                 false   true
+mirrorpush-backports   none                       mirrorpush-backports       false   true
+i18n2                  none                       i18n\ 2                    false   true
+stats                  none                       stats                      false   true
+testingsourcelist      none                       none                       false   true
+NOSTAGE                rm\ -f\ "\${LOCK_BRITNEY}" none                       false   false
+cleantransactions      none                       none                       false   false
+EOF
+        )
+
+# we need to wait for the background processes before the end of dinstall
+wait
 
 log "Daily cron scripts successful, all done"
 
@@ -518,6 +286,7 @@ GO=(
     ERR="false"
 )
 stage $GO
+state "all done"
 
 
 # Now, at the very (successful) end of dinstall, make sure we remove