X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fcommon;h=d878d5f51bc1617dd5a0f345a10a6d013c39c5b3;hb=891e1ef72e79a6b566a9230b3d471ccab7e9348f;hp=9d00cf9a444dac000e26cd040ff6938dcf1e3b8a;hpb=1792da4634920834e3f7eaf375e9dcc33ca107e9;p=dak.git

diff --git a/config/debian/common b/config/debian/common
index 9d00cf9a..d878d5f5 100644
--- a/config/debian/common
+++ b/config/debian/common
@@ -3,17 +3,17 @@
 #
 # Set $PROGRAM to a string to have it added to the output.
 function log () {
-        local prefix=${PROGRAM:-}
-        echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${prefix}[$$]: $@"
+    local prefix=${PROGRAM:-}
+    echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${prefix}[$$]: $@"
 }
 
 # log the message using log() but then also send a mail
 # to the address configured in MAILTO (if non-empty)
 function log_error () {
-        log "$@"
-        if [ -z "${MAILTO}" ]; then
-                echo "$@" | mail -a "X-Debian: DAK" -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" ${MAILTO}
-        fi
+    log "$@"
+    if [ -z "${MAILTO}" ]; then
+        echo "$@" | mail -a "X-Debian: DAK" -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" ${MAILTO}
+    fi
 }
 
 # debug log, only output when DEBUG=1
@@ -23,10 +23,59 @@ function debug () {
     fi
 }
 
+# Function that only cleans tempfiles, but does not exit or otherwise
+# care about any exit status
+function cleantempfiles() {
+    resolvetmpfiles
+    for TEMPFILE in $TMPFILES; do
+        if [ -n "${TEMPFILE}" ] && [ -f "${TEMPFILE}" ]; then
+            rm -f "${TEMPFILE}"
+        elif [ -n "${TEMPFILE}" ] && [ -d "${TEMPFILE}" ]; then
+            if [ "${TEMPFILE}" != "/" ] && [ "${TEMPFILE}" != "/*" ]; then
+                rm -rf "${TEMPFILE}"
+            fi
+        fi
+    done
+    TMPFILES=""
+}
+
+function resolvetmpfiles() {
+    # If you don't understand this better not touch the script
+    for TEMPFILE in $TEMPFILES; do
+        TMPFILES="${TMPFILES} ${!TEMPFILE:-""}"
+    done
+    TEMPFILES=""
+}
+
+# Function cleanup
+# No arguments
+# Cleans up any known tempfile.
+# Just ensure your script sets the variable
+# TEMPFILES to the names of variables of tempfiles
+# Or TMPFILES to the pathes of tempfiles
+function cleanup() {
+    ERRVAL=$?
+    trap - ERR EXIT TERM HUP INT QUIT
+
+    cleantempfiles
+
+    return $ERRVAL
+}
+TEMPFILES=${TEMPFILES:-""}
+TMPFILES=${TMPFILES:-""}
+
+# Timestamp. Used for dinstall stat graphs
+function ts() {
+    echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
+}
+
+########################################################################
+########################################################################
+
 function wbtrigger() {
-    SSHOPT="-o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240"
+    SSHOPT="-n -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240"
     if lockfile -r 3 -l 3600 "${LOCK_BUILDD}"; then
-        ssh -q -q ${SSHOPT} wbadm@buildd /org/wanna-build/trigger.often
+        ssh -q -q ${SSHOPT} wbadm@buildd /srv/wanna-build/trigger.often
     fi
     rm -f "${LOCK_BUILDD}"
 }
@@ -69,17 +118,13 @@ function punew_do() {
 
 # These versions used in dinstall
 function punew() {
-    if [ "${PROGRAM}" = "dinstall" ]; then
-        log "Doing automated p-u-new processing"
-    fi
+    log "Doing automated p-u-new processing"
     cd "${queuedir}/p-u-new"
     punew_do "$1" "${queuedir}/p-u-new" "debian-release@lists.debian.org"
 }
 
 function opunew() {
-    if [ "${PROGRAM}" = "dinstall" ]; then
-        log "Doing automated o-p-u-new processing"
-    fi
+    log "Doing automated o-p-u-new processing"
     cd "${queuedir}/o-p-u-new"
     punew_do "$1" "${queuedir}/o-p-u-new" "debian-release@lists.debian.org"
 }
@@ -89,9 +134,7 @@ function backports_policy() {
     local qdir="/srv/backports-master.debian.org/queue/policy"
     local to="backports-team@debian.org"
 
-    if [ "${PROGRAM}" = "dinstall" ]; then
-        log "Doing automated ${queue} processing"
-    fi
+    log "Doing automated ${queue} processing"
 
     cd "${qdir}"
     punew_do "${queue}" "${qdir}" "${to}"
@@ -109,17 +152,22 @@ function do_unchecked () {
     echo "$timestamp": ${changes:-"Nothing to do"}  >> $report
     dak process-upload -a ${UNCHECKED_WITHOUT_LOCK} -d "$unchecked" >> $report
     dak process-commands -d "$unchecked" >> $report
+
+    if [ ! -z "$changes" ]; then
+        sync_debbugs
+        do_buildd
+    fi
 }
 
 # process NEW policy queue
 function do_new () {
-    if [ "${PROGRAM}" = "dinstall" ]; then
-	log "Doing NEW processing"
-    fi
+    log "Doing NEW processing"
     (dak process-policy new; dak process-policy byhand) | mail -a "X-Debian: DAK" -e -s "NEW and BYHAND processing" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" ftpmaster@ftp-master.debian.org
 
+    log "Processing Backports NEW"
     dak process-policy backports-new | mail -a "X-Debian: DAK" -e -s "NEW processing for backports-new" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
 
+    log "Cleanup NEW/Backports NEW"
     dak clean-suites -a new,backports-new
 }
 
@@ -130,7 +178,7 @@ function sync_debbugs () {
     mkdir -p $queuedir/bts_version_track_archive/${timestamp}
     rsync -aq $queuedir/bts_version_track/ $queuedir/bts_version_track_archive/${timestamp}
     rmdir --ignore-fail-on-non-empty $queuedir/bts_version_track_archive/${timestamp} # remove if empty.
-    rsync -aq -e "ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30" --remove-source-files  $queuedir/bts_version_track/ bugs-sync:/org/bugs.debian.org/versions/queue/ftp-master/ 2>/dev/null && touch $lockdir/synced_bts_version || true
+    rsync -aq -e "ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30" --remove-source-files  $queuedir/bts_version_track/ bugs-sync:/srv/bugs.debian.org/versions/queue/ftp-master/ 2>/dev/null && touch $lockdir/synced_bts_version || true
     NOW=$(date +%s)
     TSTAMP=$(stat -c %Y $lockdir/synced_bts_version)
     DIFF=$(( NOW - TSTAMP ))
@@ -174,3 +222,131 @@ function get_archiveroot() {
     fi
     echo "${archiveroot}"
 }
+
+# Prepare the trees for buildds, then push wanna-build
+function do_buildd() {
+    if lockfile -r3 ${LOCK_DAILY}; then
+        TMPFILES="${TMPFILES} ${LOCK_DAILY}"
+        make_buildd_dir
+        wbtrigger
+    fi
+}
+
+# Cleanup policy queues
+function cleanpolicy() {
+    dak clean-suites -a backports-policy,policy
+}
+
+# Scan new packages for contents
+function scancontents() {
+    dak contents -l 10000 scan-binary
+    dak contents -l 1000 scan-source
+}
+
+function ddaccess() {
+    # Tell our dd accessible mirror to sync itself up. Including ftp dir.
+    log "Trigger dd accessible parts sync including ftp dir"
+    ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
+}
+
+
+
+########################################################################
+########################################################################
+########################################################################
+########################################################################
+
+# Function to save which stage we are in, so we can restart an interrupted
+# dinstall. Or even run actions in parallel, if we dare to, by simply
+# backgrounding the call to this function. But that should only really be
+# done for things we don't care much about.
+#
+# This should be called with the first argument being an array, with the
+# members
+#  - FUNC - the function name to call
+#  - ARGS - Possible arguments to hand to the function. Can be the empty string
+#  - TIME - The timestamp name. Can be the empty string
+#  - ERR  - if this is the string false, then the call will be surrounded by
+#           set +e ... set -e calls, so errors in the function do not exit
+#           dinstall. Can be the empty string, meaning true.
+#
+# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
+# ADDED FOR DINSTALL FEATURES!
+function stage() {
+    ARGS='GO[@]'
+    local "${!ARGS}"
+
+    local error=${ERR:-"true"}
+
+    ARGS=${ARGS:-""}
+
+    log "########## ${PROGRAM} BEGIN: ${FUNC} ${ARGS} ##########"
+    local STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
+    STAGEFILE=${STAGEFILE// /_}
+    if [ -f "${STAGEFILE}" ]; then
+        local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
+        local unixtime=$(date +%s)
+        local difference=$(( $unixtime - $stamptime ))
+        if [ ${difference} -ge 14400 ]; then
+            log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
+        else
+            log "Did already run ${FUNC}, not calling again..."
+        fi
+        return
+    fi
+
+    debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"
+
+    # Make sure we are always at the same place. If a function wants
+    # to be elsewhere, it has to cd first!
+    cd ${configdir}
+
+    # Now redirect the output into $STAGEFILE.log. In case it errors
+    # out somewhere our errorhandler trap can then mail the contents
+    # of $STAGEFILE.log only, instead of a whole ${PROGRAM} logfile.
+    # Short error mails ftw!
+    exec >> "${STAGEFILE}.log" 2>&1
+
+    if [ -f "${LOCK_STOP}" ]; then
+        log "${LOCK_STOP} exists, exiting immediately"
+        exit 42
+    fi
+
+    # Do we care about trouble in the function we call?
+    if [ "${error}" = "false" ]; then
+        set +e
+    fi
+    ${FUNC} ${ARGS}
+
+    # No matter what happened in the function, we make sure we have
+    # set -e default state back
+    set -e
+
+    # Make sure we are always at the same place.
+    cd ${configdir}
+
+    # We always use the same umask. If a function wants to do
+    # different, fine, but we reset.
+    umask 022
+
+    touch "${STAGEFILE}"
+
+    if [ -n "${TIME}" ]; then
+        ts "${TIME}"
+    fi
+
+    # And the output goes back to the normal logfile
+    exec >> "${LOGFILE}" 2>&1
+
+    # Now we should make sure that we have a usable ${PROGRAM}.log, so
+    # append the $STAGEFILE.log to it.
+    cat "${STAGEFILE}.log" >> "${LOGFILE}"
+    rm -f "${STAGEFILE}.log"
+
+    log "########## ${PROGRAM} END: ${FUNC} ##########"
+
+    if [ -f "${LOCK_STOP}" ]; then
+        log "${LOCK_STOP} exists, exiting immediately"
+        exit 42
+    fi
+}