Loadsa changes to make new cronscript way work

- Move functions so they are seen,
- Emit logs not only when called by dinstall
- Use correct variables (TMPFILES, not TEMPFILES)
- Define used variables
- create a scriptname link to the logfiles with datebased names
   - in the logdir, not in the configdir
- move linkmorgue from daily to weekly
- remove nonexistant tasks
- make purgeempty and fixsymlinks work on all archives, not just main
- all weekly tasks run in parallel

diff --git a/config/debian/common b/config/debian/common
index 30237dafdfe22a16b301f3c8565f259e7833bfc6..5a7ec0bf8cabe6815d3feafaacac328dde244e8a 100644 (file)
--- a/config/debian/common
+++ b/config/debian/common
@@ -59,11 +59,15 @@ function cleanup() {
 
     cleantempfiles
 
 
     cleantempfiles
 

-    exit $ERRVAL
+    return $ERRVAL

 }
 TEMPFILES=${TEMPFILES:-""}
 TMPFILES=${TMPFILES:-""}
 
 }
 TEMPFILES=${TEMPFILES:-""}
 TMPFILES=${TMPFILES:-""}
 

+# Timestamp. Used for dinstall stat graphs
+function ts() {
+    echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
+}

 
 ########################################################################
 ########################################################################
 
 ########################################################################
 ########################################################################


@@ -114,17 +118,13 @@ function punew_do() {
 
 # These versions used in dinstall
 function punew() {
 
 # These versions used in dinstall
 function punew() {

-    if [ "${PROGRAM}" = "dinstall" ]; then
-        log "Doing automated p-u-new processing"
-    fi
+    log "Doing automated p-u-new processing"

     cd "${queuedir}/p-u-new"
     punew_do "$1" "${queuedir}/p-u-new" "debian-release@lists.debian.org"
 }
 
 function opunew() {
     cd "${queuedir}/p-u-new"
     punew_do "$1" "${queuedir}/p-u-new" "debian-release@lists.debian.org"
 }
 
 function opunew() {

-    if [ "${PROGRAM}" = "dinstall" ]; then
-        log "Doing automated o-p-u-new processing"
-    fi
+    log "Doing automated o-p-u-new processing"

     cd "${queuedir}/o-p-u-new"
     punew_do "$1" "${queuedir}/o-p-u-new" "debian-release@lists.debian.org"
 }
     cd "${queuedir}/o-p-u-new"
     punew_do "$1" "${queuedir}/o-p-u-new" "debian-release@lists.debian.org"
 }


@@ -134,9 +134,7 @@ function backports_policy() {
     local qdir="/srv/backports-master.debian.org/queue/policy"
     local to="backports-team@debian.org"
 
     local qdir="/srv/backports-master.debian.org/queue/policy"
     local to="backports-team@debian.org"
 

-    if [ "${PROGRAM}" = "dinstall" ]; then
-        log "Doing automated ${queue} processing"
-    fi
+    log "Doing automated ${queue} processing"

 
     cd "${qdir}"
     punew_do "${queue}" "${qdir}" "${to}"
 
     cd "${qdir}"
     punew_do "${queue}" "${qdir}" "${to}"


@@ -163,9 +161,7 @@ function do_unchecked () {
 
 # process NEW policy queue
 function do_new () {
 
 # process NEW policy queue
 function do_new () {

-    if [ "${PROGRAM}" = "dinstall" ]; then
-       log "Doing NEW processing"
-    fi
+    log "Doing NEW processing"

     (dak process-policy new; dak process-policy byhand) | mail -a "X-Debian: DAK" -e -s "NEW and BYHAND processing" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" ftpmaster@ftp-master.debian.org
 
     dak process-policy backports-new | mail -a "X-Debian: DAK" -e -s "NEW processing for backports-new" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
     (dak process-policy new; dak process-policy byhand) | mail -a "X-Debian: DAK" -e -s "NEW and BYHAND processing" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" ftpmaster@ftp-master.debian.org
 
     dak process-policy backports-new | mail -a "X-Debian: DAK" -e -s "NEW processing for backports-new" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org


@@ -227,8 +223,8 @@ function get_archiveroot() {
 
 # Prepare the trees for buildds, then push wanna-build
 function do_buildd() {
 
 # Prepare the trees for buildds, then push wanna-build
 function do_buildd() {

-    if lockfile -r3 $NOTICE; then
-        TEMPFILES="${TEMPFILES} ${NOTICE}"
+    if lockfile -r3 ${LOCK_DAILY}; then
+        TMPFILES="${TMPFILES} ${LOCK_DAILY}"

         make_buildd_dir
         wbtrigger
     fi
         make_buildd_dir
         wbtrigger
     fi

diff --git a/config/debian/cronscript b/config/debian/cronscript
index ffaf977ea8557c344c2d3cb26f0b69ca21aa2079..fec8f93e14236632e9e27efed674674f16157220 100755 (executable)
--- a/config/debian/cronscript
+++ b/config/debian/cronscript
@@ -40,6 +40,10 @@ export LANG=C
 export LC_ALL=C
 
 ARG=${1:-"meh"}
 export LC_ALL=C
 
 ARG=${1:-"meh"}

+
+# set DEBUG if you want to see a little more logs (needs to be used more)
+DEBUG=${DEBUG:-0}
+

 # While this check can be done in the following case, some assumptions
 # down there are easier if we sorted out calls without an arg before.
 if [[ ${ARG} == meh ]]; then
 # While this check can be done in the following case, some assumptions
 # down there are easier if we sorted out calls without an arg before.
 if [[ ${ARG} == meh ]]; then


@@ -101,15 +105,14 @@ case ${ARG,,} in
             # log "aborting cron.unchecked because $LOCK_UNCHECKED has already been locked"
             exit 0
         fi
             # log "aborting cron.unchecked because $LOCK_UNCHECKED has already been locked"
             exit 0
         fi

-        TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}"
+        TMPFILES="${TMPFILES} ${LOCK_UNCHECKED}"

         ;;
     dinstall)
         ;;
     hourly)
         # Only one of me should ever run.
         FLOCKER=${FLOCKER:-""}
         ;;
     dinstall)
         ;;
     hourly)
         # Only one of me should ever run.
         FLOCKER=${FLOCKER:-""}

-        [  "${FLOCKER}"  != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0"
-        "$@" || :
+        [  "${FLOCKER}"  != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0" "$@" || :

         ;;
     daily)
         ;;
         ;;
     daily)
         ;;


@@ -124,7 +127,7 @@ case ${ARG,,} in
 esac
 
 # An easy access by name for the current log
 esac
 
 # An easy access by name for the current log

-ln -sf ${LOGFILE} ${PROGRAM}
+ln -sf ${LOGFILE} ${logdir}/${PROGRAM}

 
 # And from here, all output to the log please
 exec >> "$LOGFILE" 2>&1
 
 # And from here, all output to the log please
 exec >> "$LOGFILE" 2>&1

diff --git a/config/debian/daily.functions b/config/debian/daily.functions
index 2d52836844ac1a7adda36b14b85d8128718e0f42..e2b4b07a0087442f490ba2f1df0dba9dabbccf48 100644 (file)
--- a/config/debian/daily.functions
+++ b/config/debian/daily.functions
@@ -3,7 +3,7 @@
 # get the latest list of wnpp bugs and their source packages
 function wnppbugs() {
     TMPFILE=$( mktemp -p ${TMPDIR} )
 # get the latest list of wnpp bugs and their source packages
 function wnppbugs() {
     TMPFILE=$( mktemp -p ${TMPDIR} )

-    TEMPFILES="${TEMPFILES} ${TMPFILE}"
+    TMPFILES="${TMPFILES} ${TMPFILE}"

     wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
     chmod go+r ${TMPFILE}
     mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
     wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
     chmod go+r ${TMPFILE}
     mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm


@@ -39,7 +39,7 @@ function newstats() {
 function contributor() {
     log "Submitting data to contributors"
     TMPCNTB=$( mktemp -p ${TMPDIR} )
 function contributor() {
     log "Submitting data to contributors"
     TMPCNTB=$( mktemp -p ${TMPDIR} )

-    TEMPFILES="${TEMPFILES} ${TMPCNTB}"
+    TMPFILES="${TMPFILES} ${TMPCNTB}"

     REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
 
     # Post with curl as a workaround for #801506
     REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
 
     # Post with curl as a workaround for #801506

diff --git a/config/debian/daily.tasks b/config/debian/daily.tasks
index 7bec97e6dd69e4e8d39e744073239913fda72aa8..51c4e5d66f9cc162e7d36b87080a6342c092b495 100644 (file)
--- a/config/debian/daily.tasks
+++ b/config/debian/daily.tasks
@@ -1,5 +1,4 @@
 # FUNC                 ARGS                       TIME                       ERR     BG
 # FUNC                 ARGS                       TIME                       ERR     BG

-linkmorgue             none                       none                       false   true

 wnppbugs               none                       none                       false   false
 pushfilesdb            none                       none                       false   false
 wbdump                 none                       none                       false   false
 wnppbugs               none                       none                       false   false
 pushfilesdb            none                       none                       false   false
 wbdump                 none                       none                       false   false


@@ -8,5 +7,3 @@ clean_debbugs          none                       none                       fal
 overridedisp           none                       none                       false   false
 newstats               none                       none                       false   false
 contributor            none                       none                       false   false
 overridedisp           none                       none                       false   false
 newstats               none                       none                       false   false
 contributor            none                       none                       false   false

-finalize               none                       none                       false   false
-

diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index fcd2de356aafed0c1ad6d0fe55bb12c55695623e..f351ad4aa73258b4a4a43d7ebd9eeb01c182570f 100644 (file)
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -1,8 +1,4 @@
 # -*- mode:sh -*-
 # -*- mode:sh -*-

-# Timestamp. Used for dinstall stat graphs
-function ts() {
-        echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
-}

 
 # Remove daily lock
 # FIXME: Dead code when switched to cronscript
 
 # Remove daily lock
 # FIXME: Dead code when switched to cronscript


@@ -193,13 +189,13 @@ function mpfm() {
 
     log "Generating package / file mapping"
     for archive in "${public_archives[@]}"; do
 
     log "Generating package / file mapping"
     for archive in "${public_archives[@]}"; do

+        log "  archive: ${archive}"

         archiveroot="$(get_archiveroot "${archive}")"
         dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2"
     done
 }
 
 function packages() {
         archiveroot="$(get_archiveroot "${archive}")"
         dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2"
     done
 }
 
 function packages() {

-    log "Generating Packages and Sources files"

     for archive in "${public_archives[@]}"; do
         log "  Generating Packages/Sources for ${archive}"
         dak generate-packages-sources2 -a "${archive}"
     for archive in "${public_archives[@]}"; do
         log "  Generating Packages/Sources for ${archive}"
         dak generate-packages-sources2 -a "${archive}"


@@ -216,6 +212,7 @@ function pdiff() {
 function release() {
     log "Generating Release files"
     for archive in "${public_archives[@]}"; do
 function release() {
     log "Generating Release files"
     for archive in "${public_archives[@]}"; do

+        log "  archive: ${archive}"

         dak generate-releases -a "${archive}"
     done
 }
         dak generate-releases -a "${archive}"
     done
 }


@@ -680,9 +677,8 @@ function signotherfiles() {
 function startup() {
     touch "${DINSTALLSTART}"
     ts "startup"
 function startup() {
     touch "${DINSTALLSTART}"
     ts "startup"

-    DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"

     lockfile -l 3600 "${LOCK_DAILY}"
     trap onerror ERR
     lockfile -l 3600 "${LOCK_DAILY}"
     trap onerror ERR

-    TEMPFILES="${TEMPFILES} ${LOCK_DAILY}"

     touch "${LOCK_BRITNEY}"
     touch "${LOCK_BRITNEY}"

+    TMPFILES="${TMPFILES} ${LOCK_DAILY} ${LOCK_BRITNEY}"

 }
 }

diff --git a/config/debian/dinstall.tasks b/config/debian/dinstall.tasks
index 2185287ba2df8826e1b0ace700d72aacf363c1ac..79ae575945130e073861d88544cb311265fd084c 100644 (file)
--- a/config/debian/dinstall.tasks
+++ b/config/debian/dinstall.tasks
@@ -46,5 +46,4 @@ mirrorpush-backports   none                       mirrorpush-backports       fal
 i18n2                  none                       i18n\ 2                    false   true
 stats                  none                       stats                      false   true
 testingsourcelist      none                       none                       false   true
 i18n2                  none                       i18n\ 2                    false   true
 stats                  none                       stats                      false   true
 testingsourcelist      none                       none                       false   true

-NOSTAGE                rm\ -f\ "\${LOCK_BRITNEY}" none                       false   false

 cleantransactions      none                       none                       false   false
 cleantransactions      none                       none                       false   false

diff --git a/config/debian/dinstall.variables b/config/debian/dinstall.variables
index 6f5c9531dbaf199d933ed33106e95beff4ec4a11..c4efabebfd05e92bda5863c4ea80c6d360019f74 100644 (file)
--- a/config/debian/dinstall.variables
+++ b/config/debian/dinstall.variables
@@ -19,6 +19,9 @@ DINSTALLSTART="${lockdir}/dinstallstart"
 # Marker for dinstall end
 DINSTALLEND="${lockdir}/dinstallend"
 
 # Marker for dinstall end
 DINSTALLEND="${lockdir}/dinstallend"
 

+# Timestamp for start
+DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
+

 # Lock cron.unchecked from doing work
 LOCK_ACCEPTED="$lockdir/unchecked.lock"
 
 # Lock cron.unchecked from doing work
 LOCK_ACCEPTED="$lockdir/unchecked.lock"
 

diff --git a/config/debian/hourly.functions b/config/debian/hourly.functions
index 5a4331023b1e61135dd2c4ecdc6b6c9128383103..3e809fafece0dda5a903621e6bd6c5a8d1519c6d 100644 (file)
--- a/config/debian/hourly.functions
+++ b/config/debian/hourly.functions
@@ -7,7 +7,7 @@ function importusers() {
 function newoverview() {
     # do not run show-new and other stuff in parallel
     if lockfile -r16 $LOCK_UNCHECKED 2> /dev/null; then
 function newoverview() {
     # do not run show-new and other stuff in parallel
     if lockfile -r16 $LOCK_UNCHECKED 2> /dev/null; then

-        TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}"
+        TMPFILES="${TMPFILES} ${LOCK_UNCHECKED}"

         do_new
         dak show-new > /dev/null || true
     fi
         do_new
         dak show-new > /dev/null || true
     fi


@@ -49,7 +49,7 @@ function backportsacl() {
 # do the buildd key updates
 function builddautosigning() {
     BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX)
 # do the buildd key updates
 function builddautosigning() {
     BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX)

-    TEMPFILES="${TEMPFILEs} ${BUILDDFUN}"
+    TMPFILES="${TMPFILES} ${BUILDDFUN}"

     exec >> "${BUILDDFUN}" 2>&1
     ${scriptsdir}/buildd-remove-keys
     ${scriptsdir}/buildd-add-keys
     exec >> "${BUILDDFUN}" 2>&1
     ${scriptsdir}/buildd-remove-keys
     ${scriptsdir}/buildd-add-keys

diff --git a/config/debian/hourly.tasks b/config/debian/hourly.tasks
index b5417db534a4d3325970088fde04c7cfefdc30fc..93053a479a1521a0c2e0001671ea88ff8a81be6f 100644 (file)
--- a/config/debian/hourly.tasks
+++ b/config/debian/hourly.tasks
@@ -7,4 +7,4 @@ rss                    none                       none                       fal
 ddaccess               none                       none                       true    true
 gen_di                 none                       none                       false   false
 backportsacl           none                       none                       false   false
 ddaccess               none                       none                       true    true
 gen_di                 none                       none                       false   false
 backportsacl           none                       none                       false   false

-builddautosigning      none                       none                       false   false
\ No newline at end of file
+builddautosigning      none                       none                       false   false

diff --git a/config/debian/vars b/config/debian/vars
index c94ece03c013727ba6a12263cd63bfea59ef0dea..83cbcf1bd7c6a924728b6bf3cf723993a8dba592 100644 (file)
--- a/config/debian/vars
+++ b/config/debian/vars
@@ -43,6 +43,7 @@ GZIP='--rsyncable' ; export GZIP
 # lock cron.unchecked (it immediately exits when this exists)
 LOCK_DAILY="$lockdir/daily.lock"
 LOCK_UNCHECKED="$lockdir/unchecked.lock"
 # lock cron.unchecked (it immediately exits when this exists)
 LOCK_DAILY="$lockdir/daily.lock"
 LOCK_UNCHECKED="$lockdir/unchecked.lock"

+LOCK_STOP="$lockdir/archive.stop"

 
 # Set the database variables
 eval $(dak admin config db-shell)
 
 # Set the database variables
 eval $(dak admin config db-shell)

diff --git a/config/debian/weekly.functions b/config/debian/weekly.functions
index 55a7f23fa066700219a3a06da7a1bb79b2f62e54..893813dc7da3f8ee5824f6e0358a22ea239ab4be 100644 (file)
--- a/config/debian/weekly.functions
+++ b/config/debian/weekly.functions
@@ -1,11 +1,18 @@
 # -*- mode:sh -*-
 
 function purgeempty() {
 # -*- mode:sh -*-
 
 function purgeempty() {

-    log "Purging empty directories in $ftpdir/pool/"
+    log "Purging empty directories"
+    local archiveroot

 
 

-    if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then
-        find $ftpdir/pool/ -type d -empty | xargs rmdir;
-    fi
+    for archive in "${public_archives[@]}"; do
+        log "... archive: ${archive}"
+        archiveroot="$(get_archiveroot "${archive}")"
+
+        # FIXME: double find
+        if [ ! -z "$(find ${archiveroot}/pool/ -type d -empty)" ]; then
+            find ${archiveroot}/pool/ -type d -empty | xargs rmdir;
+        fi
+    done

 }
 
 function gitcleanup() {
 }
 
 function gitcleanup() {


@@ -18,7 +25,13 @@ function gitcleanup() {
     chmod -R g+w logs/
 }
 
     chmod -R g+w logs/
 }
 

-function symlinks() {
-    log "Fixing symlinks in $ftpdir"
-    symlinks -d -r $ftpdir
+function fixsymlinks() {
+    log "Fixing symlinks"
+    local archiveroot
+
+    for archive in "${public_archives[@]}"; do
+        log "... archive: ${archive}"
+        archiveroot="$(get_archiveroot "${archive}")"
+        symlinks -d -r ${archiveroot}
+    done

 }
 }

diff --git a/config/debian/weekly.tasks b/config/debian/weekly.tasks
index 4fb0d92dfb3eed7cb35e08a9f46b2e016abbeb85..874a1bccfdd200edfed88aed873d46d5439b38d9 100644 (file)
--- a/config/debian/weekly.tasks
+++ b/config/debian/weekly.tasks
@@ -1,4 +1,5 @@
 # FUNC                 ARGS                       TIME                       ERR     BG
 # FUNC                 ARGS                       TIME                       ERR     BG

-purgeempty             none                       none                       false   false
-gitcleanup             none                       none                       false   false
-symlinks               none                       none                       false   false
+purgeempty             none                       none                       false   true
+gitcleanup             none                       none                       false   true
+fixsymlinks            none                       none                       false   true
+linkmorgue             none                       none                       false   true