From: Joerg Jaspert Date: Tue, 29 Dec 2015 23:42:34 +0000 (+0100) Subject: Loadsa changes to make new cronscript way work X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=commitdiff_plain;h=9639afdb34ea7c5fdaf62e05a78ca0aab288f6de Loadsa changes to make new cronscript way work - Move functions so they are seen, - Emit logs not only when called by dinstall - Use correct variables (TMPFILES, not TEMPFILES) - Define used variables - create a scriptname link to the logfiles with datebased names - in the logdir, not in the configdir - move linkmorgue from daily to weekly - remove nonexistant tasks - make purgeempty and fixsymlinks work on all archives, not just main - all weekly tasks run in parallel --- diff --git a/config/debian/common b/config/debian/common index 30237daf..5a7ec0bf 100644 --- a/config/debian/common +++ b/config/debian/common @@ -59,11 +59,15 @@ function cleanup() { cleantempfiles - exit $ERRVAL + return $ERRVAL } TEMPFILES=${TEMPFILES:-""} TMPFILES=${TMPFILES:-""} +# Timestamp. Used for dinstall stat graphs +function ts() { + echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)" +} ######################################################################## ######################################################################## @@ -114,17 +118,13 @@ function punew_do() { # These versions used in dinstall function punew() { - if [ "${PROGRAM}" = "dinstall" ]; then - log "Doing automated p-u-new processing" - fi + log "Doing automated p-u-new processing" cd "${queuedir}/p-u-new" punew_do "$1" "${queuedir}/p-u-new" "debian-release@lists.debian.org" } function opunew() { - if [ "${PROGRAM}" = "dinstall" ]; then - log "Doing automated o-p-u-new processing" - fi + log "Doing automated o-p-u-new processing" cd "${queuedir}/o-p-u-new" punew_do "$1" "${queuedir}/o-p-u-new" "debian-release@lists.debian.org" } @@ -134,9 +134,7 @@ function backports_policy() { local qdir="/srv/backports-master.debian.org/queue/policy" local to="backports-team@debian.org" - if [ "${PROGRAM}" = "dinstall" ]; then - log "Doing automated ${queue} processing" - fi + log "Doing automated ${queue} processing" cd "${qdir}" punew_do "${queue}" "${qdir}" "${to}" @@ -163,9 +161,7 @@ function do_unchecked () { # process NEW policy queue function do_new () { - if [ "${PROGRAM}" = "dinstall" ]; then - log "Doing NEW processing" - fi + log "Doing NEW processing" (dak process-policy new; dak process-policy byhand) | mail -a "X-Debian: DAK" -e -s "NEW and BYHAND processing" -a "From: Debian FTP Masters " ftpmaster@ftp-master.debian.org dak process-policy backports-new | mail -a "X-Debian: DAK" -e -s "NEW processing for backports-new" -a "From: Debian FTP Masters " backports-team@debian.org @@ -227,8 +223,8 @@ function get_archiveroot() { # Prepare the trees for buildds, then push wanna-build function do_buildd() { - if lockfile -r3 $NOTICE; then - TEMPFILES="${TEMPFILES} ${NOTICE}" + if lockfile -r3 ${LOCK_DAILY}; then + TMPFILES="${TMPFILES} ${LOCK_DAILY}" make_buildd_dir wbtrigger fi diff --git a/config/debian/cronscript b/config/debian/cronscript index ffaf977e..fec8f93e 100755 --- a/config/debian/cronscript +++ b/config/debian/cronscript @@ -40,6 +40,10 @@ export LANG=C export LC_ALL=C ARG=${1:-"meh"} + +# set DEBUG if you want to see a little more logs (needs to be used more) +DEBUG=${DEBUG:-0} + # While this check can be done in the following case, some assumptions # down there are easier if we sorted out calls without an arg before. if [[ ${ARG} == meh ]]; then @@ -101,15 +105,14 @@ case ${ARG,,} in # log "aborting cron.unchecked because $LOCK_UNCHECKED has already been locked" exit 0 fi - TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}" + TMPFILES="${TMPFILES} ${LOCK_UNCHECKED}" ;; dinstall) ;; hourly) # Only one of me should ever run. FLOCKER=${FLOCKER:-""} - [ "${FLOCKER}" != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0" - "$@" || : + [ "${FLOCKER}" != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0" "$@" || : ;; daily) ;; @@ -124,7 +127,7 @@ case ${ARG,,} in esac # An easy access by name for the current log -ln -sf ${LOGFILE} ${PROGRAM} +ln -sf ${LOGFILE} ${logdir}/${PROGRAM} # And from here, all output to the log please exec >> "$LOGFILE" 2>&1 diff --git a/config/debian/daily.functions b/config/debian/daily.functions index 2d528368..e2b4b07a 100644 --- a/config/debian/daily.functions +++ b/config/debian/daily.functions @@ -3,7 +3,7 @@ # get the latest list of wnpp bugs and their source packages function wnppbugs() { TMPFILE=$( mktemp -p ${TMPDIR} ) - TEMPFILES="${TEMPFILES} ${TMPFILE}" + TMPFILES="${TMPFILES} ${TMPFILE}" wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm chmod go+r ${TMPFILE} mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm @@ -39,7 +39,7 @@ function newstats() { function contributor() { log "Submitting data to contributors" TMPCNTB=$( mktemp -p ${TMPDIR} ) - TEMPFILES="${TEMPFILES} ${TMPCNTB}" + TMPFILES="${TMPFILES} ${TMPCNTB}" REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB} # Post with curl as a workaround for #801506 diff --git a/config/debian/daily.tasks b/config/debian/daily.tasks index 7bec97e6..51c4e5d6 100644 --- a/config/debian/daily.tasks +++ b/config/debian/daily.tasks @@ -1,5 +1,4 @@ # FUNC ARGS TIME ERR BG -linkmorgue none none false true wnppbugs none none false false pushfilesdb none none false false wbdump none none false false @@ -8,5 +7,3 @@ clean_debbugs none none fal overridedisp none none false false newstats none none false false contributor none none false false -finalize none none false false - diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index fcd2de35..f351ad4a 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -1,8 +1,4 @@ # -*- mode:sh -*- -# Timestamp. Used for dinstall stat graphs -function ts() { - echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)" -} # Remove daily lock # FIXME: Dead code when switched to cronscript @@ -193,13 +189,13 @@ function mpfm() { log "Generating package / file mapping" for archive in "${public_archives[@]}"; do + log " archive: ${archive}" archiveroot="$(get_archiveroot "${archive}")" dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2" done } function packages() { - log "Generating Packages and Sources files" for archive in "${public_archives[@]}"; do log " Generating Packages/Sources for ${archive}" dak generate-packages-sources2 -a "${archive}" @@ -216,6 +212,7 @@ function pdiff() { function release() { log "Generating Release files" for archive in "${public_archives[@]}"; do + log " archive: ${archive}" dak generate-releases -a "${archive}" done } @@ -680,9 +677,8 @@ function signotherfiles() { function startup() { touch "${DINSTALLSTART}" ts "startup" - DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")" lockfile -l 3600 "${LOCK_DAILY}" trap onerror ERR - TEMPFILES="${TEMPFILES} ${LOCK_DAILY}" touch "${LOCK_BRITNEY}" + TMPFILES="${TMPFILES} ${LOCK_DAILY} ${LOCK_BRITNEY}" } diff --git a/config/debian/dinstall.tasks b/config/debian/dinstall.tasks index 2185287b..79ae5759 100644 --- a/config/debian/dinstall.tasks +++ b/config/debian/dinstall.tasks @@ -46,5 +46,4 @@ mirrorpush-backports none mirrorpush-backports fal i18n2 none i18n\ 2 false true stats none stats false true testingsourcelist none none false true -NOSTAGE rm\ -f\ "\${LOCK_BRITNEY}" none false false cleantransactions none none false false diff --git a/config/debian/dinstall.variables b/config/debian/dinstall.variables index 6f5c9531..c4efabeb 100644 --- a/config/debian/dinstall.variables +++ b/config/debian/dinstall.variables @@ -19,6 +19,9 @@ DINSTALLSTART="${lockdir}/dinstallstart" # Marker for dinstall end DINSTALLEND="${lockdir}/dinstallend" +# Timestamp for start +DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")" + # Lock cron.unchecked from doing work LOCK_ACCEPTED="$lockdir/unchecked.lock" diff --git a/config/debian/hourly.functions b/config/debian/hourly.functions index 5a433102..3e809faf 100644 --- a/config/debian/hourly.functions +++ b/config/debian/hourly.functions @@ -7,7 +7,7 @@ function importusers() { function newoverview() { # do not run show-new and other stuff in parallel if lockfile -r16 $LOCK_UNCHECKED 2> /dev/null; then - TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}" + TMPFILES="${TMPFILES} ${LOCK_UNCHECKED}" do_new dak show-new > /dev/null || true fi @@ -49,7 +49,7 @@ function backportsacl() { # do the buildd key updates function builddautosigning() { BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX) - TEMPFILES="${TEMPFILEs} ${BUILDDFUN}" + TMPFILES="${TMPFILES} ${BUILDDFUN}" exec >> "${BUILDDFUN}" 2>&1 ${scriptsdir}/buildd-remove-keys ${scriptsdir}/buildd-add-keys diff --git a/config/debian/hourly.tasks b/config/debian/hourly.tasks index b5417db5..93053a47 100644 --- a/config/debian/hourly.tasks +++ b/config/debian/hourly.tasks @@ -7,4 +7,4 @@ rss none none fal ddaccess none none true true gen_di none none false false backportsacl none none false false -builddautosigning none none false false \ No newline at end of file +builddautosigning none none false false diff --git a/config/debian/vars b/config/debian/vars index c94ece03..83cbcf1b 100644 --- a/config/debian/vars +++ b/config/debian/vars @@ -43,6 +43,7 @@ GZIP='--rsyncable' ; export GZIP # lock cron.unchecked (it immediately exits when this exists) LOCK_DAILY="$lockdir/daily.lock" LOCK_UNCHECKED="$lockdir/unchecked.lock" +LOCK_STOP="$lockdir/archive.stop" # Set the database variables eval $(dak admin config db-shell) diff --git a/config/debian/weekly.functions b/config/debian/weekly.functions index 55a7f23f..893813dc 100644 --- a/config/debian/weekly.functions +++ b/config/debian/weekly.functions @@ -1,11 +1,18 @@ # -*- mode:sh -*- function purgeempty() { - log "Purging empty directories in $ftpdir/pool/" + log "Purging empty directories" + local archiveroot - if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then - find $ftpdir/pool/ -type d -empty | xargs rmdir; - fi + for archive in "${public_archives[@]}"; do + log "... archive: ${archive}" + archiveroot="$(get_archiveroot "${archive}")" + + # FIXME: double find + if [ ! -z "$(find ${archiveroot}/pool/ -type d -empty)" ]; then + find ${archiveroot}/pool/ -type d -empty | xargs rmdir; + fi + done } function gitcleanup() { @@ -18,7 +25,13 @@ function gitcleanup() { chmod -R g+w logs/ } -function symlinks() { - log "Fixing symlinks in $ftpdir" - symlinks -d -r $ftpdir +function fixsymlinks() { + log "Fixing symlinks" + local archiveroot + + for archive in "${public_archives[@]}"; do + log "... archive: ${archive}" + archiveroot="$(get_archiveroot "${archive}")" + symlinks -d -r ${archiveroot} + done } diff --git a/config/debian/weekly.tasks b/config/debian/weekly.tasks index 4fb0d92d..874a1bcc 100644 --- a/config/debian/weekly.tasks +++ b/config/debian/weekly.tasks @@ -1,4 +1,5 @@ # FUNC ARGS TIME ERR BG -purgeempty none none false false -gitcleanup none none false false -symlinks none none false false +purgeempty none none false true +gitcleanup none none false true +fixsymlinks none none false true +linkmorgue none none false true