From e65fe609400677be4ee3429e7251184b2fbc178e Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2015 15:30:44 +0100 Subject: [PATCH] Adjust cronscripts to end up with just one --- config/debian/common | 178 ++++++++++++++++++++++ config/debian/cronscript | 235 ++++++++++++++++++++++++++++++ config/debian/daily.functions | 59 ++++++++ config/debian/daily.tasks | 12 ++ config/debian/dinstall.functions | 21 ++- config/debian/dinstall.tasks | 50 +++++++ config/debian/dinstall.variables | 6 - config/debian/hourly.functions | 67 +++++++++ config/debian/hourly.tasks | 10 ++ config/debian/monthly.functions | 19 +++ config/debian/monthly.tasks | 3 + config/debian/monthly.variables | 3 + config/debian/unchecked.tasks | 9 ++ config/debian/unchecked.variables | 3 + config/debian/vars | 4 + config/debian/weekly.functions | 24 +++ config/debian/weekly.tasks | 4 + 17 files changed, 693 insertions(+), 14 deletions(-) create mode 100755 config/debian/cronscript create mode 100644 config/debian/daily.functions create mode 100644 config/debian/daily.tasks create mode 100644 config/debian/dinstall.tasks create mode 100644 config/debian/hourly.functions create mode 100644 config/debian/hourly.tasks create mode 100644 config/debian/monthly.functions create mode 100644 config/debian/monthly.tasks create mode 100644 config/debian/monthly.variables create mode 100644 config/debian/unchecked.tasks create mode 100644 config/debian/unchecked.variables create mode 100644 config/debian/weekly.functions create mode 100644 config/debian/weekly.tasks diff --git a/config/debian/common b/config/debian/common index bb6f8e4a..30237daf 100644 --- a/config/debian/common +++ b/config/debian/common @@ -23,6 +23,51 @@ function debug () { fi } +# Function that only cleans tempfiles, but does not exit or otherwise +# care about any exit status +function cleantempfiles() { + resolvetmpfiles + for TEMPFILE in $TMPFILES; do + if [ -n "${TEMPFILE}" ] && [ -f "${TEMPFILE}" ]; then + rm -f "${TEMPFILE}" + elif [ -n "${TEMPFILE}" ] && [ -d "${TEMPFILE}" ]; then + if [ "${TEMPFILE}" != "/" ] && [ "${TEMPFILE}" != "/*" ]; then + rm -rf "${TEMPFILE}" + fi + fi + done + TMPFILES="" +} + +function resolvetmpfiles() { + # If you don't understand this better not touch the script + for TEMPFILE in $TEMPFILES; do + TMPFILES="${TMPFILES} ${!TEMPFILE:-""}" + done + TEMPFILES="" +} + +# Function cleanup +# No arguments +# Cleans up any known tempfile. +# Just ensure your script sets the variable +# TEMPFILES to the names of variables of tempfiles +# Or TMPFILES to the pathes of tempfiles +function cleanup() { + ERRVAL=$? + trap - ERR EXIT TERM HUP INT QUIT + + cleantempfiles + + exit $ERRVAL +} +TEMPFILES=${TEMPFILES:-""} +TMPFILES=${TMPFILES:-""} + + +######################################################################## +######################################################################## + function wbtrigger() { SSHOPT="-n -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240" if lockfile -r 3 -l 3600 "${LOCK_BUILDD}"; then @@ -109,6 +154,11 @@ function do_unchecked () { echo "$timestamp": ${changes:-"Nothing to do"} >> $report dak process-upload -a ${UNCHECKED_WITHOUT_LOCK} -d "$unchecked" >> $report dak process-commands -d "$unchecked" >> $report + + if [ ! -z "$changes" ]; then + sync_debbugs + do_buildd + fi } # process NEW policy queue @@ -174,3 +224,131 @@ function get_archiveroot() { fi echo "${archiveroot}" } + +# Prepare the trees for buildds, then push wanna-build +function do_buildd() { + if lockfile -r3 $NOTICE; then + TEMPFILES="${TEMPFILES} ${NOTICE}" + make_buildd_dir + wbtrigger + fi +} + +# Cleanup policy queues +function cleanpolicy() { + dak clean-suites -a backports-policy,policy +} + +# Scan new packages for contents +function scancontents() { + dak contents -l 10000 scan-binary + dak contents -l 1000 scan-source +} + +function ddaccess() { + # Tell our dd accessible mirror to sync itself up. Including ftp dir. + log "Trigger dd accessible parts sync including ftp dir" + ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync +} + + + +######################################################################## +######################################################################## +######################################################################## +######################################################################## + +# Function to save which stage we are in, so we can restart an interrupted +# dinstall. Or even run actions in parallel, if we dare to, by simply +# backgrounding the call to this function. But that should only really be +# done for things we don't care much about. +# +# This should be called with the first argument being an array, with the +# members +# - FUNC - the function name to call +# - ARGS - Possible arguments to hand to the function. Can be the empty string +# - TIME - The timestamp name. Can be the empty string +# - ERR - if this is the string false, then the call will be surrounded by +# set +e ... set -e calls, so errors in the function do not exit +# dinstall. Can be the empty string, meaning true. +# +# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES +# ADDED FOR DINSTALL FEATURES! +function stage() { + ARGS='GO[@]' + local "${!ARGS}" + + local error=${ERR:-"true"} + + ARGS=${ARGS:-""} + + log "########## ${PROGRAM} BEGIN: ${FUNC} ${ARGS} ##########" + local STAGEFILE="${stagedir}/${FUNC}_${ARGS}" + STAGEFILE=${STAGEFILE// /_} + if [ -f "${STAGEFILE}" ]; then + local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}") + local unixtime=$(date +%s) + local difference=$(( $unixtime - $stamptime )) + if [ ${difference} -ge 14400 ]; then + log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check." + else + log "Did already run ${FUNC}, not calling again..." + fi + return + fi + + debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}" + + # Make sure we are always at the same place. If a function wants + # to be elsewhere, it has to cd first! + cd ${configdir} + + # Now redirect the output into $STAGEFILE.log. In case it errors + # out somewhere our errorhandler trap can then mail the contents + # of $STAGEFILE.log only, instead of a whole ${PROGRAM} logfile. + # Short error mails ftw! + exec >> "${STAGEFILE}.log" 2>&1 + + if [ -f "${LOCK_STOP}" ]; then + log "${LOCK_STOP} exists, exiting immediately" + exit 42 + fi + + # Do we care about trouble in the function we call? + if [ "${error}" = "false" ]; then + set +e + fi + ${FUNC} ${ARGS} + + # No matter what happened in the function, we make sure we have + # set -e default state back + set -e + + # Make sure we are always at the same place. + cd ${configdir} + + # We always use the same umask. If a function wants to do + # different, fine, but we reset. + umask 022 + + touch "${STAGEFILE}" + + if [ -n "${TIME}" ]; then + ts "${TIME}" + fi + + # And the output goes back to the normal logfile + exec >> "${LOGFILE}" 2>&1 + + # Now we should make sure that we have a usable ${PROGRAM}.log, so + # append the $STAGEFILE.log to it. + cat "${STAGEFILE}.log" >> "${LOGFILE}" + rm -f "${STAGEFILE}.log" + + log "########## ${PROGRAM} END: ${FUNC} ##########" + + if [ -f "${LOCK_STOP}" ]; then + log "${LOCK_STOP} exists, exiting immediately" + exit 42 + fi +} diff --git a/config/debian/cronscript b/config/debian/cronscript new file mode 100755 index 00000000..ffaf977e --- /dev/null +++ b/config/debian/cronscript @@ -0,0 +1,235 @@ +#!/bin/bash +# No way I try to deal with a crippled sh just for POSIX foo. + +# Copyright (C) 2009-2015 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +# Homer: Are you saying you're never going to eat any animal again? What +# about bacon? +# Lisa: No. +# Homer: Ham? +# Lisa: No. +# Homer: Pork chops? +# Lisa: Dad, those all come from the same animal. +# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal. + +# exit on errors +set -e +set -o pipefail +# make sure to only use defined variables +set -u +# ERR traps should be inherited from functions too. (And command +# substitutions and subshells and whatnot, but for us the functions is +# the important part here) +set -E + +# And use one locale, no matter what the caller has set +export LANG=C +export LC_ALL=C + +ARG=${1:-"meh"} +# While this check can be done in the following case, some assumptions +# down there are easier if we sorted out calls without an arg before. +if [[ ${ARG} == meh ]]; then + cat - < /dev/null; then + # log "aborting cron.unchecked because $LOCK_UNCHECKED has already been locked" + exit 0 + fi + TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}" + ;; + dinstall) + ;; + hourly) + # Only one of me should ever run. + FLOCKER=${FLOCKER:-""} + [ "${FLOCKER}" != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0" + "$@" || : + ;; + daily) + ;; + weekly) + ;; + monthly) + ;; + *) + error "Unknown arg ${ARG}" + exit 42 + ;; +esac + +# An easy access by name for the current log +ln -sf ${LOGFILE} ${PROGRAM} + +# And from here, all output to the log please +exec >> "$LOGFILE" 2>&1 + +# The stage function uses this directory +# This amends the stagedir variable from "vars" +stagedir="${stagedir}/${PROGRAM}" +# Ensure the dir exists +mkdir -p ${stagedir} + +# This loop simply wants to be fed by a list of values (see below) +# made out of 5 columns. +# The first four are the array values for the stage function, the +# fifth tells us if we should background the stage call. +# +# - FUNC - the function name to call +# - ARGS - Possible arguments to hand to the function. Can be the empty string +# - TIME - The timestamp name. Can be the empty string +# - ERR - if this is the string false, then the call will be surrounded by +# set +e ... set -e calls, so errors in the function do not exit +# dinstall. Can be the empty string, meaning true. +# - BG - Background the function stage? +# +# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \ +# +# NOTE 1: There are two special values for the first column (FUNC). +# STATE - do not call stage function, call the state +# function to update the public statefile "where is dinstall" +# NOSTAGE - do not call stage function, call the command directly. +# +# Note 2: If you want to hand an empty value to the stage function, +# use the word "none" in the list below. +while read FUNC ARGS TIME ERR BACKGROUND; do + debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND" + + # Empty values in the value list are the string "none" (or the + # while read loop won't work). Here we ensure that variables that + # can be empty, are empty if the string none is set for them. + for var in ARGS TIME; do + if [[ ${!var} == none ]]; then + typeset ${var}='' + fi + done + + # ERR/BACKGROUND are boolean, check that they are. + for var in ERR BACKGROUND; do + if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then + error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}" + fi + done + + case ${FUNC} in + STATE) + state ${ARGS} + ;; + NOSTAGE) + ${ARGS} + ;; + *) + GO=( + FUNC=${FUNC} + TIME=${TIME} + ARGS=${ARGS} + ERR=${ERR} + ) + if [[ ${BACKGROUND} == true ]]; then + stage $GO & + else + stage $GO + fi + ;; + esac +done < <(grep -v '^#' ${TASKLIST} ) + +# we need to wait for the background processes before the end of the cron script +wait + + +# Common to all cron scripts +log "Cron script successful, all done" +# Redirect output to another file, as we want to compress our logfile +# and ensure its no longer used +exec > "$logdir/after${PROGRAM}.log" 2>&1 + +case ${ARG,,} in + unchecked) + ;; + dinstall) + logstats ${LOGFILE} + state "all done" + touch "${DINSTALLEND}" + ;; + hourly) + ;; + daily) + ;; + weekly) + ;; + monthly) + ;; +esac + +# Now, at the very (successful) end of this run, make sure we remove +# our stage files, so the next dinstall run will do it all again. +rm -f ${stagedir}/* +bzip2 -9 ${LOGFILE} +# FIXME: Mail the log when its non-empty +[[ -s "${logdir}/after${PROGRAM}.log" ]] || rm "${logdir}/after${PROGRAM}.log" diff --git a/config/debian/daily.functions b/config/debian/daily.functions new file mode 100644 index 00000000..2d528368 --- /dev/null +++ b/config/debian/daily.functions @@ -0,0 +1,59 @@ +# -*- mode:sh -*- + +# get the latest list of wnpp bugs and their source packages +function wnppbugs() { + TMPFILE=$( mktemp -p ${TMPDIR} ) + TEMPFILES="${TEMPFILES} ${TMPFILE}" + wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm + chmod go+r ${TMPFILE} + mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm +} + +# Push files over to security +function pushfilesdb() { + # The key over there should have the following set for the ssh key: + # command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity" + psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \ + xz -3 | \ + ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \ + -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync +} + +# Update wanna-build dump +function wbdump() { + log "Update wanna-build database dump" + $base/dak/scripts/nfu/get-w-b-db +} + +# Generate list of override disparities +function overridedisp() { + dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz +} + +# Generate stats about the new queue +function newstats() { + dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null +} + +# Generate the contributor data +function contributor() { + log "Submitting data to contributors" + TMPCNTB=$( mktemp -p ${TMPDIR} ) + TEMPFILES="${TEMPFILES} ${TMPCNTB}" + REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB} + + # Post with curl as a workaround for #801506 + # See https://wiki.debian.org/ServicesSSL#curl + dir=/etc/ssl/ca-debian + test -d $dir && capath="--capath $dir" + curl -s $capath https://contributors.debian.org/contributors/post \ + -F source=ftp.debian.org \ + -F auth_token="$(cat ${base}/s3kr1t/contributor.auth)" \ + -F data=@${TMPCNTB} > ${TMPCNTB}.result + cat ${TMPCNTB}.result + rm -f ${TMPCNTB}.result +} + +function linkmorgue() { + ${scriptsdir}/link_morgue.sh +} diff --git a/config/debian/daily.tasks b/config/debian/daily.tasks new file mode 100644 index 00000000..7bec97e6 --- /dev/null +++ b/config/debian/daily.tasks @@ -0,0 +1,12 @@ +# FUNC ARGS TIME ERR BG +linkmorgue none none false true +wnppbugs none none false false +pushfilesdb none none false false +wbdump none none false false +reports none none false false +clean_debbugs none none false false +overridedisp none none false false +newstats none none false false +contributor none none false false +finalize none none false false + diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index 498999eb..fcd2de35 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -5,6 +5,7 @@ function ts() { } # Remove daily lock +# FIXME: Dead code when switched to cronscript function remove_daily_lock() { rm -f $LOCK_DAILY } @@ -50,7 +51,7 @@ function onerror() { } ######################################################################## -# the actual dinstall functions follow # +# the actual functions follow # ######################################################################## # pushing merkels QA user, part one @@ -482,12 +483,6 @@ function bts() { sudo -u dak-unpriv dak bts-categorize } -function ddaccess() { - # Tell our dd accessible mirror to sync itself up. Including ftp dir. - log "Trigger dd accessible parts sync including ftp dir" - ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync -} - function mirrorpush() { log "Checking the public archive copies..." @@ -615,7 +610,6 @@ function logstats() { # save timestamp when we start function savetimestamp() { - NOW=`date "+%Y.%m.%d-%H:%M:%S"` echo ${NOW} > "${dbdir}/dinstallstart" } @@ -663,6 +657,7 @@ function changelogs() { fi } +# Generate a list of extra mirror files, sha256sum em and sign that function signotherfiles() { log "Signing extra mirror files" @@ -681,3 +676,13 @@ function signotherfiles() { rm -f ${TMPLO} done } + +function startup() { + touch "${DINSTALLSTART}" + ts "startup" + DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")" + lockfile -l 3600 "${LOCK_DAILY}" + trap onerror ERR + TEMPFILES="${TEMPFILES} ${LOCK_DAILY}" + touch "${LOCK_BRITNEY}" +} diff --git a/config/debian/dinstall.tasks b/config/debian/dinstall.tasks new file mode 100644 index 00000000..2185287b --- /dev/null +++ b/config/debian/dinstall.tasks @@ -0,0 +1,50 @@ +# FUNC ARGS TIME ERR BG +STATE startup none false false +NOSTAGE startup none false false +savetimestamp none none false false +qa1 none init true true +pg_timestamp predinstall pg_dump1 false false +updates none External\ Updates false false +i18n1 none i18n\ 1 false false +dep11 none dep11\ 1 false false +NOSTAGE lockaccepted none false false +punew stable-new p-u-new false false +opunew oldstable-new o-p-u-new false false +backports_policy none backports-policy false false +cruft none cruft false false +STATE indices none false false +dominate none dominate false false +autocruft none autocruft false false +fingerprints none import-keyring false false +overrides none overrides false false +mpfm none pkg-file-mapping false false +STATE packages/contents none false false +packages none apt-ftparchive false false +STATE dists/ none false false +pdiff none pdiff false false +release none release\ files false false +dakcleanup none cleanup false false +STATE scripts none false false +mkmaintainers none mkmaintainers false false +copyoverrides none copyoverrides false false +mklslar none mklslar false false +mkfilesindices none mkfilesindices false false +mkchecksums none mkchecksums false false +signotherfiles none signotherfiles false false +mirror none mirror\ hardlinks false false +NOSTAGE remove_locks none false false +STATE postlock none false false +ddaccess none ddaccessible\ sync true true +changelogs none changelogs false true +pg_timestamp postdinstall pg_dump2 false false +expire none expire_dumps false true +transitionsclean none transitionsclean false true +dm none none false true +bts none none false true +mirrorpush none mirrorpush false true +mirrorpush-backports none mirrorpush-backports false true +i18n2 none i18n\ 2 false true +stats none stats false true +testingsourcelist none none false true +NOSTAGE rm\ -f\ "\${LOCK_BRITNEY}" none false false +cleantransactions none none false false diff --git a/config/debian/dinstall.variables b/config/debian/dinstall.variables index 0c1db162..6f5c9531 100644 --- a/config/debian/dinstall.variables +++ b/config/debian/dinstall.variables @@ -2,9 +2,6 @@ # usually we are not using debug logs. Set to 1 if you want them. DEBUG=0 -# our name -PROGRAM="dinstall" - # where do we want mails to go? For example log entries made with error() if [ "x$(hostname -s)x" != "xfranckx" ]; then # Not our ftpmaster host @@ -22,9 +19,6 @@ DINSTALLSTART="${lockdir}/dinstallstart" # Marker for dinstall end DINSTALLEND="${lockdir}/dinstallend" -# lock cron.unchecked (it immediately exits when this exists) -LOCK_DAILY="$lockdir/daily.lock" - # Lock cron.unchecked from doing work LOCK_ACCEPTED="$lockdir/unchecked.lock" diff --git a/config/debian/hourly.functions b/config/debian/hourly.functions new file mode 100644 index 00000000..5a433102 --- /dev/null +++ b/config/debian/hourly.functions @@ -0,0 +1,67 @@ +# -*- mode:sh -*- + +function importusers() { + dak import-users-from-passwd +} + +function newoverview() { + # do not run show-new and other stuff in parallel + if lockfile -r16 $LOCK_UNCHECKED 2> /dev/null; then + TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}" + do_new + dak show-new > /dev/null || true + fi + cleantempfiles +} + +function queuereport() { + dak queue-report -n > ${webdir}/new.html + dak queue-report -n -d backports-new,backports-policy > ${webdir}/backports-new.html + dak queue-report -8 -d new,byhand,stable-new,oldstable-new,backports-new -r $webdir/stat + sudo -u dak-unpriv dak show-deferred -r ${webdir}/stat > ${webdir}/deferred.html + dak graph -n new,byhand,stable-new,oldstable-new,deferred -r ${webdir}/stat -i ${webdir}/stat -x $scriptsdir/rrd-release-freeze-dates +} + +function removalstxt() { + cd ${webdir} + cat removals-20*.txt > removals-full.txt + cat removals.txt >> removals-full.txt + cat removals-20*.822 > removals-full.822 + cat removals.822 >> removals-full.822 +} + +function rss() { + $base/dak/tools/queue_rss.py -q $queuedir/new -o ${webdir}/rss/ -d $base/misc -l $base/log/ + $base/dak/tools/removals.pl $configdir/removalsrss.rc > ${webdir}/rss/removals.rss +} + +function gen_di() { + $scriptsdir/generate-d-i +} + +function backportsacl() { + # Update backports ACL + sudo -u dak-unpriv cat /srv/backports-master.debian.org/etc/acl \ + | dak acl set-fingerprints backports \ + | mail -a "X-Debian: DAK" -e -s "backports: acl changes" -a "From: Debian FTP Masters " backports-team@debian.org +} + +# do the buildd key updates +function builddautosigning() { + BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX) + TEMPFILES="${TEMPFILEs} ${BUILDDFUN}" + exec >> "${BUILDDFUN}" 2>&1 + ${scriptsdir}/buildd-remove-keys + ${scriptsdir}/buildd-add-keys + ${scriptsdir}/buildd-prepare-dir + + for keyring in $(dak admin k list-binary); do + dak import-keyring --generate-users "%s" ${keyring} + done + exec >>/dev/null 2>&1 + + DATE=$(date -Is) + cat "${BUILDDFUN}" | mail -a "X-Debian: DAK" -e -s "[$(hostname -s)] Buildd key changes ${DATE}" -a "From: Debian FTP Masters " buildd-keys@ftp-master.debian.org + + rm -f "${BUILDDFUN}" +} diff --git a/config/debian/hourly.tasks b/config/debian/hourly.tasks new file mode 100644 index 00000000..b5417db5 --- /dev/null +++ b/config/debian/hourly.tasks @@ -0,0 +1,10 @@ +# FUNC ARGS TIME ERR BG +importusers none none false false +newoverview none none false false +queuereport none none false true +removalstxt none none false true +rss none none false true +ddaccess none none true true +gen_di none none false false +backportsacl none none false false +builddautosigning none none false false \ No newline at end of file diff --git a/config/debian/monthly.functions b/config/debian/monthly.functions new file mode 100644 index 00000000..0241a96d --- /dev/null +++ b/config/debian/monthly.functions @@ -0,0 +1,19 @@ +# -*- mode:sh -*- + +function rotatelog() { + cd ${base}/log + touch ${DATE} + ln -sf ${DATE} current + chmod g+w ${DATE} + chown dak:ftpteam ${DATE} + xz -9 ${OLDDATE} +} + +function rotatequeued() { + cd /srv/upload.debian.org/queued + sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued -k || true + mv run/log log.${OLDDATE} + sudo -u dak-unpriv touch run/log + xz -9 log.${OLDDATE} + sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued 2>/dev/null +} \ No newline at end of file diff --git a/config/debian/monthly.tasks b/config/debian/monthly.tasks new file mode 100644 index 00000000..619f58b4 --- /dev/null +++ b/config/debian/monthly.tasks @@ -0,0 +1,3 @@ +# FUNC ARGS TIME ERR BG +rotatelog none none false true +rotatequeued none none false true \ No newline at end of file diff --git a/config/debian/monthly.variables b/config/debian/monthly.variables new file mode 100644 index 00000000..da3dd1be --- /dev/null +++ b/config/debian/monthly.variables @@ -0,0 +1,3 @@ +# -*- mode:sh -*- +DATE=$(date +%Y-%m) +OLDDATE=$(date -d yesterday +%Y-%m) diff --git a/config/debian/unchecked.tasks b/config/debian/unchecked.tasks new file mode 100644 index 00000000..d8f5211a --- /dev/null +++ b/config/debian/unchecked.tasks @@ -0,0 +1,9 @@ +# FUNC ARGS TIME ERR BG +pg_timestamp preunchecked none false false +punew stable-new p-u-new false false +opunew oldstable-new o-p-u-new false false +backports_policy none backports-policy false false +cleanpolicy none cleanpolicy false false +do_unchecked none unchecked false false +scancontents none contentsscan false false +pg_timestamp postunchecked none false false diff --git a/config/debian/unchecked.variables b/config/debian/unchecked.variables new file mode 100644 index 00000000..e09c696a --- /dev/null +++ b/config/debian/unchecked.variables @@ -0,0 +1,3 @@ +# -*- mode:sh -*- +LOCKFILE="$lockdir/unchecked.lock" +LOCK_BUILDD="$lockdir/buildd.lock" diff --git a/config/debian/vars b/config/debian/vars index aaa852f1..c94ece03 100644 --- a/config/debian/vars +++ b/config/debian/vars @@ -40,5 +40,9 @@ umask 022 unset CDPATH GZIP='--rsyncable' ; export GZIP +# lock cron.unchecked (it immediately exits when this exists) +LOCK_DAILY="$lockdir/daily.lock" +LOCK_UNCHECKED="$lockdir/unchecked.lock" + # Set the database variables eval $(dak admin config db-shell) diff --git a/config/debian/weekly.functions b/config/debian/weekly.functions new file mode 100644 index 00000000..55a7f23f --- /dev/null +++ b/config/debian/weekly.functions @@ -0,0 +1,24 @@ +# -*- mode:sh -*- + +function purgeempty() { + log "Purging empty directories in $ftpdir/pool/" + + if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then + find $ftpdir/pool/ -type d -empty | xargs rmdir; + fi +} + +function gitcleanup() { + log "Doing git stuff" + cd ${public}/git/dak.git + git gc --prune + git update-server-info + # now workaround a git bug not honoring the setup in logs/* + # (fix in development, but until it reached backports.org.......) + chmod -R g+w logs/ +} + +function symlinks() { + log "Fixing symlinks in $ftpdir" + symlinks -d -r $ftpdir +} diff --git a/config/debian/weekly.tasks b/config/debian/weekly.tasks new file mode 100644 index 00000000..4fb0d92d --- /dev/null +++ b/config/debian/weekly.tasks @@ -0,0 +1,4 @@ +# FUNC ARGS TIME ERR BG +purgeempty none none false false +gitcleanup none none false false +symlinks none none false false -- 2.39.2