From: Joerg Jaspert Date: Tue, 29 Dec 2015 23:50:16 +0000 (+0100) Subject: And away with the old scripts X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=15449de742543b884babbd11a0e08ad91fa50abf;p=dak.git And away with the old scripts --- diff --git a/config/debian/cron.daily b/config/debian/cron.daily deleted file mode 100755 index 0b822bc5..00000000 --- a/config/debian/cron.daily +++ /dev/null @@ -1,86 +0,0 @@ -#! /bin/bash -# -# Run daily via cron, out of dak's crontab. - -set -e -set -o pipefail -set -u -export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars -. $SCRIPTVARS - -# common functions are "outsourced" -. "${configdir}/common" - -################################################################################ - -TMPFILE=$( mktemp -p ${TMPDIR} ) -TMPCNTB=$( mktemp -p ${TMPDIR} ) - -function cleanup { - ERRVAL=$? - rm -f ${TMPFILE} ${TMPCNTB} - exit ${ERRVAL} -} -trap cleanup SIGHUP SIGINT SIGPIPE SIGTERM EXIT ERR - -# log to dinstall's logfile instead of sending email -PROGRAM="cron.daily" -# Start logging -NOW=`date "+%Y.%m.%d-%H:%M:%S"` -LOGFILE="$logdir/daily_${NOW}.log" -exec >> "$LOGFILE" 2>&1 - -# get the latest list of wnpp bugs and their source packages -wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm -chmod go+r ${TMPFILE} -mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm - -# Push files over to security -# The key over there should have the following set for the ssh key: -# command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity" -psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \ - xz -3 | \ - ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \ - -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync - -# Update wanna-build dump -log "Update wanna-build database dump" -$base/dak/scripts/nfu/get-w-b-db - -reports - -clean_debbugs - -# Generate list of override disparities -dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz - -# Generate stats about the new queue -dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null - -# Generate the contributor data -# FIXME: In a day or three, when this worked from cron without -# failure, redirect its output to dev/null. Alternatively until then -# enrico added a --quiet and we use that. -log "Submitting data to contributors" -REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB} - -# Post with curl as a workaround for #801506 -# See https://wiki.debian.org/ServicesSSL#curl -dir=/etc/ssl/ca-debian -test -d $dir && capath="--capath $dir" -curl -s $capath https://contributors.debian.org/contributors/post \ - -F source=ftp.debian.org \ - -F auth_token="$(cat ${base}/s3kr1t/contributor.auth)" \ - -F data=@${TMPCNTB} > ${TMPCNTB}.result -cat ${TMPCNTB}.result -rm -f ${TMPCNTB}.result - - -${scriptsdir}/link_morgue.sh - -################################################################################ - -log "Finally, all is done, compressing logfile" -exec > /dev/null 2>&1 - -bzip2 -9 "$LOGFILE" diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall deleted file mode 100755 index f1a3dcf5..00000000 --- a/config/debian/cron.dinstall +++ /dev/null @@ -1,315 +0,0 @@ -#!/bin/bash -# No way I try to deal with a crippled sh just for POSIX foo. - -# Copyright (C) 2009-2015 Joerg Jaspert -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; version 2. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -# Homer: Are you saying you're never going to eat any animal again? What -# about bacon? -# Lisa: No. -# Homer: Ham? -# Lisa: No. -# Homer: Pork chops? -# Lisa: Dad, those all come from the same animal. -# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal. - -# exit on errors -set -e -set -o pipefail -# make sure to only use defined variables -set -u -# ERR traps should be inherited from functions too. (And command -# substitutions and subshells and whatnot, but for us the functions is -# the important part here) -set -E - -# import the general variable set. -export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars -. $SCRIPTVARS - -######################################################################## -# Functions # -######################################################################## -# common functions are "outsourced" -. "${configdir}/common" - -# source the dinstall functions -. "${configdir}/dinstall.functions" - -######################################################################## -######################################################################## - -# Function to save which stage we are in, so we can restart an interrupted -# dinstall. Or even run actions in parallel, if we dare to, by simply -# backgrounding the call to this function. But that should only really be -# done for things we don't care much about. -# -# This should be called with the first argument being an array, with the -# members -# - FUNC - the function name to call -# - ARGS - Possible arguments to hand to the function. Can be the empty string -# - TIME - The timestamp name. Can be the empty string -# - ERR - if this is the string false, then the call will be surrounded by -# set +e ... set -e calls, so errors in the function do not exit -# dinstall. Can be the empty string, meaning true. -# -# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES -# ADDED FOR DINSTALL FEATURES! -function stage() { - ARGS='GO[@]' - local "${!ARGS}" - - local error=${ERR:-"true"} - - ARGS=${ARGS:-""} - - log "########## DINSTALL BEGIN: ${FUNC} ${ARGS} ##########" - local STAGEFILE="${stagedir}/${FUNC}_${ARGS}" - STAGEFILE=${STAGEFILE// /_} - if [ -f "${STAGEFILE}" ]; then - local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}") - local unixtime=$(date +%s) - local difference=$(( $unixtime - $stamptime )) - if [ ${difference} -ge 14400 ]; then - log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check." - else - log "Did already run ${FUNC}, not calling again..." - fi - return - fi - - debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}" - - # Make sure we are always at the same place. If a function wants - # to be elsewhere, it has to cd first! - cd ${configdir} - - # Now redirect the output into $STAGEFILE.log. In case it errors - # out somewhere our errorhandler trap can then mail the contents - # of $STAGEFILE.log only, instead of a whole dinstall logfile. - # Short error mails ftw! - exec >> "${STAGEFILE}.log" 2>&1 - - if [ -f "${LOCK_STOP}" ]; then - log "${LOCK_STOP} exists, exiting immediately" - exit 42 - fi - - # Do we care about trouble in the function we call? - if [ "${error}" = "false" ]; then - set +e - fi - ${FUNC} ${ARGS} - - # No matter what happened in the function, we make sure we have - # set -e default state back - set -e - - # Make sure we are always at the same place. - cd ${configdir} - - # We always use the same umask. If a function wants to do - # different, fine, but we reset. - umask 022 - - touch "${STAGEFILE}" - - if [ -n "${TIME}" ]; then - ts "${TIME}" - fi - - # And the output goes back to the normal logfile - exec >> "$LOGFILE" 2>&1 - - # Now we should make sure that we have a usable dinstall.log, so - # append the $STAGEFILE.log to it. - cat "${STAGEFILE}.log" >> "${LOGFILE}" - rm -f "${STAGEFILE}.log" - - log "########## DINSTALL END: ${FUNC} ##########" - - if [ -f "${LOCK_STOP}" ]; then - log "${LOCK_STOP} exists, exiting immediately" - exit 42 - fi -} - -######################################################################## - -# We need logs. -LOGFILE="$logdir/dinstall.log" - -exec >> "$LOGFILE" 2>&1 - -# And now source our default config -. "${configdir}/dinstall.variables" - -# Make sure we start out with a sane umask setting -umask 022 - -# And use one locale, no matter what the caller has set -export LANG=C -export LC_ALL=C - -touch "${DINSTALLSTART}" -ts "startup" -DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")" -state "Startup" - -lockfile -l 3600 "${LOCK_DAILY}" -trap onerror ERR -trap remove_daily_lock EXIT TERM HUP INT QUIT - -touch "${LOCK_BRITNEY}" - -# This loop simply wants to be fed by a list of values (see below) -# made out of 5 columns. -# The first four are the array values for the stage function, the -# fifth tells us if we should background the stage call. -# -# - FUNC - the function name to call -# - ARGS - Possible arguments to hand to the function. Can be the empty string -# - TIME - The timestamp name. Can be the empty string -# - ERR - if this is the string false, then the call will be surrounded by -# set +e ... set -e calls, so errors in the function do not exit -# dinstall. Can be the empty string, meaning true. -# - BG - Background the function stage? -# -# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \ -# -# NOTE 1: There are two special values for the first column (FUNC). -# STATE - do not call stage function, call the state -# function to update the public statefile "where is dinstall" -# NOSTAGE - do not call stage function, call the command directly. -# -# Note 2: If you want to hand an empty value to the stage function, -# use the word "none" in the list below. -while read FUNC ARGS TIME ERR BACKGROUND; do - debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND" - - # Empty values in the value list are the string "none" (or the - # while read loop won't work). Here we ensure that variables that - # can be empty, are empty if the string none is set for them. - for var in ARGS TIME; do - if [[ ${!var} == none ]]; then - typeset ${var}='' - fi - done - - # ERR/BACKGROUND are boolean, check that they are. - for var in ERR BACKGROUND; do - if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then - error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}" - fi - done - - case ${FUNC} in - STATE) - state ${ARGS} - ;; - NOSTAGE) - ${ARGS} - ;; - *) - GO=( - FUNC=${FUNC} - TIME=${TIME} - ARGS=${ARGS} - ERR=${ERR} - ) - if [[ ${BACKGROUND} == true ]]; then - stage $GO & - else - stage $GO - fi - ;; - esac -done < <(cat - < "$logdir/afterdinstall.log" 2>&1 - -if [ -f "${dbdir}/dinstallstart" ]; then - NOW=$(cat "${dbdir}/dinstallstart") - mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" - logstats "$logdir/dinstall_${NOW}.log" - bzip2 -9 "$logdir/dinstall_${NOW}.log" -else - error "Problem, I don't know when dinstall started, unable to do log statistics." - NOW=`date "+%Y.%m.%d-%H:%M:%S"` - mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" - bzip2 -9 "$logdir/dinstall_${NOW}.log" -fi - -state "all done" - -# Now, at the very (successful) end of dinstall, make sure we remove -# our stage files, so the next dinstall run will do it all again. -rm -f ${stagedir}/* -touch "${DINSTALLEND}" diff --git a/config/debian/cron.hourly b/config/debian/cron.hourly deleted file mode 100755 index d2451b8a..00000000 --- a/config/debian/cron.hourly +++ /dev/null @@ -1,82 +0,0 @@ -#! /bin/bash -# -# Executed hourly via cron, out of dak's crontab. - -# Only one of me should ever run. -[ "${FLOCKER}" != "$0" ] && exec env FLOCKER="$0" flock -E 0 -en "$0" "$0" -"$@" || : - -set -e -set -o pipefail -set -u - -export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars -. $SCRIPTVARS - -PROGRAM="Hourly" -######################################################################## -# Functions # -######################################################################## -# common functions are "outsourced" -. "${configdir}/common" - -dak import-users-from-passwd - -# do not run show-new and other stuff in parallel -LOCKFILE="$lockdir/unchecked.lock" -cleanup() { - rm -f "${LOCKFILE}" -} - -if lockfile -r16 $LOCKFILE 2> /dev/null; then - trap cleanup EXIT - do_new - dak show-new > /dev/null || true - cleanup - trap - EXIT -fi - -dak queue-report -n > $webdir/new.html -dak queue-report -n -d backports-new,backports-policy > ${webdir}/backports-new.html -dak queue-report -8 -d new,byhand,stable-new,oldstable-new,backports-new -r $webdir/stat -sudo -u dak-unpriv dak show-deferred -r $webdir/stat > ${webdir}/deferred.html -dak graph -n new,byhand,stable-new,oldstable-new,deferred -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates - -cd $webdir -cat removals-20*.txt > removals-full.txt -cat removals.txt >> removals-full.txt -cat removals-20*.822 > removals-full.822 -cat removals.822 >> removals-full.822 - -$base/dak/tools/queue_rss.py -q $queuedir/new -o $webdir/rss/ -d $base/misc -l $base/log/ -$base/dak/tools/removals.pl $configdir/removalsrss.rc > $webdir/rss/removals.rss - -# cd $masterdir -# timeout -k 180 120 /usr/bin/epydoc -q --html --graph all --css blue -n DAK -o $webdir/epydoc --include-log --graph all --show-imports $masterdir/dak/ $masterdir/dakweb/ || true - -# Tell the public mirror to sync its tree -${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync - -$scriptsdir/generate-d-i - -# Update backports ACL -sudo -u dak-unpriv cat /srv/backports-master.debian.org/etc/acl \ - | dak acl set-fingerprints backports \ - | mail -a "X-Debian: DAK" -e -s "backports: acl changes" -a "From: Debian FTP Masters " backports-team@debian.org - -# do the buildd key updates -BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX) -exec >> "${BUILDDFUN}" 2>&1 -${scriptsdir}/buildd-remove-keys -${scriptsdir}/buildd-add-keys -${scriptsdir}/buildd-prepare-dir - -for keyring in $(dak admin k list-binary); do - dak import-keyring --generate-users "%s" ${keyring} -done -exec >>/dev/null 2>&1 - -DATE=$(date -Is) -cat "${BUILDDFUN}" | mail -a "X-Debian: DAK" -e -s "[$(hostname -s)] Buildd key changes ${DATE}" -a "From: Debian FTP Masters " buildd-keys@ftp-master.debian.org - -rm -f "${BUILDDFUN}" diff --git a/config/debian/cron.monthly b/config/debian/cron.monthly deleted file mode 100755 index 1f92f914..00000000 --- a/config/debian/cron.monthly +++ /dev/null @@ -1,31 +0,0 @@ -#! /bin/bash -# -# Run at the beginning of the month via cron, out of dak's crontab. - -set -e -set -o pipefail -set -u -export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars -. $SCRIPTVARS - -################################################################################ - -DATE=`date -d yesterday +%y%m` - -DATE=$(date +%Y-%m) -OLDDATE=$(date -d yesterday +%Y-%m) -cd ${base}/log -touch $DATE -ln -sf $DATE current -chmod g+w $DATE -chown dak:ftpteam $DATE -xz -9 ${OLDDATE} - -cd /srv/upload.debian.org/queued -sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued -k || true -mv run/log log.${OLDDATE} -sudo -u dak-unpriv touch run/log -xz -9 log.${OLDDATE} -sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued 2>/dev/null - -################################################################################ diff --git a/config/debian/cron.unchecked b/config/debian/cron.unchecked deleted file mode 100755 index 916d4e5d..00000000 --- a/config/debian/cron.unchecked +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -# No way I try to deal with a crippled sh just for POSIX foo. - -# Copyright (C) 2009 Joerg Jaspert -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; version 2. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -# exit on errors -set -e -set -o pipefail -# make sure to only use defined variables -set -u -# ERR traps should be inherited from functions too. (And command -# substitutions and subshells and whatnot, but for us the functions is -# the important part here) -set -E - -# import the general variable set. -export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars -. $SCRIPTVARS - -LOCKDAILY="" -LOCKFILE="$lockdir/unchecked.lock" -NOTICE="$lockdir/daily.lock" -LOCK_BUILDD="$lockdir/buildd.lock" - -# our name -PROGRAM="unchecked" - -if [ -e $NOTICE ]; then - exit 0; -fi - -######################################################################## -# Functions # -######################################################################## -# common functions are "outsourced" -. "${configdir}/common" - -STAMP=$(date "+%Y%m%d%H%M") - -cleanup() { - rm -f "$LOCKFILE" - if [ ! -z "$LOCKDAILY" ]; then - rm -f "$NOTICE" - fi -} - -function do_buildd () { - if lockfile -r3 $NOTICE; then - LOCKDAILY="YES" - make_buildd_dir - wbtrigger - fi -} - -######################################################################## -# the actual unchecked functions follow # -######################################################################## - -# And use one locale, no matter what the caller has set -export LANG=C -export LC_ALL=C - -# only run one cron.unchecked -if ! lockfile -r8 $LOCKFILE 2> /dev/null; then - # echo "aborting cron.unchecked because $LOCKFILE has already been locked" - exit 0 -fi -trap cleanup 0 - - -pg_timestamp preunchecked >/dev/null - -# Process policy queues -punew stable-new -opunew oldstable-new -backports_policy -dak clean-suites -a backports-policy,policy - -# Finally deal with unchecked -do_unchecked - -if [ ! -z "$changes" ]; then - sync_debbugs - do_buildd -fi - -dak contents -l 10000 scan-binary -dak contents -l 1000 scan-source -pg_timestamp postunchecked >/dev/null diff --git a/config/debian/cron.weekly b/config/debian/cron.weekly deleted file mode 100755 index 7d884b6a..00000000 --- a/config/debian/cron.weekly +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# -# Run once a week via cron, out of dak's crontab. - -set -e -set -o pipefail -set -u -# ERR traps should be inherited from functions too. (And command -# substitutions and subshells and whatnot, but for us the functions is -# the important part here) -set -E -export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars -. $SCRIPTVARS - -# Start logging -NOW=`date "+%Y.%m.%d-%H:%M:%S"` -LOGFILE="$logdir/weekly_${NOW}.log" -exec > "$LOGFILE" 2>&1 - -cleanup() { - echo "Cleanup" - rm -f "$LOGFILE" -} -trap cleanup 0 - -################################################################################ - -# Purge empty directories -echo "Purging empty directories in $ftpdir/pool/" - -if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then - find $ftpdir/pool/ -type d -empty | xargs rmdir; -fi - -# Do git cleanup stuff -echo "Doing git stuff" -cd /srv/ftp.debian.org/git/dak.git -git gc --prune -git update-server-info -# now workaround a git bug not honoring the setup in logs/* -# (fix in development, but until it reached backports.org.......) -chmod -R g+w logs/ - -echo "Fixing symlinks in $ftpdir" -symlinks -d -r $ftpdir - -echo "Finally, all is done, compressing logfile" -exec > /dev/null 2>&1 - -bzip2 -9 "$LOGFILE" - - -################################################################################