+++ /dev/null
-#! /bin/bash
-#
-# Run daily via cron, out of dak's crontab.
-
-set -e
-set -o pipefail
-set -u
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-# common functions are "outsourced"
-. "${configdir}/common"
-
-################################################################################
-
-TMPFILE=$( mktemp -p ${TMPDIR} )
-TMPCNTB=$( mktemp -p ${TMPDIR} )
-
-function cleanup {
- ERRVAL=$?
- rm -f ${TMPFILE} ${TMPCNTB}
- exit ${ERRVAL}
-}
-trap cleanup SIGHUP SIGINT SIGPIPE SIGTERM EXIT ERR
-
-# log to dinstall's logfile instead of sending email
-PROGRAM="cron.daily"
-# Start logging
-NOW=`date "+%Y.%m.%d-%H:%M:%S"`
-LOGFILE="$logdir/daily_${NOW}.log"
-exec >> "$LOGFILE" 2>&1
-
-# get the latest list of wnpp bugs and their source packages
-wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
-chmod go+r ${TMPFILE}
-mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
-
-# Push files over to security
-# The key over there should have the following set for the ssh key:
-# command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity"
-psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \
- xz -3 | \
- ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \
- -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync
-
-# Update wanna-build dump
-log "Update wanna-build database dump"
-$base/dak/scripts/nfu/get-w-b-db
-
-reports
-
-clean_debbugs
-
-# Generate list of override disparities
-dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz
-
-# Generate stats about the new queue
-dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null
-
-# Generate the contributor data
-# FIXME: In a day or three, when this worked from cron without
-# failure, redirect its output to dev/null. Alternatively until then
-# enrico added a --quiet and we use that.
-log "Submitting data to contributors"
-REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
-
-# Post with curl as a workaround for #801506
-# See https://wiki.debian.org/ServicesSSL#curl
-dir=/etc/ssl/ca-debian
-test -d $dir && capath="--capath $dir"
-curl -s $capath https://contributors.debian.org/contributors/post \
- -F source=ftp.debian.org \
- -F auth_token="$(cat ${base}/s3kr1t/contributor.auth)" \
- -F data=@${TMPCNTB} > ${TMPCNTB}.result
-cat ${TMPCNTB}.result
-rm -f ${TMPCNTB}.result
-
-
-${scriptsdir}/link_morgue.sh
-
-################################################################################
-
-log "Finally, all is done, compressing logfile"
-exec > /dev/null 2>&1
-
-bzip2 -9 "$LOGFILE"
+++ /dev/null
-#!/bin/bash
-# No way I try to deal with a crippled sh just for POSIX foo.
-
-# Copyright (C) 2009-2015 Joerg Jaspert <joerg@debian.org>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; version 2.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-# Homer: Are you saying you're never going to eat any animal again? What
-# about bacon?
-# Lisa: No.
-# Homer: Ham?
-# Lisa: No.
-# Homer: Pork chops?
-# Lisa: Dad, those all come from the same animal.
-# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
-
-# exit on errors
-set -e
-set -o pipefail
-# make sure to only use defined variables
-set -u
-# ERR traps should be inherited from functions too. (And command
-# substitutions and subshells and whatnot, but for us the functions is
-# the important part here)
-set -E
-
-# import the general variable set.
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-########################################################################
-# Functions #
-########################################################################
-# common functions are "outsourced"
-. "${configdir}/common"
-
-# source the dinstall functions
-. "${configdir}/dinstall.functions"
-
-########################################################################
-########################################################################
-
-# Function to save which stage we are in, so we can restart an interrupted
-# dinstall. Or even run actions in parallel, if we dare to, by simply
-# backgrounding the call to this function. But that should only really be
-# done for things we don't care much about.
-#
-# This should be called with the first argument being an array, with the
-# members
-# - FUNC - the function name to call
-# - ARGS - Possible arguments to hand to the function. Can be the empty string
-# - TIME - The timestamp name. Can be the empty string
-# - ERR - if this is the string false, then the call will be surrounded by
-# set +e ... set -e calls, so errors in the function do not exit
-# dinstall. Can be the empty string, meaning true.
-#
-# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
-# ADDED FOR DINSTALL FEATURES!
-function stage() {
- ARGS='GO[@]'
- local "${!ARGS}"
-
- local error=${ERR:-"true"}
-
- ARGS=${ARGS:-""}
-
- log "########## DINSTALL BEGIN: ${FUNC} ${ARGS} ##########"
- local STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
- STAGEFILE=${STAGEFILE// /_}
- if [ -f "${STAGEFILE}" ]; then
- local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
- local unixtime=$(date +%s)
- local difference=$(( $unixtime - $stamptime ))
- if [ ${difference} -ge 14400 ]; then
- log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
- else
- log "Did already run ${FUNC}, not calling again..."
- fi
- return
- fi
-
- debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"
-
- # Make sure we are always at the same place. If a function wants
- # to be elsewhere, it has to cd first!
- cd ${configdir}
-
- # Now redirect the output into $STAGEFILE.log. In case it errors
- # out somewhere our errorhandler trap can then mail the contents
- # of $STAGEFILE.log only, instead of a whole dinstall logfile.
- # Short error mails ftw!
- exec >> "${STAGEFILE}.log" 2>&1
-
- if [ -f "${LOCK_STOP}" ]; then
- log "${LOCK_STOP} exists, exiting immediately"
- exit 42
- fi
-
- # Do we care about trouble in the function we call?
- if [ "${error}" = "false" ]; then
- set +e
- fi
- ${FUNC} ${ARGS}
-
- # No matter what happened in the function, we make sure we have
- # set -e default state back
- set -e
-
- # Make sure we are always at the same place.
- cd ${configdir}
-
- # We always use the same umask. If a function wants to do
- # different, fine, but we reset.
- umask 022
-
- touch "${STAGEFILE}"
-
- if [ -n "${TIME}" ]; then
- ts "${TIME}"
- fi
-
- # And the output goes back to the normal logfile
- exec >> "$LOGFILE" 2>&1
-
- # Now we should make sure that we have a usable dinstall.log, so
- # append the $STAGEFILE.log to it.
- cat "${STAGEFILE}.log" >> "${LOGFILE}"
- rm -f "${STAGEFILE}.log"
-
- log "########## DINSTALL END: ${FUNC} ##########"
-
- if [ -f "${LOCK_STOP}" ]; then
- log "${LOCK_STOP} exists, exiting immediately"
- exit 42
- fi
-}
-
-########################################################################
-
-# We need logs.
-LOGFILE="$logdir/dinstall.log"
-
-exec >> "$LOGFILE" 2>&1
-
-# And now source our default config
-. "${configdir}/dinstall.variables"
-
-# Make sure we start out with a sane umask setting
-umask 022
-
-# And use one locale, no matter what the caller has set
-export LANG=C
-export LC_ALL=C
-
-touch "${DINSTALLSTART}"
-ts "startup"
-DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
-state "Startup"
-
-lockfile -l 3600 "${LOCK_DAILY}"
-trap onerror ERR
-trap remove_daily_lock EXIT TERM HUP INT QUIT
-
-touch "${LOCK_BRITNEY}"
-
-# This loop simply wants to be fed by a list of values (see below)
-# made out of 5 columns.
-# The first four are the array values for the stage function, the
-# fifth tells us if we should background the stage call.
-#
-# - FUNC - the function name to call
-# - ARGS - Possible arguments to hand to the function. Can be the empty string
-# - TIME - The timestamp name. Can be the empty string
-# - ERR - if this is the string false, then the call will be surrounded by
-# set +e ... set -e calls, so errors in the function do not exit
-# dinstall. Can be the empty string, meaning true.
-# - BG - Background the function stage?
-#
-# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \
-#
-# NOTE 1: There are two special values for the first column (FUNC).
-# STATE - do not call stage function, call the state
-# function to update the public statefile "where is dinstall"
-# NOSTAGE - do not call stage function, call the command directly.
-#
-# Note 2: If you want to hand an empty value to the stage function,
-# use the word "none" in the list below.
-while read FUNC ARGS TIME ERR BACKGROUND; do
- debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND"
-
- # Empty values in the value list are the string "none" (or the
- # while read loop won't work). Here we ensure that variables that
- # can be empty, are empty if the string none is set for them.
- for var in ARGS TIME; do
- if [[ ${!var} == none ]]; then
- typeset ${var}=''
- fi
- done
-
- # ERR/BACKGROUND are boolean, check that they are.
- for var in ERR BACKGROUND; do
- if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then
- error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}"
- fi
- done
-
- case ${FUNC} in
- STATE)
- state ${ARGS}
- ;;
- NOSTAGE)
- ${ARGS}
- ;;
- *)
- GO=(
- FUNC=${FUNC}
- TIME=${TIME}
- ARGS=${ARGS}
- ERR=${ERR}
- )
- if [[ ${BACKGROUND} == true ]]; then
- stage $GO &
- else
- stage $GO
- fi
- ;;
- esac
-done < <(cat - <<EOF
-savetimestamp none none false false
-qa1 none init true true
-pg_timestamp predinstall pg_dump1 false false
-updates none External\ Updates false false
-i18n1 none i18n\ 1 false false
-dep11 none dep11\ 1 false false
-NOSTAGE lockaccepted none false false
-punew stable-new p-u-new false false
-opunew oldstable-new o-p-u-new false false
-backports_policy none backports-policy false false
-cruft none cruft false false
-STATE indices none false false
-dominate none dominate false false
-autocruft none autocruft false false
-fingerprints none import-keyring false false
-overrides none overrides false false
-mpfm none pkg-file-mapping false false
-STATE packages/contents none false false
-packages none apt-ftparchive false false
-STATE dists/ none false false
-pdiff none pdiff false false
-release none release\ files false false
-dakcleanup none cleanup false false
-STATE scripts none false false
-mkmaintainers none mkmaintainers false false
-copyoverrides none copyoverrides false false
-mklslar none mklslar false false
-mkfilesindices none mkfilesindices false false
-mkchecksums none mkchecksums false false
-signotherfiles none signotherfiles false false
-mirror none mirror\ hardlinks false false
-NOSTAGE remove_locks none false false
-STATE postlock none false false
-ddaccess none ddaccessible\ sync true true
-changelogs none changelogs false true
-pg_timestamp postdinstall pg_dump2 false false
-expire none expire_dumps false true
-transitionsclean none transitionsclean false true
-dm none none false true
-bts none none false true
-mirrorpush none mirrorpush false true
-mirrorpush-backports none mirrorpush-backports false true
-i18n2 none i18n\ 2 false true
-stats none stats false true
-testingsourcelist none none false true
-NOSTAGE rm\ -f\ "\${LOCK_BRITNEY}" none false false
-cleantransactions none none false false
-EOF
- )
-# FUNC ARGS TIME ERR BG
-
-# we need to wait for the background processes before the end of dinstall
-wait
-
-log "Daily cron scripts successful, all done"
-
-exec > "$logdir/afterdinstall.log" 2>&1
-
-if [ -f "${dbdir}/dinstallstart" ]; then
- NOW=$(cat "${dbdir}/dinstallstart")
- mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
- logstats "$logdir/dinstall_${NOW}.log"
- bzip2 -9 "$logdir/dinstall_${NOW}.log"
-else
- error "Problem, I don't know when dinstall started, unable to do log statistics."
- NOW=`date "+%Y.%m.%d-%H:%M:%S"`
- mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
- bzip2 -9 "$logdir/dinstall_${NOW}.log"
-fi
-
-state "all done"
-
-# Now, at the very (successful) end of dinstall, make sure we remove
-# our stage files, so the next dinstall run will do it all again.
-rm -f ${stagedir}/*
-touch "${DINSTALLEND}"
+++ /dev/null
-#! /bin/bash
-#
-# Executed hourly via cron, out of dak's crontab.
-
-# Only one of me should ever run.
-[ "${FLOCKER}" != "$0" ] && exec env FLOCKER="$0" flock -E 0 -en "$0" "$0"
-"$@" || :
-
-set -e
-set -o pipefail
-set -u
-
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-PROGRAM="Hourly"
-########################################################################
-# Functions #
-########################################################################
-# common functions are "outsourced"
-. "${configdir}/common"
-
-dak import-users-from-passwd
-
-# do not run show-new and other stuff in parallel
-LOCKFILE="$lockdir/unchecked.lock"
-cleanup() {
- rm -f "${LOCKFILE}"
-}
-
-if lockfile -r16 $LOCKFILE 2> /dev/null; then
- trap cleanup EXIT
- do_new
- dak show-new > /dev/null || true
- cleanup
- trap - EXIT
-fi
-
-dak queue-report -n > $webdir/new.html
-dak queue-report -n -d backports-new,backports-policy > ${webdir}/backports-new.html
-dak queue-report -8 -d new,byhand,stable-new,oldstable-new,backports-new -r $webdir/stat
-sudo -u dak-unpriv dak show-deferred -r $webdir/stat > ${webdir}/deferred.html
-dak graph -n new,byhand,stable-new,oldstable-new,deferred -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates
-
-cd $webdir
-cat removals-20*.txt > removals-full.txt
-cat removals.txt >> removals-full.txt
-cat removals-20*.822 > removals-full.822
-cat removals.822 >> removals-full.822
-
-$base/dak/tools/queue_rss.py -q $queuedir/new -o $webdir/rss/ -d $base/misc -l $base/log/
-$base/dak/tools/removals.pl $configdir/removalsrss.rc > $webdir/rss/removals.rss
-
-# cd $masterdir
-# timeout -k 180 120 /usr/bin/epydoc -q --html --graph all --css blue -n DAK -o $webdir/epydoc --include-log --graph all --show-imports $masterdir/dak/ $masterdir/dakweb/ || true
-
-# Tell the public mirror to sync its tree
-${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
-
-$scriptsdir/generate-d-i
-
-# Update backports ACL
-sudo -u dak-unpriv cat /srv/backports-master.debian.org/etc/acl \
- | dak acl set-fingerprints backports \
- | mail -a "X-Debian: DAK" -e -s "backports: acl changes" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
-
-# do the buildd key updates
-BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX)
-exec >> "${BUILDDFUN}" 2>&1
-${scriptsdir}/buildd-remove-keys
-${scriptsdir}/buildd-add-keys
-${scriptsdir}/buildd-prepare-dir
-
-for keyring in $(dak admin k list-binary); do
- dak import-keyring --generate-users "%s" ${keyring}
-done
-exec >>/dev/null 2>&1
-
-DATE=$(date -Is)
-cat "${BUILDDFUN}" | mail -a "X-Debian: DAK" -e -s "[$(hostname -s)] Buildd key changes ${DATE}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" buildd-keys@ftp-master.debian.org
-
-rm -f "${BUILDDFUN}"
+++ /dev/null
-#!/bin/bash
-# No way I try to deal with a crippled sh just for POSIX foo.
-
-# Copyright (C) 2009 Joerg Jaspert <joerg@debian.org>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; version 2.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-# exit on errors
-set -e
-set -o pipefail
-# make sure to only use defined variables
-set -u
-# ERR traps should be inherited from functions too. (And command
-# substitutions and subshells and whatnot, but for us the functions is
-# the important part here)
-set -E
-
-# import the general variable set.
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-LOCKDAILY=""
-LOCKFILE="$lockdir/unchecked.lock"
-NOTICE="$lockdir/daily.lock"
-LOCK_BUILDD="$lockdir/buildd.lock"
-
-# our name
-PROGRAM="unchecked"
-
-if [ -e $NOTICE ]; then
- exit 0;
-fi
-
-########################################################################
-# Functions #
-########################################################################
-# common functions are "outsourced"
-. "${configdir}/common"
-
-STAMP=$(date "+%Y%m%d%H%M")
-
-cleanup() {
- rm -f "$LOCKFILE"
- if [ ! -z "$LOCKDAILY" ]; then
- rm -f "$NOTICE"
- fi
-}
-
-function do_buildd () {
- if lockfile -r3 $NOTICE; then
- LOCKDAILY="YES"
- make_buildd_dir
- wbtrigger
- fi
-}
-
-########################################################################
-# the actual unchecked functions follow #
-########################################################################
-
-# And use one locale, no matter what the caller has set
-export LANG=C
-export LC_ALL=C
-
-# only run one cron.unchecked
-if ! lockfile -r8 $LOCKFILE 2> /dev/null; then
- # echo "aborting cron.unchecked because $LOCKFILE has already been locked"
- exit 0
-fi
-trap cleanup 0
-
-
-pg_timestamp preunchecked >/dev/null
-
-# Process policy queues
-punew stable-new
-opunew oldstable-new
-backports_policy
-dak clean-suites -a backports-policy,policy
-
-# Finally deal with unchecked
-do_unchecked
-
-if [ ! -z "$changes" ]; then
- sync_debbugs
- do_buildd
-fi
-
-dak contents -l 10000 scan-binary
-dak contents -l 1000 scan-source
-pg_timestamp postunchecked >/dev/null
projects / dak.git / commitdiff