#
# Set $PROGRAM to a string to have it added to the output.
function log () {
- local prefix=${PROGRAM:-}
- echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${prefix}[$$]: $@"
+ local prefix=${PROGRAM:-}
+ echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${prefix}[$$]: $@"
}
# log the message using log() but then also send a mail
# to the address configured in MAILTO (if non-empty)
function log_error () {
- log "$@"
- if [ -z "${MAILTO}" ]; then
- echo "$@" | mail -a "X-Debian: DAK" -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" ${MAILTO}
- fi
+ log "$@"
+ if [ -z "${MAILTO}" ]; then
+ echo "$@" | mail -a "X-Debian: DAK" -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" ${MAILTO}
+ fi
}
# debug log, only output when DEBUG=1
fi
}
+# Function that only cleans tempfiles, but does not exit or otherwise
+# care about any exit status
+function cleantempfiles() {
+ resolvetmpfiles
+ for TEMPFILE in $TMPFILES; do
+ if [ -n "${TEMPFILE}" ] && [ -f "${TEMPFILE}" ]; then
+ rm -f "${TEMPFILE}"
+ elif [ -n "${TEMPFILE}" ] && [ -d "${TEMPFILE}" ]; then
+ if [ "${TEMPFILE}" != "/" ] && [ "${TEMPFILE}" != "/*" ]; then
+ rm -rf "${TEMPFILE}"
+ fi
+ fi
+ done
+ TMPFILES=""
+}
+
+function resolvetmpfiles() {
+ # If you don't understand this better not touch the script
+ for TEMPFILE in $TEMPFILES; do
+ TMPFILES="${TMPFILES} ${!TEMPFILE:-""}"
+ done
+ TEMPFILES=""
+}
+
+# Function cleanup
+# No arguments
+# Cleans up any known tempfile.
+# Just ensure your script sets the variable
+# TEMPFILES to the names of variables of tempfiles
+# Or TMPFILES to the pathes of tempfiles
+function cleanup() {
+ ERRVAL=$?
+ trap - ERR EXIT TERM HUP INT QUIT
+
+ cleantempfiles
+
+ return $ERRVAL
+}
+TEMPFILES=${TEMPFILES:-""}
+TMPFILES=${TMPFILES:-""}
+
+# Timestamp. Used for dinstall stat graphs
+function ts() {
+ echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
+}
+
+########################################################################
+########################################################################
+
function wbtrigger() {
SSHOPT="-n -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240"
if lockfile -r 3 -l 3600 "${LOCK_BUILDD}"; then
# These versions used in dinstall
function punew() {
- if [ "${PROGRAM}" = "dinstall" ]; then
- log "Doing automated p-u-new processing"
- fi
+ log "Doing automated p-u-new processing"
cd "${queuedir}/p-u-new"
punew_do "$1" "${queuedir}/p-u-new" "debian-release@lists.debian.org"
}
function opunew() {
- if [ "${PROGRAM}" = "dinstall" ]; then
- log "Doing automated o-p-u-new processing"
- fi
+ log "Doing automated o-p-u-new processing"
cd "${queuedir}/o-p-u-new"
punew_do "$1" "${queuedir}/o-p-u-new" "debian-release@lists.debian.org"
}
local qdir="/srv/backports-master.debian.org/queue/policy"
local to="backports-team@debian.org"
- if [ "${PROGRAM}" = "dinstall" ]; then
- log "Doing automated ${queue} processing"
- fi
+ log "Doing automated ${queue} processing"
cd "${qdir}"
punew_do "${queue}" "${qdir}" "${to}"
echo "$timestamp": ${changes:-"Nothing to do"} >> $report
dak process-upload -a ${UNCHECKED_WITHOUT_LOCK} -d "$unchecked" >> $report
dak process-commands -d "$unchecked" >> $report
+
+ if [ ! -z "$changes" ]; then
+ sync_debbugs
+ do_buildd
+ fi
}
# process NEW policy queue
function do_new () {
- if [ "${PROGRAM}" = "dinstall" ]; then
- log "Doing NEW processing"
- fi
+ log "Doing NEW processing"
(dak process-policy new; dak process-policy byhand) | mail -a "X-Debian: DAK" -e -s "NEW and BYHAND processing" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" ftpmaster@ftp-master.debian.org
dak process-policy backports-new | mail -a "X-Debian: DAK" -e -s "NEW processing for backports-new" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
fi
echo "${archiveroot}"
}
+
+# Prepare the trees for buildds, then push wanna-build
+function do_buildd() {
+ if lockfile -r3 ${LOCK_DAILY}; then
+ TMPFILES="${TMPFILES} ${LOCK_DAILY}"
+ make_buildd_dir
+ wbtrigger
+ fi
+}
+
+# Cleanup policy queues
+function cleanpolicy() {
+ dak clean-suites -a backports-policy,policy
+}
+
+# Scan new packages for contents
+function scancontents() {
+ dak contents -l 10000 scan-binary
+ dak contents -l 1000 scan-source
+}
+
+function ddaccess() {
+ # Tell our dd accessible mirror to sync itself up. Including ftp dir.
+ log "Trigger dd accessible parts sync including ftp dir"
+ ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
+}
+
+
+
+########################################################################
+########################################################################
+########################################################################
+########################################################################
+
+# Function to save which stage we are in, so we can restart an interrupted
+# dinstall. Or even run actions in parallel, if we dare to, by simply
+# backgrounding the call to this function. But that should only really be
+# done for things we don't care much about.
+#
+# This should be called with the first argument being an array, with the
+# members
+# - FUNC - the function name to call
+# - ARGS - Possible arguments to hand to the function. Can be the empty string
+# - TIME - The timestamp name. Can be the empty string
+# - ERR - if this is the string false, then the call will be surrounded by
+# set +e ... set -e calls, so errors in the function do not exit
+# dinstall. Can be the empty string, meaning true.
+#
+# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
+# ADDED FOR DINSTALL FEATURES!
+function stage() {
+ ARGS='GO[@]'
+ local "${!ARGS}"
+
+ local error=${ERR:-"true"}
+
+ ARGS=${ARGS:-""}
+
+ log "########## ${PROGRAM} BEGIN: ${FUNC} ${ARGS} ##########"
+ local STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
+ STAGEFILE=${STAGEFILE// /_}
+ if [ -f "${STAGEFILE}" ]; then
+ local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
+ local unixtime=$(date +%s)
+ local difference=$(( $unixtime - $stamptime ))
+ if [ ${difference} -ge 14400 ]; then
+ log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
+ else
+ log "Did already run ${FUNC}, not calling again..."
+ fi
+ return
+ fi
+
+ debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"
+
+ # Make sure we are always at the same place. If a function wants
+ # to be elsewhere, it has to cd first!
+ cd ${configdir}
+
+ # Now redirect the output into $STAGEFILE.log. In case it errors
+ # out somewhere our errorhandler trap can then mail the contents
+ # of $STAGEFILE.log only, instead of a whole ${PROGRAM} logfile.
+ # Short error mails ftw!
+ exec >> "${STAGEFILE}.log" 2>&1
+
+ if [ -f "${LOCK_STOP}" ]; then
+ log "${LOCK_STOP} exists, exiting immediately"
+ exit 42
+ fi
+
+ # Do we care about trouble in the function we call?
+ if [ "${error}" = "false" ]; then
+ set +e
+ fi
+ ${FUNC} ${ARGS}
+
+ # No matter what happened in the function, we make sure we have
+ # set -e default state back
+ set -e
+
+ # Make sure we are always at the same place.
+ cd ${configdir}
+
+ # We always use the same umask. If a function wants to do
+ # different, fine, but we reset.
+ umask 022
+
+ touch "${STAGEFILE}"
+
+ if [ -n "${TIME}" ]; then
+ ts "${TIME}"
+ fi
+
+ # And the output goes back to the normal logfile
+ exec >> "${LOGFILE}" 2>&1
+
+ # Now we should make sure that we have a usable ${PROGRAM}.log, so
+ # append the $STAGEFILE.log to it.
+ cat "${STAGEFILE}.log" >> "${LOGFILE}"
+ rm -f "${STAGEFILE}.log"
+
+ log "########## ${PROGRAM} END: ${FUNC} ##########"
+
+ if [ -f "${LOCK_STOP}" ]; then
+ log "${LOCK_STOP} exists, exiting immediately"
+ exit 42
+ fi
+}
+++ /dev/null
-#! /bin/bash
-#
-# Run daily via cron, out of dak's crontab.
-
-set -e
-set -o pipefail
-set -u
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-# common functions are "outsourced"
-. "${configdir}/common"
-
-################################################################################
-
-TMPFILE=$( mktemp -p ${TMPDIR} )
-TMPCNTB=$( mktemp -p ${TMPDIR} )
-
-function cleanup {
- ERRVAL=$?
- rm -f ${TMPFILE} ${TMPCNTB}
- exit ${ERRVAL}
-}
-trap cleanup SIGHUP SIGINT SIGPIPE SIGTERM EXIT ERR
-
-# log to dinstall's logfile instead of sending email
-PROGRAM="cron.daily"
-# Start logging
-NOW=`date "+%Y.%m.%d-%H:%M:%S"`
-LOGFILE="$logdir/daily_${NOW}.log"
-exec >> "$LOGFILE" 2>&1
-
-# get the latest list of wnpp bugs and their source packages
-wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
-chmod go+r ${TMPFILE}
-mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
-
-# Push files over to security
-# The key over there should have the following set for the ssh key:
-# command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity"
-psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \
- xz -3 | \
- ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \
- -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync
-
-# Update wanna-build dump
-log "Update wanna-build database dump"
-$base/dak/scripts/nfu/get-w-b-db
-
-reports
-
-clean_debbugs
-
-# Generate list of override disparities
-dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz
-
-# Generate stats about the new queue
-dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null
-
-# Generate the contributor data
-# FIXME: In a day or three, when this worked from cron without
-# failure, redirect its output to dev/null. Alternatively until then
-# enrico added a --quiet and we use that.
-log "Submitting data to contributors"
-REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
-
-# Post with curl as a workaround for #801506
-# See https://wiki.debian.org/ServicesSSL#curl
-dir=/etc/ssl/ca-debian
-test -d $dir && capath="--capath $dir"
-curl -s $capath https://contributors.debian.org/contributors/post \
- -F source=ftp.debian.org \
- -F auth_token="$(cat ${base}/s3kr1t/contributor.auth)" \
- -F data=@${TMPCNTB} > ${TMPCNTB}.result
-cat ${TMPCNTB}.result
-rm -f ${TMPCNTB}.result
-
-
-${scriptsdir}/link_morgue.sh
-
-################################################################################
-
-log "Finally, all is done, compressing logfile"
-exec > /dev/null 2>&1
-
-bzip2 -9 "$LOGFILE"
+++ /dev/null
-#!/bin/bash
-# No way I try to deal with a crippled sh just for POSIX foo.
-
-# Copyright (C) 2009-2015 Joerg Jaspert <joerg@debian.org>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; version 2.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-# Homer: Are you saying you're never going to eat any animal again? What
-# about bacon?
-# Lisa: No.
-# Homer: Ham?
-# Lisa: No.
-# Homer: Pork chops?
-# Lisa: Dad, those all come from the same animal.
-# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
-
-# exit on errors
-set -e
-set -o pipefail
-# make sure to only use defined variables
-set -u
-# ERR traps should be inherited from functions too. (And command
-# substitutions and subshells and whatnot, but for us the functions is
-# the important part here)
-set -E
-
-# import the general variable set.
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-########################################################################
-# Functions #
-########################################################################
-# common functions are "outsourced"
-. "${configdir}/common"
-
-# source the dinstall functions
-. "${configdir}/dinstall.functions"
-
-########################################################################
-########################################################################
-
-# Function to save which stage we are in, so we can restart an interrupted
-# dinstall. Or even run actions in parallel, if we dare to, by simply
-# backgrounding the call to this function. But that should only really be
-# done for things we don't care much about.
-#
-# This should be called with the first argument being an array, with the
-# members
-# - FUNC - the function name to call
-# - ARGS - Possible arguments to hand to the function. Can be the empty string
-# - TIME - The timestamp name. Can be the empty string
-# - ERR - if this is the string false, then the call will be surrounded by
-# set +e ... set -e calls, so errors in the function do not exit
-# dinstall. Can be the empty string, meaning true.
-#
-# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
-# ADDED FOR DINSTALL FEATURES!
-function stage() {
- ARGS='GO[@]'
- local "${!ARGS}"
-
- local error=${ERR:-"true"}
-
- ARGS=${ARGS:-""}
-
- log "########## DINSTALL BEGIN: ${FUNC} ${ARGS} ##########"
- local STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
- STAGEFILE=${STAGEFILE// /_}
- if [ -f "${STAGEFILE}" ]; then
- local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
- local unixtime=$(date +%s)
- local difference=$(( $unixtime - $stamptime ))
- if [ ${difference} -ge 14400 ]; then
- log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
- else
- log "Did already run ${FUNC}, not calling again..."
- fi
- return
- fi
-
- debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"
-
- # Make sure we are always at the same place. If a function wants
- # to be elsewhere, it has to cd first!
- cd ${configdir}
-
- # Now redirect the output into $STAGEFILE.log. In case it errors
- # out somewhere our errorhandler trap can then mail the contents
- # of $STAGEFILE.log only, instead of a whole dinstall logfile.
- # Short error mails ftw!
- exec >> "${STAGEFILE}.log" 2>&1
-
- if [ -f "${LOCK_STOP}" ]; then
- log "${LOCK_STOP} exists, exiting immediately"
- exit 42
- fi
-
- # Do we care about trouble in the function we call?
- if [ "${error}" = "false" ]; then
- set +e
- fi
- ${FUNC} ${ARGS}
-
- # No matter what happened in the function, we make sure we have
- # set -e default state back
- set -e
-
- # Make sure we are always at the same place.
- cd ${configdir}
-
- # We always use the same umask. If a function wants to do
- # different, fine, but we reset.
- umask 022
-
- touch "${STAGEFILE}"
-
- if [ -n "${TIME}" ]; then
- ts "${TIME}"
- fi
-
- # And the output goes back to the normal logfile
- exec >> "$LOGFILE" 2>&1
-
- # Now we should make sure that we have a usable dinstall.log, so
- # append the $STAGEFILE.log to it.
- cat "${STAGEFILE}.log" >> "${LOGFILE}"
- rm -f "${STAGEFILE}.log"
-
- log "########## DINSTALL END: ${FUNC} ##########"
-
- if [ -f "${LOCK_STOP}" ]; then
- log "${LOCK_STOP} exists, exiting immediately"
- exit 42
- fi
-}
-
-########################################################################
-
-# We need logs.
-LOGFILE="$logdir/dinstall.log"
-
-exec >> "$LOGFILE" 2>&1
-
-# And now source our default config
-. "${configdir}/dinstall.variables"
-
-# Make sure we start out with a sane umask setting
-umask 022
-
-# And use one locale, no matter what the caller has set
-export LANG=C
-export LC_ALL=C
-
-touch "${DINSTALLSTART}"
-ts "startup"
-DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
-state "Startup"
-
-lockfile -l 3600 "${LOCK_DAILY}"
-trap onerror ERR
-trap remove_daily_lock EXIT TERM HUP INT QUIT
-
-touch "${LOCK_BRITNEY}"
-
-# This loop simply wants to be fed by a list of values (see below)
-# made out of 5 columns.
-# The first four are the array values for the stage function, the
-# fifth tells us if we should background the stage call.
-#
-# - FUNC - the function name to call
-# - ARGS - Possible arguments to hand to the function. Can be the empty string
-# - TIME - The timestamp name. Can be the empty string
-# - ERR - if this is the string false, then the call will be surrounded by
-# set +e ... set -e calls, so errors in the function do not exit
-# dinstall. Can be the empty string, meaning true.
-# - BG - Background the function stage?
-#
-# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \
-#
-# NOTE 1: There are two special values for the first column (FUNC).
-# STATE - do not call stage function, call the state
-# function to update the public statefile "where is dinstall"
-# NOSTAGE - do not call stage function, call the command directly.
-#
-# Note 2: If you want to hand an empty value to the stage function,
-# use the word "none" in the list below.
-while read FUNC ARGS TIME ERR BACKGROUND; do
- debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND"
-
- # Empty values in the value list are the string "none" (or the
- # while read loop won't work). Here we ensure that variables that
- # can be empty, are empty if the string none is set for them.
- for var in ARGS TIME; do
- if [[ ${!var} == none ]]; then
- typeset ${var}=''
- fi
- done
-
- # ERR/BACKGROUND are boolean, check that they are.
- for var in ERR BACKGROUND; do
- if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then
- error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}"
- fi
- done
-
- case ${FUNC} in
- STATE)
- state ${ARGS}
- ;;
- NOSTAGE)
- ${ARGS}
- ;;
- *)
- GO=(
- FUNC=${FUNC}
- TIME=${TIME}
- ARGS=${ARGS}
- ERR=${ERR}
- )
- if [[ ${BACKGROUND} == true ]]; then
- stage $GO &
- else
- stage $GO
- fi
- ;;
- esac
-done < <(cat - <<EOF
-savetimestamp none none false false
-qa1 none init true true
-pg_timestamp predinstall pg_dump1 false false
-updates none External\ Updates false false
-i18n1 none i18n\ 1 false false
-dep11 none dep11\ 1 false false
-NOSTAGE lockaccepted none false false
-punew stable-new p-u-new false false
-opunew oldstable-new o-p-u-new false false
-backports_policy none backports-policy false false
-cruft none cruft false false
-STATE indices none false false
-dominate none dominate false false
-autocruft none autocruft false false
-fingerprints none import-keyring false false
-overrides none overrides false false
-mpfm none pkg-file-mapping false false
-STATE packages/contents none false false
-packages none apt-ftparchive false false
-STATE dists/ none false false
-pdiff none pdiff false false
-release none release\ files false false
-dakcleanup none cleanup false false
-STATE scripts none false false
-mkmaintainers none mkmaintainers false false
-copyoverrides none copyoverrides false false
-mklslar none mklslar false false
-mkfilesindices none mkfilesindices false false
-mkchecksums none mkchecksums false false
-signotherfiles none signotherfiles false false
-mirror none mirror\ hardlinks false false
-NOSTAGE remove_locks none false false
-STATE postlock none false false
-ddaccess none ddaccessible\ sync true true
-changelogs none changelogs false true
-pg_timestamp postdinstall pg_dump2 false false
-expire none expire_dumps false true
-transitionsclean none transitionsclean false true
-dm none none false true
-bts none none false true
-mirrorpush none mirrorpush false true
-mirrorpush-backports none mirrorpush-backports false true
-i18n2 none i18n\ 2 false true
-stats none stats false true
-testingsourcelist none none false true
-NOSTAGE rm\ -f\ "\${LOCK_BRITNEY}" none false false
-cleantransactions none none false false
-EOF
- )
-# FUNC ARGS TIME ERR BG
-
-# we need to wait for the background processes before the end of dinstall
-wait
-
-log "Daily cron scripts successful, all done"
-
-exec > "$logdir/afterdinstall.log" 2>&1
-
-if [ -f "${dbdir}/dinstallstart" ]; then
- NOW=$(cat "${dbdir}/dinstallstart")
- mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
- logstats "$logdir/dinstall_${NOW}.log"
- bzip2 -9 "$logdir/dinstall_${NOW}.log"
-else
- error "Problem, I don't know when dinstall started, unable to do log statistics."
- NOW=`date "+%Y.%m.%d-%H:%M:%S"`
- mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
- bzip2 -9 "$logdir/dinstall_${NOW}.log"
-fi
-
-state "all done"
-
-# Now, at the very (successful) end of dinstall, make sure we remove
-# our stage files, so the next dinstall run will do it all again.
-rm -f ${stagedir}/*
-touch "${DINSTALLEND}"
+++ /dev/null
-#! /bin/bash
-#
-# Executed hourly via cron, out of dak's crontab.
-
-# Only one of me should ever run.
-[ "${FLOCKER}" != "$0" ] && exec env FLOCKER="$0" flock -E 0 -en "$0" "$0"
-"$@" || :
-
-set -e
-set -o pipefail
-set -u
-
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-PROGRAM="Hourly"
-########################################################################
-# Functions #
-########################################################################
-# common functions are "outsourced"
-. "${configdir}/common"
-
-dak import-users-from-passwd
-
-# do not run show-new and other stuff in parallel
-LOCKFILE="$lockdir/unchecked.lock"
-cleanup() {
- rm -f "${LOCKFILE}"
-}
-
-if lockfile -r16 $LOCKFILE 2> /dev/null; then
- trap cleanup EXIT
- do_new
- dak show-new > /dev/null || true
- cleanup
- trap - EXIT
-fi
-
-dak queue-report -n > $webdir/new.html
-dak queue-report -n -d backports-new,backports-policy > ${webdir}/backports-new.html
-dak queue-report -8 -d new,byhand,stable-new,oldstable-new,backports-new -r $webdir/stat
-sudo -u dak-unpriv dak show-deferred -r $webdir/stat > ${webdir}/deferred.html
-dak graph -n new,byhand,stable-new,oldstable-new,deferred -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates
-
-cd $webdir
-cat removals-20*.txt > removals-full.txt
-cat removals.txt >> removals-full.txt
-cat removals-20*.822 > removals-full.822
-cat removals.822 >> removals-full.822
-
-$base/dak/tools/queue_rss.py -q $queuedir/new -o $webdir/rss/ -d $base/misc -l $base/log/
-$base/dak/tools/removals.pl $configdir/removalsrss.rc > $webdir/rss/removals.rss
-
-# cd $masterdir
-# timeout -k 180 120 /usr/bin/epydoc -q --html --graph all --css blue -n DAK -o $webdir/epydoc --include-log --graph all --show-imports $masterdir/dak/ $masterdir/dakweb/ || true
-
-# Tell the public mirror to sync its tree
-${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
-
-$scriptsdir/generate-d-i
-
-# Update backports ACL
-sudo -u dak-unpriv cat /srv/backports-master.debian.org/etc/acl \
- | dak acl set-fingerprints backports \
- | mail -a "X-Debian: DAK" -e -s "backports: acl changes" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
-
-# do the buildd key updates
-BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX)
-exec >> "${BUILDDFUN}" 2>&1
-${scriptsdir}/buildd-remove-keys
-${scriptsdir}/buildd-add-keys
-${scriptsdir}/buildd-prepare-dir
-
-for keyring in $(dak admin k list-binary); do
- dak import-keyring --generate-users "%s" ${keyring}
-done
-exec >>/dev/null 2>&1
-
-DATE=$(date -Is)
-cat "${BUILDDFUN}" | mail -a "X-Debian: DAK" -e -s "[$(hostname -s)] Buildd key changes ${DATE}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" buildd-keys@ftp-master.debian.org
-
-rm -f "${BUILDDFUN}"
+++ /dev/null
-#! /bin/bash
-#
-# Run at the beginning of the month via cron, out of dak's crontab.
-
-set -e
-set -o pipefail
-set -u
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-################################################################################
-
-DATE=`date -d yesterday +%y%m`
-
-DATE=$(date +%Y-%m)
-OLDDATE=$(date -d yesterday +%Y-%m)
-cd ${base}/log
-touch $DATE
-ln -sf $DATE current
-chmod g+w $DATE
-chown dak:ftpteam $DATE
-xz -9 ${OLDDATE}
-
-cd /srv/upload.debian.org/queued
-sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued -k || true
-mv run/log log.${OLDDATE}
-sudo -u dak-unpriv touch run/log
-xz -9 log.${OLDDATE}
-sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued 2>/dev/null
-
-################################################################################
+++ /dev/null
-#!/bin/bash
-# No way I try to deal with a crippled sh just for POSIX foo.
-
-# Copyright (C) 2009 Joerg Jaspert <joerg@debian.org>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; version 2.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-# exit on errors
-set -e
-set -o pipefail
-# make sure to only use defined variables
-set -u
-# ERR traps should be inherited from functions too. (And command
-# substitutions and subshells and whatnot, but for us the functions is
-# the important part here)
-set -E
-
-# import the general variable set.
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-LOCKDAILY=""
-LOCKFILE="$lockdir/unchecked.lock"
-NOTICE="$lockdir/daily.lock"
-LOCK_BUILDD="$lockdir/buildd.lock"
-
-# our name
-PROGRAM="unchecked"
-
-if [ -e $NOTICE ]; then
- exit 0;
-fi
-
-########################################################################
-# Functions #
-########################################################################
-# common functions are "outsourced"
-. "${configdir}/common"
-
-STAMP=$(date "+%Y%m%d%H%M")
-
-cleanup() {
- rm -f "$LOCKFILE"
- if [ ! -z "$LOCKDAILY" ]; then
- rm -f "$NOTICE"
- fi
-}
-
-function do_buildd () {
- if lockfile -r3 $NOTICE; then
- LOCKDAILY="YES"
- make_buildd_dir
- wbtrigger
- fi
-}
-
-########################################################################
-# the actual unchecked functions follow #
-########################################################################
-
-# And use one locale, no matter what the caller has set
-export LANG=C
-export LC_ALL=C
-
-# only run one cron.unchecked
-if ! lockfile -r8 $LOCKFILE 2> /dev/null; then
- # echo "aborting cron.unchecked because $LOCKFILE has already been locked"
- exit 0
-fi
-trap cleanup 0
-
-
-pg_timestamp preunchecked >/dev/null
-
-# Process policy queues
-punew stable-new
-opunew oldstable-new
-backports_policy
-dak clean-suites -a backports-policy,policy
-
-# Finally deal with unchecked
-do_unchecked
-
-if [ ! -z "$changes" ]; then
- sync_debbugs
- do_buildd
-fi
-
-dak contents -l 10000 scan-binary
-dak contents -l 1000 scan-source
-pg_timestamp postunchecked >/dev/null
+++ /dev/null
-#!/bin/bash
-#
-# Run once a week via cron, out of dak's crontab.
-
-set -e
-set -o pipefail
-set -u
-# ERR traps should be inherited from functions too. (And command
-# substitutions and subshells and whatnot, but for us the functions is
-# the important part here)
-set -E
-export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
-. $SCRIPTVARS
-
-# Start logging
-NOW=`date "+%Y.%m.%d-%H:%M:%S"`
-LOGFILE="$logdir/weekly_${NOW}.log"
-exec > "$LOGFILE" 2>&1
-
-cleanup() {
- echo "Cleanup"
- rm -f "$LOGFILE"
-}
-trap cleanup 0
-
-################################################################################
-
-# Purge empty directories
-echo "Purging empty directories in $ftpdir/pool/"
-
-if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then
- find $ftpdir/pool/ -type d -empty | xargs rmdir;
-fi
-
-# Do git cleanup stuff
-echo "Doing git stuff"
-cd /srv/ftp.debian.org/git/dak.git
-git gc --prune
-git update-server-info
-# now workaround a git bug not honoring the setup in logs/*
-# (fix in development, but until it reached backports.org.......)
-chmod -R g+w logs/
-
-echo "Fixing symlinks in $ftpdir"
-symlinks -d -r $ftpdir
-
-echo "Finally, all is done, compressing logfile"
-exec > /dev/null 2>&1
-
-bzip2 -9 "$LOGFILE"
-
-
-################################################################################
--- /dev/null
+#!/bin/bash
+# No way I try to deal with a crippled sh just for POSIX foo.
+
+# Copyright (C) 2009-2015 Joerg Jaspert <joerg@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# Homer: Are you saying you're never going to eat any animal again? What
+# about bacon?
+# Lisa: No.
+# Homer: Ham?
+# Lisa: No.
+# Homer: Pork chops?
+# Lisa: Dad, those all come from the same animal.
+# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
+
+# exit on errors
+set -e
+# A pipeline's return status is the value of the last (rightmost)
+# command to exit with a non-zero status, or zero if all commands exit
+# successfully.
+set -o pipefail
+# make sure to only use defined variables
+set -u
+# ERR traps should be inherited from functions too. (And command
+# substitutions and subshells and whatnot, but for us the functions is
+# the important part here)
+set -E
+
+# And use one locale, no matter what the caller has set
+export LANG=C.UTF-8
+export LC_ALL=C.UTF-8
+
+# One arg please
+declare -lr ARG=${1:-"meh"}
+
+# set DEBUG if you want to see a little more logs (needs to be used more)
+DEBUG=${DEBUG:-0}
+
+# This doesn't catch calling us with an unknown argument, but it
+# catches missing args and saves a good bunch of processing time
+# (reading the scriptvars later is slow)
+if [[ ${ARG} == meh ]]; then
+ cat - <<EOF
+This is the FTPMaster cronscript. It needs an argument or it won't do
+anything for you.
+
+Currently accepted Arguments:
+
+ unchecked - Process the unchecked queue
+ dinstall - Run a dinstall
+ hourly, daily, weekly - Run that part
+
+EOF
+ exit 0
+fi
+
+# import the general variable set.
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
+. $SCRIPTVARS
+
+# common functions are "outsourced"
+. "${configdir}/common"
+
+# program name is the (lower cased) first argument.
+PROGRAM="${ARG}"
+
+# Timestamp when we started
+NOW=$(date "+%Y.%m.%d-%H:%M:%S")
+
+# Which list of tasks should we run?
+declare -r TASKLIST="${configdir}/${PROGRAM}.tasks"
+
+# A logfile for every cron script
+LOGFILE="${logdir}/${PROGRAM}_${NOW}.log"
+
+# Each "cronscript" may have a variables and a functions file
+# that we source
+for what in variables functions; do
+ if [[ -f ${configdir}/${PROGRAM}.${what} ]]; then
+ . ${configdir}/${PROGRAM}.${what}
+ fi
+done
+
+# Get rid of tempfiles at the end
+trap cleanup EXIT TERM HUP INT QUIT
+
+case ${ARG} in
+ unchecked)
+ # Do not run during dinstall
+ if [[ -e ${LOCK_DAILY} ]]; then
+ exit 0;
+ fi
+ # only run one cron.unchecked and also lock against hourly (newoverview)
+ if ! lockfile -r8 ${LOCK_UNCHECKED} 2> /dev/null; then
+ # log "aborting cron.unchecked because $LOCK_UNCHECKED has already been locked"
+ exit 0
+ fi
+ TMPFILES="${TMPFILES} ${LOCK_UNCHECKED}"
+ ;;
+ dinstall)
+ ;;
+ hourly)
+ # Only one of me should ever run.
+ FLOCKER=${FLOCKER:-""}
+ [ "${FLOCKER}" != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0" "$@" || :
+ ;;
+ daily)
+ ;;
+ weekly)
+ ;;
+ monthly)
+ ;;
+ *)
+ error "Unknown arg ${ARG}"
+ exit 42
+ ;;
+esac
+
+# An easy access by name for the current log
+ln -sf ${LOGFILE} ${logdir}/${PROGRAM}
+
+# And from here, all output to the log please
+exec >> "$LOGFILE" 2>&1
+
+# The stage function uses this directory
+# This amends the stagedir variable from "vars"
+stagedir="${stagedir}/${PROGRAM}"
+# Ensure the dir exists
+mkdir -p ${stagedir}
+
+# This loop simply wants to be fed by a list of values (see below)
+# made out of 5 columns.
+# The first four are the array values for the stage function, the
+# fifth tells us if we should background the stage call.
+#
+# - FUNC - the function name to call
+# - ARGS - Possible arguments to hand to the function. Can be the empty string
+# - TIME - The timestamp name. Can be the empty string
+# - ERR - if this is the string false, then the call will be surrounded by
+# set +e ... set -e calls, so errors in the function do not exit
+# dinstall. Can be the empty string, meaning true.
+# - BG - Background the function stage?
+#
+# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \
+#
+# NOTE 1: There are two special values for the first column (FUNC).
+# STATE - do not call stage function, call the state
+# function to update the public statefile "where is dinstall"
+# NOSTAGE - do not call stage function, call the command directly.
+#
+# Note 2: If you want to hand an empty value to the stage function,
+# use the word "none" in the list below.
+while read FUNC ARGS TIME ERR BACKGROUND; do
+ debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND"
+
+ # Empty values in the value list are the string "none" (or the
+ # while read loop won't work). Here we ensure that variables that
+ # can be empty, are empty if the string none is set for them.
+ for var in ARGS TIME; do
+ if [[ ${!var} == none ]]; then
+ typeset ${var}=''
+ fi
+ done
+
+ # ERR/BACKGROUND are boolean, check that they are.
+ for var in ERR BACKGROUND; do
+ if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then
+ error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}"
+ fi
+ done
+
+ case ${FUNC} in
+ STATE)
+ state ${ARGS}
+ ;;
+ NOSTAGE)
+ ${ARGS}
+ ;;
+ *)
+ GO=(
+ FUNC=${FUNC}
+ TIME=${TIME}
+ ARGS=${ARGS}
+ ERR=${ERR}
+ )
+ if [[ ${BACKGROUND} == true ]]; then
+ stage $GO &
+ else
+ stage $GO
+ fi
+ ;;
+ esac
+done < <(grep -v '^#' ${TASKLIST} )
+
+# we need to wait for the background processes before the end of the cron script
+wait
+
+
+# Common to all cron scripts
+log "Cron script successful, all done"
+# Redirect output to another file, as we want to compress our logfile
+# and ensure its no longer used
+exec > "$logdir/after${PROGRAM}.log" 2>&1
+
+case ${ARG} in
+ unchecked)
+ ;;
+ dinstall)
+ logstats ${LOGFILE}
+ state "all done"
+ touch "${DINSTALLEND}"
+ ;;
+ hourly)
+ ;;
+ daily)
+ ;;
+ weekly)
+ ;;
+ monthly)
+ ;;
+esac
+
+# Now, at the very (successful) end of this run, make sure we remove
+# our stage files, so the next dinstall run will do it all again.
+rm -f ${stagedir}/*
+bzip2 -9 ${LOGFILE}
+
+# Logfile should be gone, remove the symlink
+[[ -L ${logdir}/${PROGRAM} ]] && [[ ! -f ${logdir}/${PROGRAM} ]] && rm -f ${logdir}/${PROGRAM} || log "Logfile still exists or symlink gone already? Something fishy going on"
+
+# FIXME: Mail the log when its non-empty
+[[ -s "${logdir}/after${PROGRAM}.log" ]] || rm "${logdir}/after${PROGRAM}.log"
MAILTO=cron@ftp-master.debian.org
PATH=/usr/local/bin:/usr/bin:/bin
# m h dom mon dow command
-2,17,32,47 * * * * /srv/ftp-master.debian.org/dak/config/debian/cron.unchecked
-0 * * * * /srv/ftp-master.debian.org/dak/config/debian/cron.hourly
-52 1,7,13,19 * * * /srv/ftp-master.debian.org/dak/config/debian/cron.dinstall
-#### And an alternate line, for the times the release team wants it half an hour late.
-####22 2,8,14,20 * * * /srv/ftp-master.debian.org/dak/config/debian/cron.dinstall
-3 9 * * * /srv/ftp-master.debian.org/dak/config/debian/cron.daily
-0 12 * * 0 /srv/ftp-master.debian.org/dak/config/debian/cron.weekly
-6 0 1 * * /srv/ftp-master.debian.org/dak/config/debian/cron.monthly
-@reboot /srv/ftp-master.debian.org/dak/config/debian/cron.reboot
+2,17,32,47 * * * * flock -s -E 0 -n /var/run/reboot-lock /srv/ftp-master.debian.org/dak/config/debian/cronscript unchecked
+0 * * * * flock -s -E 0 -n /var/run/reboot-lock /srv/ftp-master.debian.org/dak/config/debian/cronscript hourly
+52 1,7,13,19 * * * flock -s -E 0 -n /var/run/reboot-lock /srv/ftp-master.debian.org/dak/config/debian/cronscript dinstall
+##### And an alternate line, for the times the release team wants it half an hour late.
+#####22 2,9,14,20 * * * flock -s -E 0 -n /var/run/reboot-lock /srv/ftp-master.debian.org/dak/config/debian/cronscript dinstall
+# And for release time, turn off cron after final dinstall
+#45 9 * * * /home/dak/bin/cronoff
+3 9 * * * flock -s -E 0 -n /var/run/reboot-lock /srv/ftp-master.debian.org/dak/config/debian/cronscript daily
+0 12 * * 0 flock -s -E 0 -n /var/run/reboot-lock /srv/ftp-master.debian.org/dak/config/debian/cronscript weekly
+6 0 1 * * flock -s -E 0 -n /var/run/reboot-lock /srv/ftp-master.debian.org/dak/config/debian/cronscript monthly
+
+# Reboot cron doesn't use cronscript
+@reboot flock -s -E 0 -n /var/run/reboot-lock /srv/ftp-master.debian.org/dak/config/debian/cron.reboot
0 0 * * * crontab -l > ~/crontab.out
# We do like our queued and make sure it will always be there.
# Luckily it checks itself and doesn't start if it already runs
-@reboot sudo -u dak-unpriv /srv/queued/debianqueued-0.9/debianqueued
-*/10 * * * * sudo -u dak-unpriv /srv/queued/debianqueued-0.9/debianqueued 2>/dev/null
+@reboot sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued
+*/10 * * * * sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued 2>/dev/null
--- /dev/null
+# -*- mode:sh -*-
+
+# get the latest list of wnpp bugs and their source packages
+function wnppbugs() {
+ log "Fetching latest list of wnpp bugs"
+ TMPLIST=$( mktemp -p ${TMPDIR} )
+ TMPFILES="${TMPFILES} ${TMPLIST}"
+ wget -q -O${TMPLIST} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
+ chmod go+r ${TMPLIST}
+ mv ${TMPLIST} ${scriptdir}/masterfiles/wnpp_rm
+}
+
+# Push files over to security
+function pushfilesdb() {
+ log "Pushing files table to security"
+ # The key over there should have the following set for the ssh key:
+ # command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity"
+ psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \
+ xz -3 | \
+ ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \
+ -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync
+}
+
+# Update wanna-build dump
+function wbdump() {
+ log "Update wanna-build database dump"
+ ${masterdir}/scripts/nfu/get-w-b-db
+}
+
+
+# Generate list of override disparities
+function overridedisp() {
+ log "Generating list of override disparities"
+ dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz
+}
+
+# Generate stats about the new queue
+function newstats() {
+ log "Generating stats about the new queue"
+ dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null
+}
+
+# Generate the contributor data
+function contributor() {
+ log "Submitting data to contributors"
+ TMPCNTB=$( mktemp -p ${TMPDIR} )
+ TMPFILES="${TMPFILES} ${TMPCNTB}"
+ REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
+
+ # Post with curl as a workaround for #801506
+ # See https://wiki.debian.org/ServicesSSL#curl
+ dir=/etc/ssl/ca-debian
+ test -d ${dir} && capath="--capath ${dir}"
+ curl -s ${capath} https://contributors.debian.org/contributors/post \
+ -F source=ftp.debian.org \
+ -F auth_token="$(cat ${base}/s3kr1t/contributor.auth)" \
+ -F data=@${TMPCNTB} > ${TMPCNTB}.result
+ log "Results:"
+ cat ${TMPCNTB}.result
+ log "----"
+ rm -f ${TMPCNTB}.result
+}
--- /dev/null
+# FUNC ARGS TIME ERR BG
+wnppbugs none none false false
+pushfilesdb none none false false
+wbdump none none false false
+reports none none false false
+clean_debbugs none none false false
+overridedisp none none false false
+newstats none none false false
+contributor none none false false
# -*- mode:sh -*-
-# Timestamp. Used for dinstall stat graphs
-function ts() {
- echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
-}
# Remove daily lock
+# FIXME: Dead code when switched to cronscript
function remove_daily_lock() {
rm -f $LOCK_DAILY
}
}
########################################################################
-# the actual dinstall functions follow #
+# the actual functions follow #
########################################################################
# pushing merkels QA user, part one
log "Generating package / file mapping"
for archive in "${public_archives[@]}"; do
+ log " archive: ${archive}"
archiveroot="$(get_archiveroot "${archive}")"
dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2"
done
}
function packages() {
- log "Generating Packages and Sources files"
for archive in "${public_archives[@]}"; do
log " Generating Packages/Sources for ${archive}"
dak generate-packages-sources2 -a "${archive}"
function release() {
log "Generating Release files"
for archive in "${public_archives[@]}"; do
+ log " archive: ${archive}"
dak generate-releases -a "${archive}"
done
}
ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
for a in $ARCHES; do
(sed -n "s/|$a$//p" $ARCHLIST
- sed -n 's/|all$//p' $ARCHLIST
+ sed -n 's/|all$//p' $ARCHLIST
- cd $base/ftp
- find ./dists -maxdepth 1 \! -type d
- find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
+ cd $base/ftp
+ find ./dists -maxdepth 1 \! -type d
+ find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
) | sort -u | gzip -9 > arch-$a.list.gz
done
}
psql -F' ' -At -c "SELECT id, suite_name FROM suite" |
- while read id suite; do
- [ -e $base/ftp/dists/$suite ] || continue
- (
- (cd $base/ftp
- distname=$(cd dists; readlink $suite || echo $suite)
- find ./dists/$distname \! -type d
- for distdir in ./dists/*; do
- [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
- done
- )
- suite_list $id
- ) | sort -u | gzip -9 > suite-${suite}.list.gz
- done
+ while read id suite; do
+ [ -e $base/ftp/dists/$suite ] || continue
+ (
+ (cd $base/ftp
+ distname=$(cd dists; readlink $suite || echo $suite)
+ find ./dists/$distname \! -type d
+ for distdir in ./dists/*; do
+ [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
+ done
+ )
+ suite_list $id
+ ) | sort -u | gzip -9 > suite-${suite}.list.gz
+ done
log "Finding everything on the ftp site to generate sundries"
(cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
rm -f sundries.list
zcat *.list.gz | cat - *.list | sort -u |
- diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
+ diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
log "Generating files list"
for a in $ARCHES; do
(echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
- cat - sundries.list dists.list project.list docs.list indices.list |
- sort -u | poolfirst > ../arch-$a.files
+ cat - sundries.list dists.list project.list docs.list indices.list |
+ sort -u | poolfirst > ../arch-$a.files
done
(cd $base/ftp/
- for dist in sid jessie stretch; do
- find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
- done
+ for dist in sid jessie stretch; do
+ find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
+ done
)
(cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz ; zcat translation-stretch.list.gz) |
- sort -u | poolfirst > ../typical.files
+ sort -u | poolfirst > ../typical.files
rm -f $ARCHLIST
log "Done!"
sudo -u dak-unpriv dak bts-categorize
}
-function ddaccess() {
- # Tell our dd accessible mirror to sync itself up. Including ftp dir.
- log "Trigger dd accessible parts sync including ftp dir"
- ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
-}
-
function mirrorpush() {
log "Checking the public archive copies..."
# save timestamp when we start
function savetimestamp() {
- NOW=`date "+%Y.%m.%d-%H:%M:%S"`
echo ${NOW} > "${dbdir}/dinstallstart"
}
fi
}
+# Generate a list of extra mirror files, sha256sum em and sign that
function signotherfiles() {
log "Signing extra mirror files"
rm -f ${TMPLO}
done
}
+
+function startup() {
+ touch "${DINSTALLSTART}"
+ ts "startup"
+ lockfile -l 3600 "${LOCK_DAILY}"
+ trap onerror ERR
+ touch "${LOCK_BRITNEY}"
+ TMPFILES="${TMPFILES} ${LOCK_DAILY} ${LOCK_BRITNEY}"
+}
--- /dev/null
+# FUNC ARGS TIME ERR BG
+STATE startup none false false
+NOSTAGE startup none false false
+savetimestamp none none false false
+qa1 none init true true
+pg_timestamp predinstall pg_dump1 false false
+updates none External\ Updates false false
+i18n1 none i18n\ 1 false false
+dep11 none dep11\ 1 false false
+NOSTAGE lockaccepted none false false
+punew stable-new p-u-new false false
+opunew oldstable-new o-p-u-new false false
+backports_policy none backports-policy false false
+cruft none cruft false false
+STATE indices none false false
+dominate none dominate false false
+autocruft none autocruft false false
+fingerprints none import-keyring false false
+overrides none overrides false false
+mpfm none pkg-file-mapping false false
+STATE packages/contents none false false
+packages none apt-ftparchive false false
+STATE dists/ none false false
+pdiff none pdiff false false
+release none release\ files false false
+dakcleanup none cleanup false false
+STATE scripts none false false
+mkmaintainers none mkmaintainers false false
+copyoverrides none copyoverrides false false
+mklslar none mklslar false false
+mkfilesindices none mkfilesindices false false
+mkchecksums none mkchecksums false false
+signotherfiles none signotherfiles false false
+mirror none mirror\ hardlinks false false
+NOSTAGE remove_locks none false false
+STATE postlock none false false
+ddaccess none ddaccessible\ sync true true
+changelogs none changelogs false true
+pg_timestamp postdinstall pg_dump2 false false
+expire none expire_dumps false true
+transitionsclean none transitionsclean false true
+dm none none false true
+bts none none false true
+mirrorpush none mirrorpush false true
+mirrorpush-backports none mirrorpush-backports false true
+i18n2 none i18n\ 2 false true
+stats none stats false true
+testingsourcelist none none false true
+cleantransactions none none false false
# usually we are not using debug logs. Set to 1 if you want them.
DEBUG=0
-# our name
-PROGRAM="dinstall"
-
# where do we want mails to go? For example log entries made with error()
if [ "x$(hostname -s)x" != "xfranckx" ]; then
# Not our ftpmaster host
# Marker for dinstall end
DINSTALLEND="${lockdir}/dinstallend"
-# lock cron.unchecked (it immediately exits when this exists)
-LOCK_DAILY="$lockdir/daily.lock"
+# Timestamp for start
+DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
# Lock cron.unchecked from doing work
LOCK_ACCEPTED="$lockdir/unchecked.lock"
--- /dev/null
+# -*- mode:sh -*-
+
+function importusers() {
+ dak import-users-from-passwd
+}
+
+function newoverview() {
+ # do not run show-new and other stuff in parallel
+ if lockfile -r16 $LOCK_UNCHECKED 2> /dev/null; then
+ log "Creating new overview"
+ TMPFILES="${TMPFILES} ${LOCK_UNCHECKED}"
+ do_new
+ dak show-new > /dev/null || true
+ cleantempfiles
+ fi
+}
+
+function queuereport() {
+ log "Creating queue reports, new/backports-new, 822 format"
+ dak queue-report -n > ${webdir}/new.html
+ dak queue-report -n -d backports-new,backports-policy > ${webdir}/backports-new.html
+ dak queue-report -8 -d new,byhand,stable-new,oldstable-new,backports-new -r $webdir/stat
+ log "Deferred queue overview"
+ sudo -u dak-unpriv dak show-deferred -r ${webdir}/stat > ${webdir}/deferred.html
+ log "Graphs about the queues"
+ dak graph -n new,byhand,stable-new,oldstable-new,deferred,backports-new -r ${webdir}/stat -i ${webdir}/stat -x $scriptsdir/rrd-release-freeze-dates
+}
+
+function removalstxt() {
+ cd ${webdir}
+ cat removals-20*.txt > removals-full.txt
+ cat removals.txt >> removals-full.txt
+ cat removals-20*.822 > removals-full.822
+ cat removals.822 >> removals-full.822
+}
+
+function rss() {
+ log "NEW RSS feed"
+ $base/dak/tools/queue_rss.py -q $queuedir/new -o ${webdir}/rss/ -d $base/misc -l $base/log/
+ log "Removals RSS feed"
+ $base/dak/tools/removals.pl $configdir/removalsrss.rc > ${webdir}/rss/removals.rss
+}
+
+function gen_di() {
+ $scriptsdir/generate-d-i
+}
+
+function backportsacl() {
+ # Update backports ACL
+ sudo -u dak-unpriv cat /srv/backports-master.debian.org/etc/acl \
+ | dak acl set-fingerprints backports \
+ | mail -a "X-Debian: DAK" -e -s "backports: acl changes" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
+}
+
+# do the buildd key updates
+function builddautosigning() {
+ BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX)
+ TMPFILES="${TMPFILES} ${BUILDDFUN}"
+ exec >> "${BUILDDFUN}" 2>&1
+ ${scriptsdir}/buildd-remove-keys
+ ${scriptsdir}/buildd-add-keys
+ ${scriptsdir}/buildd-prepare-dir
+
+ for keyring in $(dak admin k list-binary); do
+ dak import-keyring --generate-users "%s" ${keyring}
+ done
+ exec >>/dev/null 2>&1
+
+ DATE=$(date -Is)
+ cat "${BUILDDFUN}" | mail -a "X-Debian: DAK" -e -s "[$(hostname -s)] Buildd key changes ${DATE}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" buildd-keys@ftp-master.debian.org
+
+ rm -f "${BUILDDFUN}"
+}
--- /dev/null
+# FUNC ARGS TIME ERR BG
+importusers none none false false
+newoverview none none false false
+queuereport none none false true
+removalstxt none none false true
+rss none none false true
+ddaccess none none true true
+gen_di none none false false
+backportsacl none none false false
+builddautosigning none none false false
--- /dev/null
+# -*- mode:sh -*-
+
+function rotatelog() {
+ cd ${base}/log
+ touch ${DATE}
+ ln -sf ${DATE} current
+ chmod g+w ${DATE}
+ chown dak:ftpteam ${DATE}
+ xz -9 ${OLDDATE}
+}
+
+function rotatequeued() {
+ cd /srv/upload.debian.org/queued
+ sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued -k || true
+ mv run/log log.${OLDDATE}
+ sudo -u dak-unpriv touch run/log
+ xz -9 log.${OLDDATE}
+ sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued 2>/dev/null
+}
\ No newline at end of file
--- /dev/null
+# FUNC ARGS TIME ERR BG
+rotatelog none none false true
+rotatequeued none none false true
\ No newline at end of file
--- /dev/null
+# -*- mode:sh -*-
+DATE=$(date +%Y-%m)
+OLDDATE=$(date -d yesterday +%Y-%m)
--- /dev/null
+# FUNC ARGS TIME ERR BG
+pg_timestamp preunchecked none false false
+punew stable-new p-u-new false false
+opunew oldstable-new o-p-u-new false false
+backports_policy none backports-policy false false
+cleanpolicy none cleanpolicy false false
+do_unchecked none unchecked false false
+scancontents none contentsscan false false
+pg_timestamp postunchecked none false false
--- /dev/null
+# -*- mode:sh -*-
+LOCKFILE="$lockdir/unchecked.lock"
+LOCK_BUILDD="$lockdir/buildd.lock"
# -*- mode:sh -*-
-# locations used by many scripts
+umask 022
+unset CDPATH
+GZIP='--rsyncable' ; export GZIP
+
+# locations used by many scripts
base=/srv/ftp-master.debian.org
-public=/srv/ftp.debian.org
-bindir=$base/bin
-ftpdir=$base/ftp
-webdir=$public/web
-indices=$ftpdir/indices
-archs=$(dak admin a list | tr '\n' ' ')
+bindir=${base}/bin
+masterdir=${base}/dak/
+queuedir=${base}/queue/
+scriptdir=${base}/scripts
+
+configdir=${masterdir}/config/debian/
+dbdir=${base}/database/
+exportdir=${base}/export/
+extoverridedir=${scriptdir}/external-overrides
+ftpdir=${base}/ftp
+incoming=${base}/public/incoming.debian.org/
+indices=${ftpdir}/indices
+lockdir=${base}/lock/
+logdir=${base}/log/cron/
+mirrordir=${base}/mirror/
+
+accepted=${queuedir}/accepted/
+newstage=${queuedir}/newstage/
+overridedir=${scriptdir}/override
+scriptsdir=${masterdir}/scripts/debian/
+stagedir=${lockdir}/stages
+unchecked=${queuedir}/unchecked/
-scriptdir=$base/scripts
-masterdir=$base/dak/
-configdir=$base/dak/config/debian/
-scriptsdir=$base/dak/scripts/debian/
-dbdir=$base/database/
-lockdir=$base/lock/
-stagedir=$lockdir/stages
-overridedir=$scriptdir/override
-extoverridedir=$scriptdir/external-overrides
-logdir=$base/log/cron/
-
-queuedir=$base/queue/
-unchecked=$queuedir/unchecked/
-accepted=$queuedir/accepted/
-mirrordir=$base/mirror/
-incoming=$base/public/incoming.debian.org/
-newstage=$queuedir/newstage/
-exportdir=$base/export/
-exportpublic=$public/rsync/export/
+public=/srv/ftp.debian.org
+webdir=${public}/web
+exportpublic=${public}/rsync/export/
ftpgroup=debadmin
-
public_archives=(ftp-master backports debian-debug)
TMPDIR=${base}/tmp
+PATH=${masterdir}:${PATH}
-PATH=$masterdir:$PATH
-umask 022
-unset CDPATH
-GZIP='--rsyncable' ; export GZIP
+# Various lockfiles used by multiple cron scripts
+LOCK_DAILY="${lockdir}/daily.lock"
+LOCK_UNCHECKED="${lockdir}/unchecked.lock"
+LOCK_STOP="${lockdir}/archive.stop"
+
+archs=$(dak admin a list | tr '\n' ' ')
# Set the database variables
eval $(dak admin config db-shell)
--- /dev/null
+# -*- mode:sh -*-
+
+function purgeempty() {
+ log "Purging empty directories"
+ local archiveroot
+
+ for archive in "${public_archives[@]}"; do
+ log "... archive: ${archive}"
+ archiveroot="$(get_archiveroot "${archive}")"
+
+ # FIXME: double find
+ if [ ! -z "$(find ${archiveroot}/pool/ -type d -empty)" ]; then
+ find ${archiveroot}/pool/ -type d -empty | xargs rmdir;
+ fi
+ done
+}
+
+function gitcleanup() {
+ log "Doing git stuff"
+ cd ${public}/git/dak.git
+ git gc --prune
+ git update-server-info
+ # now workaround a git bug not honoring the setup in logs/*
+ # (fix in development, but until it reached backports.org.......)
+ chmod -R g+w logs/
+}
+
+function fixsymlinks() {
+ log "Fixing symlinks"
+ local archiveroot
+
+ for archive in "${public_archives[@]}"; do
+ log "... archive: ${archive}"
+ archiveroot="$(get_archiveroot "${archive}")"
+ symlinks -d -r ${archiveroot}
+ done
+}
+
+function linkmorgue() {
+ ${scriptsdir}/link_morgue.sh
+}
--- /dev/null
+# FUNC ARGS TIME ERR BG
+purgeempty none none false true
+gitcleanup none none false true
+fixsymlinks none none false true
+linkmorgue none none false true
log "Processing ${PROCESSDIR}"
find ${PROCESSDIR} -type f |
while read mfile; do
+ if [[ -f ${mfile}.nosnapshot ]]; then
+ # We know this file does not exist on snapshot, don't check again
+ continue
+ fi
+
# Get the files sha1sum
mshasum=$(sha1sum ${mfile})
mshasum=${mshasum%% *}
# Yay for tons of dangling symlinks, but when this is done a rsync
# will run and transfer the whole shitload of links over to the morgue host.
ln -sf "${FARMBASE}/${LVL1}/${LVL2}/${mshasum}" "${mfile}"
+ else
+ touch "${mfile}.nosnapshot"
fi
fi
done # for mfile in...
projects / dak.git / commitdiff