From: Mark Hymers Date: Sat, 4 May 2013 10:10:40 +0000 (+0000) Subject: Remove unused backports configuration X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=6c171cdabb9a83e73b69dc16aeb256a88d465ed7;p=dak.git Remove unused backports configuration Signed-off-by: Mark Hymers --- diff --git a/config/backports/Contents.top b/config/backports/Contents.top deleted file mode 100644 index e03f7a64..00000000 --- a/config/backports/Contents.top +++ /dev/null @@ -1,32 +0,0 @@ -This file maps each file available in the backports.debian.org archive -system to the package from which it originates. It includes packages -from the DIST distribution for the ARCH architecture. - -You can use this list to determine which package contains a specific -file, or whether or not a specific file is available. The list is -updated weekly, each architecture on a different day. - -When a file is contained in more than one package, all packages are -listed. When a directory is contained in more than one package, only -the first is listed. - -The best way to search quickly for a file is with the Unix `grep' -utility, as in `grep CONTENTS': - - $ grep nose Contents - etc/nosendfile net/sendfile - usr/X11R6/bin/noseguy x11/xscreensaver - usr/X11R6/man/man1/noseguy.1x.gz x11/xscreensaver - usr/doc/examples/ucbmpeg/mpeg_encode/nosearch.param graphics/ucbmpeg - usr/lib/cfengine/bin/noseyparker admin/cfengine - -This list contains files in all packages, even though not all of the -packages are installed on an actual system at once. If you want to -find out which packages on an installed Debian system provide a -particular file, you can use `dpkg --search ': - - $ dpkg --search /usr/bin/dselect - dpkg: /usr/bin/dselect - - -FILE LOCATION diff --git a/config/backports/apt.conf b/config/backports/apt.conf deleted file mode 100644 index e9a1e9fd..00000000 --- a/config/backports/apt.conf +++ /dev/null @@ -1,44 +0,0 @@ -Dir -{ - ArchiveDir "/srv/backports-master.debian.org/ftp/"; - OverrideDir "/srv/backports-master.debian.org/scripts/override/"; - CacheDir "/srv/backports-master.debian.org/database/"; -}; - -Default -{ - Packages::Compress "gzip bzip2"; - Sources::Compress "gzip bzip2"; - DeLinkLimit 0; - FileMode 0664; - Contents::Compress "gzip"; - MaxContentsChange 12000; -}; - -TreeDefault -{ - Contents::Header "/srv/backports-master.debian.org/dak/config/backports/Contents.top"; -}; - -tree "dists/squeeze-backports" -{ - FileList "/srv/backports-master.debian.org/database/dists/squeeze-backports_$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/srv/backports-master.debian.org/database/dists/squeeze-backports_$(SECTION)_source.list"; - Sections "main contrib non-free"; - Architectures "amd64 armel i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 sparc source"; - BinOverride "override.squeeze-backports.$(SECTION)"; - ExtraOverride "override.squeeze-backports.extra.$(SECTION)"; - SrcOverride "override.squeeze-backports.$(SECTION).src"; -}; - -tree "dists/squeeze-backports/main" -{ - FileList "/srv/backports-master.debian.org/database/dists/squeeze-backports_main_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "amd64 armel i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 sparc source"; - BinOverride "override.squeeze-backports.main.$(SECTION)"; - SrcOverride "override.squeeze-backports.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - Contents "$(DIST)/../Contents-udeb"; -}; diff --git a/config/backports/common b/config/backports/common deleted file mode 100644 index dd9876a7..00000000 --- a/config/backports/common +++ /dev/null @@ -1,109 +0,0 @@ -# -*- mode:sh -*- -# log something (basically echo it together with a timestamp) -# -# Set $PROGRAM to a string to have it added to the output. -function log () { - if [ -z "${PROGRAM}" ]; then - echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@" - else - echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@" - fi -} - -# log the message using log() but then also send a mail -# to the address configured in MAILTO (if non-empty) -function log_error () { - log "$@" - if [ -z "${MAILTO}" ]; then - echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO} - fi -} - -# debug log, only output when DEBUG=1 -function debug () { - if [ $DEBUG -eq 1 ]; then - log "$*" - fi -} - -function wbtrigger() { - SSHOPT="-o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240" - if lockfile -r 3 -l 3600 "${LOCK_BUILDD}"; then - ssh -q -q ${SSHOPT} wbadm@buildd /org/wanna-build/trigger.often - fi - rm -f "${LOCK_BUILDD}" -} - -# used by cron.dinstall *and* cron.unchecked. -function make_buildd_dir () { - dak manage-build-queues -a - - for dist in $(ls -1 ${incoming}/dists/); do - cd ${incoming}/dists/${dist} - mkdir -p tree/${STAMP} - cp -al ${incoming}/dists/${dist}/buildd/. tree/${STAMP}/ - ln -sfT tree/${STAMP} ${incoming}/dists/${dist}/current - find ./tree -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf - done - -} - -# Do the unchecked processing, in case we have files. -function do_unchecked () { - cd $unchecked - - changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs) - report=$queuedir/REPORT - timestamp=$(date "+%Y-%m-%d %H:%M") - UNCHECKED_WITHOUT_LOCK=${UNCHECKED_WITHOUT_LOCK:-""} - - echo "$timestamp": ${changes:-"Nothing to do"} >> $report - dak process-upload -a ${UNCHECKED_WITHOUT_LOCK} -d "$unchecked" >> $report -} - -# Do the newstage processing, in case we have files. -function do_newstage () { - cd $newstage - - changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs) - report=$queuedir/REPORT - timestamp=$(date "+%Y-%m-%d %H:%M") - UNCHECKED_WITHOUT_LOCK=${UNCHECKED_WITHOUT_LOCK:-""} - - echo "$timestamp": ${changes:-"Nothing to do in newstage"} >> $report - dak process-upload -a ${UNCHECKED_WITHOUT_LOCK} -d "$newstage" >> $report -} - -function sync_debbugs () { - # sync with debbugs - echo "--" >> $report - timestamp=$(date "+%Y-%m-%d-%H:%M") - mkdir -p $queuedir/bts_version_track_archive/${timestamp} - rsync -aq --remove-source-files $queuedir/bts_version_track/ $queuedir/bts_version_track_archive/${timestamp} - rmdir --ignore-fail-on-non-empty $queuedir/bts_version_track_archive/${timestamp} # remove if empty. - return 0 - rsync -aq -e "ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30" --remove-source-files $queuedir/bts_version_track/ bugs-sync:/org/bugs.debian.org/versions/queue/ftp-master/ 2>/dev/null && touch $lockdir/synced_bts_version || true - NOW=$(date +%s) - TSTAMP=$(stat -c %Y $lockdir/synced_bts_version) - DIFF=$(( NOW - TSTAMP )) - if [ $DIFF -ge 259200 ]; then - log "Kids, you tried your best and you failed miserably. The lesson is, never try. (Homer Simpson)" - fi -} - -function reports() { - # Send a report on NEW/BYHAND packages - log "Nagging ftpteam about NEW/BYHAND packages" - dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" team@backports.debian.org - # and one on crufty packages - log "Sending information about crufty packages" - dak cruft-report -R > $webdir/cruft-report-daily.txt -# dak cruft-report -R -s experimental >> $webdir/cruft-report-daily.txt - cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" team@backports.debian.org -} - -function pg_timestamp() { - tsname=${1:-"unknown"} - log "Saving postgres transaction id for ${tsname}" - psql -tAc 'select txid_current();' > $base/backup/txid_${tsname}_$(date +%Y.%m.%d-%H:%M:%S) -} diff --git a/config/backports/cron.daily b/config/backports/cron.daily deleted file mode 100755 index d5775243..00000000 --- a/config/backports/cron.daily +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/bash -# - -set -e -set -o pipefail -set -u - -export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars -. $SCRIPTVARS - -################################################################################ -# Clean out old packages -dak clean-suites -m 10000 -dak clean-queues - -# Send a report on NEW/BYHAND packages -dak queue-report -d new,proposedupdates | mail -e -s "NEW and BYHAND on $(date +%D)" team@backports.debian.org -# and one on crufty packages - -dak cruft-report -R -m bdo -s squeeze-backports > $webdir/cruft-report-daily.txt -cat $webdir/cruft-report-daily.txt | mail -e -s "Debian backports archive cruft report for $(date +%D)" team@backports.debian.org - -echo Daily cron scripts successful. diff --git a/config/backports/cron.dinstall b/config/backports/cron.dinstall deleted file mode 100755 index 4d3bef8f..00000000 --- a/config/backports/cron.dinstall +++ /dev/null @@ -1,437 +0,0 @@ -#!/bin/bash -# No way I try to deal with a crippled sh just for POSIX foo. - -# Copyright (C) 2009, 2010 Joerg Jaspert -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; version 2. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -# Homer: Are you saying you're never going to eat any animal again? What -# about bacon? -# Lisa: No. -# Homer: Ham? -# Lisa: No. -# Homer: Pork chops? -# Lisa: Dad, those all come from the same animal. -# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal. - -# exit on errors -set -e -set -o pipefail -# make sure to only use defined variables -set -u -# ERR traps should be inherited from functions too. (And command -# substitutions and subshells and whatnot, but for us the functions is -# the important part here) -set -E - -# import the general variable set. -export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars -. $SCRIPTVARS - -######################################################################## -# Functions # -######################################################################## -# common functions are "outsourced" -. "${configdir}/common" - -# source the dinstall functions -. "${configdir}/dinstall.functions" - -######################################################################## -######################################################################## - -# Function to save which stage we are in, so we can restart an interrupted -# dinstall. Or even run actions in parallel, if we dare to, by simply -# backgrounding the call to this function. But that should only really be -# done for things we don't care much about. -# -# This should be called with the first argument being an array, with the -# members -# - FUNC - the function name to call -# - ARGS - Possible arguments to hand to the function. Can be the empty string -# - TIME - The timestamp name. Can be the empty string -# - ERR - if this is the string false, then the call will be surrounded by -# set +e ... set -e calls, so errors in the function do not exit -# dinstall. Can be the empty string, meaning true. -# -# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES -# ADDED FOR DINSTALL FEATURES! -function stage() { - ARGS='GO[@]' - local "${!ARGS}" - - error=${ERR:-"true"} - - STAGEFILE="${stagedir}/${FUNC}" - if [ -f "${STAGEFILE}" ]; then - stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}") - unixtime=$(date +%s) - difference=$(( $unixtime - $stamptime )) - if [ ${difference} -ge 14400 ]; then - log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check." - else - log "Did already run ${FUNC}, not calling again..." - fi - return - fi - - debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}" - - # Make sure we are always at the same place. If a function wants to be elsewhere, - # it has to cd first! - cd ${configdir} - - # Now redirect the output into $STAGEFILE.log. In case it errors out somewhere our - # errorhandler trap can then mail the contents of $STAGEFILE.log only, instead of a whole - # dinstall logfile. Short error mails ftw! - exec >> "${STAGEFILE}.log" 2>&1 - - if [ -f "${LOCK_STOP}" ]; then - log "${LOCK_STOP} exists, exiting immediately" - exit 42 - fi - - if [ "${error}" = "false" ]; then - set +e - fi - ${FUNC} ${ARGS} - - # No matter what happened in the function, we make sure we have set -e default state back - set -e - - # Make sure we are always at the same place. - cd ${configdir} - - # We always use the same umask. If a function wants to do different, fine, but we reset. - umask 022 - - touch "${STAGEFILE}" - - if [ -n "${TIME}" ]; then - ts "${TIME}" - fi - - # And the output goes back to the normal logfile - exec >> "$LOGFILE" 2>&1 - - # Now we should make sure that we have a usable dinstall.log, so append the $STAGEFILE.log - # to it. - cat "${STAGEFILE}.log" >> "${LOGFILE}" - rm -f "${STAGEFILE}.log" - - if [ -f "${LOCK_STOP}" ]; then - log "${LOCK_STOP} exists, exiting immediately" - exit 42 - fi -} - -######################################################################## - -# We need logs. -LOGFILE="$logdir/dinstall.log" - -exec >> "$LOGFILE" 2>&1 - -# And now source our default config -. "${configdir}/dinstall.variables" - -# Make sure we start out with a sane umask setting -umask 022 - -# And use one locale, no matter what the caller has set -export LANG=C -export LC_ALL=C - -# If we did not install new packages, we dont want to run. -if ! [ -f "${DINSTALLPACKAGES}" ]; then - log "nothing to do" - exit 0 -fi -rm -f "${DINSTALLPACKAGES}" - -touch "${DINSTALLSTART}" -ts "startup" -DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")" -state "Startup" - -lockfile -l 3600 "${LOCK_DAILY}" -trap onerror ERR -trap cleanup EXIT TERM HUP INT QUIT - -touch "${LOCK_BRITNEY}" - -GO=( - FUNC="savetimestamp" - TIME="" - ARGS="" - ERR="false" -) -stage $GO - -GO=( - FUNC="pg_timestamp" - TIME="pg_dump1" - ARGS="predinstall" - ERR="" -) -stage $GO - -lockfile "$LOCK_ACCEPTED" -lockfile "$LOCK_NEW" - -GO=( - FUNC="punew" - TIME="p-u-new" - ARGS="proposedupdates" - ERR="false" -) -stage $GO - -GO=( - FUNC="newstage" - TIME="newstage" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="cruft" - TIME="cruft" - ARGS="" - ERR="" -) -stage $GO - -state "indices" - -GO=( - FUNC="dominate" - TIME="dominate" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="filelist" - TIME="generate-filelist" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="fingerprints" - TIME="import-keyring" - ARGS="" - ERR="false" -) -stage $GO - -GO=( - FUNC="overrides" - TIME="overrides" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="mpfm" - TIME="pkg-file-mapping" - ARGS="" - ERR="false" -) -stage $GO - -state "packages/contents" -GO=( - FUNC="packages" - TIME="apt-ftparchive" - ARGS="" - ERR="" -) -# Careful: When we ever go and remove this monster-long thing, we have to check the backgrounded -# functions before it. We no longer have a 1.5hour sync point then. -stage $GO - -state "dists/" -GO=( - FUNC="pdiff" - TIME="pdiff" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="release" - TIME="release files" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="dakcleanup" - TIME="cleanup" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="buildd_dir" - TIME="buildd_dir" - ARGS="" - ERR="" -) -stage $GO - -state "scripts" -GO=( - FUNC="mkmaintainers" - TIME="mkmaintainers" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="copyoverrides" - TIME="copyoverrides" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="mklslar" - TIME="mklslar" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="mkchecksums" - TIME="mkchecksums" - ARGS="" - ERR="" -) -stage $GO - -GO=( - FUNC="mirror" - TIME="mirror hardlinks" - ARGS="" - ERR="" -) -stage $GO - -rm -f "$LOCK_ACCEPTED" -rm -f "$LOCK_NEW" -rm -f "${LOCK_DAILY}" - -ts "locked part finished" -state "postlock" - -GO=( - FUNC="changelogs" - TIME="changelogs" - ARGS="" - ERR="false" -) -stage $GO & - -GO=( - FUNC="pg_timestamp" - TIME="pg_dump2" - ARGS="postdinstall" - ERR="" -) -stage $GO & - -GO=( - FUNC="expire" - TIME="expire_dumps" - ARGS="" - ERR="" -) -#stage $GO & - -# GO=( -# FUNC="dm" -# TIME="" -# ARGS="" -# ERR="" -# ) -# stage $GO & - -GO=( - FUNC="mirrorpush" - TIME="mirrorpush" - ARGS="" - ERR="false" -) -stage $GO & - -GO=( - FUNC="stats" - TIME="stats" - ARGS="" - ERR="false" -) -stage $GO & - -rm -f "${LOCK_BRITNEY}" - -GO=( - FUNC="cleantransactions" - TIME="" - ARGS="" - ERR="" -) -stage $GO & - -# GO=( -# FUNC="aptftpcleanup" -# TIME="apt-ftparchive cleanup" -# ARGS="" -# ERR="false" -# ) -# stage $GO - -# we need to wait for the background processes before the end of dinstall -wait - -log "Daily cron scripts successful, all done" - -exec > "$logdir/afterdinstall.log" 2>&1 - -GO=( - FUNC="renamelogfile" - TIME="" - ARGS="" - ERR="false" -) -stage $GO -state "all done" - - -# Now, at the very (successful) end of dinstall, make sure we remove -# our stage files, so the next dinstall run will do it all again. -rm -f ${stagedir}/* -touch "${DINSTALLEND}" diff --git a/config/backports/cron.hourly b/config/backports/cron.hourly deleted file mode 100755 index 627c8b09..00000000 --- a/config/backports/cron.hourly +++ /dev/null @@ -1,44 +0,0 @@ -#! /bin/bash -# -# Executed hourly via cron, out of dak's crontab. - -set -e -set -o pipefail -set -u - -export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars -. $SCRIPTVARS - -dak import-users-from-passwd -dak queue-report -n > $webdir/new.html -dak queue-report -8 -d new,byhand,proposedupdates,oldproposedupdates -r $webdir/stat -#dak show-deferred -r $webdir/stat > ${webdir}/deferred.html -dak graph -n new,byhand,proposedupdates,oldproposedupdates,deferred -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates -dak show-new > /dev/null - -# cd $webdir -# cat removals-20*.txt > removals-full.txt -# cat removals.txt >> removals-full.txt -# cat removals-20*.822 > removals-full.822 -# cat removals.822 >> removals-full.822 - -#$base/dak/tools/queue_rss.py -q $queuedir/new -o $webdir/rss/ -d $base/misc -l $base/log/ -$base/dak/tools/removals.pl $configdir/removalsrss.rc > $webdir/removals.rss - -#$scriptsdir/generate-di - -# do the buildd key updates -BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX) -exec >> "${BUILDDFUN}" 2>&1 -#${scriptsdir}/buildd-remove-keys -#${scriptsdir}/buildd-add-keys -#${scriptsdir}/buildd-prepare-dir -for keyring in $(dak admin k list-binary); do - dak import-keyring --generate-users "%s" ${keyring} -done -exec >>/dev/null 2>&1 - -DATE=$(date -Is) -cat "${BUILDDFUN}" | mail -a "X-Debian: DAK" -e -s "[$(hostname -s)] Buildd key changes ${DATE}" buildd-keys@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org - -rm -f "${BUILDDFUN}" diff --git a/config/backports/cron.monthly b/config/backports/cron.monthly deleted file mode 100755 index 34593a3e..00000000 --- a/config/backports/cron.monthly +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# - -set -e -set -o pipefail -set -u - -export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars -. $SCRIPTVARS - -################################################################################ - -DATE=`date -d yesterday +%y%m` - -cd ${base}/mail/archive -for m in mail import; do - if [ -f $m ]; then - mv $m ${m}-$DATE - sleep 20 - xz -9 ${m}-$DATE - chgrp backports ${m}-$DATE.xz - chmod 660 ${m}-$DATE.xz - fi; -done - -DATE=`date +%Y-%m` -cd ${base}/log -touch $DATE -ln -sf $DATE current -chmod g+w $DATE -chown dak:backports $DATE - -################################################################################ diff --git a/config/backports/cron.reboot b/config/backports/cron.reboot deleted file mode 100755 index 1e2e8d98..00000000 --- a/config/backports/cron.reboot +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -# No way I try to deal with a crippled sh just for POSIX foo. - -# Copyright (C) 2009 Joerg Jaspert -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; version 2. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -# exit on errors -set -e -set -o pipefail -# make sure to only use defined variables -set -u -# ERR traps should be inherited from functions too. (And command -# substitutions and subshells and whatnot, but for us the functions is -# the important part here) -set -E - -# import the general variable set. -export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars -. $SCRIPTVARS - -# common functions are "outsourced" -. "${configdir}/common" - -# usually we are not using debug logs. Set to 1 if you want them. -DEBUG=0 - -# our name -PROGRAM="dinstall_reboot" - -# where do we want mails to go? For example log entries made with error() -if [ "x$(hostname -s)x" != "xfranckx" ]; then - # Not our ftpmaster host - MAILTO=${MAILTO:-"root"} -else - # Yay, ftpmaster - MAILTO=${MAILTO:-"ftpmaster@backports.debian.org"} -fi - -# Marker for dinstall start -DINSTALLSTART="${lockdir}/dinstallstart" -# Marker for dinstall end -DINSTALLEND="${lockdir}/dinstallend" - -set +e -starttime=$(/usr/bin/stat -c %Z "${DINSTALLSTART}") -endtime=$(/usr/bin/stat -c %Z "${DINSTALLEND}") -set -e - -if [ ${endtime} -gt ${starttime} ]; then - # Great, last dinstall run did seem to end without trouble, no need to rerun - log "Last dinstall run did end without trouble, not rerunning" - exit 0 -else - # Hrm, it looks like we did not successfully end the last run. - # This either means dinstall did abort due to an error, or we had a reboot - # No way to tell, so lets restart and see what happens. - - # Make sure we are not fooled by some random touching of the files, only - # really restart if we have the first stage stampfile there, indicating that - # dinstall got started - if [ -f "${stagedir}/savetimestamp" ]; then - log "Seems we have to restart a dinstall run after reboot" - ${configdir}/cron.dinstall - fi -fi diff --git a/config/backports/cron.unchecked b/config/backports/cron.unchecked deleted file mode 100755 index 13be8d59..00000000 --- a/config/backports/cron.unchecked +++ /dev/null @@ -1,108 +0,0 @@ -#! /bin/bash - -# Copyright (C) 2009 Joerg Jaspert -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; version 2. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -# exit on errors -set -e -set -o pipefail -# make sure to only use defined variables -set -u -# ERR traps should be inherited from functions too. (And command -# substitutions and subshells and whatnot, but for us the functions is -# the important part here) -set -E - -export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars -. $SCRIPTVARS - - -LOCKDAILY="" -LOCKFILE="$lockdir/unchecked.lock" -LOCK_NEW="$lockdir/processnew.lock" -NOTICE="$lockdir/daily.lock" -LOCK_BUILDD="$lockdir/buildd.lock" -# The state file telling us we have something new to do -DINSTALLPACKAGES="${lockdir}/dinstall.packages" - -# our name -PROGRAM="unchecked" - -if [ -e $NOTICE ]; then - exit 0; -fi - -######################################################################## -# Functions # -######################################################################## -# common functions are "outsourced" -. "${configdir}/common" - -STAMP=$(date "+%Y%m%d%H%M") - -cleanup() { - rm -f "$LOCKFILE" - if [ ! -z "$LOCKDAILY" ]; then - rm -f "$NOTICE" - fi -} - -function do_buildd () { - if lockfile -r3 $NOTICE; then - LOCKDAILY="YES" - cd $overridedir - dak make-overrides &>/dev/null - rm -f override.sid.all3 override.sid.all3.src - for i in main contrib non-free main.debian-installer; do - cat override.squeeze-backports.$i >> override.sid.all3 - if [ "$i" != "main.debian-installer" ]; then - cat override.squeeze-backports.$i.src >> override.sid.all3.src - fi - done - make_buildd_dir - wbtrigger - fi -} - -function do_dists () { - cd $configdir - dak generate-filelist - dak generate-packages-sources -} - -######################################################################## -# the actual unchecked functions follow # -######################################################################## - -# And use one locale, no matter what the caller has set -export LANG=C -export LC_ALL=C - -# only run one cron.unchecked -lockfile -r3 $LOCKFILE || exit 0 -trap cleanup 0 - -do_newstage -do_unchecked - -if [ ! -z "$changes" ]; then - touch ${DINSTALLPACKAGES} - sync_debbugs - do_buildd - -# echo "Starting g-p-s: $(date +%H:%M:%S)" -# do_dists -# echo "Done with g-p-s: $(date +%H:%M:%S)" -fi diff --git a/config/backports/cron.weekly b/config/backports/cron.weekly deleted file mode 100755 index ee57d39f..00000000 --- a/config/backports/cron.weekly +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# - -set -e -set -o pipefail -set -u -# ERR traps should be inherited from functions too. (And command -# substitutions and subshells and whatnot, but for us the functions is -# the important part here) -set -E -export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars -. $SCRIPTVARS - -# Start logging -NOW=`date "+%Y.%m.%d-%H:%M:%S"` -LOGFILE="$logdir/weekly_${NOW}.log" -exec > "$LOGFILE" 2>&1 - -cleanup() { - echo "Cleanup" - rm -f "$LOGFILE" -} -trap cleanup 0 - -################################################################################ - -# Purge empty directories -echo "Purging empty directories in $ftpdir/pool/" -if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then - find $ftpdir/pool/ -type d -empty | xargs rmdir; -fi - -# Clean up apt-ftparchive's databases -cd $configdir -#echo "Cleanup apt-ftparchive's database" -#apt-ftparchive -q clean apt.conf - -echo "Fixing symlinks in $ftpdir" -symlinks -d -r $ftpdir - -echo "Finally, all is done, compressing logfile" -exec > /dev/null 2>&1 - -bzip2 -9 "$LOGFILE" - - -################################################################################ diff --git a/config/backports/dak.conf b/config/backports/dak.conf deleted file mode 100644 index 890cd6b3..00000000 --- a/config/backports/dak.conf +++ /dev/null @@ -1,361 +0,0 @@ -Dinstall -{ - // To sign the release files. Adjust the keyid! - // Note: Key must be without a passphrase or it wont work automagically! - SigningKeyring "/srv/backports-master.debian.org/s3kr1t/dot-gnupg/secring.gpg"; - SigningPubKeyring "/srv/backports-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg"; - SendmailCommand "/usr/sbin/sendmail -oi -t -f envelope@backports.debian.org"; - MyEmailAddress "Backports Debian FTP Masters "; - MyAdminAddress "ftpmaster@backports.debian.org"; - MyHost "backports.debian.org"; // used for generating user@my_host addresses in e.g. manual_reject() - MyDistribution "backports.debian.org archive"; // Used in emails - // Alicia and melanie can use it - BugServer "bugs.debian.org"; - // melanie uses the packages server. - // PackagesServer "packages.test.backports.org"; - // If defined then the package@this.server gets a copy of most of the - // actions related to the package. For an example look at - // packages.qa.debian.org - // TrackingServer "packages.qa.test.backports.org"; - // If defined this address gets a bcc of all mails. - // FIXME: Einrichten wenn das hier produktiv geht! - Bcc "debian-backports-changes@lists.debian.org"; - GroupOverrideFilename "override.group-maint"; - FutureTimeTravelGrace 28800; // 8 hours - PastCutoffYear "1984"; - SkipTime 300; - // If defined then mails to close bugs are sent to the bugserver. - CloseBugs "false"; - OverrideDisparityCheck "false"; - DefaultSuite "squeeze-backports"; - ReleaseTransitions "/srv/backports-master.debian.org/hints/transitions.yaml"; - // If set, only send mails to addresses listed there. - // format of entries: one entry per line. Either an email address directly, or a regular expression, - // prefixed by "RE:". Examples: "jane.doe@domain.com" or "RE:jane[^@]@domain.com", where the first will - // only allow to mail jane.doe@domain.com while the second will mail all of jane*@domain.com - MailWhiteList "/srv/backports-master.debian.org/dak/config/backports/mail-whitelist"; -}; - -Generate-Index-Diffs -{ - Options - { - TempDir "/srv/backports-master.debian.org/tiffani"; - MaxDiffs { Default 50; }; - }; -}; - -Add-User -{ -// Should we sent a mail to newly added users? - SendEmail "true"; - -// Should we create an account so they can login? -// Account will be created with the defaults from adduser, so adjust -// it's configuration to fit your needs. -// NOTE: This requires that your dak user has a sudo entry, allowing -// to run /usr/sbin/useradd! - CreateAccount "false"; - -// Note: This is a comma separated list of additional groupnames to -// which uma should add the user. NO spaces between the groupnames or -// useradd will die. -// Disable it if you dont want or need that feature. -// GID "debuser"; - -}; - -Show-New -{ - HTMLPath "/srv/backports-web.debian.org/underlay/new/"; -} - -Check-Overrides -{ - OverrideSuites - { - squeeze-backports - { - Process "1"; -// OriginSuite "Unstable"; - }; - - }; -}; - - -Import-Users-From-Passwd -{ - // The Primary GID of your users. Using uma it is the gid from group users. - ValidGID "800"; - // Comma separated list of users who are in Postgres but not the passwd file - KnownPostgres "postgres,katie,dak,www-data,qa,guest,repuser"; -}; - -Queue-Report -{ - ReportLocations - { - 822Location "/srv/backports-web.debian.org/underlay/new.822"; - }; -}; - -Clean-Queues -{ - Options - { - Days 14; - }; - MorgueSubDir "queues"; -}; - -Control-Overrides -{ - Options - { - Component "main"; - Suite "squeeze-backports"; - Type "deb"; - }; -}; - -Rm -{ - Options - { - Suite "squeeze-backports"; - }; - - LogFile "/srv/backports-web.debian.org/underlay/removals.txt"; - LogFile822 "/srv/backports-web.debian.org/underlay/removals.822"; -}; - -Import-Archive -{ - ExportDir "/srv/backports-master.debian.org/dak/import-archive-files/"; -}; - -Clean-Suites -{ - // How long (in seconds) dead packages are left before being killed - StayOfExecution 1209600; // 14 days - AcceptedAutoBuildStayOfExecution 86400; // 24 hours - MorgueSubDir "pool"; -}; - -Process-New -{ - DinstallLockFile "/srv/backports-master.debian.org/lock/processnew.lock"; - LockDir "/srv/backports-master.debian.org/lock/new/"; -}; - -Suite -{ - squeeze-backports - { - Components - { - main; - contrib; - non-free; - }; - }; - -}; - -Dir -{ - Base "/srv/backports-master.debian.org/"; - Root "/srv/backports-master.debian.org/ftp/"; - Pool "/srv/backports-master.debian.org/ftp/pool/"; - Export "/srv/backports-master.debian.org/export/"; - Templates "/srv/backports-master.debian.org/dak/templates/"; - Lists "/srv/backports-master.debian.org/database/dists/"; - Cache "/srv/backports-master.debian.org/database/"; - Log "/srv/backports-master.debian.org/log/"; - Lock "/srv/backports-master.debian.org/lock"; - Morgue "/srv/backports-master.debian.org/morgue/"; - Override "/srv/backports-master.debian.org/scripts/override/"; - UrgencyLog "/srv/backports-master.debian.org/testing/urgencies/"; - TempPath "/srv/backports-master.debian.org/tmp"; - BTSVersionTrack "/srv/backports-master.debian.org/queue/bts_version_track/"; - Holding "/srv/backports-master.debian.org/queue/holding/"; - Done "/srv/backports-master.debian.org/queue/done/"; - Reject "/srv/backports-master.debian.org/queue/reject/"; - - Queue - { - Byhand "/srv/backports-master.debian.org/queue/byhand/"; - New "/srv/backports-master.debian.org/queue/new/"; - Unchecked "/srv/backports-master.debian.org/queue/unchecked/"; - Newstage "/srv/backports-master.debian.org/queue/newstage/"; - Embargoed "/srv/backports-master.debian.org/queue/Embargoed/"; - Unembargoed "/srv/backports-master.debian.org/queue/Unembargoed/"; - ProposedUpdates "/srv/backports-master.debian.org/queue/p-u-new/"; - OldProposedUpdates "/srv/backports-master.debian.org/queue/Unembargoed/"; - }; -}; - -DB -{ - Service "backports"; - // PoolSize should be at least ThreadCount + 1 - PoolSize 5; - // MaxOverflow shouldn't exceed postgresql.conf's max_connections - PoolSize - MaxOverflow 13; - // should be false for encoding == SQL_ASCII - Unicode "false" -}; - -SuiteMappings -{ - "map squeeze squeeze-backports"; - "map squeeze-bpo squeeze-backports"; -}; - -Architectures -{ - source "Source"; - all "Architecture Independent"; - alpha "DEC Alpha"; - amd64 "AMD x86_64 (AMD64)"; - hurd-i386 "Intel ia32 running the HURD"; - hppa "HP PA RISC"; - arm "ARM"; - armel "ARM EABI"; - i386 "Intel ia32"; - ia64 "Intel ia64"; - m68k "Motorola Mc680x0"; - mips "MIPS (Big Endian)"; - mipsel "MIPS (Little Endian)"; - powerpc "PowerPC"; - s390 "IBM S/390"; - sh "Hitatchi SuperH"; - sparc "Sun SPARC/UltraSPARC"; - kfreebsd-i386 "GNU/kFreeBSD i386"; - kfreebsd-amd64 "GNU/kFreeBSD amd64"; -}; - -Archive -{ - backports - { - OriginServer "backports.debian.org"; - PrimaryMirror "backports.debian.org"; - Description "Master Archive for backports.debian.org archive"; - }; -}; - -Section -{ - admin; - cli-mono; - comm; - database; - debian-installer; - debug; - devel; - doc; - editors; - embedded; - electronics; - fonts; - games; - gnome; - graphics; - gnu-r; - gnustep; - hamradio; - haskell; - httpd; - interpreters; - java; - kde; - kernel; - libdevel; - libs; - lisp; - localization; - mail; - math; - misc; - net; - news; - ocaml; - oldlibs; - otherosfs; - perl; - php; - python; - ruby; - science; - shells; - sound; - tex; - text; - utils; - web; - vcs; - video; - x11; - xfce; - zope; -}; - -Priority -{ - required 1; - important 2; - standard 3; - optional 4; - extra 5; - source 0; // i.e. unused -}; - -Urgency -{ - Default "low"; - Valid - { - low; - medium; - high; - emergency; - critical; - }; -}; - -Common -{ - // The default number of threads for multithreading parts of dak: - ThreadCount 16; -} - -Import-LDAP-Fingerprints -{ - LDAPDn "ou=users,dc=debian,dc=org"; - LDAPServer "db.debian.org"; - ExtraKeyrings - { - "/srv/keyring.debian.org/keyrings/removed-keys.pgp"; - "/srv/keyring.debian.org/keyrings/removed-keys.gpg"; - "/srv/keyring.debian.org/keyrings/extra-keys.pgp"; - }; - KeyServer "wwwkeys.eu.pgp.net"; -}; - -Generate-Releases -{ - MD5Sum - { - squeeze-backports; - }; - SHA1 - { - squeeze-backports; - }; - SHA256 - { - squeeze-backports; - }; -} diff --git a/config/backports/dak.conf-etc b/config/backports/dak.conf-etc deleted file mode 100644 index bf7d26c6..00000000 --- a/config/backports/dak.conf-etc +++ /dev/null @@ -1,22 +0,0 @@ -Config -{ - // FQDN hostname - morricone.debian.org - { - - // Optional hostname as it appears in the database (if it differs - // from the FQDN hostname). - DatabaseHostname "backports"; - - // Optional filename of katie's config file; if not present, this - // file is assumed to contain katie config info. - DakConfig "/srv/backports-master.debian.org/dak/config/backports/dak.conf"; - - // Optional filename of apt-ftparchive's config file; if not - // present, the file is assumed to be 'apt.conf' in the same - // directory as this file. - AptConfig "/srv/backports-master.debian.org/dak/config/backports/apt.conf"; - } - -} - diff --git a/config/backports/dinstall.functions b/config/backports/dinstall.functions deleted file mode 100644 index cf84cae8..00000000 --- a/config/backports/dinstall.functions +++ /dev/null @@ -1,571 +0,0 @@ -# -*- mode:sh -*- -# Timestamp. Used for dinstall stat graphs -function ts() { - echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)" -} - -# Cleanup actions -function cleanup() { - rm -f ${LOCK_DAILY} - rm -f ${LOCK_ACCEPTED} -} - -# If we error out this one is called, *FOLLOWED* by cleanup above -function onerror() { - ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S") - - subject="ATTENTION ATTENTION!" - if [ "${error}" = "false" ]; then - subject="${subject} (continued)" - else - subject="${subject} (interrupted)" - fi - subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)" - - if [ -r "${STAGEFILE}.log" ]; then - cat "${STAGEFILE}.log" - else - echo "file ${STAGEFILE}.log does not exist, sorry" - fi | mail -s "${subject}" -a "X-Debian: DAK" cron@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org -} - -######################################################################## -# the actual dinstall functions follow # -######################################################################## - -function qa1() { - log "Telling QA user that we start dinstall" - ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1 - } - -# Updating various files -function updates() { - log "Updating Bugs docu, Mirror list and mailing-lists.txt" - cd $configdir - $scriptsdir/update-bugdoctxt - $scriptsdir/update-mirrorlists - $scriptsdir/update-mailingliststxt - $scriptsdir/update-pseudopackages.sh -} - -# Process (oldstable)-proposed-updates "NEW" queue -function punew_do() { - date -u -R >> REPORT - dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" team@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org - echo >> REPORT -} -function punew() { - log "Doing automated p-u-new processing" - cd "${queuedir}/p-u-new" - punew_do "$1" -} -function opunew() { - log "Doing automated o-p-u-new processing" - cd "${queuedir}/o-p-u-new" - punew_do "$1" -} - -# The first i18n one, syncing new descriptions -function i18n1() { - log "Synchronizing i18n package descriptions" - # First sync their newest data - cd ${scriptdir}/i18nsync - rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true - - # Now check if we still know about the packages for which they created the files - # is the timestamp signed by us? - if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then - # now read it. As its signed by us we are sure the content is what we expect, no need - # to do more here. And we only test -d a directory on it anyway. - TSTAMP=$(cat timestamp) - # do we have the dir still? - if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then - # Lets check! - if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then - # Yay, worked, lets copy around - for dir in wheezy sid; do - if [ -d dists/${dir}/ ]; then - cd dists/${dir}/main/i18n - rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/. - fi - cd ${scriptdir}/i18nsync - done - else - echo "ARRRR, bad guys, wrong files, ARRR" - echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org - fi - else - echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR" - echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org - fi - else - echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR." - echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org - fi -} - -function cruft() { - log "Checking for cruft in overrides" - dak check-overrides -} - -function dominate() { - log "Removing obsolete source and binary associations" - dak dominate -} - -function filelist() { - log "Generating file lists for apt-ftparchive" - dak generate-filelist -} - -function fingerprints() { - log "Updating fingerprints" - dak import-keyring -L /srv/backports-master.debian.org/keyrings/keyring.gpg - - OUTFILE=$(mktemp) - dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}" - - if [ -s "${OUTFILE}" ]; then - /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org < -To: -Subject: Debian Maintainers Keyring changes -Content-Type: text/plain; charset=utf-8 -X-Debian: DAK -MIME-Version: 1.0 - -The following changes to the debian-maintainers keyring have just been activated: - -$(cat $OUTFILE) - -Debian distribution maintenance software, -on behalf of the Keyring maintainers - -EOF - fi - rm -f "$OUTFILE" -} - -function overrides() { - log "Writing overrides into text files" - cd $overridedir - dak make-overrides - - # FIXME - rm -f override.sid.all3 - for i in main contrib non-free main.debian-installer; do cat override.squeeze-backports.$i >> override.sid.all3; done -} - -function mpfm() { - log "Generating package / file mapping" - dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2 -} - -function packages() { - log "Generating Packages and Sources files" - cd $configdir - apt-ftparchive generate apt.conf - #dak generate-packages-sources -} - -function pdiff() { - log "Generating pdiff files" - dak generate-index-diffs -} - -function release() { - log "Generating Release files" - dak generate-releases -} - -function dakcleanup() { - log "Cleanup old packages/files" - dak clean-suites -m 10000 - dak clean-queues -} - -function buildd_dir() { - # Rebuilt the buildd dir to avoid long times of 403 - log "Regenerating the buildd incoming dir" - STAMP=$(date "+%Y%m%d%H%M") - make_buildd_dir -} - -function mklslar() { - cd $ftpdir - - FILENAME=ls-lR - - log "Removing any core files ..." - find -type f -name core -print0 | xargs -0r rm -v - - log "Checking permissions on files in the FTP tree ..." - find -type f \( \! -perm -444 -o -perm +002 \) -ls - find -type d \( \! -perm -555 -o -perm +002 \) -ls - - log "Checking symlinks ..." - symlinks -rd . - - log "Creating recursive directory listing ... " - rm -f .${FILENAME}.new - TZ=UTC ls -lR > .${FILENAME}.new - - if [ -r ${FILENAME}.gz ] ; then - mv -f ${FILENAME}.gz ${FILENAME}.old.gz - mv -f .${FILENAME}.new ${FILENAME} - rm -f ${FILENAME}.patch.gz - zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz - rm -f ${FILENAME}.old.gz - else - mv -f .${FILENAME}.new ${FILENAME} - fi - - gzip -9cfN ${FILENAME} >${FILENAME}.gz - rm -f ${FILENAME} -} - -function mkmaintainers() { - log 'Creating Maintainers index ... ' - - cd $indices - dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers - gzip -9v --rsyncable Maintainers.gz - gzip -9v --rsyncable Uploaders.gz -} - -function copyoverrides() { - log 'Copying override files into public view ...' - - for ofile in $copyoverrides ; do - cd $overridedir - chmod g+w override.$ofile - - cd $indices - - newofile=override.$ofile.gz - rm -f .newover-$ofile.gz - pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`" - if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then - log " installing new $newofile $pc" - mv -f .newover-$ofile.gz $newofile - chmod g+w $newofile - else - rm -f .newover-$ofile.gz - fi - done -} - -function mkfilesindices() { - umask 002 - cd $base/ftp/indices/files/components - - ARCHLIST=$(tempfile) - - log "Querying postgres" - local query=" - SELECT CONCAT('./pool/', c.name, '/', f.filename) AS path, a.arch_string AS arch_string - FROM files f - JOIN files_archive_map af ON f.id = af.file_id - JOIN component c ON af.component_id = c.id - JOIN archive ON af.archive_id = archive.id - LEFT OUTER JOIN - (binaries b - JOIN architecture a ON b.architecture = a.id) - ON f.id = b.file - WHERE archive.name = 'ftp-master' - ORDER BY path, arch_string - " - psql -At -c "$query" >$ARCHLIST - - includedirs () { - perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }' - } - poolfirst () { - perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);' - } - - log "Generating sources list" - ( - sed -n 's/|$//p' $ARCHLIST - cd $base/ftp - find ./dists -maxdepth 1 \! -type d - find ./dists \! -type d | grep "/source/" - ) | sort -u | gzip -9 > source.list.gz - - log "Generating arch lists" - - ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u) - for a in $ARCHES; do - (sed -n "s/|$a$//p" $ARCHLIST - sed -n 's/|all$//p' $ARCHLIST - - cd $base/ftp - find ./dists -maxdepth 1 \! -type d - find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" - ) | sort -u | gzip -9 > arch-$a.list.gz - done - - log "Generating suite lists" - - suite_list () { - local suite_id="$(printf %d $1)" - local query - query=" - SELECT DISTINCT CONCAT('./pool/', c.name, '/', f.filename) - FROM - (SELECT sa.source AS source - FROM src_associations sa - WHERE sa.suite = $suite_id - UNION - SELECT esr.src_id - FROM extra_src_references esr - JOIN bin_associations ba ON esr.bin_id = ba.bin - WHERE ba.suite = $suite_id - UNION - SELECT b.source AS source - FROM bin_associations ba - JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s - JOIN dsc_files df ON s.source = df.source - JOIN files f ON df.file = f.id - JOIN files_archive_map af ON f.id = af.file_id - JOIN component c ON af.component_id = c.id - JOIN archive ON af.archive_id = archive.id - WHERE archive.name = 'ftp-master' - " - psql -F' ' -A -t -c "$query" - - query=" - SELECT CONCAT('./pool/', c.name, '/', f.filename) - FROM bin_associations ba - JOIN binaries b ON ba.bin = b.id - JOIN files f ON b.file = f.id - JOIN files_archive_map af ON f.id = af.file_id - JOIN component c ON af.component_id = c.id - JOIN archive ON af.archive_id = archive.id - WHERE ba.suite = $suite_id AND archive.name = 'ftp-master' - " - psql -F' ' -A -t -c "$query" - } - - psql -F' ' -At -c 'SELECT id, suite_name FROM suite' | - while read id suite; do - [ -e $base/ftp/dists/$suite ] || continue - ( - (cd $base/ftp - distname=$(cd dists; readlink $suite || echo $suite) - find ./dists/$distname \! -type d - for distdir in ./dists/*; do - [ "$(readlink $distdir)" != "$distname" ] || echo $distdir - done - ) - suite_list $id - ) | sort -u | gzip -9 > suite-${suite}.list.gz - done - - log "Finding everything on the ftp site to generate sundries" - (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST - - rm -f sundries.list - zcat *.list.gz | cat - *.list | sort -u | - diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list - - log "Generating files list" - - for a in $ARCHES; do - (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) | - cat - sundries.list dists.list project.list docs.list indices.list | - sort -u | poolfirst > ../arch-$a.files - done - - (cd $base/ftp/ - for dist in sid wheezy; do - find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz - done - ) - - (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) | - sort -u | poolfirst > ../typical.files - - rm -f $ARCHLIST - log "Done!" -} - -function mkchecksums() { - dsynclist=$dbdir/dsync.list - md5list=$indices/md5sums - - log -n "Creating md5 / dsync index file ... " - - cd "$ftpdir" - ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5 - ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz - ${bindir}/dsync-flist -q link-dups $dsynclist || true -} - -function mirror() { - log "Regenerating \"public\" mirror/ hardlink fun" - DATE_SERIAL=$(date +"%Y%m%d01") - FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} ) - if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then - SERIAL="$DATE_SERIAL" - else - SERIAL="$FILESOAPLUS1" - fi - date -u > ${TRACEFILE} - echo "Using dak v1" >> ${TRACEFILE} - echo "Running on host: $(hostname -f)" >> ${TRACEFILE} - echo "Archive serial: ${SERIAL}" >> ${TRACEFILE} - cd ${mirrordir} - rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. . -} - -function expire() { - log "Expiring old database dumps..." - cd $base/backup - $scriptsdir/expire_dumps -d . -p -f "dump_*" -} - -function transitionsclean() { - log "Removing out of date transitions..." - cd $base - dak transitions -c -a -} - -function dm() { - log "Updating DM html page" - #$scriptsdir/dm-monitor >$webdir/dm-uploaders.html -} - -function bts() { - log "Categorizing uncategorized bugs filed against ftp.debian.org" - dak bts-categorize -} - -function ddaccess() { - # Tell our dd accessible mirror to sync itself up. Including ftp dir. - log "Trigger dd accessible parts sync including ftp dir" - ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool -} - -function mirrorpush() { - log "Starting the mirrorpush" - date -u > /srv/backports-web.debian.org/underlay/mirrorstart - echo "Using dak v1" >> /srv/backports-web.debian.org/underlay/mirrorstart - echo "Running on host $(hostname -f)" >> /srv/backports-web.debian.org/underlay/mirrorstart - sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 & -} - -function i18n2() { - log "Exporting package data foo for i18n project" - STAMP=$(date "+%Y%m%d%H%M") - mkdir -p ${scriptdir}/i18n/${STAMP} - cd ${scriptdir}/i18n/${STAMP} - dak control-suite -l stable > squeeze - dak control-suite -l testing > wheezy - dak control-suite -l unstable > sid - echo "${STAMP}" > timestamp - gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --detach-sign -o timestamp.gpg timestamp - rm -f md5sum - md5sum * > md5sum - cd ${webdir}/ - ln -sfT ${scriptdir}/i18n/${STAMP} i18n - - cd ${scriptdir} - find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf -} - -function stats() { - log "Updating stats data" - cd $configdir - #$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data - #R --slave --vanilla < $base/misc/ftpstats.R - dak stats arch-space > $webdir/arch-space - dak stats pkg-nums > $webdir/pkg-nums -} - -function aptftpcleanup() { - log "Clean up apt-ftparchive's databases" - cd $configdir - apt-ftparchive -q clean apt.conf -} - -function cleantransactions() { - log "Cleanup transaction ids older than 3 months" - cd $base/backup/ - find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm -} - -function logstats() { - $masterdir/tools/logs.py "$1" -} - -# save timestamp when we start -function savetimestamp() { - NOW=`date "+%Y.%m.%d-%H:%M:%S"` - echo ${NOW} > "${dbdir}/dinstallstart" -} - -function maillogfile() { - cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org -} - -function renamelogfile() { - if [ -f "${dbdir}/dinstallstart" ]; then - NOW=$(cat "${dbdir}/dinstallstart") -# maillogfile - mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" -# logstats "$logdir/dinstall_${NOW}.log" - bzip2 -9 "$logdir/dinstall_${NOW}.log" - else - error "Problem, I don't know when dinstall started, unable to do log statistics." - NOW=`date "+%Y.%m.%d-%H:%M:%S"` -# maillogfile - mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" - bzip2 -9 "$logdir/dinstall_${NOW}.log" - fi -} - -function testingsourcelist() { - dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list -} - -# do a last run of process-unchecked before dinstall is on. -function process_unchecked() { - log "Processing the unchecked queue" - UNCHECKED_WITHOUT_LOCK="-p" - do_unchecked - sync_debbugs -} - -# do a run of newstage only before dinstall is on. -function newstage() { - log "Processing the newstage queue" - UNCHECKED_WITHOUT_LOCK="-p" - do_newstage -} - -# Function to update a "statefile" telling people what we are doing -# (more or less). -# -# This should be called with the argument(s) -# - Status name we want to show. -# -function state() { - RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")" - cat >"${DINSTALLSTATE}" < ~dak/runmirrors-metadata.log 2>&1 & -} diff --git a/config/backports/dinstall.variables b/config/backports/dinstall.variables deleted file mode 100644 index d6d7bb3a..00000000 --- a/config/backports/dinstall.variables +++ /dev/null @@ -1,53 +0,0 @@ -# -*- mode:sh -*- -# usually we are not using debug logs. Set to 1 if you want them. -DEBUG=0 - -# our name -PROGRAM="dinstall" - -# where do we want mails to go? For example log entries made with error() -if [ "x$(hostname -s)x" != "xmorriconex" ]; then - # Not our ftpmaster host - MAILTO=${MAILTO:-"root"} -else - # Yay, ftpmaster - MAILTO=${MAILTO:-"ftpmaster@backports.debian.org"} -fi - -# How many logfiles to keep -LOGROTATE=${LOGROTATE:-400} - -# Marker for dinstall start -DINSTALLSTART="${lockdir}/dinstallstart" -# Marker for dinstall end -DINSTALLEND="${lockdir}/dinstallend" - -# lock cron.unchecked (it immediately exits when this exists) -LOCK_DAILY="$lockdir/daily.lock" - -# Lock cron.unchecked from doing work -LOCK_ACCEPTED="$lockdir/unchecked.lock" - -# Lock process-new from doing work -LOCK_NEW="$lockdir/processnew.lock" - -# This file is simply used to indicate to britney whether or not -# the Packages file updates completed sucessfully. It's not a lock -# from our point of view -LOCK_BRITNEY="$lockdir/britney.lock" - -# If this file exists we exit immediately after the currently running -# function is done -LOCK_STOP="$lockdir/archive.stop" - -# Lock buildd updates -LOCK_BUILDD="$lockdir/buildd.lock" - -# Statefile for the users -DINSTALLSTATE="${webdir}/dinstall.status" - -# The mirror trace file -TRACEFILE="${ftpdir}/project/trace/backports-master.debian.org" - -# The state file telling us we have something new to do -DINSTALLPACKAGES="${lockdir}/dinstall.packages" diff --git a/config/backports/removalsrss.rc b/config/backports/removalsrss.rc deleted file mode 100644 index 92395b24..00000000 --- a/config/backports/removalsrss.rc +++ /dev/null @@ -1,12 +0,0 @@ -input: /srv/backports-web.debian.org/underlay/removals.txt -items: 16 -titlelength: 80 - -title: Removals from Debian Backports -link: http://backports-master.debian.org/dak/removals.txt -description: List of all the removals from Debian's Backport archives -subject: Removals from Debian Backports -creator: tfheen@debian.org -publisher: joerg@debian.org -rights: Copyright 2005, Tollef Fog Heen -language: en-us diff --git a/config/backports/vars b/config/backports/vars deleted file mode 100644 index 5a1ca15b..00000000 --- a/config/backports/vars +++ /dev/null @@ -1,59 +0,0 @@ -# locations used by many scripts - -base=/srv/backports-master.debian.org -public=/srv/backports-web.debian.org -bindir=$base/bin -ftpdir=$base/ftp/ -indices=$ftpdir/indices -webdir=$public/underlay/ -scriptdir=$base/scripts - -archs="alpha amd64 arm armel hppa hurd-i386 i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64 " - -masterdir=$base/dak/ -overridedir=$base/scripts/override -extoverridedir=$scriptdir/external-overrides -configdir=$base/dak/config/backports/ -scriptsdir=$base/dak/scripts/debian/ -dbdir=$base/database/ - -queuedir=$base/queue -unchecked=$queuedir/unchecked/ -accepted=$queuedir/accepted/ -done=$queuedir/done/ -over=$base/over/ -lockdir=$base/lock/ -stagedir=$lockdir/stages -incoming=$base/incoming -logdir=$base/log/cron/ -mirrordir=$base/mirror/ -newstage=$queuedir/newstage/ -exportdir=$base/export/ -exportpublic=$public/export/ - - -ftpgroup=debadmin - -copyoverrides="squeeze-backports.contrib squeeze-backports.contrib.src squeeze-backports.main squeeze-backports.main.debian-installer squeeze-backports.main.src squeeze-backports.extra.contrib squeeze-backports.extra.main" - -# Change this to your hostname -uploadhost=localhost -uploaddir=/pub/UploadQueue/ - -# What components to support -components="main contrib non-free" -suites="squeeze-backports" -override_types="deb dsc udeb" - -# export TMP=/srv/backports-master.debian.org/tmp -# export TEMP=/srv/backports-master.debian.org/tmp - -TMPDIR=$base/tmp - -PATH=$masterdir:$PATH -umask 022 -unset CDPATH -GZIP='--rsyncable' ; export GZIP - -# Set the database variables -eval $(dak admin config db-shell)