From: Mark Hymers Date: Tue, 22 Mar 2011 15:07:29 +0000 (+0000) Subject: Merge remote branch 'mhy/master' into merge X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=1d6c35970efaaa35a4511976510661b548e19514;hp=58f226d25d6c19c1e1283fddc2f81120839d296f;p=dak.git Merge remote branch 'mhy/master' into merge Signed-off-by: Mark Hymers --- diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall index 40451c81..27200dd1 100755 --- a/config/debian/cron.dinstall +++ b/config/debian/cron.dinstall @@ -172,7 +172,7 @@ GO=( stage $GO GO=( - FUNC="merkel1" + FUNC="qa1" TIME="init" ARGS="" ERR="false" @@ -436,14 +436,6 @@ GO=( ) stage $GO & -GO=( - FUNC="merkel2" - TIME="merkel projectb push" - ARGS="" - ERR="false" -) -stage $GO & - GO=( FUNC="mirrorpush" TIME="mirrorpush" @@ -479,8 +471,8 @@ stage $GO & rm -f "${LOCK_BRITNEY}" GO=( - FUNC="merkel3" - TIME="merkel ddaccessible sync" + FUNC="ddaccess" + TIME="ddaccessible sync" ARGS="" ERR="false" ) diff --git a/config/debian/cron.hourly b/config/debian/cron.hourly index 73ba6afc..c20f4d1d 100755 --- a/config/debian/cron.hourly +++ b/config/debian/cron.hourly @@ -30,4 +30,7 @@ cat removals.822 >> removals-full.822 $base/dak/tools/queue_rss.py -q $queuedir/new -o $webdir/rss/ -d $base/misc -l $base/log/ $base/dak/tools/removals.pl $configdir/removalsrss.rc > $webdir/rss/removals.rss +# Tell ries to sync its tree +ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org sync + $scriptsdir/generate-di diff --git a/config/debian/dak.conf b/config/debian/dak.conf index a9eb3fb4..b888a606 100644 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@ -453,7 +453,7 @@ DB { Service "projectb"; // PoolSize should be at least ThreadCount + 1 - PoolSize 17; + PoolSize 5; // MaxOverflow shouldn't exceed postgresql.conf's max_connections - PoolSize MaxOverflow 13; // should be false for encoding == SQL_ASCII diff --git a/config/debian/dak.conf-etc b/config/debian/dak.conf-etc index b0f6a50e..5336a8df 100644 --- a/config/debian/dak.conf-etc +++ b/config/debian/dak.conf-etc @@ -7,5 +7,12 @@ Config DakConfig "/srv/ftp-master.debian.org/dak/config/debian/dak.conf"; AptConfig "/srv/ftp-master.debian.org/dak/config/debian/apt.conf"; } + ries.debian.org + { + AllowLocalConfig "false"; + DatabaseHostname "ftp-master"; + DakConfig "/srv/ftp-master.debian.org/dak/config/debian/dak.conf"; + AptConfig "/srv/ftp-master.debian.org/dak/config/debian/apt.conf"; + } } diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index abece803..e4de479e 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -38,27 +38,11 @@ function onerror() { ######################################################################## # pushing merkels QA user, part one -function merkel1() { - log "Telling merkels QA user that we start dinstall" +function qa1() { + log "Telling QA user that we start dinstall" ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1 } -# Create the postgres dump files -function pgdump_pre() { - log "Creating pre-daily-cron-job backup of $PGDATABASE database..." - pg_dump > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S) -} - -function pgdump_post() { - log "Creating post-daily-cron-job backup of $PGDATABASE database..." - cd $base/backup - POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S) - pg_dump > $base/backup/dump_$POSTDUMP - #pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP - ln -sf $base/backup/dump_$POSTDUMP current - #ln -sf $base/backup/dumpall_$POSTDUMP currentall -} - # Updating various files function updates() { log "Updating Bugs docu, Mirror list and mailing-lists.txt" @@ -444,18 +428,10 @@ function bts() { dak bts-categorize } -function merkel2() { - # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached - log "Trigger merkel/flotows $PGDATABASE sync" - ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1 - # Also trigger flotow, the ftpmaster test box - ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1 -} - -function merkel3() { - # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached - log "Trigger merkels dd accessible parts sync" - ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1 +function ddaccess() { + # Tell our dd accessible mirror to sync itself up. Including ftp dir. + log "Trigger dd accessible parts sync including ftp dir" + ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool } function mirrorpush() { diff --git a/config/homedir/syncdd.sh b/config/homedir/syncdd.sh new file mode 100755 index 00000000..652c29a0 --- /dev/null +++ b/config/homedir/syncdd.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# Copyright (C) 2011 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + + +set -e +set -u +set -E + +export LANG=C +export LC_ALL=C + +export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars +. $SCRIPTVARS + +EXTRA="" + +check_commandline() { + while [ $# -gt 0 ]; do + case "$1" in + sync) + EXTRA="--exclude ftp/" + ;; + pool) + ;; + *) + echo "Unknown option ${1} ignored" + ;; + esac + shift # Check next set of parameters. + done +} + +if [ $# -gt 0 ]; then + ORIGINAL_COMMAND=$* +else + ORIGINAL_COMMAND="" +fi + +SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""} +if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then + set "nothing" "${SSH_ORIGINAL_COMMAND}" + shift + check_commandline $* +fi + +if [ -n "${ORIGINAL_COMMAND}" ]; then + set ${ORIGINAL_COMMAND} + check_commandline $* +fi + + +cleanup() { + rm -f "${HOME}/sync.lock" +} +trap cleanup EXIT TERM HUP INT QUIT + +# not using $lockdir as thats inside the rsync dir, and --delete would +# kick the lock away. Yes we could exclude it, but wth bother? +# +# Also, NEVER use --delete-excluded! +if lockfile -r3 ${HOME}/sync.lock; then + cd $base/ + rsync -aH -B8192 \ + --exclude backup/*.xz \ + --exclude backup/dump* \ + ${EXTRA} \ + --exclude mirror \ + --exclude morgue/ \ + --exclude=lost+found/ \ + --exclude .da-backup.trace \ + --delete \ + --delete-after \ + --timeout 3600 \ + -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ + ftpmaster-sync:/srv/ftp-master.debian.org/ . + + cd $public/ + rsync -aH -B8192 \ + --exclude mirror \ + --exclude rsync/ \ + --exclude=lost+found/ \ + --exclude .da-backup.trace \ + --exclude web-users/ \ + --delete \ + --delete-after \ + --timeout 3600 \ + -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ + ftpmaster-sync2:/srv/ftp.debian.org/ . + +else + echo "Couldn't get the lock, not syncing" + exit 0 +fi + + +## ftpmaster-sync is defined in .ssh/config as: +# Host ftpmaster-sync +# Hostname franck.debian.org +# User dak +# IdentityFile ~dak/.ssh/syncftpmaster +# ForwardX11 no +# ForwardAgent no +# StrictHostKeyChecking yes +# PasswordAuthentication no +# BatchMode yes + +## ftpmaster-sync2 is the same, just a second ssh key diff --git a/dak/admin.py b/dak/admin.py index 663b196b..218eea57 100755 --- a/dak/admin.py +++ b/dak/admin.py @@ -359,6 +359,7 @@ def show_config(command): connstr += "?port=%s" % cnf["DB::Port"] print connstr elif mode == 'db-shell': + e = [] if cnf.has_key("DB::Service"): e.append('PGSERVICE') print "PGSERVICE=%s" % cnf["DB::Service"]