From 33233a41d742249ceb0c816c15f2850970ba3634 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 1 Dec 2014 12:10:28 +0100 Subject: [PATCH] Adjust way of syncing the dd-accessible copy - Take out ries, hostname is coccia for a long time (use dd...) - Allow sync-dd to push only the dak/ code directory - Fixup whitespace in that script - Take out the distinction of sync|pool push - that wasn't done for a long time, ftp/ is never ever synced instead. Signed-off-by: Joerg Jaspert --- config/debian/cron.hourly | 2 +- config/debian/dinstall.functions | 2 +- config/homedir/ssh/ftpmaster-config | 9 +- scripts/debian/sync-dd | 128 +++++++++++++++------------- 4 files changed, 78 insertions(+), 63 deletions(-) diff --git a/config/debian/cron.hourly b/config/debian/cron.hourly index 1a98bace..87fad61c 100755 --- a/config/debian/cron.hourly +++ b/config/debian/cron.hourly @@ -49,7 +49,7 @@ $base/dak/tools/removals.pl $configdir/removalsrss.rc > $webdir/rss/removals.rss # Tell ries to sync its tree -${scriptsdir}/sync-dd ries-sync ries-sync1 ries-sync2 sync +${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync $scriptsdir/generate-d-i diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index 1414986e..bd57a3a8 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -444,7 +444,7 @@ function bts() { function ddaccess() { # Tell our dd accessible mirror to sync itself up. Including ftp dir. log "Trigger dd accessible parts sync including ftp dir" - ${scriptsdir}/sync-dd ries-sync ries-sync1 ries-sync2 pool + ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 pool } function mirrorpush() { diff --git a/config/homedir/ssh/ftpmaster-config b/config/homedir/ssh/ftpmaster-config index d90b5a4c..5db7c72a 100644 --- a/config/homedir/ssh/ftpmaster-config +++ b/config/homedir/ssh/ftpmaster-config @@ -22,16 +22,21 @@ Host morgue-sync User dak IdentityFile /srv/ftp-master.debian.org/s3kr1t/push_morgue -Host ries-sync1 +Host dd-sync1 Hostname coccia.debian.org User dak IdentityFile /srv/ftp-master.debian.org/scripts/s3kr1t/ssh/push_dd1 -Host ries-sync2 +Host dd-sync2 Hostname coccia.debian.org User dak IdentityFile /srv/ftp-master.debian.org/scripts/s3kr1t/ssh/push_dd2 +Host dd-push-git + Hostname coccia.debian.org + User dak + IdentityFile /srv/ftp-master.debian.org/scripts/s3kr1t/ssh/push_coccia_dak + Host external-security Hostname chopin.debian.org User dak diff --git a/scripts/debian/sync-dd b/scripts/debian/sync-dd index 748c28c4..6c8f0f8e 100755 --- a/scripts/debian/sync-dd +++ b/scripts/debian/sync-dd @@ -27,16 +27,17 @@ export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars . $SCRIPTVARS usage() { - echo "usage: $0 sync|pool" - echo - echo "sync dd-accessible copy of the archive" - echo - echo "arguments:" - echo " lock: file used for locking" - echo " host1: hostname for syncing /srv/ftp-master.debian.org" - echo " host2: hostname for syncing /srv/ftp.debian.org" - echo " sync|pool: sync excludes ftp/, pool syncs ftp/ too" - exit ${1:-0} + echo "usage: $0 sync|code" + echo + echo "sync dd-accessible copy of the archive" + echo + echo "arguments:" + echo " lock: file used for locking" + echo " host1: hostname for syncing /srv/ftp-master.debian.org" + echo " host2: hostname for syncing /srv/ftp.debian.org" + echo " sync|code: sync syncs everything" + echo " code only syncs the dak/ directory (uses host1 value as target)" + exit ${1:-0} } if [ $# -ne 4 ]; then @@ -51,62 +52,71 @@ mode="${4}" # extra options for rsync of /srv/ftp-master.debian.org extra1="" -case "${mode}" in - pool|sync) - ;; - *) - echo "Unknown mode ${mode}." >&2 - exit 1 - ;; -esac - cleanup() { - rm -f "${lockfile}" + rm -f "${lockfile}" } trap cleanup EXIT TERM HUP INT QUIT # Also, NEVER use --delete-excluded! if lockfile -r3 "${lockfile}"; then - rsync -aH -B8192 \ - ${extra1} \ - --exclude "/.nobackup" \ - --exclude "/backup/*.xz" \ - --exclude "/backup/dump*" \ - --exclude "/build-queues/" \ - --exclude "/database/*.db" \ - --exclude ".da-backup.trace" \ - --exclude "/export/changelogs/tmp*/" \ - --exclude "/ftp" \ - --exclude "lost+found" \ - --exclude "/lock/" \ - --exclude "/mirror" \ - --exclude "/morgue/" \ - --exclude "/queue/bts_version_track/" \ - --exclude "/queue/unchecked/" \ - --exclude "/s3kr1t" \ - --exclude "/scripts/s3kr1t" \ - --exclude "/tmp/" \ - --exclude "/public/incoming.debian.org" \ - --delete --delete-after \ - --timeout 3600 \ - -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ - /srv/ftp-master.debian.org/ "${host1}:/srv/ftp-master.debian.org/" - # command for the remote side: - # command="rsync --server -lHogDtpre.iLsf -B8192 --timeout=3600 --delete-after . /srv/ftp-master.debian.org/" + case "${mode}" in + sync) + rsync -aH -B8192 \ + ${extra1} \ + --exclude "/.nobackup" \ + --exclude "/backup/*.xz" \ + --exclude "/backup/dump*" \ + --exclude "/build-queues/" \ + --exclude "/database/*.db" \ + --exclude ".da-backup.trace" \ + --exclude "/export/changelogs/tmp*/" \ + --exclude "/ftp" \ + --exclude "lost+found" \ + --exclude "/lock/" \ + --exclude "/mirror" \ + --exclude "/morgue/" \ + --exclude "/queue/bts_version_track/" \ + --exclude "/queue/unchecked/" \ + --exclude "/s3kr1t" \ + --exclude "/scripts/s3kr1t" \ + --exclude "/tmp/" \ + --exclude "/public/incoming.debian.org" \ + --delete --delete-after \ + --timeout 3600 \ + -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ + /srv/ftp-master.debian.org/ "${host1}:/srv/ftp-master.debian.org/" + # command for the remote side: + # command="rsync --server -lHogDtpre.iLsf -B8192 --timeout=3600 --delete-after . /srv/ftp-master.debian.org/" - rsync -aH -B8192 \ - --exclude "/.nobackup" \ - --exclude mirror \ - --exclude rsync/ \ - --exclude lost+found \ - --exclude .da-backup.trace \ - --exclude web-users/ \ - --delete --delete-after \ - --timeout 3600 \ - -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ - /srv/ftp.debian.org/ "${host2}:/srv/ftp.debian.org/" - # command for the remote side: - # command="rsync --server -lHogDtpre.iLsf -B8192 --timeout=3600 --delete-after . /srv/ftp.debian.org/" + rsync -aH -B8192 \ + --exclude "/.nobackup" \ + --exclude mirror \ + --exclude rsync/ \ + --exclude lost+found \ + --exclude .da-backup.trace \ + --exclude web-users/ \ + --delete --delete-after \ + --timeout 3600 \ + -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ + /srv/ftp.debian.org/ "${host2}:/srv/ftp.debian.org/" + # command for the remote side: + # command="rsync --server -lHogDtpre.iLsf -B8192 --timeout=3600 --delete-after . /srv/ftp.debian.org/" + ;; + code) + rsync -aH -B8192 \ + --exclude "/.nobackup" \ + --delete --delete-after \ + --timeout 3600 \ + -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ + /srv/ftp-master.debian.org/dak/ "${host1}:/srv/ftp-master.debian.org/dak/" + # command for the remote side: + # command="rsync --server -lHogDtpre.iLsf -B8192 --timeout=3600 --delete-after . /srv/ftp-master.debian.org/dak/" + ;; + *) + echo "Unknown mode ${mode}." >&2 + exit 1 + ;; + esac else echo "Couldn't get the lock, not syncing" exit 0 -- 2.39.2