echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
}
-# Cleanup actions
-function cleanup() {
- rm -f ${LOCK_DAILY}
- rm -f ${LOCK_ACCEPTED}
+# Remove daily lock
+function remove_daily_lock() {
+ rm -f $LOCK_DAILY
+}
+
+# Remove all locks
+function remove_all_locks() {
+ rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
}
# If we error out this one is called, *FOLLOWED* by cleanup above
fi
subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
- cat "${STAGEFILE}.log" | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org
+ if [ -r "${STAGEFILE}.log" ]; then
+ cat "${STAGEFILE}.log"
+ else
+ echo "file ${STAGEFILE}.log does not exist, sorry"
+ fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
}
########################################################################
########################################################################
# pushing merkels QA user, part one
-function merkel1() {
- log "Telling merkels QA user that we start dinstall"
- ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1
-}
-
-# Create the postgres dump files
-function pgdump_pre() {
- log "Creating pre-daily-cron-job backup of projectb database..."
- pg_dump projectb > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
-}
-
-function pgdump_post() {
- log "Creating post-daily-cron-job backup of projectb database..."
- cd $base/backup
- POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
- pg_dump projectb > $base/backup/dump_$POSTDUMP
- #pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
- ln -sf $base/backup/dump_$POSTDUMP current
- #ln -sf $base/backup/dumpall_$POSTDUMP currentall
-}
-
-# Load the dak-dev projectb
-function pgdakdev() {
- cd $base/backup
- echo "drop database projectb" | psql -p 5433 template1
- #cat currentall | psql -p 5433 template1
- createdb -p 5433 -T template1 projectb
- fgrep -v '\connect' current | psql -p 5433 projectb
+function qa1() {
+ log "Telling QA user that we start dinstall"
+ ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
}
# Updating various files
# Process (oldstable)-proposed-updates "NEW" queue
function punew_do() {
date -u -R >> REPORT
- dak process-policy $1 | tee -a REPORT | mail -e -s "NEW changes in $1" debian-release@lists.debian.org
+ dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" debian-release@lists.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
echo >> REPORT
}
function punew() {
# Lets check!
if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
# Yay, worked, lets copy around
- for dir in squeeze sid; do
+ for dir in wheezy sid; do
if [ -d dists/${dir}/ ]; then
cd dists/${dir}/main/i18n
rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
done
else
echo "ARRRR, bad guys, wrong files, ARRR"
- echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
+ echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
fi
else
echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
- echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
+ echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
fi
else
echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
- echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
+ echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
fi
}
}
function fingerprints() {
- log "Not updating fingerprints - scripts needs checking"
-
log "Updating fingerprints"
dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
To: <debian-project@lists.debian.org>
Subject: Debian Maintainers Keyring changes
Content-Type: text/plain; charset=utf-8
+X-Debian: DAK
MIME-Version: 1.0
The following changes to the debian-maintainers keyring have just been activated:
function packages() {
log "Generating Packages and Sources files"
cd $configdir
- GZIP='--rsyncable' ; export GZIP
- apt-ftparchive generate apt.conf
+ #apt-ftparchive generate apt.conf
+ dak generate-packages-sources
+ dak contents generate
}
function pdiff() {
mv -f ${FILENAME}.gz ${FILENAME}.old.gz
mv -f .${FILENAME}.new ${FILENAME}
rm -f ${FILENAME}.patch.gz
- zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip --rsyncable -9cfn - >${FILENAME}.patch.gz
+ zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
rm -f ${FILENAME}.old.gz
else
mv -f .${FILENAME}.new ${FILENAME}
fi
- gzip --rsyncable -9cfN ${FILENAME} >${FILENAME}.gz
+ gzip -9cfN ${FILENAME} >${FILENAME}.gz
rm -f ${FILENAME}
}
if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then
log "installing Maintainers ... "
mv -f .new-maintainers Maintainers
- gzip --rsyncable -9v <Maintainers >.new-maintainers.gz
+ gzip -9v <Maintainers >.new-maintainers.gz
mv -f .new-maintainers.gz Maintainers.gz
else
rm -f .new-maintainers
if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then
log "installing Uploaders ... "
mv -f .new-uploaders Uploaders
- gzip --rsyncable -9v <Uploaders >.new-uploaders.gz
+ gzip -9v <Uploaders >.new-uploaders.gz
mv -f .new-uploaders.gz Uploaders.gz
else
rm -f .new-uploaders
ARCHLIST=$(tempfile)
- log "Querying projectb..."
- echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql projectb -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
+ log "Querying postgres"
+ echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
includedirs () {
perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
cd $base/ftp
find ./dists -maxdepth 1 \! -type d
find ./dists \! -type d | grep "/source/"
- ) | sort -u | gzip --rsyncable -9 > source.list.gz
+ ) | sort -u | gzip -9 > source.list.gz
log "Generating arch lists"
cd $base/ftp
find ./dists -maxdepth 1 \! -type d
find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
- ) | sort -u | gzip --rsyncable -9 > arch-$a.list.gz
+ ) | sort -u | gzip -9 > arch-$a.list.gz
done
log "Generating suite lists"
suite_list () {
- printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t projectb
+ printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
- printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t projectb
+ printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
}
- printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At projectb |
+ printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
while read id suite; do
[ -e $base/ftp/dists/$suite ] || continue
(
done
)
suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
- ) | sort -u | gzip --rsyncable -9 > suite-${suite}.list.gz
+ ) | sort -u | gzip -9 > suite-${suite}.list.gz
done
log "Finding everything on the ftp site to generate sundries"
done
(cd $base/ftp/
- for dist in sid squeeze; do
- find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip --rsyncable -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
+ for dist in sid wheezy; do
+ find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
done
)
- (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-squeeze.list.gz) |
+ (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
sort -u | poolfirst > ../typical.files
rm -f $ARCHLIST
cd "$ftpdir"
${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
- ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n --rsyncable > ${md5list}.gz
+ ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
${bindir}/dsync-flist -q link-dups $dsynclist || true
}
echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
cd ${mirrordir}
- rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
+ rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
}
function expire() {
dak transitions -c -a
}
-function reports() {
- # Send a report on NEW/BYHAND packages
- log "Nagging ftpteam about NEW/BYHAND packages"
- dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
- # and one on crufty packages
- log "Sending information about crufty packages"
- dak cruft-report > $webdir/cruft-report-daily.txt
- dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
- cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
-}
-
function dm() {
log "Updating DM html page"
$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
dak bts-categorize
}
-function merkel2() {
- # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
- log "Trigger merkel/flotows projectb sync"
- ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
- # Also trigger flotow, the ftpmaster test box
- ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
-}
-
-function merkel3() {
- # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
- log "Trigger merkels dd accessible parts sync"
- ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
+function ddaccess() {
+ # Tell our dd accessible mirror to sync itself up. Including ftp dir.
+ log "Trigger dd accessible parts sync including ftp dir"
+ ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
}
function mirrorpush() {
log "Starting the mirrorpush"
- date -u > /srv/ftp-master.debian.org/web/mirrorstart
- echo "Using dak v1" >> /srv/ftp-master.debian.org/web/mirrorstart
- echo "Running on host $(hostname -f)" >> /srv/ftp-master.debian.org/web/mirrorstart
+ date -u > /srv/ftp.debian.org/web/mirrorstart
+ echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
+ echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
}
STAMP=$(date "+%Y%m%d%H%M")
mkdir -p ${scriptdir}/i18n/${STAMP}
cd ${scriptdir}/i18n/${STAMP}
- dak control-suite -l stable > lenny
- dak control-suite -l testing > squeeze
+ dak control-suite -l stable > squeeze
+ dak control-suite -l testing > wheezy
dak control-suite -l unstable > sid
echo "${STAMP}" > timestamp
gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
apt-ftparchive -q clean apt.conf
}
-function compress() {
- log "Compress old psql backups"
+function cleantransactions() {
+ log "Cleanup transaction ids older than 3 months"
cd $base/backup/
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
-
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
- while read dumpname; do
- echo "Compressing $dumpname"
- bzip2 -9fv "$dumpname"
- done
- find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
- while read dumpname; do
- echo "Compressing $dumpname"
- bzip2 -9fv "$dumpname"
- done
- finddup -l -d $base/backup
+ find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
}
function logstats() {
}
function maillogfile() {
- cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
+ cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
}
function renamelogfile() {
Action start: ${RIGHTNOW}
EOF
}
+
+# extract changelogs and stuff
+function changelogs() {
+ log "Extracting changelogs"
+ dak make-changelog -e
+ mkdir -p ${exportpublic}/changelogs
+ cd ${exportpublic}/changelogs
+ rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
+ sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &
+}