X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fdinstall.functions;h=b7679dcfa50edd7b59abf82813ce9a0499f4410a;hb=e13fd3c48245d48b0335649d294aacaee666bab5;hp=c5aeed131656094cdd76e318d19e91d6afb2f9a8;hpb=cf5d3c03acf3065f46ac19e67521f9aaf5ca5bad;p=dak.git diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index c5aeed13..b7679dcf 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -4,10 +4,14 @@ function ts() { echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)" } -# Cleanup actions -function cleanup() { - rm -f ${LOCK_DAILY} - rm -f ${LOCK_ACCEPTED} +# Remove daily lock +function remove_daily_lock() { + rm -f $LOCK_DAILY +} + +# Remove all locks +function remove_all_locks() { + rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW } # If we error out this one is called, *FOLLOWED* by cleanup above @@ -22,7 +26,11 @@ function onerror() { fi subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)" - cat "${STAGEFILE}.log" | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org + if [ -r "${STAGEFILE}.log" ]; then + cat "${STAGEFILE}.log" + else + echo "file ${STAGEFILE}.log does not exist, sorry" + fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org } ######################################################################## @@ -30,46 +38,11 @@ function onerror() { ######################################################################## # pushing merkels QA user, part one -function merkel1() { - log "Telling merkels QA user that we start dinstall" +function qa1() { + log "Telling QA user that we start dinstall" ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1 } -# Create the postgres dump files -function pgdump_pre() { - log "Creating pre-daily-cron-job backup of $PGDATABASE database..." - pg_dump > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S) -} - -function pgdump_post() { - log "Creating post-daily-cron-job backup of $PGDATABASE database..." - cd $base/backup - POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S) - pg_dump > $base/backup/dump_$POSTDUMP - #pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP - ln -sf $base/backup/dump_$POSTDUMP current - #ln -sf $base/backup/dumpall_$POSTDUMP currentall -} - -# Load the dak-dev projectb -function pgdakdev() { - # Make sure to unset any possible psql variables so we don't drop the wrong - # f****** database by accident - local PGDATABASE - unset PGDATABASE - local PGHOST - unset PGHOST - local PGPORT - unset PGPORT - local PGUSER - unset PGUSER - cd $base/backup - echo "drop database projectb" | psql -p 5434 template1 - #cat currentall | psql -p 5433 template1 - createdb -p 5434 -T template1 projectb - fgrep -v '\connect' current | psql -p 5434 projectb -} - # Updating various files function updates() { log "Updating Bugs docu, Mirror list and mailing-lists.txt" @@ -80,23 +53,6 @@ function updates() { $scriptsdir/update-pseudopackages.sh } -# Process (oldstable)-proposed-updates "NEW" queue -function punew_do() { - date -u -R >> REPORT - dak process-policy $1 | tee -a REPORT | mail -e -s "NEW changes in $1" debian-release@lists.debian.org - echo >> REPORT -} -function punew() { - log "Doing automated p-u-new processing" - cd "${queuedir}/p-u-new" - punew_do "$1" -} -function opunew() { - log "Doing automated o-p-u-new processing" - cd "${queuedir}/o-p-u-new" - punew_do "$1" -} - # The first i18n one, syncing new descriptions function i18n1() { log "Synchronizing i18n package descriptions" @@ -115,24 +71,24 @@ function i18n1() { # Lets check! if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then # Yay, worked, lets copy around - for dir in squeeze sid; do + for dir in wheezy sid; do if [ -d dists/${dir}/ ]; then cd dists/${dir}/main/i18n - rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/. + rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/. fi cd ${scriptdir}/i18nsync done else echo "ARRRR, bad guys, wrong files, ARRR" - echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org + echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org fi else echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR" - echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org + echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org fi else echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR." - echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org + echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org fi } @@ -152,8 +108,6 @@ function filelist() { } function fingerprints() { - log "Not updating fingerprints - scripts needs checking" - log "Updating fingerprints" dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg @@ -166,6 +120,7 @@ From: Debian FTP Masters To: Subject: Debian Maintainers Keyring changes Content-Type: text/plain; charset=utf-8 +X-Debian: DAK MIME-Version: 1.0 The following changes to the debian-maintainers keyring have just been activated: @@ -197,10 +152,8 @@ function mpfm() { function packages() { log "Generating Packages and Sources files" - cd $configdir - GZIP='--rsyncable' ; export GZIP - #apt-ftparchive generate apt.conf - dak generate-packages-sources + dak generate-packages-sources2 + dak contents generate } function pdiff() { @@ -209,6 +162,14 @@ function pdiff() { } function release() { + # XXX: disable once we can remove i18n/Index (#649314) + log "Generating i18n/Index" + ( + cd "$ftpdir/dists"; + for dist in testing unstable experimental proposed-updates testing-proposed-updates; do + $scriptsdir/generate-i18n-Index $dist; + done + ) log "Generating Release files" dak generate-releases } @@ -249,13 +210,13 @@ function mklslar() { mv -f ${FILENAME}.gz ${FILENAME}.old.gz mv -f .${FILENAME}.new ${FILENAME} rm -f ${FILENAME}.patch.gz - zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip --rsyncable -9cfn - >${FILENAME}.patch.gz + zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz rm -f ${FILENAME}.old.gz else mv -f .${FILENAME}.new ${FILENAME} fi - gzip --rsyncable -9cfN ${FILENAME} >${FILENAME}.gz + gzip -9cfN ${FILENAME} >${FILENAME}.gz rm -f ${FILENAME} } @@ -263,57 +224,18 @@ function mkmaintainers() { log 'Creating Maintainers index ... ' cd $indices - dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers | \ - sed -e "s/~[^ ]*\([ ]\)/\1/" | \ - awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-maintainers - - if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then - log "installing Maintainers ... " - mv -f .new-maintainers Maintainers - gzip --rsyncable -9v .new-maintainers.gz - mv -f .new-maintainers.gz Maintainers.gz - else - rm -f .new-maintainers - fi -} - -function mkuploaders() { - log 'Creating Uploaders index ... ' - - cd $indices - dak make-maintainers -u ${scriptdir}/masterfiles/pseudo-packages.maintainers | \ - sed -e "s/~[^ ]*\([ ]\)/\1/" | \ - awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-uploaders - - if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then - log "installing Uploaders ... " - mv -f .new-uploaders Uploaders - gzip --rsyncable -9v .new-uploaders.gz - mv -f .new-uploaders.gz Uploaders.gz - else - rm -f .new-uploaders - fi + dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers + gzip -9v --rsyncable Maintainers.gz + gzip -9v --rsyncable Uploaders.gz } function copyoverrides() { log 'Copying override files into public view ...' - for ofile in $copyoverrides ; do - cd $overridedir - chmod g+w override.$ofile - - cd $indices - - newofile=override.$ofile.gz - rm -f .newover-$ofile.gz - pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`" - if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then - log " installing new $newofile $pc" - mv -f .newover-$ofile.gz $newofile - chmod g+w $newofile - else - rm -f .newover-$ofile.gz - fi + for ofile in ${overridedir}/override.{squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do + bname=${ofile##*/} + gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz + chmod g+w ${indices}/${bname}.gz done } @@ -323,7 +245,7 @@ function mkfilesindices() { ARCHLIST=$(tempfile) - log "Querying $PGDATABASE..." + log "Querying postgres" echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST includedirs () { @@ -339,7 +261,7 @@ function mkfilesindices() { cd $base/ftp find ./dists -maxdepth 1 \! -type d find ./dists \! -type d | grep "/source/" - ) | sort -u | gzip --rsyncable -9 > source.list.gz + ) | sort -u | gzip -9 > source.list.gz log "Generating arch lists" @@ -351,7 +273,7 @@ function mkfilesindices() { cd $base/ftp find ./dists -maxdepth 1 \! -type d find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" - ) | sort -u | gzip --rsyncable -9 > arch-$a.list.gz + ) | sort -u | gzip -9 > arch-$a.list.gz done log "Generating suite lists" @@ -374,7 +296,7 @@ function mkfilesindices() { done ) suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,' - ) | sort -u | gzip --rsyncable -9 > suite-${suite}.list.gz + ) | sort -u | gzip -9 > suite-${suite}.list.gz done log "Finding everything on the ftp site to generate sundries" @@ -393,12 +315,12 @@ function mkfilesindices() { done (cd $base/ftp/ - for dist in sid squeeze; do - find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip --rsyncable -9 > $base/ftp/indices/files/components/translation-$dist.list.gz + for dist in sid wheezy; do + find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz done ) - (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-squeeze.list.gz) | + (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) | sort -u | poolfirst > ../typical.files rm -f $ARCHLIST @@ -413,7 +335,7 @@ function mkchecksums() { cd "$ftpdir" ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5 - ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n --rsyncable > ${md5list}.gz + ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz ${bindir}/dsync-flist -q link-dups $dsynclist || true } @@ -431,7 +353,7 @@ function mirror() { echo "Running on host: $(hostname -f)" >> ${TRACEFILE} echo "Archive serial: ${SERIAL}" >> ${TRACEFILE} cd ${mirrordir} - rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. . + rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. . } function expire() { @@ -446,17 +368,6 @@ function transitionsclean() { dak transitions -c -a } -function reports() { - # Send a report on NEW/BYHAND packages - log "Nagging ftpteam about NEW/BYHAND packages" - dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org - # and one on crufty packages - log "Sending information about crufty packages" - dak cruft-report > $webdir/cruft-report-daily.txt - dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt - cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org -} - function dm() { log "Updating DM html page" $scriptsdir/dm-monitor >$webdir/dm-uploaders.html @@ -467,21 +378,60 @@ function bts() { dak bts-categorize } -function merkel2() { - # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached - log "Trigger merkel/flotows $PGDATABASE sync" - ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1 - # Also trigger flotow, the ftpmaster test box - ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1 -} - -function merkel3() { - # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached - log "Trigger merkels dd accessible parts sync" - ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1 +function ddaccess() { + # Tell our dd accessible mirror to sync itself up. Including ftp dir. + log "Trigger dd accessible parts sync including ftp dir" + ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool } function mirrorpush() { + log "Checking the public archive copy" + cd ${mirrordir}/dists + + broken=0 + for release in $(find . -name "InRelease"); do + echo "Processing: ${release}" + subdir=${release%/InRelease} + while read SHASUM SIZE NAME; do + if ! [ -f "${subdir}/${NAME}" ]; then + bname=$(basename ${NAME}) + if [[ "${bname}" =~ ^(Packages|Sources|Translation-[a-zA-Z_]+)$ ]]; then + # We don't keep unpacked files, don't check for their existance. + # We might want to go and check their unpacked shasum, but right now + # I don't care. I believe it should be enough if all the packed shasums + # match. + continue + fi + broken=$(( broken + 1 )) + echo "File ${subdir}/${NAME} is missing" + continue + fi + + # We do have symlinks in the tree (see the contents files currently). + # So we use "readlink -f" to check the size of the target, as thats basically + # what gen-releases does + fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}")) + if [ ${fsize} -ne ${SIZE} ]; then + broken=$(( broken + 1 )) + echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}" + continue + fi + + fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}")) + fshasum=${fshasum%% *} + if [ "${fshasum}" != "${SHASUM}" ]; then + broken=$(( broken + 1 )) + echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}" + continue + fi + done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d') + done + + if [ $broken -gt 0 ]; then + log_error "Trouble with the public mirror, found ${broken} errors" + return 21 + fi + log "Starting the mirrorpush" date -u > /srv/ftp.debian.org/web/mirrorstart echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart @@ -494,9 +444,12 @@ function i18n2() { STAMP=$(date "+%Y%m%d%H%M") mkdir -p ${scriptdir}/i18n/${STAMP} cd ${scriptdir}/i18n/${STAMP} - dak control-suite -l stable > lenny - dak control-suite -l testing > squeeze - dak control-suite -l unstable > sid + for suite in stable testing unstable; do + codename=$(dak admin s show ${suite}|grep '^Codename') + codename=${codename##* } + echo "Codename is ${codename}" + dak control-suite -l ${suite} >${codename} + done echo "${STAMP}" > timestamp gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp rm -f md5sum @@ -523,22 +476,10 @@ function aptftpcleanup() { apt-ftparchive -q clean apt.conf } -function compress() { - log "Compress old psql backups" +function cleantransactions() { + log "Cleanup transaction ids older than 3 months" cd $base/backup/ - find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm - - find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 | - while read dumpname; do - echo "Compressing $dumpname" - bzip2 -9fv "$dumpname" - done - find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin +720 | - while read dumpname; do - echo "Compressing $dumpname" - bzip2 -9fv "$dumpname" - done - finddup -l -d $base/backup + find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm } function logstats() { @@ -552,7 +493,7 @@ function savetimestamp() { } function maillogfile() { - cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org + cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org } function renamelogfile() { @@ -612,4 +553,38 @@ function changelogs() { mkdir -p ${exportpublic}/changelogs cd ${exportpublic}/changelogs rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. . -} + sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 & +} + +function gitpdiff() { + # Might be that we want to change this to have more than one git repository. + # Advantage of one is that we do not need much space in terms of storage in git itself, + # git gc is pretty good on our input. + # But it might be faster. Well, lets test. + log "Adjusting the git tree for pdiffs" + cd ${dbdir}/git/git/ + + # The regex needs the architectures seperated with \| + garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q') + + # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we + # want to work with. + # Also, we only want contents, packages and sources. + for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do + log "${file}" + basen=${file%%.gz}; + basen=${basen##../}; + dir=${basen%/*}; + mkdir -p $dir; + zcat $file > $basen; + done + + # Second, add all there is into git + cd dists + git add . + # Maybe we want to make this the same for tag and commit? But well, shouldn't matter + COMD=$(date -Is) + TAGD=$(date +%Y-%m-%d-%H-%M) + git commit -m "Commit of ${COMD}" + git tag "${TAGD}" + }