X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fdinstall.functions;h=2ee713c98347a9ddbd65f4ce459db50d7dabbd04;hb=b2d06df451fcf23b97f572a4253ccfbd26d58f45;hp=9aadf836059925e76e56db028784318a341ce6e1;hpb=705fde638084f64c52b161a7bf8703716772ca3f;p=dak.git diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index 9aadf836..2ee713c9 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -1,10 +1,7 @@ # -*- mode:sh -*- -# Timestamp. Used for dinstall stat graphs -function ts() { - echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)" -} # Remove daily lock +# FIXME: Dead code when switched to cronscript function remove_daily_lock() { rm -f $LOCK_DAILY } @@ -50,7 +47,7 @@ function onerror() { } ######################################################################## -# the actual dinstall functions follow # +# the actual functions follow # ######################################################################## # pushing merkels QA user, part one @@ -59,14 +56,48 @@ function qa1() { ssh -n -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1 } +function mirrorlists() { + local mldir="${scriptdir}/mirrorlist" + local masterlist=${mldir}/Mirrors.masterlist + + cd ${mldir} + + [[ -f ${HOME}/.cvspass ]] || touch ${HOME}/.cvspass + + grep -q "anonscm.debian.org:/cvs/webwml" ~/.cvspass || \ + echo ":pserver:anonymous@anonscm.debian.org:/cvs/webwml A" >> ${HOME}/.cvspass + + cvs update + + if [[ ! -f ${ftpdir}/README.mirrors.html ]] || [[ ${masterlist} -nt ${ftpdir}/README.mirrors.html ]]; then + rm -f ${ftpdir}/README.mirrors.{html,txt} + ${mldir}/mirror_list.pl -m ${masterlist} -t html > ${ftpdir}/README.mirrors.html + ${mldir}/mirror_list.pl -m ${masterlist} -t text > ${ftpdir}/README.mirrors.txt + log Updated archive version of mirrors file + fi +} + +function mailingliststxt() { + cd ${ftpdir}/doc + + log "Updating archive version of mailing-lists.txt" + wget ${wgetopts} https://www.debian.org/misc/mailing-lists.txt +} + +function pseudopackages() { + cd ${scriptdir}/masterfiles + + log "Updating archive version of pseudo-packages" + for file in maintainers description; do + wget ${wgetopts} https://bugs.debian.org/pseudopackages/pseudo-packages.${file} + done +} + # Updating various files -function updates() { - log "Updating Bugs docu, Mirror list and mailing-lists.txt" +function bugdoctxt() { + log "Updating Bugs docu" cd $configdir $scriptsdir/update-bugdoctxt - $scriptsdir/update-mirrorlists - $scriptsdir/update-mailingliststxt - $scriptsdir/update-pseudopackages.sh } # The first i18n one, syncing new descriptions @@ -192,13 +223,13 @@ function mpfm() { log "Generating package / file mapping" for archive in "${public_archives[@]}"; do + log " archive: ${archive}" archiveroot="$(get_archiveroot "${archive}")" dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2" done } function packages() { - log "Generating Packages and Sources files" for archive in "${public_archives[@]}"; do log " Generating Packages/Sources for ${archive}" dak generate-packages-sources2 -a "${archive}" @@ -215,6 +246,7 @@ function pdiff() { function release() { log "Generating Release files" for archive in "${public_archives[@]}"; do + log " archive: ${archive}" dak generate-releases -a "${archive}" done } @@ -321,11 +353,11 @@ function mkfilesindices() { ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u) for a in $ARCHES; do (sed -n "s/|$a$//p" $ARCHLIST - sed -n 's/|all$//p' $ARCHLIST + sed -n 's/|all$//p' $ARCHLIST - cd $base/ftp - find ./dists -maxdepth 1 \! -type d - find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" + cd $base/ftp + find ./dists -maxdepth 1 \! -type d + find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" ) | sort -u | gzip -9 > arch-$a.list.gz done @@ -372,43 +404,43 @@ function mkfilesindices() { } psql -F' ' -At -c "SELECT id, suite_name FROM suite" | - while read id suite; do - [ -e $base/ftp/dists/$suite ] || continue - ( - (cd $base/ftp - distname=$(cd dists; readlink $suite || echo $suite) - find ./dists/$distname \! -type d - for distdir in ./dists/*; do - [ "$(readlink $distdir)" != "$distname" ] || echo $distdir - done - ) - suite_list $id - ) | sort -u | gzip -9 > suite-${suite}.list.gz - done + while read id suite; do + [ -e $base/ftp/dists/$suite ] || continue + ( + (cd $base/ftp + distname=$(cd dists; readlink $suite || echo $suite) + find ./dists/$distname \! -type d + for distdir in ./dists/*; do + [ "$(readlink $distdir)" != "$distname" ] || echo $distdir + done + ) + suite_list $id + ) | sort -u | gzip -9 > suite-${suite}.list.gz + done log "Finding everything on the ftp site to generate sundries" (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST rm -f sundries.list zcat *.list.gz | cat - *.list | sort -u | - diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list + diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list log "Generating files list" for a in $ARCHES; do (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) | - cat - sundries.list dists.list project.list docs.list indices.list | - sort -u | poolfirst > ../arch-$a.files + cat - sundries.list dists.list project.list docs.list indices.list | + sort -u | poolfirst > ../arch-$a.files done (cd $base/ftp/ - for dist in sid jessie stretch; do - find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz - done + for dist in sid jessie stretch; do + find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz + done ) (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz ; zcat translation-stretch.list.gz) | - sort -u | poolfirst > ../typical.files + sort -u | poolfirst > ../typical.files rm -f $ARCHLIST log "Done!" @@ -482,12 +514,6 @@ function bts() { sudo -u dak-unpriv dak bts-categorize } -function ddaccess() { - # Tell our dd accessible mirror to sync itself up. Including ftp dir. - log "Trigger dd accessible parts sync including ftp dir" - ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync -} - function mirrorpush() { log "Checking the public archive copies..." @@ -615,7 +641,6 @@ function logstats() { # save timestamp when we start function savetimestamp() { - NOW=`date "+%Y.%m.%d-%H:%M:%S"` echo ${NOW} > "${dbdir}/dinstallstart" } @@ -663,6 +688,7 @@ function changelogs() { fi } +# Generate a list of extra mirror files, sha256sum em and sign that function signotherfiles() { log "Signing extra mirror files" @@ -676,8 +702,23 @@ function signotherfiles() { cd ${archiveroot} rm -f extrafiles - sha256sum $(find * -type f | egrep -v '(pool|i18n|dep11|source)/|Contents-.*\.(gz|diff)|installer|binary-|(In)?Release(.gpg)?|\.changes') > ${TMPLO} + sha256sum $(find * -type f | egrep -v '(pool|i18n|dep11|source)/|Contents-.*\.(gz|diff)|installer|binary-|(In)?Release(.gpg)?|\.changes' | sort) > ${TMPLO} gpg --no-options --batch --no-tty --armour --personal-digest-preferences=SHA256 --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --clearsign --output ${archiveroot}/extrafiles ${TMPLO} rm -f ${TMPLO} done } + +function startup() { + touch "${DINSTALLSTART}" + ts "startup" + lockfile -l 3600 "${LOCK_DAILY}" + trap onerror ERR + touch "${LOCK_BRITNEY}" + TMPFILES="${TMPFILES} ${LOCK_DAILY} ${LOCK_BRITNEY}" +} + +function postcronscript() { + logstats ${LOGFILE} + state "all done" + touch "${DINSTALLEND}" +} \ No newline at end of file