X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fdinstall.functions;h=f4a1ef3e5460552c7514c9030d62bc9cac41a944;hb=deb1e850a2fb83b0eff6d0761ea4a4aafb406112;hp=6e28b815f74cb6f7049a012abbb8d932a5c8f2bf;hpb=fe2e3f8f2d030fd8c6f495693fa6a2d4a9a6118c;p=dak.git diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index 6e28b815..f4a1ef3e 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -1,10 +1,7 @@ # -*- mode:sh -*- -# Timestamp. Used for dinstall stat graphs -function ts() { - echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)" -} # Remove daily lock +# FIXME: Dead code when switched to cronscript function remove_daily_lock() { rm -f $LOCK_DAILY } @@ -50,7 +47,7 @@ function onerror() { } ######################################################################## -# the actual dinstall functions follow # +# the actual functions follow # ######################################################################## # pushing merkels QA user, part one @@ -192,13 +189,13 @@ function mpfm() { log "Generating package / file mapping" for archive in "${public_archives[@]}"; do + log " archive: ${archive}" archiveroot="$(get_archiveroot "${archive}")" dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2" done } function packages() { - log "Generating Packages and Sources files" for archive in "${public_archives[@]}"; do log " Generating Packages/Sources for ${archive}" dak generate-packages-sources2 -a "${archive}" @@ -215,6 +212,7 @@ function pdiff() { function release() { log "Generating Release files" for archive in "${public_archives[@]}"; do + log " archive: ${archive}" dak generate-releases -a "${archive}" done } @@ -321,11 +319,11 @@ function mkfilesindices() { ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u) for a in $ARCHES; do (sed -n "s/|$a$//p" $ARCHLIST - sed -n 's/|all$//p' $ARCHLIST + sed -n 's/|all$//p' $ARCHLIST - cd $base/ftp - find ./dists -maxdepth 1 \! -type d - find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" + cd $base/ftp + find ./dists -maxdepth 1 \! -type d + find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" ) | sort -u | gzip -9 > arch-$a.list.gz done @@ -372,43 +370,43 @@ function mkfilesindices() { } psql -F' ' -At -c "SELECT id, suite_name FROM suite" | - while read id suite; do - [ -e $base/ftp/dists/$suite ] || continue - ( - (cd $base/ftp - distname=$(cd dists; readlink $suite || echo $suite) - find ./dists/$distname \! -type d - for distdir in ./dists/*; do - [ "$(readlink $distdir)" != "$distname" ] || echo $distdir - done - ) - suite_list $id - ) | sort -u | gzip -9 > suite-${suite}.list.gz - done + while read id suite; do + [ -e $base/ftp/dists/$suite ] || continue + ( + (cd $base/ftp + distname=$(cd dists; readlink $suite || echo $suite) + find ./dists/$distname \! -type d + for distdir in ./dists/*; do + [ "$(readlink $distdir)" != "$distname" ] || echo $distdir + done + ) + suite_list $id + ) | sort -u | gzip -9 > suite-${suite}.list.gz + done log "Finding everything on the ftp site to generate sundries" (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST rm -f sundries.list zcat *.list.gz | cat - *.list | sort -u | - diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list + diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list log "Generating files list" for a in $ARCHES; do (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) | - cat - sundries.list dists.list project.list docs.list indices.list | - sort -u | poolfirst > ../arch-$a.files + cat - sundries.list dists.list project.list docs.list indices.list | + sort -u | poolfirst > ../arch-$a.files done (cd $base/ftp/ - for dist in sid jessie stretch; do - find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz - done + for dist in sid jessie stretch; do + find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz + done ) (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz ; zcat translation-stretch.list.gz) | - sort -u | poolfirst > ../typical.files + sort -u | poolfirst > ../typical.files rm -f $ARCHLIST log "Done!" @@ -482,12 +480,6 @@ function bts() { sudo -u dak-unpriv dak bts-categorize } -function ddaccess() { - # Tell our dd accessible mirror to sync itself up. Including ftp dir. - log "Trigger dd accessible parts sync including ftp dir" - ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync -} - function mirrorpush() { log "Checking the public archive copies..." @@ -615,7 +607,6 @@ function logstats() { # save timestamp when we start function savetimestamp() { - NOW=`date "+%Y.%m.%d-%H:%M:%S"` echo ${NOW} > "${dbdir}/dinstallstart" } @@ -663,35 +654,31 @@ function changelogs() { fi } -function gitpdiff() { - # Might be that we want to change this to have more than one git repository. - # Advantage of one is that we do not need much space in terms of storage in git itself, - # git gc is pretty good on our input. - # But it might be faster. Well, lets test. - log "Adjusting the git tree for pdiffs" - cd ${dbdir}/git/git/ - - # The regex needs the architectures seperated with \| - garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q') - - # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we - # want to work with. - # Also, we only want contents, packages and sources. - for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do - log "${file}" - basen=${file%%.gz}; - basen=${basen##../}; - dir=${basen%/*}; - mkdir -p $dir; - zcat $file > $basen; +# Generate a list of extra mirror files, sha256sum em and sign that +function signotherfiles() { + log "Signing extra mirror files" + + local archiveroot + + for archive in "${public_archives[@]}"; do + log "... archive: ${archive}" + archiveroot="$(get_archiveroot "${archive}")" + local TMPLO=$( mktemp -p ${TMPDIR} ) + trap "rm -f ${TMPLO}" ERR EXIT TERM HUP INT QUIT + + cd ${archiveroot} + rm -f extrafiles + sha256sum $(find * -type f | egrep -v '(pool|i18n|dep11|source)/|Contents-.*\.(gz|diff)|installer|binary-|(In)?Release(.gpg)?|\.changes' | sort) > ${TMPLO} + gpg --no-options --batch --no-tty --armour --personal-digest-preferences=SHA256 --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --clearsign --output ${archiveroot}/extrafiles ${TMPLO} + rm -f ${TMPLO} done +} - # Second, add all there is into git - cd dists - git add . - # Maybe we want to make this the same for tag and commit? But well, shouldn't matter - COMD=$(date -Is) - TAGD=$(date +%Y-%m-%d-%H-%M) - git commit -m "Commit of ${COMD}" - git tag "${TAGD}" - } +function startup() { + touch "${DINSTALLSTART}" + ts "startup" + lockfile -l 3600 "${LOCK_DAILY}" + trap onerror ERR + touch "${LOCK_BRITNEY}" + TMPFILES="${TMPFILES} ${LOCK_DAILY} ${LOCK_BRITNEY}" +}