X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fdinstall.functions;h=bd57a3a8de757a1b193e034f891add2c3a3b8cbb;hb=33233a41d742249ceb0c816c15f2850970ba3634;hp=535f109fa784b8e16b9c1fd0c32a3321073b6490;hpb=e92f1f90ce6541d49eb69051257cf513a20be518;p=dak.git diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index 535f109f..bd57a3a8 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -9,9 +9,14 @@ function remove_daily_lock() { rm -f $LOCK_DAILY } +# Remove changelog lock +function remove_changelog_lock() { + rm -f $LOCK_CHANGELOG +} + # Remove all locks function remove_all_locks() { - rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW + rm -f $LOCK_DAILY $LOCK_ACCEPTED } # If we error out this one is called, *FOLLOWED* by cleanup above @@ -71,7 +76,7 @@ function i18n1() { # Lets check! if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then # Yay, worked, lets copy around - for dir in wheezy sid; do + for dir in jessie sid; do if [ -d dists/${dir}/ ]; then cd dists/${dir}/main/i18n rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/. @@ -146,14 +151,23 @@ function overrides() { } function mpfm() { + local archiveroot + log "Generating package / file mapping" - dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2 + for archive in "${public_archives[@]}"; do + archiveroot="$(get_archiveroot "${archive}")" + dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2" + done } function packages() { log "Generating Packages and Sources files" - dak generate-packages-sources2 - dak contents generate + for archive in "${public_archives[@]}"; do + log " Generating Packages/Sources for ${archive}" + dak generate-packages-sources2 -a "${archive}" + log " Generating Contents for ${archive}" + dak contents generate -a "${archive}" + done } function pdiff() { @@ -162,60 +176,62 @@ function pdiff() { } function release() { - # XXX: disable once we can remove i18n/Index (#649314) - log "Generating i18n/Index" - ( - cd "$ftpdir/dists"; - for dist in testing unstable experimental proposed-updates testing-proposed-updates; do - $scriptsdir/generate-i18n-Index $dist; - done - ) log "Generating Release files" - dak generate-releases + for archive in "${public_archives[@]}"; do + dak generate-releases -a "${archive}" + done } function dakcleanup() { log "Cleanup old packages/files" dak clean-suites -m 10000 - dak clean-queues -} - -function buildd_dir() { - # Rebuilt the buildd dir to avoid long times of 403 - log "Regenerating the buildd incoming dir" - STAMP=$(date "+%Y%m%d%H%M") - make_buildd_dir + dak clean-queues -i "$unchecked" } function mklslar() { - cd $ftpdir + local archiveroot + local FILENAME=ls-lR - FILENAME=ls-lR + for archive in "${public_archives[@]}"; do + archiveroot="$(get_archiveroot "${archive}")" + cd "${archiveroot}" - log "Removing any core files ..." - find -type f -name core -print -delete + log "Removing any core files ..." + find -type f -name core -print -delete - log "Checking symlinks ..." - symlinks -rd . + log "Checking symlinks ..." + symlinks -rd . - log "Creating recursive directory listing ... " - rm -f ${FILENAME}.gz - TZ=UTC ls -lR | gzip -9c --rsyncable > ${FILENAME}.gz + log "Creating recursive directory listing ... " + rm -f ${FILENAME}.gz + TZ=UTC ls -lR | gzip -9c --rsyncable > ${FILENAME}.gz + done } function mkmaintainers() { + local archiveroot + local indices + log 'Creating Maintainers index ... ' - cd $indices - dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers - gzip -9v --rsyncable Maintainers.gz - gzip -9v --rsyncable Uploaders.gz + for archive in "${public_archives[@]}"; do + archiveroot="$(get_archiveroot "${archive}")" + indices="${archiveroot}/indices" + if ! [ -d "${indices}" ]; then + mkdir "${indices}" + fi + cd "${indices}" + + dak make-maintainers -a "${archive}" ${scriptdir}/masterfiles/pseudo-packages.maintainers + gzip -9v --rsyncable Maintainers.gz + gzip -9v --rsyncable Uploaders.gz + done } function copyoverrides() { log 'Copying override files into public view ...' - for ofile in ${overridedir}/override.{squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do + for ofile in ${overridedir}/override.{squeeze,wheezy,jessie,sid}.{,extra.}{main,contrib,non-free}*; do bname=${ofile##*/} gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz chmod g+w ${indices}/${bname}.gz @@ -223,14 +239,27 @@ function copyoverrides() { } function mkfilesindices() { + set +o pipefail umask 002 cd $base/ftp/indices/files/components ARCHLIST=$(tempfile) log "Querying postgres" - local query='SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' - psql -At -c "$query" | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST + local query=" + SELECT './pool/' || c.name || '/' || f.filename AS path, a.arch_string AS arch_string + FROM files f + JOIN files_archive_map af ON f.id = af.file_id + JOIN component c ON af.component_id = c.id + JOIN archive ON af.archive_id = archive.id + LEFT OUTER JOIN + (binaries b + JOIN architecture a ON b.architecture = a.id) + ON f.id = b.file + WHERE archive.name = 'ftp-master' + ORDER BY path, arch_string + " + psql -At -c "$query" >$ARCHLIST includedirs () { perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }' @@ -263,11 +292,42 @@ function mkfilesindices() { log "Generating suite lists" suite_list () { + local suite_id="$(printf %d $1)" local query - query="$(printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)' $1 $1)" + query=" + SELECT DISTINCT './pool/' || c.name || '/' || f.filename + FROM + (SELECT sa.source AS source + FROM src_associations sa + WHERE sa.suite = $suite_id + UNION + SELECT esr.src_id + FROM extra_src_references esr + JOIN bin_associations ba ON esr.bin_id = ba.bin + WHERE ba.suite = $suite_id + UNION + SELECT b.source AS source + FROM bin_associations ba + JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s + JOIN dsc_files df ON s.source = df.source + JOIN files f ON df.file = f.id + JOIN files_archive_map af ON f.id = af.file_id + JOIN component c ON af.component_id = c.id + JOIN archive ON af.archive_id = archive.id + WHERE archive.name = 'ftp-master' + " psql -F' ' -A -t -c "$query" - query="$(printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d' $1)" + query=" + SELECT './pool/' || c.name || '/' || f.filename + FROM bin_associations ba + JOIN binaries b ON ba.bin = b.id + JOIN files f ON b.file = f.id + JOIN files_archive_map af ON f.id = af.file_id + JOIN component c ON af.component_id = c.id + JOIN archive ON af.archive_id = archive.id + WHERE ba.suite = $suite_id AND archive.name = 'ftp-master' + " psql -F' ' -A -t -c "$query" } @@ -282,7 +342,7 @@ function mkfilesindices() { [ "$(readlink $distdir)" != "$distname" ] || echo $distdir done ) - suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,' + suite_list $id ) | sort -u | gzip -9 > suite-${suite}.list.gz done @@ -302,31 +362,39 @@ function mkfilesindices() { done (cd $base/ftp/ - for dist in sid wheezy; do + for dist in sid jessie; do find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz done ) - (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) | + (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz) | sort -u | poolfirst > ../typical.files rm -f $ARCHLIST log "Done!" + set -o pipefail } function mkchecksums() { - dsynclist=$dbdir/dsync.list - md5list=$indices/md5sums + local archiveroot dsynclist md5list + + for archive in "${public_archives[@]}"; do + archiveroot="$(get_archiveroot "${archive}")" + dsynclist=$dbdir/dsync.${archive}.list + md5list=${archiveroot}/indices/md5sums - log -n "Creating md5 / dsync index file ... " + log -n "Creating md5 / dsync index file for ${archive}... " - cd "$ftpdir" - ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5 - ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz - ${bindir}/dsync-flist -q link-dups $dsynclist || true + cd "$archiveroot" + ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5 + ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz + ${bindir}/dsync-flist -q link-dups $dsynclist || true + done } function mirror() { + local archiveroot + log "Regenerating \"public\" mirror/ hardlink fun" DATE_SERIAL=$(date +"%Y%m%d01") FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} ) @@ -339,8 +407,16 @@ function mirror() { echo "Using dak v1" >> ${TRACEFILE} echo "Running on host: $(hostname -f)" >> ${TRACEFILE} echo "Archive serial: ${SERIAL}" >> ${TRACEFILE} - cd ${mirrordir} - rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. . + + # Ugly "hack", but hey, it does what we want. + cp ${TRACEFILE} ${TRACEFILE_BDO} + + for archive in "${public_archives[@]}"; do + archiveroot="$(get_archiveroot "${archive}")" + mirrordir="${archiveroot}/../mirror" + cd ${mirrordir} + rsync -aH --link-dest ${archiveroot} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${archiveroot}/. . + done } function expire() { @@ -356,8 +432,8 @@ function transitionsclean() { } function dm() { - log "Updating DM html page" - $scriptsdir/dm-monitor >$webdir/dm-uploaders.html + log "Updating DM permissions page" + dak acl export-per-source dm >$exportdir/dm.txt } function bts() { @@ -368,7 +444,7 @@ function bts() { function ddaccess() { # Tell our dd accessible mirror to sync itself up. Including ftp dir. log "Trigger dd accessible parts sync including ftp dir" - ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool + ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 pool } function mirrorpush() { @@ -424,6 +500,12 @@ function mirrorpush() { echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 & + sudo -H -u archvsync /home/archvsync/runmirrors -a backports > ~dak/runmirrorsbpo.log 2>&1 & +} + +function mirrorpush-backports() { + log "Syncing backports mirror" + sudo -u backports /home/backports/bin/update-archive } function i18n2() { @@ -438,7 +520,7 @@ function i18n2() { dak control-suite -l ${suite} >${codename} done echo "${STAMP}" > timestamp - gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp + gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --detach-sign -o timestamp.gpg timestamp rm -f md5sum md5sum * > md5sum cd ${webdir}/ @@ -457,12 +539,6 @@ function stats() { dak stats pkg-nums > $webdir/pkg-nums } -function aptftpcleanup() { - log "Clean up apt-ftparchive's databases" - cd $configdir - apt-ftparchive -q clean apt.conf -} - function cleantransactions() { log "Cleanup transaction ids older than 3 months" cd $base/backup/ @@ -511,13 +587,6 @@ function process_unchecked() { sync_debbugs } -# do a run of newstage only before dinstall is on. -function newstage() { - log "Processing the newstage queue" - UNCHECKED_WITHOUT_LOCK="-p" - do_newstage -} - # Function to update a "statefile" telling people what we are doing # (more or less). # @@ -535,12 +604,23 @@ EOF # extract changelogs and stuff function changelogs() { - log "Extracting changelogs" - dak make-changelog -e - mkdir -p ${exportpublic}/changelogs - cd ${exportpublic}/changelogs - rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. . - sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 & + if lockfile -r3 $LOCK_CHANGELOG; then + log "Extracting changelogs" + dak make-changelog -e -a ftp-master + [ -f ${exportdir}/changelogs/filelist.yaml ] && xz -f ${exportdir}/changelogs/filelist.yaml + mkdir -p ${exportpublic}/changelogs + cd ${exportpublic}/changelogs + rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. . + sudo -H -u staticsync /usr/local/bin/static-update-component metadata.ftp-master.debian.org >/dev/null 2>&1 & + + dak make-changelog -e -a backports + [ -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml ] && xz -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml + mkdir -p /srv/backports-master.debian.org/rsync/export/changelogs + cd /srv/backports-master.debian.org/rsync/export/changelogs + rsync -aHW --delete --delete-after --ignore-errors /srv/backports-master.debian.org/export/changelogs/. . + remove_changelog_lock + trap remove_changelog_lock EXIT TERM HUP INT QUIT + fi } function gitpdiff() {