# -*- mode:sh -*- # Timestamp. Used for dinstall stat graphs function ts() { echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)" } # Remove daily lock function remove_daily_lock() { rm -f $LOCK_DAILY } # Remove changelog lock function remove_changelog_lock() { rm -f $LOCK_CHANGELOG } # Remove all locks function remove_all_locks() { rm -f $LOCK_DAILY $LOCK_ACCEPTED } # If we error out this one is called, *FOLLOWED* by cleanup above function onerror() { ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S") subject="ATTENTION ATTENTION!" if [ "${error}" = "false" ]; then subject="${subject} (continued)" else subject="${subject} (interrupted)" fi subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)" if [ -r "${STAGEFILE}.log" ]; then cat "${STAGEFILE}.log" else echo "file ${STAGEFILE}.log does not exist, sorry" fi | mail -s "${subject}" -a "X-Debian: DAK" -a "From: Debian FTP Masters " cron@ftp-master.debian.org } ######################################################################## # the actual dinstall functions follow # ######################################################################## # pushing merkels QA user, part one function qa1() { log "Telling QA user that we start dinstall" ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1 } # Updating various files function updates() { log "Updating Bugs docu, Mirror list and mailing-lists.txt" cd $configdir $scriptsdir/update-bugdoctxt $scriptsdir/update-mirrorlists $scriptsdir/update-mailingliststxt $scriptsdir/update-pseudopackages.sh } # The first i18n one, syncing new descriptions function i18n1() { log "Synchronizing i18n package descriptions" # First sync their newest data cd ${scriptdir}/i18nsync rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true # Now check if we still know about the packages for which they created the files # is the timestamp signed by us? if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then # now read it. As its signed by us we are sure the content is what we expect, no need # to do more here. And we only test -d a directory on it anyway. TSTAMP=$(cat timestamp) # do we have the dir still? if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then # Lets check! if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then # Yay, worked, lets copy around for dir in stretch sid; do if [ -d dists/${dir}/ ]; then cd dists/${dir}/main/i18n rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/. fi cd ${scriptdir}/i18nsync done else echo "ARRRR, bad guys, wrong files, ARRR" echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters " debian-l10n-devel@lists.alioth.debian.org fi else echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR" echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." -a "From: Debian FTP Masters " debian-l10n-devel@lists.alioth.debian.org fi else echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR." echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." -a "From: Debian FTP Masters " debian-l10n-devel@lists.alioth.debian.org fi } function cruft() { log "Checking for cruft in overrides" dak check-overrides } function dominate() { log "Removing obsolete source and binary associations" dak dominate } function filelist() { log "Generating file lists for apt-ftparchive" dak generate-filelist } function fingerprints() { log "Updating fingerprints" dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg OUTFILE=$(mktemp) dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}" if [ -s "${OUTFILE}" ]; then /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org < To: Subject: Debian Maintainers Keyring changes Content-Type: text/plain; charset=utf-8 X-Debian: DAK MIME-Version: 1.0 The following changes to the debian-maintainers keyring have just been activated: $(cat $OUTFILE) Debian distribution maintenance software, on behalf of the Keyring maintainers EOF fi rm -f "$OUTFILE" } function overrides() { log "Writing overrides into text files" cd $overridedir dak make-overrides } function mpfm() { local archiveroot log "Generating package / file mapping" for archive in "${public_archives[@]}"; do archiveroot="$(get_archiveroot "${archive}")" dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2" done } function packages() { log "Generating Packages and Sources files" for archive in "${public_archives[@]}"; do log " Generating Packages/Sources for ${archive}" dak generate-packages-sources2 -a "${archive}" log " Generating Contents for ${archive}" dak contents generate -a "${archive}" done } function pdiff() { log "Generating pdiff files" dak generate-index-diffs } function release() { log "Generating Release files" for archive in "${public_archives[@]}"; do dak generate-releases -a "${archive}" done } function dakcleanup() { log "Cleanup old packages/files" dak clean-suites -m 10000 dak clean-queues -i "$unchecked" } function mklslar() { local archiveroot local FILENAME=ls-lR for archive in "${public_archives[@]}"; do archiveroot="$(get_archiveroot "${archive}")" cd "${archiveroot}" log "Removing any core files ..." find -type f -name core -print -delete log "Checking symlinks ..." symlinks -rd . log "Creating recursive directory listing ... " rm -f ${FILENAME}.gz TZ=UTC ls -lR | gzip -9c --rsyncable > ${FILENAME}.gz done } function mkmaintainers() { local archiveroot local indices log 'Creating Maintainers index ... ' for archive in "${public_archives[@]}"; do archiveroot="$(get_archiveroot "${archive}")" indices="${archiveroot}/indices" if ! [ -d "${indices}" ]; then mkdir "${indices}" fi cd "${indices}" dak make-maintainers -a "${archive}" ${scriptdir}/masterfiles/pseudo-packages.maintainers gzip -9v --rsyncable Maintainers.gz gzip -9v --rsyncable Uploaders.gz done } function copyoverrides() { log 'Copying override files into public view ...' for ofile in ${overridedir}/override.{squeeze,wheezy,jessie,stretch,sid}.{,extra.}{main,contrib,non-free}*; do bname=${ofile##*/} gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz chmod g+w ${indices}/${bname}.gz done } function mkfilesindices() { set +o pipefail umask 002 cd $base/ftp/indices/files/components ARCHLIST=$(tempfile) log "Querying postgres" local query=" SELECT './pool/' || c.name || '/' || f.filename AS path, a.arch_string AS arch_string FROM files f JOIN files_archive_map af ON f.id = af.file_id JOIN component c ON af.component_id = c.id JOIN archive ON af.archive_id = archive.id LEFT OUTER JOIN (binaries b JOIN architecture a ON b.architecture = a.id) ON f.id = b.file WHERE archive.name = 'ftp-master' ORDER BY path, arch_string " psql -At -c "$query" >$ARCHLIST includedirs () { perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }' } poolfirst () { perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);' } log "Generating sources list" ( sed -n 's/|$//p' $ARCHLIST cd $base/ftp find ./dists -maxdepth 1 \! -type d find ./dists \! -type d | grep "/source/" ) | sort -u | gzip -9 > source.list.gz log "Generating arch lists" ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u) for a in $ARCHES; do (sed -n "s/|$a$//p" $ARCHLIST sed -n 's/|all$//p' $ARCHLIST cd $base/ftp find ./dists -maxdepth 1 \! -type d find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" ) | sort -u | gzip -9 > arch-$a.list.gz done log "Generating suite lists" suite_list () { local suite_id="$(printf %d $1)" local query query=" SELECT DISTINCT './pool/' || c.name || '/' || f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = $suite_id UNION SELECT esr.src_id FROM extra_src_references esr JOIN bin_associations ba ON esr.bin_id = ba.bin WHERE ba.suite = $suite_id UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s JOIN dsc_files df ON s.source = df.source JOIN files f ON df.file = f.id JOIN files_archive_map af ON f.id = af.file_id JOIN component c ON af.component_id = c.id JOIN archive ON af.archive_id = archive.id WHERE archive.name = 'ftp-master' " psql -F' ' -A -t -c "$query" query=" SELECT './pool/' || c.name || '/' || f.filename FROM bin_associations ba JOIN binaries b ON ba.bin = b.id JOIN files f ON b.file = f.id JOIN files_archive_map af ON f.id = af.file_id JOIN component c ON af.component_id = c.id JOIN archive ON af.archive_id = archive.id WHERE ba.suite = $suite_id AND archive.name = 'ftp-master' " psql -F' ' -A -t -c "$query" } psql -F' ' -At -c "SELECT id, suite_name FROM suite" | while read id suite; do [ -e $base/ftp/dists/$suite ] || continue ( (cd $base/ftp distname=$(cd dists; readlink $suite || echo $suite) find ./dists/$distname \! -type d for distdir in ./dists/*; do [ "$(readlink $distdir)" != "$distname" ] || echo $distdir done ) suite_list $id ) | sort -u | gzip -9 > suite-${suite}.list.gz done log "Finding everything on the ftp site to generate sundries" (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST rm -f sundries.list zcat *.list.gz | cat - *.list | sort -u | diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list log "Generating files list" for a in $ARCHES; do (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) | cat - sundries.list dists.list project.list docs.list indices.list | sort -u | poolfirst > ../arch-$a.files done (cd $base/ftp/ for dist in sid jessie stretch; do find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz done ) (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz ; zcat translation-stretch.list.gz) | sort -u | poolfirst > ../typical.files rm -f $ARCHLIST log "Done!" set -o pipefail } function mkchecksums() { local archiveroot dsynclist md5list for archive in "${public_archives[@]}"; do archiveroot="$(get_archiveroot "${archive}")" dsynclist=$dbdir/dsync.${archive}.list md5list=${archiveroot}/indices/md5sums log -n "Creating md5 / dsync index file for ${archive}... " cd "$archiveroot" ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz ${bindir}/dsync-flist -q link-dups $dsynclist || true done } function mirror() { local archiveroot mirrordir log "Regenerating \"public\" mirror/ hardlink fun" DATE_SERIAL=$(date +"%Y%m%d01") FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} ) if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then SERIAL="$DATE_SERIAL" else SERIAL="$FILESOAPLUS1" fi date -u > ${TRACEFILE} echo "Using dak v1" >> ${TRACEFILE} echo "Running on host: $(hostname -f)" >> ${TRACEFILE} echo "Archive serial: ${SERIAL}" >> ${TRACEFILE} # Ugly "hack", but hey, it does what we want. cp ${TRACEFILE} ${TRACEFILE_BDO} for archive in "${public_archives[@]}"; do archiveroot="$(get_archiveroot "${archive}")" mirrordir="${archiveroot}/../mirror" cd ${mirrordir} rsync -aH --link-dest ${archiveroot} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${archiveroot}/. . done } function expire() { log "Expiring old database dumps..." cd $base/backup $scriptsdir/expire_dumps -d . -p -f "dump_*" } function transitionsclean() { log "Removing out of date transitions..." cd $base dak transitions -c -a } function dm() { log "Updating DM permissions page" dak acl export-per-source dm >$exportdir/dm.txt } function bts() { log "Categorizing uncategorized bugs filed against ftp.debian.org" sudo -u dak-unpriv dak bts-categorize } function ddaccess() { # Tell our dd accessible mirror to sync itself up. Including ftp dir. log "Trigger dd accessible parts sync including ftp dir" ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync } function mirrorpush() { log "Checking the public archive copy" cd ${mirrordir}/dists broken=0 for release in $(find . -name "InRelease"); do echo "Processing: ${release}" subdir=${release%/InRelease} while read SHASUM SIZE NAME; do if ! [ -f "${subdir}/${NAME}" ]; then bname=$(basename ${NAME}) if [[ "${bname}" =~ ^(Packages|Sources|Translation-[a-zA-Z_]+)$ ]]; then # We don't keep unpacked files, don't check for their existance. # We might want to go and check their unpacked shasum, but right now # I don't care. I believe it should be enough if all the packed shasums # match. continue fi broken=$(( broken + 1 )) echo "File ${subdir}/${NAME} is missing" continue fi # We do have symlinks in the tree (see the contents files currently). # So we use "readlink -f" to check the size of the target, as thats basically # what gen-releases does fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}")) if [ ${fsize} -ne ${SIZE} ]; then broken=$(( broken + 1 )) echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}" continue fi fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}")) fshasum=${fshasum%% *} if [ "${fshasum}" != "${SHASUM}" ]; then broken=$(( broken + 1 )) echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}" continue fi done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d') done if [ $broken -gt 0 ]; then log_error "Trouble with the public mirror, found ${broken} errors" return 21 fi log "Starting the mirrorpush" date -u > /srv/ftp.debian.org/web/mirrorstart echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 & sudo -H -u archvsync /home/archvsync/runmirrors -a backports > ~dak/runmirrorsbpo.log 2>&1 & } function mirrorpush-backports() { log "Syncing backports mirror" sudo -u backports /home/backports/bin/update-archive } function i18n2() { log "Exporting package data foo for i18n project" STAMP=$(date "+%Y%m%d%H%M") mkdir -p ${scriptdir}/i18n/${STAMP} cd ${scriptdir}/i18n/${STAMP} for suite in stable testing unstable; do codename=$(dak admin s show ${suite}|grep '^Codename') codename=${codename##* } echo "Codename is ${codename}" dak control-suite -l ${suite} >${codename} done echo "${STAMP}" > timestamp gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --detach-sign -o timestamp.gpg timestamp rm -f md5sum md5sum * > md5sum cd ${webdir}/ ln -sfT ${scriptdir}/i18n/${STAMP} i18n cd ${scriptdir} find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf } function stats() { log "Updating stats data" cd $configdir $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data R --slave --vanilla < $base/misc/ftpstats.R dak stats arch-space > $webdir/arch-space dak stats pkg-nums > $webdir/pkg-nums } function cleantransactions() { log "Cleanup transaction ids older than 3 months" cd $base/backup/ find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm } function logstats() { $masterdir/tools/logs.py "$1" } # save timestamp when we start function savetimestamp() { NOW=`date "+%Y.%m.%d-%H:%M:%S"` echo ${NOW} > "${dbdir}/dinstallstart" } function maillogfile() { cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" -a "From: Debian FTP Masters " cron@ftp-master.debian.org } function renamelogfile() { if [ -f "${dbdir}/dinstallstart" ]; then NOW=$(cat "${dbdir}/dinstallstart") # maillogfile mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" logstats "$logdir/dinstall_${NOW}.log" bzip2 -9 "$logdir/dinstall_${NOW}.log" else error "Problem, I don't know when dinstall started, unable to do log statistics." NOW=`date "+%Y.%m.%d-%H:%M:%S"` # maillogfile mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" bzip2 -9 "$logdir/dinstall_${NOW}.log" fi } function testingsourcelist() { dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list } # do a last run of process-unchecked before dinstall is on. function process_unchecked() { log "Processing the unchecked queue" UNCHECKED_WITHOUT_LOCK="-p" do_unchecked sync_debbugs } # Function to update a "statefile" telling people what we are doing # (more or less). # # This should be called with the argument(s) # - Status name we want to show. # function state() { RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")" cat >"${DINSTALLSTATE}" </dev/null 2>&1 & dak make-changelog -e -a backports [ -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml ] && xz -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml mkdir -p /srv/backports-master.debian.org/rsync/export/changelogs cd /srv/backports-master.debian.org/rsync/export/changelogs rsync -aHW --delete --delete-after --ignore-errors /srv/backports-master.debian.org/export/changelogs/. . remove_changelog_lock trap remove_changelog_lock EXIT TERM HUP INT QUIT fi } function gitpdiff() { # Might be that we want to change this to have more than one git repository. # Advantage of one is that we do not need much space in terms of storage in git itself, # git gc is pretty good on our input. # But it might be faster. Well, lets test. log "Adjusting the git tree for pdiffs" cd ${dbdir}/git/git/ # The regex needs the architectures seperated with \| garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q') # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we # want to work with. # Also, we only want contents, packages and sources. for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do log "${file}" basen=${file%%.gz}; basen=${basen##../}; dir=${basen%/*}; mkdir -p $dir; zcat $file > $basen; done # Second, add all there is into git cd dists git add . # Maybe we want to make this the same for tag and commit? But well, shouldn't matter COMD=$(date -Is) TAGD=$(date +%Y-%m-%d-%H-%M) git commit -m "Commit of ${COMD}" git tag "${TAGD}" }