4 # FIXME: Dead code when switched to cronscript
5 function remove_daily_lock() {
9 # Remove changelog lock
10 function remove_changelog_lock() {
15 function remove_all_locks() {
16 rm -f $LOCK_DAILY $LOCK_ACCEPTED
19 function remove_locks {
21 trap - EXIT TERM HUP INT QUIT
22 ts "locked part finished"
25 function lockaccepted {
26 lockfile "$LOCK_ACCEPTED"
27 trap remove_all_locks EXIT TERM HUP INT QUIT
30 # If we error out this one is called, *FOLLOWED* by cleanup above
32 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
34 subject="ATTENTION ATTENTION!"
35 if [ "${error}" = "false" ]; then
36 subject="${subject} (continued)"
38 subject="${subject} (interrupted)"
40 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
42 if [ -r "${STAGEFILE}.log" ]; then
43 cat "${STAGEFILE}.log"
45 echo "file ${STAGEFILE}.log does not exist, sorry"
46 fi | mail -s "${subject}" -a "X-Debian: DAK" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" cron@ftp-master.debian.org
49 ########################################################################
50 # the actual functions follow #
51 ########################################################################
53 # pushing merkels QA user, part one
55 log "Telling QA user that we start dinstall"
56 ssh -n -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
59 # Updating various files
61 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
63 $scriptsdir/update-bugdoctxt
64 $scriptsdir/update-mirrorlists
65 $scriptsdir/update-mailingliststxt
66 $scriptsdir/update-pseudopackages.sh
69 # The first i18n one, syncing new descriptions
71 log "Synchronizing i18n package descriptions"
72 # First sync their newest data
73 cd ${scriptdir}/i18nsync
74 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
76 # Now check if we still know about the packages for which they created the files
77 # is the timestamp signed by us?
78 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
79 # now read it. As its signed by us we are sure the content is what we expect, no need
80 # to do more here. And we only test -d a directory on it anyway.
81 TSTAMP=$(cat timestamp)
82 # do we have the dir still?
83 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
85 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
86 # Yay, worked, lets copy around
87 for dir in ${extimportdists}; do
88 if [ -d dists/${dir}/ ]; then
89 cd dists/${dir}/main/i18n
90 rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/.
92 cd ${scriptdir}/i18nsync
95 echo "ARRRR, bad guys, wrong files, ARRR"
96 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" debian-l10n-devel@lists.alioth.debian.org
99 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
100 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" debian-l10n-devel@lists.alioth.debian.org
103 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
104 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" debian-l10n-devel@lists.alioth.debian.org
108 # Syncing AppStream/DEP-11 data
110 log "Synchronizing AppStream metadata"
111 # First sync their newest data
112 local dep11dir="${scriptdir}/dep11"
115 rsync -aq --delete --delete-after dep11-sync:/does/not/matter . || true
118 if ${scriptsdir}/dep11-basic-validate.py . ${dep11dir}/; then
119 # Yay, worked, lets copy around
120 for dir in ${extimportdists}; do
121 if [ -d ${dir}/ ]; then
122 for comp in main contrib non-free; do
123 mkdir -p ${ftpdir}/dists/${dir}/${comp}/dep11
125 rsync -aq --delete --delete-after --exclude *.tmp . ${ftpdir}/dists/${dir}/${comp}/dep11/.
131 echo "ARRRR, bad guys, wrong files, ARRR"
132 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" mak@debian.org
137 log "Checking for cruft in overrides"
141 function dominate() {
142 log "Removing obsolete source and binary associations"
144 dak manage-debug-suites unstable-debug experimental-debug
147 function autocruft() {
148 log "Check for obsolete binary packages"
149 dak auto-decruft -s unstable
150 dak auto-decruft -s experimental --if-newer-version-in unstable --if-newer-version-in-rm-msg "NVIU"
153 function fingerprints() {
154 log "Updating fingerprints"
155 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
158 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
160 if [ -s "${OUTFILE}" ]; then
161 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
162 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
163 To: <debian-project@lists.debian.org>
164 Subject: Debian Maintainers Keyring changes
165 Content-Type: text/plain; charset=utf-8
169 The following changes to the debian-maintainers keyring have just been activated:
173 Debian distribution maintenance software,
174 on behalf of the Keyring maintainers
181 function overrides() {
182 log "Writing overrides into text files"
190 log "Generating package / file mapping"
191 for archive in "${public_archives[@]}"; do
192 log " archive: ${archive}"
193 archiveroot="$(get_archiveroot "${archive}")"
194 dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2"
198 function packages() {
199 for archive in "${public_archives[@]}"; do
200 log " Generating Packages/Sources for ${archive}"
201 dak generate-packages-sources2 -a "${archive}"
202 log " Generating Contents for ${archive}"
203 dak contents generate -a "${archive}"
208 log "Generating pdiff files"
209 dak generate-index-diffs
213 log "Generating Release files"
214 for archive in "${public_archives[@]}"; do
215 log " archive: ${archive}"
216 dak generate-releases -a "${archive}"
220 function dakcleanup() {
221 log "Cleanup old packages/files"
222 dak clean-suites -m 10000
223 dak clean-queues -i "$unchecked"
230 for archive in "${public_archives[@]}"; do
231 archiveroot="$(get_archiveroot "${archive}")"
234 log "Removing any core files ..."
235 find -type f -name core -print -delete
237 log "Checking symlinks ..."
240 log "Creating recursive directory listing ... "
242 TZ=UTC ls -lR | gzip -9c --rsyncable > ${FILENAME}.gz
246 function mkmaintainers() {
250 log 'Creating Maintainers index ... '
252 for archive in "${public_archives[@]}"; do
253 archiveroot="$(get_archiveroot "${archive}")"
254 indices="${archiveroot}/indices"
255 if ! [ -d "${indices}" ]; then
260 dak make-maintainers -a "${archive}" ${scriptdir}/masterfiles/pseudo-packages.maintainers
261 gzip -9v --rsyncable <Maintainers >Maintainers.gz
262 gzip -9v --rsyncable <Uploaders >Uploaders.gz
266 function copyoverrides() {
267 log 'Copying override files into public view ...'
271 for ofile in ${overridedir}/override.{squeeze,wheezy,jessie,stretch,sid}.{,extra.}{main,contrib,non-free}*; do
273 gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
274 chmod g+w ${indices}/${bname}.gz
279 function mkfilesindices() {
282 cd $base/ftp/indices/files/components
286 log "Querying postgres"
288 SELECT './pool/' || c.name || '/' || f.filename AS path, a.arch_string AS arch_string
290 JOIN files_archive_map af ON f.id = af.file_id
291 JOIN component c ON af.component_id = c.id
292 JOIN archive ON af.archive_id = archive.id
295 JOIN architecture a ON b.architecture = a.id)
297 WHERE archive.name = 'ftp-master'
298 ORDER BY path, arch_string
300 psql -At -c "$query" >$ARCHLIST
303 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
306 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
309 log "Generating sources list"
311 sed -n 's/|$//p' $ARCHLIST
313 find ./dists -maxdepth 1 \! -type d
314 find ./dists \! -type d | grep "/source/"
315 ) | sort -u | gzip -9 > source.list.gz
317 log "Generating arch lists"
319 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
321 (sed -n "s/|$a$//p" $ARCHLIST
322 sed -n 's/|all$//p' $ARCHLIST
325 find ./dists -maxdepth 1 \! -type d
326 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
327 ) | sort -u | gzip -9 > arch-$a.list.gz
330 log "Generating suite lists"
333 local suite_id="$(printf %d $1)"
336 SELECT DISTINCT './pool/' || c.name || '/' || f.filename
338 (SELECT sa.source AS source
339 FROM src_associations sa
340 WHERE sa.suite = $suite_id
343 FROM extra_src_references esr
344 JOIN bin_associations ba ON esr.bin_id = ba.bin
345 WHERE ba.suite = $suite_id
347 SELECT b.source AS source
348 FROM bin_associations ba
349 JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s
350 JOIN dsc_files df ON s.source = df.source
351 JOIN files f ON df.file = f.id
352 JOIN files_archive_map af ON f.id = af.file_id
353 JOIN component c ON af.component_id = c.id
354 JOIN archive ON af.archive_id = archive.id
355 WHERE archive.name = 'ftp-master'
357 psql -F' ' -A -t -c "$query"
360 SELECT './pool/' || c.name || '/' || f.filename
361 FROM bin_associations ba
362 JOIN binaries b ON ba.bin = b.id
363 JOIN files f ON b.file = f.id
364 JOIN files_archive_map af ON f.id = af.file_id
365 JOIN component c ON af.component_id = c.id
366 JOIN archive ON af.archive_id = archive.id
367 WHERE ba.suite = $suite_id AND archive.name = 'ftp-master'
369 psql -F' ' -A -t -c "$query"
372 psql -F' ' -At -c "SELECT id, suite_name FROM suite" |
373 while read id suite; do
374 [ -e $base/ftp/dists/$suite ] || continue
377 distname=$(cd dists; readlink $suite || echo $suite)
378 find ./dists/$distname \! -type d
379 for distdir in ./dists/*; do
380 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
384 ) | sort -u | gzip -9 > suite-${suite}.list.gz
387 log "Finding everything on the ftp site to generate sundries"
388 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
391 zcat *.list.gz | cat - *.list | sort -u |
392 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
394 log "Generating files list"
397 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
398 cat - sundries.list dists.list project.list docs.list indices.list |
399 sort -u | poolfirst > ../arch-$a.files
403 for dist in sid jessie stretch; do
404 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
408 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz ; zcat translation-stretch.list.gz) |
409 sort -u | poolfirst > ../typical.files
416 function mkchecksums() {
417 local archiveroot dsynclist md5list
419 for archive in "${public_archives[@]}"; do
420 archiveroot="$(get_archiveroot "${archive}")"
421 dsynclist=$dbdir/dsync.${archive}.list
422 md5list=${archiveroot}/indices/md5sums
424 log -n "Creating md5 / dsync index file for ${archive}... "
427 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
428 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
429 ${bindir}/dsync-flist -q link-dups $dsynclist || true
434 local archiveroot targetpath TRACEFILE
436 for archive in "${public_archives[@]}"; do
437 archiveroot="$(get_archiveroot "${archive}")"
438 targetpath="${mirrordir}/${archive}"
439 TRACEFILE="${archiveroot}/project/trace/ftp-master.debian.org"
440 mkdir -p "${archiveroot}/project/trace/"
442 log "Regenerating \"public\" mirror/${archive} hardlink fun"
443 DATE_SERIAL=$(date +"%Y%m%d01")
444 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} || echo ${DATE_SERIAL} )
445 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
446 SERIAL="$DATE_SERIAL"
448 SERIAL="$FILESOAPLUS1"
450 date -u > ${TRACEFILE}
451 echo "Using dak v1" >> ${TRACEFILE}
452 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
453 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
455 mkdir -p ${targetpath}
457 rsync -aH --link-dest ${archiveroot} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${archiveroot}/. .
462 log "Expiring old database dumps..."
464 $scriptsdir/expire_dumps -d . -p -f "dump_*"
467 function transitionsclean() {
468 log "Removing out of date transitions..."
470 dak transitions -c -a
474 log "Updating DM permissions page"
475 dak acl export-per-source dm >$exportdir/dm.txt
479 log "Categorizing uncategorized bugs filed against ftp.debian.org"
480 sudo -u dak-unpriv dak bts-categorize
483 function mirrorpush() {
484 log "Checking the public archive copies..."
486 local archiveroot targetpath
488 for archive in "${public_archives[@]}"; do
489 log "... archive: ${archive}"
490 archiveroot="$(get_archiveroot "${archive}")"
491 targetpath="${mirrordir}/${archive}"
492 cd ${archiveroot}/dists
495 for release in $(find . -name "InRelease"); do
496 echo "Processing: ${release}"
497 subdir=${release%/InRelease}
498 while read SHASUM SIZE NAME; do
499 if ! [ -f "${subdir}/${NAME}" ]; then
500 bname=$(basename ${NAME})
501 if [[ "${bname}" =~ ^(Packages|Sources|Contents-[a-zA-Z0-9-]+|Translation-[a-zA-Z_]+|Components-(amd64|i386)\.yml|icons-(128x128|64x64)\.tar)$ ]]; then
503 # We don't keep unpacked files, don't check for their existance.
504 # We might want to go and check their unpacked shasum, but right now
505 # I don't care. I believe it should be enough if all the packed shasums
509 broken=$(( broken + 1 ))
510 echo "File ${subdir}/${NAME} is missing"
514 # We do have symlinks in the tree (see the contents files currently).
515 # So we use "readlink -f" to check the size of the target, as thats basically
516 # what gen-releases does
517 fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
518 if [ ${fsize} -ne ${SIZE} ]; then
519 broken=$(( broken + 1 ))
520 echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
524 fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
525 fshasum=${fshasum%% *}
526 if [ "${fshasum}" != "${SHASUM}" ]; then
527 broken=$(( broken + 1 ))
528 echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
531 done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
534 if [ $broken -gt 0 ]; then
535 log_error "Trouble with the public mirror for ${archive}, found ${broken} errors"
538 log "Starting the mirrorpush for ${archive}"
548 pusharg="-a backports"
551 fname="mirrorstart.${archive}"
554 date -u > /srv/ftp.debian.org/web/${fname}
555 echo "Using dak v1" >> /srv/ftp.debian.org/web/${fname}
556 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/${fname}
557 sudo -H -u archvsync /home/archvsync/runmirrors ${pusharg} > ~dak/runmirrors-${archive}.log 2>&1 &
562 function mirrorpush-backports() {
563 log "Syncing backports mirror"
564 sudo -u backports /home/backports/bin/update-archive
568 log "Exporting package data foo for i18n project"
569 STAMP=$(date "+%Y%m%d%H%M")
570 mkdir -p ${scriptdir}/i18n/${STAMP}
571 cd ${scriptdir}/i18n/${STAMP}
572 for suite in stable testing unstable; do
573 codename=$(dak admin s show ${suite}|grep '^Codename')
574 codename=${codename##* }
575 echo "Codename is ${codename}"
576 dak control-suite -l ${suite} >${codename}
578 echo "${STAMP}" > timestamp
579 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --detach-sign -o timestamp.gpg timestamp
583 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
586 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
590 log "Updating stats data"
592 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
593 R --slave --vanilla < $base/misc/ftpstats.R
594 dak stats arch-space > $webdir/arch-space
595 dak stats pkg-nums > $webdir/pkg-nums
598 function cleantransactions() {
599 log "Cleanup transaction ids older than 3 months"
601 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -delete
604 function logstats() {
605 $masterdir/tools/logs.py "$1"
608 # save timestamp when we start
609 function savetimestamp() {
610 echo ${NOW} > "${dbdir}/dinstallstart"
613 function maillogfile() {
614 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" cron@ftp-master.debian.org
617 function testingsourcelist() {
618 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
621 # Function to update a "statefile" telling people what we are doing
624 # This should be called with the argument(s)
625 # - Status name we want to show.
628 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
629 cat >"${DINSTALLSTATE}" <<EOF
630 Dinstall start: ${DINSTALLBEGIN}
632 Action start: ${RIGHTNOW}
636 # extract changelogs and stuff
637 function changelogs() {
638 if lockfile -r3 $LOCK_CHANGELOG; then
639 log "Extracting changelogs"
640 dak make-changelog -e -a ftp-master
641 [ -f ${exportdir}/changelogs/filelist.yaml ] && xz -f ${exportdir}/changelogs/filelist.yaml
642 mkdir -p ${exportpublic}/changelogs
643 cd ${exportpublic}/changelogs
644 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
645 sudo -H -u staticsync /usr/local/bin/static-update-component metadata.ftp-master.debian.org >/dev/null 2>&1 &
647 dak make-changelog -e -a backports
648 [ -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml ] && xz -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml
649 mkdir -p /srv/backports-master.debian.org/rsync/export/changelogs
650 cd /srv/backports-master.debian.org/rsync/export/changelogs
651 rsync -aHW --delete --delete-after --ignore-errors /srv/backports-master.debian.org/export/changelogs/. .
652 remove_changelog_lock
653 trap remove_changelog_lock EXIT TERM HUP INT QUIT
657 # Generate a list of extra mirror files, sha256sum em and sign that
658 function signotherfiles() {
659 log "Signing extra mirror files"
663 for archive in "${public_archives[@]}"; do
664 log "... archive: ${archive}"
665 archiveroot="$(get_archiveroot "${archive}")"
666 local TMPLO=$( mktemp -p ${TMPDIR} )
667 trap "rm -f ${TMPLO}" ERR EXIT TERM HUP INT QUIT
671 sha256sum $(find * -type f | egrep -v '(pool|i18n|dep11|source)/|Contents-.*\.(gz|diff)|installer|binary-|(In)?Release(.gpg)?|\.changes' | sort) > ${TMPLO}
672 gpg --no-options --batch --no-tty --armour --personal-digest-preferences=SHA256 --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --clearsign --output ${archiveroot}/extrafiles ${TMPLO}
678 touch "${DINSTALLSTART}"
680 lockfile -l 3600 "${LOCK_DAILY}"
682 touch "${LOCK_BRITNEY}"
683 TMPFILES="${TMPFILES} ${LOCK_DAILY} ${LOCK_BRITNEY}"