2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 # FIXME: Dead code when switched to cronscript
9 function remove_daily_lock() {
13 # Remove changelog lock
14 function remove_changelog_lock() {
19 function remove_all_locks() {
20 rm -f $LOCK_DAILY $LOCK_ACCEPTED
23 function remove_locks {
25 trap - EXIT TERM HUP INT QUIT
26 ts "locked part finished"
29 function lockaccepted {
30 lockfile "$LOCK_ACCEPTED"
31 trap remove_all_locks EXIT TERM HUP INT QUIT
34 # If we error out this one is called, *FOLLOWED* by cleanup above
36 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
38 subject="ATTENTION ATTENTION!"
39 if [ "${error}" = "false" ]; then
40 subject="${subject} (continued)"
42 subject="${subject} (interrupted)"
44 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
46 if [ -r "${STAGEFILE}.log" ]; then
47 cat "${STAGEFILE}.log"
49 echo "file ${STAGEFILE}.log does not exist, sorry"
50 fi | mail -s "${subject}" -a "X-Debian: DAK" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" cron@ftp-master.debian.org
53 ########################################################################
54 # the actual functions follow #
55 ########################################################################
57 # pushing merkels QA user, part one
59 log "Telling QA user that we start dinstall"
60 ssh -n -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
63 # Updating various files
65 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
67 $scriptsdir/update-bugdoctxt
68 $scriptsdir/update-mirrorlists
69 $scriptsdir/update-mailingliststxt
70 $scriptsdir/update-pseudopackages.sh
73 # The first i18n one, syncing new descriptions
75 log "Synchronizing i18n package descriptions"
76 # First sync their newest data
77 cd ${scriptdir}/i18nsync
78 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
80 # Now check if we still know about the packages for which they created the files
81 # is the timestamp signed by us?
82 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
83 # now read it. As its signed by us we are sure the content is what we expect, no need
84 # to do more here. And we only test -d a directory on it anyway.
85 TSTAMP=$(cat timestamp)
86 # do we have the dir still?
87 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
89 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
90 # Yay, worked, lets copy around
91 for dir in ${extimportdists}; do
92 if [ -d dists/${dir}/ ]; then
93 cd dists/${dir}/main/i18n
94 rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/.
96 cd ${scriptdir}/i18nsync
99 echo "ARRRR, bad guys, wrong files, ARRR"
100 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" debian-l10n-devel@lists.alioth.debian.org
103 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
104 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" debian-l10n-devel@lists.alioth.debian.org
107 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
108 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" debian-l10n-devel@lists.alioth.debian.org
112 # Syncing AppStream/DEP-11 data
114 log "Synchronizing AppStream metadata"
115 # First sync their newest data
116 local dep11dir="${scriptdir}/dep11"
119 rsync -aq --delete --delete-after dep11-sync:/does/not/matter . || true
122 if ${scriptsdir}/dep11-basic-validate.py . ${dep11dir}/; then
123 # Yay, worked, lets copy around
124 for dir in ${extimportdists}; do
125 if [ -d ${dir}/ ]; then
126 for comp in main contrib non-free; do
127 mkdir -p ${ftpdir}/dists/${dir}/${comp}/dep11
129 rsync -aq --delete --delete-after --exclude *.tmp . ${ftpdir}/dists/${dir}/${comp}/dep11/.
135 echo "ARRRR, bad guys, wrong files, ARRR"
136 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" mak@debian.org
141 log "Checking for cruft in overrides"
145 function dominate() {
146 log "Removing obsolete source and binary associations"
148 dak manage-debug-suites unstable-debug experimental-debug
151 function autocruft() {
152 log "Check for obsolete binary packages"
153 dak auto-decruft -s unstable
154 dak auto-decruft -s experimental --if-newer-version-in unstable --if-newer-version-in-rm-msg "NVIU"
157 function fingerprints() {
158 log "Updating fingerprints"
159 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
162 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
164 if [ -s "${OUTFILE}" ]; then
165 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
166 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
167 To: <debian-project@lists.debian.org>
168 Subject: Debian Maintainers Keyring changes
169 Content-Type: text/plain; charset=utf-8
173 The following changes to the debian-maintainers keyring have just been activated:
177 Debian distribution maintenance software,
178 on behalf of the Keyring maintainers
185 function overrides() {
186 log "Writing overrides into text files"
194 log "Generating package / file mapping"
195 for archive in "${public_archives[@]}"; do
196 archiveroot="$(get_archiveroot "${archive}")"
197 dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2"
201 function packages() {
202 log "Generating Packages and Sources files"
203 for archive in "${public_archives[@]}"; do
204 log " Generating Packages/Sources for ${archive}"
205 dak generate-packages-sources2 -a "${archive}"
206 log " Generating Contents for ${archive}"
207 dak contents generate -a "${archive}"
212 log "Generating pdiff files"
213 dak generate-index-diffs
217 log "Generating Release files"
218 for archive in "${public_archives[@]}"; do
219 dak generate-releases -a "${archive}"
223 function dakcleanup() {
224 log "Cleanup old packages/files"
225 dak clean-suites -m 10000
226 dak clean-queues -i "$unchecked"
233 for archive in "${public_archives[@]}"; do
234 archiveroot="$(get_archiveroot "${archive}")"
237 log "Removing any core files ..."
238 find -type f -name core -print -delete
240 log "Checking symlinks ..."
243 log "Creating recursive directory listing ... "
245 TZ=UTC ls -lR | gzip -9c --rsyncable > ${FILENAME}.gz
249 function mkmaintainers() {
253 log 'Creating Maintainers index ... '
255 for archive in "${public_archives[@]}"; do
256 archiveroot="$(get_archiveroot "${archive}")"
257 indices="${archiveroot}/indices"
258 if ! [ -d "${indices}" ]; then
263 dak make-maintainers -a "${archive}" ${scriptdir}/masterfiles/pseudo-packages.maintainers
264 gzip -9v --rsyncable <Maintainers >Maintainers.gz
265 gzip -9v --rsyncable <Uploaders >Uploaders.gz
269 function copyoverrides() {
270 log 'Copying override files into public view ...'
274 for ofile in ${overridedir}/override.{squeeze,wheezy,jessie,stretch,sid}.{,extra.}{main,contrib,non-free}*; do
276 gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
277 chmod g+w ${indices}/${bname}.gz
282 function mkfilesindices() {
285 cd $base/ftp/indices/files/components
289 log "Querying postgres"
291 SELECT './pool/' || c.name || '/' || f.filename AS path, a.arch_string AS arch_string
293 JOIN files_archive_map af ON f.id = af.file_id
294 JOIN component c ON af.component_id = c.id
295 JOIN archive ON af.archive_id = archive.id
298 JOIN architecture a ON b.architecture = a.id)
300 WHERE archive.name = 'ftp-master'
301 ORDER BY path, arch_string
303 psql -At -c "$query" >$ARCHLIST
306 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
309 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
312 log "Generating sources list"
314 sed -n 's/|$//p' $ARCHLIST
316 find ./dists -maxdepth 1 \! -type d
317 find ./dists \! -type d | grep "/source/"
318 ) | sort -u | gzip -9 > source.list.gz
320 log "Generating arch lists"
322 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
324 (sed -n "s/|$a$//p" $ARCHLIST
325 sed -n 's/|all$//p' $ARCHLIST
328 find ./dists -maxdepth 1 \! -type d
329 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
330 ) | sort -u | gzip -9 > arch-$a.list.gz
333 log "Generating suite lists"
336 local suite_id="$(printf %d $1)"
339 SELECT DISTINCT './pool/' || c.name || '/' || f.filename
341 (SELECT sa.source AS source
342 FROM src_associations sa
343 WHERE sa.suite = $suite_id
346 FROM extra_src_references esr
347 JOIN bin_associations ba ON esr.bin_id = ba.bin
348 WHERE ba.suite = $suite_id
350 SELECT b.source AS source
351 FROM bin_associations ba
352 JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s
353 JOIN dsc_files df ON s.source = df.source
354 JOIN files f ON df.file = f.id
355 JOIN files_archive_map af ON f.id = af.file_id
356 JOIN component c ON af.component_id = c.id
357 JOIN archive ON af.archive_id = archive.id
358 WHERE archive.name = 'ftp-master'
360 psql -F' ' -A -t -c "$query"
363 SELECT './pool/' || c.name || '/' || f.filename
364 FROM bin_associations ba
365 JOIN binaries b ON ba.bin = b.id
366 JOIN files f ON b.file = f.id
367 JOIN files_archive_map af ON f.id = af.file_id
368 JOIN component c ON af.component_id = c.id
369 JOIN archive ON af.archive_id = archive.id
370 WHERE ba.suite = $suite_id AND archive.name = 'ftp-master'
372 psql -F' ' -A -t -c "$query"
375 psql -F' ' -At -c "SELECT id, suite_name FROM suite" |
376 while read id suite; do
377 [ -e $base/ftp/dists/$suite ] || continue
380 distname=$(cd dists; readlink $suite || echo $suite)
381 find ./dists/$distname \! -type d
382 for distdir in ./dists/*; do
383 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
387 ) | sort -u | gzip -9 > suite-${suite}.list.gz
390 log "Finding everything on the ftp site to generate sundries"
391 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
394 zcat *.list.gz | cat - *.list | sort -u |
395 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
397 log "Generating files list"
400 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
401 cat - sundries.list dists.list project.list docs.list indices.list |
402 sort -u | poolfirst > ../arch-$a.files
406 for dist in sid jessie stretch; do
407 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
411 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz ; zcat translation-stretch.list.gz) |
412 sort -u | poolfirst > ../typical.files
419 function mkchecksums() {
420 local archiveroot dsynclist md5list
422 for archive in "${public_archives[@]}"; do
423 archiveroot="$(get_archiveroot "${archive}")"
424 dsynclist=$dbdir/dsync.${archive}.list
425 md5list=${archiveroot}/indices/md5sums
427 log -n "Creating md5 / dsync index file for ${archive}... "
430 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
431 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
432 ${bindir}/dsync-flist -q link-dups $dsynclist || true
437 local archiveroot targetpath TRACEFILE
439 for archive in "${public_archives[@]}"; do
440 archiveroot="$(get_archiveroot "${archive}")"
441 targetpath="${mirrordir}/${archive}"
442 TRACEFILE="${archiveroot}/project/trace/ftp-master.debian.org"
443 mkdir -p "${archiveroot}/project/trace/"
445 log "Regenerating \"public\" mirror/${archive} hardlink fun"
446 DATE_SERIAL=$(date +"%Y%m%d01")
447 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} || echo ${DATE_SERIAL} )
448 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
449 SERIAL="$DATE_SERIAL"
451 SERIAL="$FILESOAPLUS1"
453 date -u > ${TRACEFILE}
454 echo "Using dak v1" >> ${TRACEFILE}
455 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
456 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
458 mkdir -p ${targetpath}
460 rsync -aH --link-dest ${archiveroot} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${archiveroot}/. .
465 log "Expiring old database dumps..."
467 $scriptsdir/expire_dumps -d . -p -f "dump_*"
470 function transitionsclean() {
471 log "Removing out of date transitions..."
473 dak transitions -c -a
477 log "Updating DM permissions page"
478 dak acl export-per-source dm >$exportdir/dm.txt
482 log "Categorizing uncategorized bugs filed against ftp.debian.org"
483 sudo -u dak-unpriv dak bts-categorize
486 function mirrorpush() {
487 log "Checking the public archive copies..."
489 local archiveroot targetpath
491 for archive in "${public_archives[@]}"; do
492 log "... archive: ${archive}"
493 archiveroot="$(get_archiveroot "${archive}")"
494 targetpath="${mirrordir}/${archive}"
495 cd ${archiveroot}/dists
498 for release in $(find . -name "InRelease"); do
499 echo "Processing: ${release}"
500 subdir=${release%/InRelease}
501 while read SHASUM SIZE NAME; do
502 if ! [ -f "${subdir}/${NAME}" ]; then
503 bname=$(basename ${NAME})
504 if [[ "${bname}" =~ ^(Packages|Sources|Contents-[a-zA-Z0-9-]+|Translation-[a-zA-Z_]+|Components-(amd64|i386)\.yml|icons-(128x128|64x64)\.tar)$ ]]; then
506 # We don't keep unpacked files, don't check for their existance.
507 # We might want to go and check their unpacked shasum, but right now
508 # I don't care. I believe it should be enough if all the packed shasums
512 broken=$(( broken + 1 ))
513 echo "File ${subdir}/${NAME} is missing"
517 # We do have symlinks in the tree (see the contents files currently).
518 # So we use "readlink -f" to check the size of the target, as thats basically
519 # what gen-releases does
520 fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
521 if [ ${fsize} -ne ${SIZE} ]; then
522 broken=$(( broken + 1 ))
523 echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
527 fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
528 fshasum=${fshasum%% *}
529 if [ "${fshasum}" != "${SHASUM}" ]; then
530 broken=$(( broken + 1 ))
531 echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
534 done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
537 if [ $broken -gt 0 ]; then
538 log_error "Trouble with the public mirror for ${archive}, found ${broken} errors"
541 log "Starting the mirrorpush for ${archive}"
551 pusharg="-a backports"
554 fname="mirrorstart.${archive}"
557 date -u > /srv/ftp.debian.org/web/${fname}
558 echo "Using dak v1" >> /srv/ftp.debian.org/web/${fname}
559 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/${fname}
560 sudo -H -u archvsync /home/archvsync/runmirrors ${pusharg} > ~dak/runmirrors-${archive}.log 2>&1 &
565 function mirrorpush-backports() {
566 log "Syncing backports mirror"
567 sudo -u backports /home/backports/bin/update-archive
571 log "Exporting package data foo for i18n project"
572 STAMP=$(date "+%Y%m%d%H%M")
573 mkdir -p ${scriptdir}/i18n/${STAMP}
574 cd ${scriptdir}/i18n/${STAMP}
575 for suite in stable testing unstable; do
576 codename=$(dak admin s show ${suite}|grep '^Codename')
577 codename=${codename##* }
578 echo "Codename is ${codename}"
579 dak control-suite -l ${suite} >${codename}
581 echo "${STAMP}" > timestamp
582 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --detach-sign -o timestamp.gpg timestamp
586 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
589 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
593 log "Updating stats data"
595 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
596 R --slave --vanilla < $base/misc/ftpstats.R
597 dak stats arch-space > $webdir/arch-space
598 dak stats pkg-nums > $webdir/pkg-nums
601 function cleantransactions() {
602 log "Cleanup transaction ids older than 3 months"
604 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -delete
607 function logstats() {
608 $masterdir/tools/logs.py "$1"
611 # save timestamp when we start
612 function savetimestamp() {
613 echo ${NOW} > "${dbdir}/dinstallstart"
616 function maillogfile() {
617 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" cron@ftp-master.debian.org
620 function testingsourcelist() {
621 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
624 # Function to update a "statefile" telling people what we are doing
627 # This should be called with the argument(s)
628 # - Status name we want to show.
631 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
632 cat >"${DINSTALLSTATE}" <<EOF
633 Dinstall start: ${DINSTALLBEGIN}
635 Action start: ${RIGHTNOW}
639 # extract changelogs and stuff
640 function changelogs() {
641 if lockfile -r3 $LOCK_CHANGELOG; then
642 log "Extracting changelogs"
643 dak make-changelog -e -a ftp-master
644 [ -f ${exportdir}/changelogs/filelist.yaml ] && xz -f ${exportdir}/changelogs/filelist.yaml
645 mkdir -p ${exportpublic}/changelogs
646 cd ${exportpublic}/changelogs
647 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
648 sudo -H -u staticsync /usr/local/bin/static-update-component metadata.ftp-master.debian.org >/dev/null 2>&1 &
650 dak make-changelog -e -a backports
651 [ -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml ] && xz -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml
652 mkdir -p /srv/backports-master.debian.org/rsync/export/changelogs
653 cd /srv/backports-master.debian.org/rsync/export/changelogs
654 rsync -aHW --delete --delete-after --ignore-errors /srv/backports-master.debian.org/export/changelogs/. .
655 remove_changelog_lock
656 trap remove_changelog_lock EXIT TERM HUP INT QUIT
660 # Generate a list of extra mirror files, sha256sum em and sign that
661 function signotherfiles() {
662 log "Signing extra mirror files"
666 for archive in "${public_archives[@]}"; do
667 log "... archive: ${archive}"
668 archiveroot="$(get_archiveroot "${archive}")"
669 local TMPLO=$( mktemp -p ${TMPDIR} )
670 trap "rm -f ${TMPLO}" ERR EXIT TERM HUP INT QUIT
674 sha256sum $(find * -type f | egrep -v '(pool|i18n|dep11|source)/|Contents-.*\.(gz|diff)|installer|binary-|(In)?Release(.gpg)?|\.changes' | sort) > ${TMPLO}
675 gpg --no-options --batch --no-tty --armour --personal-digest-preferences=SHA256 --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --clearsign --output ${archiveroot}/extrafiles ${TMPLO}
681 touch "${DINSTALLSTART}"
683 DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
684 lockfile -l 3600 "${LOCK_DAILY}"
686 TEMPFILES="${TEMPFILES} ${LOCK_DAILY}"
687 touch "${LOCK_BRITNEY}"