2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Updating various files
48 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
50 $scriptsdir/update-bugdoctxt
51 $scriptsdir/update-mirrorlists
52 $scriptsdir/update-mailingliststxt
53 $scriptsdir/update-pseudopackages.sh
56 # The first i18n one, syncing new descriptions
58 log "Synchronizing i18n package descriptions"
59 # First sync their newest data
60 cd ${scriptdir}/i18nsync
61 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
63 # Now check if we still know about the packages for which they created the files
64 # is the timestamp signed by us?
65 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
66 # now read it. As its signed by us we are sure the content is what we expect, no need
67 # to do more here. And we only test -d a directory on it anyway.
68 TSTAMP=$(cat timestamp)
69 # do we have the dir still?
70 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
72 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
73 # Yay, worked, lets copy around
74 for dir in wheezy sid; do
75 if [ -d dists/${dir}/ ]; then
76 cd dists/${dir}/main/i18n
77 rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/.
79 cd ${scriptdir}/i18nsync
82 echo "ARRRR, bad guys, wrong files, ARRR"
83 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
86 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
87 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
90 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
91 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
96 log "Checking for cruft in overrides"
100 function dominate() {
101 log "Removing obsolete source and binary associations"
105 function filelist() {
106 log "Generating file lists for apt-ftparchive"
107 dak generate-filelist
110 function fingerprints() {
111 log "Updating fingerprints"
112 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
115 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
117 if [ -s "${OUTFILE}" ]; then
118 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
119 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
120 To: <debian-project@lists.debian.org>
121 Subject: Debian Maintainers Keyring changes
122 Content-Type: text/plain; charset=utf-8
126 The following changes to the debian-maintainers keyring have just been activated:
130 Debian distribution maintenance software,
131 on behalf of the Keyring maintainers
138 function overrides() {
139 log "Writing overrides into text files"
144 rm -f override.sid.all3
145 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
149 log "Generating package / file mapping"
150 dak make-pkg-file-mapping ftp-master | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
153 function packages() {
154 log "Generating Packages and Sources files"
155 dak generate-packages-sources2 -a ftp-master
156 dak contents generate -a ftp-master
160 log "Generating pdiff files"
161 dak generate-index-diffs
165 # XXX: disable once we can remove i18n/Index (#649314)
166 log "Generating i18n/Index"
169 for dist in testing unstable experimental proposed-updates testing-proposed-updates; do
170 $scriptsdir/generate-i18n-Index $dist;
173 log "Generating Release files"
174 dak generate-releases -a ftp-master
177 function dakcleanup() {
178 log "Cleanup old packages/files"
179 dak clean-suites -m 10000
180 dak clean-queues -i "$unchecked"
183 function buildd_dir() {
184 # Rebuilt the buildd dir to avoid long times of 403
185 log "Regenerating the buildd incoming dir"
186 STAMP=$(date "+%Y%m%d%H%M")
195 log "Removing any core files ..."
196 find -type f -name core -print -delete
198 log "Checking symlinks ..."
201 log "Creating recursive directory listing ... "
203 TZ=UTC ls -lR | gzip -9c --rsyncable > ${FILENAME}.gz
206 function mkmaintainers() {
207 log 'Creating Maintainers index ... '
210 dak make-maintainers -a ftp-master ${scriptdir}/masterfiles/pseudo-packages.maintainers
211 gzip -9v --rsyncable <Maintainers >Maintainers.gz
212 gzip -9v --rsyncable <Uploaders >Uploaders.gz
215 function copyoverrides() {
216 log 'Copying override files into public view ...'
218 for ofile in ${overridedir}/override.{squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do
220 gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
221 chmod g+w ${indices}/${bname}.gz
225 function mkfilesindices() {
228 cd $base/ftp/indices/files/components
232 log "Querying postgres"
234 SELECT './pool/' || c.name || '/' || f.filename AS path, a.arch_string AS arch_string
236 JOIN files_archive_map af ON f.id = af.file_id
237 JOIN component c ON af.component_id = c.id
238 JOIN archive ON af.archive_id = archive.id
241 JOIN architecture a ON b.architecture = a.id)
243 WHERE archive.name = 'ftp-master'
244 ORDER BY path, arch_string
246 psql -At -c "$query" >$ARCHLIST
249 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
252 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
255 log "Generating sources list"
257 sed -n 's/|$//p' $ARCHLIST
259 find ./dists -maxdepth 1 \! -type d
260 find ./dists \! -type d | grep "/source/"
261 ) | sort -u | gzip -9 > source.list.gz
263 log "Generating arch lists"
265 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
267 (sed -n "s/|$a$//p" $ARCHLIST
268 sed -n 's/|all$//p' $ARCHLIST
271 find ./dists -maxdepth 1 \! -type d
272 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
273 ) | sort -u | gzip -9 > arch-$a.list.gz
276 log "Generating suite lists"
279 local suite_id="$(printf %d $1)"
282 SELECT DISTINCT './pool/' || c.name || '/' || f.filename
284 (SELECT sa.source AS source
285 FROM src_associations sa
286 WHERE sa.suite = $suite_id
289 FROM extra_src_references esr
290 JOIN bin_associations ba ON esr.bin_id = ba.bin
291 WHERE ba.suite = $suite_id
293 SELECT b.source AS source
294 FROM bin_associations ba
295 JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s
296 JOIN dsc_files df ON s.source = df.source
297 JOIN files f ON df.file = f.id
298 JOIN files_archive_map af ON f.id = af.file_id
299 JOIN component c ON af.component_id = c.id
300 JOIN archive ON af.archive_id = archive.id
301 WHERE archive.name = 'ftp-master'
303 psql -F' ' -A -t -c "$query"
306 SELECT './pool/' || c.name || '/' || f.filename
307 FROM bin_associations ba
308 JOIN binaries b ON ba.bin = b.id
309 JOIN files f ON b.file = f.id
310 JOIN files_archive_map af ON f.id = af.file_id
311 JOIN component c ON af.component_id = c.id
312 JOIN archive ON af.archive_id = archive.id
313 WHERE ba.suite = $suite_id AND archive.name = 'ftp-master'
315 psql -F' ' -A -t -c "$query"
318 psql -F' ' -At -c "SELECT id, suite_name FROM suite" |
319 while read id suite; do
320 [ -e $base/ftp/dists/$suite ] || continue
323 distname=$(cd dists; readlink $suite || echo $suite)
324 find ./dists/$distname \! -type d
325 for distdir in ./dists/*; do
326 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
330 ) | sort -u | gzip -9 > suite-${suite}.list.gz
333 log "Finding everything on the ftp site to generate sundries"
334 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
337 zcat *.list.gz | cat - *.list | sort -u |
338 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
340 log "Generating files list"
343 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
344 cat - sundries.list dists.list project.list docs.list indices.list |
345 sort -u | poolfirst > ../arch-$a.files
349 for dist in sid wheezy; do
350 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
354 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
355 sort -u | poolfirst > ../typical.files
362 function mkchecksums() {
363 dsynclist=$dbdir/dsync.list
364 md5list=$indices/md5sums
366 log -n "Creating md5 / dsync index file ... "
369 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
370 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
371 ${bindir}/dsync-flist -q link-dups $dsynclist || true
375 log "Regenerating \"public\" mirror/ hardlink fun"
376 DATE_SERIAL=$(date +"%Y%m%d01")
377 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
378 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
379 SERIAL="$DATE_SERIAL"
381 SERIAL="$FILESOAPLUS1"
383 date -u > ${TRACEFILE}
384 echo "Using dak v1" >> ${TRACEFILE}
385 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
386 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
388 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
392 log "Expiring old database dumps..."
394 $scriptsdir/expire_dumps -d . -p -f "dump_*"
397 function transitionsclean() {
398 log "Removing out of date transitions..."
400 dak transitions -c -a
404 log "Updating DM html page"
405 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
409 log "Categorizing uncategorized bugs filed against ftp.debian.org"
413 function ddaccess() {
414 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
415 log "Trigger dd accessible parts sync including ftp dir"
416 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
419 function mirrorpush() {
420 log "Checking the public archive copy"
421 cd ${mirrordir}/dists
424 for release in $(find . -name "InRelease"); do
425 echo "Processing: ${release}"
426 subdir=${release%/InRelease}
427 while read SHASUM SIZE NAME; do
428 if ! [ -f "${subdir}/${NAME}" ]; then
429 bname=$(basename ${NAME})
430 if [[ "${bname}" =~ ^(Packages|Sources|Translation-[a-zA-Z_]+)$ ]]; then
431 # We don't keep unpacked files, don't check for their existance.
432 # We might want to go and check their unpacked shasum, but right now
433 # I don't care. I believe it should be enough if all the packed shasums
437 broken=$(( broken + 1 ))
438 echo "File ${subdir}/${NAME} is missing"
442 # We do have symlinks in the tree (see the contents files currently).
443 # So we use "readlink -f" to check the size of the target, as thats basically
444 # what gen-releases does
445 fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
446 if [ ${fsize} -ne ${SIZE} ]; then
447 broken=$(( broken + 1 ))
448 echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
452 fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
453 fshasum=${fshasum%% *}
454 if [ "${fshasum}" != "${SHASUM}" ]; then
455 broken=$(( broken + 1 ))
456 echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
459 done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
462 if [ $broken -gt 0 ]; then
463 log_error "Trouble with the public mirror, found ${broken} errors"
467 log "Starting the mirrorpush"
468 date -u > /srv/ftp.debian.org/web/mirrorstart
469 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
470 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
471 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
475 log "Exporting package data foo for i18n project"
476 STAMP=$(date "+%Y%m%d%H%M")
477 mkdir -p ${scriptdir}/i18n/${STAMP}
478 cd ${scriptdir}/i18n/${STAMP}
479 for suite in stable testing unstable; do
480 codename=$(dak admin s show ${suite}|grep '^Codename')
481 codename=${codename##* }
482 echo "Codename is ${codename}"
483 dak control-suite -l ${suite} >${codename}
485 echo "${STAMP}" > timestamp
486 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
490 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
493 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
497 log "Updating stats data"
499 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
500 R --slave --vanilla < $base/misc/ftpstats.R
501 dak stats arch-space > $webdir/arch-space
502 dak stats pkg-nums > $webdir/pkg-nums
505 function aptftpcleanup() {
506 log "Clean up apt-ftparchive's databases"
508 apt-ftparchive -q clean apt.conf
511 function cleantransactions() {
512 log "Cleanup transaction ids older than 3 months"
514 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
517 function logstats() {
518 $masterdir/tools/logs.py "$1"
521 # save timestamp when we start
522 function savetimestamp() {
523 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
524 echo ${NOW} > "${dbdir}/dinstallstart"
527 function maillogfile() {
528 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
531 function renamelogfile() {
532 if [ -f "${dbdir}/dinstallstart" ]; then
533 NOW=$(cat "${dbdir}/dinstallstart")
535 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
536 logstats "$logdir/dinstall_${NOW}.log"
537 bzip2 -9 "$logdir/dinstall_${NOW}.log"
539 error "Problem, I don't know when dinstall started, unable to do log statistics."
540 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
542 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
543 bzip2 -9 "$logdir/dinstall_${NOW}.log"
547 function testingsourcelist() {
548 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
551 # do a last run of process-unchecked before dinstall is on.
552 function process_unchecked() {
553 log "Processing the unchecked queue"
554 UNCHECKED_WITHOUT_LOCK="-p"
559 # Function to update a "statefile" telling people what we are doing
562 # This should be called with the argument(s)
563 # - Status name we want to show.
566 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
567 cat >"${DINSTALLSTATE}" <<EOF
568 Dinstall start: ${DINSTALLBEGIN}
570 Action start: ${RIGHTNOW}
574 # extract changelogs and stuff
575 function changelogs() {
576 log "Extracting changelogs"
577 dak make-changelog -e
578 mkdir -p ${exportpublic}/changelogs
579 cd ${exportpublic}/changelogs
580 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
581 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &
584 function gitpdiff() {
585 # Might be that we want to change this to have more than one git repository.
586 # Advantage of one is that we do not need much space in terms of storage in git itself,
587 # git gc is pretty good on our input.
588 # But it might be faster. Well, lets test.
589 log "Adjusting the git tree for pdiffs"
592 # The regex needs the architectures seperated with \|
593 garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q')
595 # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we
597 # Also, we only want contents, packages and sources.
598 for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do
607 # Second, add all there is into git
610 # Maybe we want to make this the same for tag and commit? But well, shouldn't matter
612 TAGD=$(date +%Y-%m-%d-%H-%M)
613 git commit -m "Commit of ${COMD}"