2 # No way I try to deal with a crippled sh just for POSIX foo.
4 # Copyright (C) 2009 Joerg Jaspert <joerg@debian.org>
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License as
8 # published by the Free Software Foundation; version 2.
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 # Homer: Are you saying you're never going to eat any animal again? What
25 # Lisa: Dad, those all come from the same animal.
26 # Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
30 # make sure to only use defined variables
32 # ERR traps should be inherited from functions too. (And command
33 # substitutions and subshells and whatnot, but for us the functions is
34 # the important part here)
37 # import the general variable set.
38 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
41 ########################################################################
43 ########################################################################
44 # common functions are "outsourced"
45 . "${configdir}/common"
47 # Timestamp. Used for dinstall stat graphs
50 echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)"
56 rm -f ${LOCK_ACCEPTED}
59 # If we error out this one is called, *FOLLOWED* by cleanup above
61 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
62 cat "${STAGEFILE}.log" | mail -s "ATTENTION ATTENTION! dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)" cron@ftp-master.debian.org
65 ########################################################################
66 # the actual dinstall functions follow #
67 ########################################################################
69 # Setup the notice file to tell bad mirrors they used the wrong time
73 Packages are currently being installed and indices rebuilt.
74 Maintenance is automatic, starting at 01|07|13|19:52 UTC,
75 and ending about an hour later. This file is then removed.
77 You should not mirror the archive during this period. If you find this
78 file on a Debian mirror please have a nice talk with the admin. They
79 are doing something wrong.
83 # pushing merkels QA user, part one
85 log "Telling merkels QA user that we start dinstall"
86 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1
89 # Create the postgres dump files
90 function pgdump_pre() {
91 log "Creating pre-daily-cron-job backup of projectb database..."
92 pg_dump projectb > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
95 function pgdump_post() {
96 log "Creating post-daily-cron-job backup of projectb database..."
98 POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
99 pg_dump projectb > $base/backup/dump_$POSTDUMP
100 pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
101 ln -sf $base/backup/dump_$POSTDUMP current
102 ln -sf $base/backup/dumpall_$POSTDUMP currentall
105 # Load the dak-dev projectb
106 function pgdakdev() {
108 echo "drop database projectb" | psql -p 5433 template1
109 cat currentall | psql -p 5433 template1
110 createdb -p 5433 -T template0 projectb
111 fgrep -v '\connect' current | psql -p 5433 projectb
114 # Updating various files
116 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
118 $scriptsdir/update-bugdoctxt
119 $scriptsdir/update-mirrorlists
120 $scriptsdir/update-mailingliststxt
121 $scriptsdir/update-pseudopackages.sh
124 # Process (oldstable)-proposed-updates "NEW" queue
125 function punew_do() {
126 cd "${queuedir}/${1}"
128 dak process-new -a -C COMMENTS >> REPORT || true
132 log "Doing automated p-u-new processing"
136 log "Doing automated o-p-u-new processing"
140 # The first i18n one, syncing new descriptions
142 log "Synchronizing i18n package descriptions"
143 # First sync their newest data
144 cd ${scriptdir}/i18nsync
145 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
147 # Now check if we still know about the packages for which they created the files
148 # is the timestamp signed by us?
149 if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
150 # now read it. As its signed by us we are sure the content is what we expect, no need
151 # to do more here. And we only test -d a directory on it anyway.
152 TSTAMP=$(cat timestamp)
153 # do we have the dir still?
154 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
156 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
157 # Yay, worked, lets copy around
158 for dir in squeeze sid; do
159 if [ -d dists/${dir}/ ]; then
160 cd dists/${dir}/main/i18n
161 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
163 cd ${scriptdir}/i18nsync
166 echo "ARRRR, bad guys, wrong files, ARRR"
167 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
170 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
171 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
174 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
175 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
179 # Process the accepted queue
180 function accepted() {
181 log "Processing queue/accepted"
182 rm -f "$accepted/REPORT"
183 dak process-accepted -pa -d "$accepted" > "$accepted/REPORT"
184 cat "$accepted/REPORT" | mail -s "Install for $(date +"%D - %R")" ftpmaster@ftp-master.debian.org
185 chgrp debadmin "$accepted/REPORT"
186 chmod 664 "$accepted/REPORT"
190 log "Checking for cruft in overrides"
193 log "Fixing symlinks in $ftpdir"
194 symlinks -d -r $ftpdir
198 log "Generating suite file lists for apt-ftparchive"
199 dak make-suite-file-list
202 function fingerprints() {
203 log "Updating fingerprints"
204 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
207 function overrides() {
208 log "Writing overrides into text files"
213 rm -f override.sid.all3
214 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
218 log "Generating package / file mapping"
219 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
222 function packages() {
223 log "Generating Packages and Sources files"
225 apt-ftparchive generate apt.conf
229 log "Generating pdiff files"
230 dak generate-index-diffs
234 log "Generating Release files"
235 dak generate-releases
238 function dakcleanup() {
239 log "Cleanup old packages/files"
240 dak clean-suites -m 10000
245 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
246 # update this before wanna-build is updated.
247 log "Regenerating wanna-build/buildd information"
248 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
249 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
250 apt-ftparchive generate apt.conf.buildd
253 function buildd_dir() {
254 # Rebuilt the buildd dir to avoid long times of 403
255 log "Regenerating the buildd incoming dir"
256 STAMP=$(date "+%Y%m%d%H%M")
261 log "Running various scripts from $scriptsdir"
271 echo "Regenerating \"public\" mirror/ hardlink fun"
273 rsync -aH --link-dest ${ftpdir} --exclude Archive_Maintenance_In_Progress --delete --delete-after --ignore-errors ${ftpdir}/. .
277 log "Trigger daily wanna-build run"
278 ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
282 log "Expiring old database dumps..."
284 $scriptsdir/expire_dumps -d . -p -f "dump_*"
287 function transitionsclean() {
288 log "Removing out of date transitions..."
290 dak transitions -c -a
294 # Send a report on NEW/BYHAND packages
295 log "Nagging ftpteam about NEW/BYHAND packages"
296 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
297 # and one on crufty packages
298 log "Sending information about crufty packages"
299 dak cruft-report > $webdir/cruft-report-daily.txt
300 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
301 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
305 log "Updating DM html page"
306 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
310 log "Categorizing uncategorized bugs filed against ftp.debian.org"
315 # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
316 log "Trigger merkel/flotows projectb sync"
317 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
318 # Also trigger flotow, the ftpmaster test box
319 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
323 # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
324 log "Trigger merkels dd accessible parts sync"
325 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
328 function runparts() {
329 log "Using run-parts to run scripts in $base/scripts/distmnt"
330 run-parts --report $base/scripts/distmnt
334 log "Exporting package data foo for i18n project"
335 STAMP=$(date "+%Y%m%d%H%M")
336 mkdir -p ${scriptdir}/i18n/${STAMP}
337 cd ${scriptdir}/i18n/${STAMP}
338 dak control-suite -l stable > lenny
339 dak control-suite -l testing > squeeze
340 dak control-suite -l unstable > sid
341 echo "${STAMP}" > timestamp
342 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
346 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
349 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
353 log "Updating stats data"
355 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
356 R --slave --vanilla < $base/misc/ftpstats.R
357 dak stats arch-space > $webdir/arch-space
358 dak stats pkg-nums > $webdir/pkg-nums
361 function aptftpcleanup() {
362 log "Clean up apt-ftparchive's databases"
364 apt-ftparchive -q clean apt.conf
367 function compress() {
368 log "Compress old psql backups"
370 find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
372 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
373 while read dumpname; do
374 echo "Compressing $dumpname"
375 bzip2 -9fv "$dumpname"
377 find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
378 while read dumpname; do
379 echo "Compressing $dumpname"
380 bzip2 -9fv "$dumpname"
382 finddup -l -d $base/backup
385 function logstats() {
386 $masterdir/tools/logs.py "$1"
389 # save timestamp when we start
390 function savetimestamp() {
391 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
392 echo ${NOW} > "${dbdir}/dinstallstart"
395 function maillogfile() {
396 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
399 function renamelogfile() {
400 if [ -f "${dbdir}/dinstallstart" ]; then
401 NOW=$(cat "${dbdir}/dinstallstart")
403 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
404 logstats "$logdir/dinstall_${NOW}.log"
405 bzip2 -9 "$logdir/dinstall_${NOW}.log"
407 error "Problem, I don't know when dinstall started, unable to do log statistics."
408 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
410 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
411 bzip2 -9 "$logdir/dinstall_${NOW}.log"
415 function testingsourcelist() {
416 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
419 # do a last run of process-unchecked before dinstall is on.
420 function process_unchecked() {
421 log "Processing the unchecked queue"
423 UNCHECKED_WITHOUT_LOCK="-p"
428 ########################################################################
429 ########################################################################
431 # Function to save which stage we are in, so we can restart an interrupted
432 # dinstall. Or even run actions in parallel, if we dare to, by simply
433 # backgrounding the call to this function. But that should only really be
434 # done for things we don't care much about.
436 # This should be called with the first argument being an array, with the
438 # - FUNC - the function name to call
439 # - ARGS - Possible arguments to hand to the function. Can be the empty string
440 # - TS - The timestamp name. Can be the empty string
441 # - ERR - if this is the string false, then the call will be surrounded by
442 # set +e ... set -e calls, so errors in the function do not exit
443 # dinstall. Can be the empty string, meaning true.
445 # MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
446 # ADDED FOR DINSTALL FEATURES!
451 STAGEFILE="${stagedir}/${FUNC}"
452 if [ -f "${STAGEFILE}" ]; then
453 stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
455 difference=$(( $unixtime - $stamptime ))
456 if [ ${difference} -ge 14400 ]; then
457 log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
459 log "Did already run ${FUNC}, not calling again..."
464 debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TS}"
466 # Make sure we are always at the same place. If a function wants to be elsewhere,
467 # it has to cd first!
470 # Now redirect the output into $STAGEFILE.log. In case it errors out somewhere our
471 # errorhandler trap can then mail the contents of $STAGEFILE.log only, instead of a whole
472 # dinstall logfile. Short error mails ftw!
473 exec >> "${STAGEFILE}.log" 2>&1
475 if [ -f "${LOCK_STOP}" ]; then
476 log "${LOCK_STOP} exists, exiting immediately"
480 if [ "${ERR}" = "false" ]; then
485 # No matter what happened in the function, we make sure we have set -e default state back
488 # Make sure we are always at the same place.
493 if [ -n "${TIME}" ]; then
497 # And the output goes back to the normal logfile
498 exec >> "$LOGFILE" 2>&1
500 # Now we should make sure that we have a usable dinstall.log, so append the $STAGEFILE.log
502 cat "${STAGEFILE}.log" >> "${LOGFILE}"
503 rm -f "${STAGEFILE}.log"
505 if [ -f "${LOCK_STOP}" ]; then
506 log "${LOCK_STOP} exists, exiting immediately"
511 ########################################################################
514 LOGFILE="$logdir/dinstall.log"
516 exec >> "$LOGFILE" 2>&1
518 # usually we are not using debug logs. Set to 1 if you want them.
524 # where do we want mails to go? For example log entries made with error()
525 if [ "x$(hostname -s)x" != "xriesx" ]; then
526 # Not our ftpmaster host
527 MAILTO=${MAILTO:-"root"}
530 MAILTO=${MAILTO:-"ftpmaster@debian.org"}
533 # How many logfiles to keep
534 LOGROTATE=${LOGROTATE:-400}
536 # Marker for dinstall start
537 DINSTALLSTART="${lockdir}/dinstallstart"
538 # Marker for dinstall end
539 DINSTALLEND="${lockdir}/dinstallend"
541 # Timestamps start at -1. so first gets 0
543 touch "${DINSTALLSTART}"
546 # Tell everyone we are doing some work
547 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
549 # lock cron.unchecked (it immediately exits when this exists)
550 LOCK_DAILY="$lockdir/daily.lock"
552 # Lock cron.unchecked from doing work
553 LOCK_ACCEPTED="$lockdir/unchecked.lock"
555 # Lock process-new from doing work
556 LOCK_NEW="$lockdir/processnew.lock"
558 # This file is simply used to indicate to britney whether or not
559 # the Packages file updates completed sucessfully. It's not a lock
560 # from our point of view
561 LOCK_BRITNEY="$lockdir/britney.lock"
563 # If this file exists we exit immediately after the currently running
565 LOCK_STOP="$lockdir/archive.stop"
567 lockfile -l 3600 "${LOCK_DAILY}"
569 trap cleanup EXIT TERM HUP INT QUIT
571 touch "${LOCK_BRITNEY}"
607 TIME="External Updates"
637 lockfile "$LOCK_ACCEPTED"
641 FUNC="process_unchecked"
673 rm -f "$LOCK_ACCEPTED"
678 TIME="make-suite-file-list"
686 TIME="import-keyring"
702 TIME="pkg-file-mapping"
710 TIME="apt-ftparchive"
758 TIME="mirror hardlinks"
773 rm -f "${LOCK_DAILY}"
775 ts "locked part finished"
794 FUNC="transitionsclean"
795 TIME="transitionsclean"
827 TIME="merkel projectb push"
858 FUNC="testingsourcelist"
865 rm -f ${LOCK_BRITNEY}
877 TIME="apt-ftparchive cleanup"
885 TIME="merkel ddaccessible sync"
899 log "Daily cron scripts successful, all done"
901 exec > "$logdir/afterdinstall.log" 2>&1
912 # Now, at the very (successful) end of dinstall, make sure we remove
913 # our stage files, so the next dinstall run will do it all again.
915 touch "${DINSTALLEND}"