2 # No way I try to deal with a crippled sh just for POSIX foo.
4 # Copyright (C) 2009 Joerg Jaspert <joerg@debian.org>
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License as
8 # published by the Free Software Foundation; version 2.
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 # Homer: Are you saying you're never going to eat any animal again? What
25 # Lisa: Dad, those all come from the same animal.
26 # Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
30 # make sure to only use defined variables
32 # ERR traps should be inherited from functions too. (And command
33 # substitutions and subshells and whatnot, but for us the functions is
34 # the important part here)
37 # import the general variable set.
38 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
41 ########################################################################
43 ########################################################################
44 # common functions are "outsourced"
45 . "${configdir}/common"
47 # Timestamp. Used for dinstall stat graphs
49 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
55 rm -f ${LOCK_ACCEPTED}
58 # If we error out this one is called, *FOLLOWED* by cleanup above
60 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
61 cat "${STAGEFILE}.log" | mail -s "ATTENTION ATTENTION! dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)" cron@ftp-master.debian.org
64 ########################################################################
65 # the actual dinstall functions follow #
66 ########################################################################
68 # Setup the notice file to tell bad mirrors they used the wrong time
72 Packages are currently being installed and indices rebuilt.
73 Maintenance is automatic, starting at 01|07|13|19:52 UTC,
74 and ending about an hour later. This file is then removed.
76 You should not mirror the archive during this period. If you find this
77 file on a Debian mirror please have a nice talk with the admin. They
78 are doing something wrong.
82 # pushing merkels QA user, part one
84 log "Telling merkels QA user that we start dinstall"
85 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1
88 # Create the postgres dump files
89 function pgdump_pre() {
90 log "Creating pre-daily-cron-job backup of projectb database..."
91 pg_dump projectb > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
94 function pgdump_post() {
95 log "Creating post-daily-cron-job backup of projectb database..."
97 POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
98 pg_dump projectb > $base/backup/dump_$POSTDUMP
99 pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
100 ln -sf $base/backup/dump_$POSTDUMP current
101 ln -sf $base/backup/dumpall_$POSTDUMP currentall
104 # Load the dak-dev projectb
105 function pgdakdev() {
107 echo "drop database projectb" | psql -p 5433 template1
108 cat currentall | psql -p 5433 template1
109 createdb -p 5433 -T template0 projectb
110 fgrep -v '\connect' current | psql -p 5433 projectb
113 # Updating various files
115 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
117 $scriptsdir/update-bugdoctxt
118 $scriptsdir/update-mirrorlists
119 $scriptsdir/update-mailingliststxt
120 $scriptsdir/update-pseudopackages.sh
123 # Process (oldstable)-proposed-updates "NEW" queue
124 function punew_do() {
125 cd "${queuedir}/${1}"
127 dak process-new -a -C COMMENTS >> REPORT || true
131 log "Doing automated p-u-new processing"
135 log "Doing automated o-p-u-new processing"
139 # The first i18n one, syncing new descriptions
141 log "Synchronizing i18n package descriptions"
142 # First sync their newest data
143 cd ${scriptdir}/i18nsync
144 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
146 # Now check if we still know about the packages for which they created the files
147 # is the timestamp signed by us?
148 if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
149 # now read it. As its signed by us we are sure the content is what we expect, no need
150 # to do more here. And we only test -d a directory on it anyway.
151 TSTAMP=$(cat timestamp)
152 # do we have the dir still?
153 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
155 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
156 # Yay, worked, lets copy around
157 for dir in squeeze sid; do
158 if [ -d dists/${dir}/ ]; then
159 cd dists/${dir}/main/i18n
160 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
162 cd ${scriptdir}/i18nsync
165 echo "ARRRR, bad guys, wrong files, ARRR"
166 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
169 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
170 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
173 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
174 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
178 # Process the accepted queue
179 function accepted() {
180 log "Processing queue/accepted"
181 rm -f "$accepted/REPORT"
182 dak process-accepted -pa -d "$accepted" > "$accepted/REPORT"
183 cat "$accepted/REPORT" | mail -s "Install for $(date +"%D - %R")" ftpmaster@ftp-master.debian.org
184 chgrp debadmin "$accepted/REPORT"
185 chmod 664 "$accepted/REPORT"
189 log "Checking for cruft in overrides"
192 log "Fixing symlinks in $ftpdir"
193 symlinks -d -r $ftpdir
197 log "Generating suite file lists for apt-ftparchive"
198 dak make-suite-file-list
201 function fingerprints() {
202 log "Updating fingerprints"
203 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
206 function overrides() {
207 log "Writing overrides into text files"
212 rm -f override.sid.all3
213 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
217 log "Generating package / file mapping"
218 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
221 function packages() {
222 log "Generating Packages and Sources files"
224 apt-ftparchive generate apt.conf
228 log "Generating pdiff files"
229 dak generate-index-diffs
233 log "Generating Release files"
234 dak generate-releases
237 function dakcleanup() {
238 log "Cleanup old packages/files"
239 dak clean-suites -m 10000
244 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
245 # update this before wanna-build is updated.
246 log "Regenerating wanna-build/buildd information"
247 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
248 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
249 apt-ftparchive generate apt.conf.buildd
252 function buildd_dir() {
253 # Rebuilt the buildd dir to avoid long times of 403
254 log "Regenerating the buildd incoming dir"
255 STAMP=$(date "+%Y%m%d%H%M")
260 log "Running various scripts from $scriptsdir"
270 echo "Regenerating \"public\" mirror/ hardlink fun"
272 rsync -aH --link-dest ${ftpdir} --exclude Archive_Maintenance_In_Progress --delete --delete-after --ignore-errors ${ftpdir}/. .
276 log "Trigger daily wanna-build run"
277 ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
281 log "Expiring old database dumps..."
283 $scriptsdir/expire_dumps -d . -p -f "dump_*"
286 function transitionsclean() {
287 log "Removing out of date transitions..."
289 dak transitions -c -a
293 # Send a report on NEW/BYHAND packages
294 log "Nagging ftpteam about NEW/BYHAND packages"
295 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
296 # and one on crufty packages
297 log "Sending information about crufty packages"
298 dak cruft-report > $webdir/cruft-report-daily.txt
299 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
300 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
304 log "Updating DM html page"
305 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
309 log "Categorizing uncategorized bugs filed against ftp.debian.org"
314 # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
315 log "Trigger merkel/flotows projectb sync"
316 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
317 # Also trigger flotow, the ftpmaster test box
318 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
322 # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
323 log "Trigger merkels dd accessible parts sync"
324 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
327 function runparts() {
328 log "Using run-parts to run scripts in $base/scripts/distmnt"
329 run-parts --report $base/scripts/distmnt
333 log "Exporting package data foo for i18n project"
334 STAMP=$(date "+%Y%m%d%H%M")
335 mkdir -p ${scriptdir}/i18n/${STAMP}
336 cd ${scriptdir}/i18n/${STAMP}
337 dak control-suite -l stable > lenny
338 dak control-suite -l testing > squeeze
339 dak control-suite -l unstable > sid
340 echo "${STAMP}" > timestamp
341 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
345 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
348 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
352 log "Updating stats data"
354 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
355 R --slave --vanilla < $base/misc/ftpstats.R
356 dak stats arch-space > $webdir/arch-space
357 dak stats pkg-nums > $webdir/pkg-nums
360 function aptftpcleanup() {
361 log "Clean up apt-ftparchive's databases"
363 apt-ftparchive -q clean apt.conf
366 function compress() {
367 log "Compress old psql backups"
369 find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
371 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
372 while read dumpname; do
373 echo "Compressing $dumpname"
374 bzip2 -9fv "$dumpname"
376 find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
377 while read dumpname; do
378 echo "Compressing $dumpname"
379 bzip2 -9fv "$dumpname"
381 finddup -l -d $base/backup
384 function logstats() {
385 $masterdir/tools/logs.py "$1"
388 # save timestamp when we start
389 function savetimestamp() {
390 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
391 echo ${NOW} > "${dbdir}/dinstallstart"
394 function maillogfile() {
395 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
398 function renamelogfile() {
399 if [ -f "${dbdir}/dinstallstart" ]; then
400 NOW=$(cat "${dbdir}/dinstallstart")
402 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
403 logstats "$logdir/dinstall_${NOW}.log"
404 bzip2 -9 "$logdir/dinstall_${NOW}.log"
406 error "Problem, I don't know when dinstall started, unable to do log statistics."
407 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
409 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
410 bzip2 -9 "$logdir/dinstall_${NOW}.log"
414 function testingsourcelist() {
415 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
418 # do a last run of process-unchecked before dinstall is on.
419 function process_unchecked() {
420 log "Processing the unchecked queue"
422 UNCHECKED_WITHOUT_LOCK="-p"
427 ########################################################################
428 ########################################################################
430 # Function to save which stage we are in, so we can restart an interrupted
431 # dinstall. Or even run actions in parallel, if we dare to, by simply
432 # backgrounding the call to this function. But that should only really be
433 # done for things we don't care much about.
435 # This should be called with the first argument being an array, with the
437 # - FUNC - the function name to call
438 # - ARGS - Possible arguments to hand to the function. Can be the empty string
439 # - TS - The timestamp name. Can be the empty string
440 # - ERR - if this is the string false, then the call will be surrounded by
441 # set +e ... set -e calls, so errors in the function do not exit
442 # dinstall. Can be the empty string, meaning true.
444 # MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
445 # ADDED FOR DINSTALL FEATURES!
450 STAGEFILE="${stagedir}/${FUNC}"
451 if [ -f "${STAGEFILE}" ]; then
452 stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
454 difference=$(( $unixtime - $stamptime ))
455 if [ ${difference} -ge 14400 ]; then
456 log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
458 log "Did already run ${FUNC}, not calling again..."
463 debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TS}"
465 # Make sure we are always at the same place. If a function wants to be elsewhere,
466 # it has to cd first!
469 # Now redirect the output into $STAGEFILE.log. In case it errors out somewhere our
470 # errorhandler trap can then mail the contents of $STAGEFILE.log only, instead of a whole
471 # dinstall logfile. Short error mails ftw!
472 exec >> "${STAGEFILE}.log" 2>&1
474 if [ -f "${LOCK_STOP}" ]; then
475 log "${LOCK_STOP} exists, exiting immediately"
479 if [ "${ERR}" = "false" ]; then
484 # No matter what happened in the function, we make sure we have set -e default state back
487 # Make sure we are always at the same place.
492 if [ -n "${TIME}" ]; then
496 # And the output goes back to the normal logfile
497 exec >> "$LOGFILE" 2>&1
499 # Now we should make sure that we have a usable dinstall.log, so append the $STAGEFILE.log
501 cat "${STAGEFILE}.log" >> "${LOGFILE}"
502 rm -f "${STAGEFILE}.log"
504 if [ -f "${LOCK_STOP}" ]; then
505 log "${LOCK_STOP} exists, exiting immediately"
510 ########################################################################
513 LOGFILE="$logdir/dinstall.log"
515 exec >> "$LOGFILE" 2>&1
517 # usually we are not using debug logs. Set to 1 if you want them.
523 # where do we want mails to go? For example log entries made with error()
524 if [ "x$(hostname -s)x" != "xriesx" ]; then
525 # Not our ftpmaster host
526 MAILTO=${MAILTO:-"root"}
529 MAILTO=${MAILTO:-"ftpmaster@debian.org"}
532 # How many logfiles to keep
533 LOGROTATE=${LOGROTATE:-400}
535 # Marker for dinstall start
536 DINSTALLSTART="${lockdir}/dinstallstart"
537 # Marker for dinstall end
538 DINSTALLEND="${lockdir}/dinstallend"
540 touch "${DINSTALLSTART}"
543 # Tell everyone we are doing some work
544 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
546 # lock cron.unchecked (it immediately exits when this exists)
547 LOCK_DAILY="$lockdir/daily.lock"
549 # Lock cron.unchecked from doing work
550 LOCK_ACCEPTED="$lockdir/unchecked.lock"
552 # Lock process-new from doing work
553 LOCK_NEW="$lockdir/processnew.lock"
555 # This file is simply used to indicate to britney whether or not
556 # the Packages file updates completed sucessfully. It's not a lock
557 # from our point of view
558 LOCK_BRITNEY="$lockdir/britney.lock"
560 # If this file exists we exit immediately after the currently running
562 LOCK_STOP="$lockdir/archive.stop"
564 lockfile -l 3600 "${LOCK_DAILY}"
566 trap cleanup EXIT TERM HUP INT QUIT
568 touch "${LOCK_BRITNEY}"
604 TIME="External Updates"
634 lockfile "$LOCK_ACCEPTED"
638 FUNC="process_unchecked"
670 rm -f "$LOCK_ACCEPTED"
675 TIME="make-suite-file-list"
683 TIME="import-keyring"
699 TIME="pkg-file-mapping"
707 TIME="apt-ftparchive"
755 TIME="mirror hardlinks"
770 rm -f "${LOCK_DAILY}"
772 ts "locked part finished"
791 FUNC="transitionsclean"
792 TIME="transitionsclean"
824 TIME="merkel projectb push"
855 FUNC="testingsourcelist"
862 rm -f ${LOCK_BRITNEY}
874 TIME="apt-ftparchive cleanup"
882 TIME="merkel ddaccessible sync"
896 log "Daily cron scripts successful, all done"
898 exec > "$logdir/afterdinstall.log" 2>&1
909 # Now, at the very (successful) end of dinstall, make sure we remove
910 # our stage files, so the next dinstall run will do it all again.
912 touch "${DINSTALLEND}"