2 # No way I try to deal with a crippled sh just for POSIX foo.
4 # Copyright (C) 2009 Joerg Jaspert <joerg@debian.org>
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License as
8 # published by the Free Software Foundation; version 2.
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 # Homer: Are you saying you're never going to eat any animal again? What
25 # Lisa: Dad, those all come from the same animal.
26 # Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
30 # make sure to only use defined variables
32 # ERR traps should be inherited from functions too. (And command
33 # substitutions and subshells and whatnot, but for us the functions is
34 # the important part here)
37 # import the general variable set.
38 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
41 ########################################################################
43 ########################################################################
44 # common functions are "outsourced"
45 . "${configdir}/common"
47 # Timestamp. Used for dinstall stat graphs
49 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
55 rm -f ${LOCK_ACCEPTED}
58 # If we error out this one is called, *FOLLOWED* by cleanup above
60 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
62 subject="ATTENTION ATTENTION!"
63 if [ "${error}" = "false" ]; then
64 subject="${subject} (continued)"
66 subject="${subject} (interrupted)"
68 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
70 cat "${STAGEFILE}.log" | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org
73 ########################################################################
74 # the actual dinstall functions follow #
75 ########################################################################
77 # Setup the notice file to tell bad mirrors they used the wrong time
81 Packages are currently being installed and indices rebuilt.
82 Maintenance is automatic, starting at 01|07|13|19:52 UTC,
83 and ending about an hour later. This file is then removed.
85 You should not mirror the archive during this period. If you find this
86 file on a Debian mirror please have a nice talk with the admin. They
87 are doing something wrong.
91 # pushing merkels QA user, part one
93 log "Telling merkels QA user that we start dinstall"
94 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1
97 # Create the postgres dump files
98 function pgdump_pre() {
99 log "Creating pre-daily-cron-job backup of projectb database..."
100 pg_dump projectb > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
103 function pgdump_post() {
104 log "Creating post-daily-cron-job backup of projectb database..."
106 POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
107 pg_dump projectb > $base/backup/dump_$POSTDUMP
108 pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
109 ln -sf $base/backup/dump_$POSTDUMP current
110 ln -sf $base/backup/dumpall_$POSTDUMP currentall
113 # Load the dak-dev projectb
114 function pgdakdev() {
116 echo "drop database projectb" | psql -p 5433 template1
117 cat currentall | psql -p 5433 template1
118 createdb -p 5433 -T template0 projectb
119 fgrep -v '\connect' current | psql -p 5433 projectb
122 # Updating various files
124 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
126 $scriptsdir/update-bugdoctxt
127 $scriptsdir/update-mirrorlists
128 $scriptsdir/update-mailingliststxt
129 $scriptsdir/update-pseudopackages.sh
132 # Process (oldstable)-proposed-updates "NEW" queue
133 function punew_do() {
134 cd "${queuedir}/${1}"
136 dak process-new -a -C COMMENTS >> REPORT || true
140 log "Doing automated p-u-new processing"
144 log "Doing automated o-p-u-new processing"
148 # The first i18n one, syncing new descriptions
150 log "Synchronizing i18n package descriptions"
151 # First sync their newest data
152 cd ${scriptdir}/i18nsync
153 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
155 # Now check if we still know about the packages for which they created the files
156 # is the timestamp signed by us?
157 if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
158 # now read it. As its signed by us we are sure the content is what we expect, no need
159 # to do more here. And we only test -d a directory on it anyway.
160 TSTAMP=$(cat timestamp)
161 # do we have the dir still?
162 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
164 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
165 # Yay, worked, lets copy around
166 for dir in squeeze sid; do
167 if [ -d dists/${dir}/ ]; then
168 cd dists/${dir}/main/i18n
169 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
171 cd ${scriptdir}/i18nsync
174 echo "ARRRR, bad guys, wrong files, ARRR"
175 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
178 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
179 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
182 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
183 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
187 # Process the accepted queue
188 function accepted() {
189 log "Processing queue/accepted"
190 rm -f "$accepted/REPORT"
191 dak process-accepted -pa -d "$accepted" > "$accepted/REPORT"
192 cat "$accepted/REPORT" | mail -s "Install for $(date +"%D - %R")" ftpmaster@ftp-master.debian.org
193 chgrp debadmin "$accepted/REPORT"
194 chmod 664 "$accepted/REPORT"
198 log "Checking for cruft in overrides"
203 log "Generating suite file lists for apt-ftparchive"
204 dak make-suite-file-list
207 function fingerprints() {
208 log "Updating fingerprints"
209 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
212 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
214 if [ -s "${OUTFILE}" ]; then
215 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
216 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
217 To: <debian-project@lists.debian.org>
218 Subject: Debian Maintainers Keyring changes
219 Content-Type: text/plain; charset=utf-8
222 The following changes to the debian-maintainers keyring have just been activated:
226 Debian distribution maintenance software,
227 on behalf of the Keyring maintainers
234 function overrides() {
235 log "Writing overrides into text files"
240 rm -f override.sid.all3
241 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
245 log "Generating package / file mapping"
246 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
249 function packages() {
250 log "Generating Packages and Sources files"
252 GZIP='--rsyncable' ; export GZIP
253 apt-ftparchive generate apt.conf
257 log "Generating pdiff files"
258 dak generate-index-diffs
262 log "Generating Release files"
263 dak generate-releases
266 function dakcleanup() {
267 log "Cleanup old packages/files"
268 dak clean-suites -m 10000
273 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
274 # update this before wanna-build is updated.
275 log "Regenerating wanna-build/buildd information"
276 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
277 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
278 apt-ftparchive generate apt.conf.buildd
281 function buildd_dir() {
282 # Rebuilt the buildd dir to avoid long times of 403
283 log "Regenerating the buildd incoming dir"
284 STAMP=$(date "+%Y%m%d%H%M")
293 log "Removing any core files ..."
294 find -type f -name core -print0 | xargs -0r rm -v
296 log "Checking permissions on files in the FTP tree ..."
297 find -type f \( \! -perm -444 -o -perm +002 \) -ls
298 find -type d \( \! -perm -555 -o -perm +002 \) -ls
300 log "Checking symlinks ..."
303 log "Creating recursive directory listing ... "
304 rm -f .${FILENAME}.new
305 TZ=UTC ls -lR > .${FILENAME}.new
307 if [ -r ${FILENAME}.gz ] ; then
308 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
309 mv -f .${FILENAME}.new ${FILENAME}
310 rm -f ${FILENAME}.patch.gz
311 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip --rsyncable -9cfn - >${FILENAME}.patch.gz
312 rm -f ${FILENAME}.old.gz
314 mv -f .${FILENAME}.new ${FILENAME}
317 gzip --rsyncable -9cfN ${FILENAME} >${FILENAME}.gz
321 function mkmaintainers() {
322 log -n 'Creating Maintainers index ... '
325 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
326 sed -e "s/~[^ ]*\([ ]\)/\1/" | awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-maintainers
329 cmp .new-maintainers Maintainers >/dev/null
332 if [ $rc = 1 ] || [ ! -f Maintainers ] ; then
333 log -n "installing Maintainers ... "
334 mv -f .new-maintainers Maintainers
335 gzip --rsyncable -9v <Maintainers >.new-maintainers.gz
336 mv -f .new-maintainers.gz Maintainers.gz
337 elif [ $rc = 0 ] ; then
338 log '(same as before)'
339 rm -f .new-maintainers
347 log "Running various scripts from $scriptsdir"
357 echo "Regenerating \"public\" mirror/ hardlink fun"
359 rsync -aH --link-dest ${ftpdir} --exclude Archive_Maintenance_In_Progress --delete --delete-after --ignore-errors ${ftpdir}/. .
363 log "Trigger daily wanna-build run"
364 ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
368 log "Expiring old database dumps..."
370 $scriptsdir/expire_dumps -d . -p -f "dump_*"
373 function transitionsclean() {
374 log "Removing out of date transitions..."
376 dak transitions -c -a
380 # Send a report on NEW/BYHAND packages
381 log "Nagging ftpteam about NEW/BYHAND packages"
382 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
383 # and one on crufty packages
384 log "Sending information about crufty packages"
385 dak cruft-report > $webdir/cruft-report-daily.txt
386 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
387 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
391 log "Updating DM html page"
392 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
396 log "Categorizing uncategorized bugs filed against ftp.debian.org"
401 # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
402 log "Trigger merkel/flotows projectb sync"
403 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
404 # Also trigger flotow, the ftpmaster test box
405 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
409 # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
410 log "Trigger merkels dd accessible parts sync"
411 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
414 function mirrorpush() {
415 log "Starting the mirrorpush"
416 date -u > /srv/ftp.debian.org/web/mirrorstart
417 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
418 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
419 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
423 log "Exporting package data foo for i18n project"
424 STAMP=$(date "+%Y%m%d%H%M")
425 mkdir -p ${scriptdir}/i18n/${STAMP}
426 cd ${scriptdir}/i18n/${STAMP}
427 dak control-suite -l stable > lenny
428 dak control-suite -l testing > squeeze
429 dak control-suite -l unstable > sid
430 echo "${STAMP}" > timestamp
431 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
435 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
438 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
442 log "Updating stats data"
444 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
445 R --slave --vanilla < $base/misc/ftpstats.R
446 dak stats arch-space > $webdir/arch-space
447 dak stats pkg-nums > $webdir/pkg-nums
450 function aptftpcleanup() {
451 log "Clean up apt-ftparchive's databases"
453 apt-ftparchive -q clean apt.conf
456 function compress() {
457 log "Compress old psql backups"
459 find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
461 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
462 while read dumpname; do
463 echo "Compressing $dumpname"
464 bzip2 -9fv "$dumpname"
466 find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
467 while read dumpname; do
468 echo "Compressing $dumpname"
469 bzip2 -9fv "$dumpname"
471 finddup -l -d $base/backup
474 function logstats() {
475 $masterdir/tools/logs.py "$1"
478 # save timestamp when we start
479 function savetimestamp() {
480 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
481 echo ${NOW} > "${dbdir}/dinstallstart"
484 function maillogfile() {
485 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
488 function renamelogfile() {
489 if [ -f "${dbdir}/dinstallstart" ]; then
490 NOW=$(cat "${dbdir}/dinstallstart")
492 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
493 logstats "$logdir/dinstall_${NOW}.log"
494 bzip2 -9 "$logdir/dinstall_${NOW}.log"
496 error "Problem, I don't know when dinstall started, unable to do log statistics."
497 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
499 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
500 bzip2 -9 "$logdir/dinstall_${NOW}.log"
504 function testingsourcelist() {
505 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
508 # do a last run of process-unchecked before dinstall is on.
509 function process_unchecked() {
510 log "Processing the unchecked queue"
512 UNCHECKED_WITHOUT_LOCK="-p"
517 ########################################################################
518 ########################################################################
520 # Function to save which stage we are in, so we can restart an interrupted
521 # dinstall. Or even run actions in parallel, if we dare to, by simply
522 # backgrounding the call to this function. But that should only really be
523 # done for things we don't care much about.
525 # This should be called with the first argument being an array, with the
527 # - FUNC - the function name to call
528 # - ARGS - Possible arguments to hand to the function. Can be the empty string
529 # - TIME - The timestamp name. Can be the empty string
530 # - ERR - if this is the string false, then the call will be surrounded by
531 # set +e ... set -e calls, so errors in the function do not exit
532 # dinstall. Can be the empty string, meaning true.
534 # MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
535 # ADDED FOR DINSTALL FEATURES!
542 STAGEFILE="${stagedir}/${FUNC}"
543 if [ -f "${STAGEFILE}" ]; then
544 stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
546 difference=$(( $unixtime - $stamptime ))
547 if [ ${difference} -ge 14400 ]; then
548 log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
550 log "Did already run ${FUNC}, not calling again..."
555 debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"
557 # Make sure we are always at the same place. If a function wants to be elsewhere,
558 # it has to cd first!
561 # Now redirect the output into $STAGEFILE.log. In case it errors out somewhere our
562 # errorhandler trap can then mail the contents of $STAGEFILE.log only, instead of a whole
563 # dinstall logfile. Short error mails ftw!
564 exec >> "${STAGEFILE}.log" 2>&1
566 if [ -f "${LOCK_STOP}" ]; then
567 log "${LOCK_STOP} exists, exiting immediately"
571 if [ "${error}" = "false" ]; then
576 # No matter what happened in the function, we make sure we have set -e default state back
579 # Make sure we are always at the same place.
584 if [ -n "${TIME}" ]; then
588 # And the output goes back to the normal logfile
589 exec >> "$LOGFILE" 2>&1
591 # Now we should make sure that we have a usable dinstall.log, so append the $STAGEFILE.log
593 cat "${STAGEFILE}.log" >> "${LOGFILE}"
594 rm -f "${STAGEFILE}.log"
596 if [ -f "${LOCK_STOP}" ]; then
597 log "${LOCK_STOP} exists, exiting immediately"
602 ########################################################################
605 LOGFILE="$logdir/dinstall.log"
607 exec >> "$LOGFILE" 2>&1
609 # usually we are not using debug logs. Set to 1 if you want them.
615 # where do we want mails to go? For example log entries made with error()
616 if [ "x$(hostname -s)x" != "xriesx" ]; then
617 # Not our ftpmaster host
618 MAILTO=${MAILTO:-"root"}
621 MAILTO=${MAILTO:-"ftpmaster@debian.org"}
624 # How many logfiles to keep
625 LOGROTATE=${LOGROTATE:-400}
627 # Marker for dinstall start
628 DINSTALLSTART="${lockdir}/dinstallstart"
629 # Marker for dinstall end
630 DINSTALLEND="${lockdir}/dinstallend"
632 touch "${DINSTALLSTART}"
635 # Tell everyone we are doing some work
636 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
638 # lock cron.unchecked (it immediately exits when this exists)
639 LOCK_DAILY="$lockdir/daily.lock"
641 # Lock cron.unchecked from doing work
642 LOCK_ACCEPTED="$lockdir/unchecked.lock"
644 # Lock process-new from doing work
645 LOCK_NEW="$lockdir/processnew.lock"
647 # This file is simply used to indicate to britney whether or not
648 # the Packages file updates completed sucessfully. It's not a lock
649 # from our point of view
650 LOCK_BRITNEY="$lockdir/britney.lock"
652 # If this file exists we exit immediately after the currently running
654 LOCK_STOP="$lockdir/archive.stop"
656 lockfile -l 3600 "${LOCK_DAILY}"
658 trap cleanup EXIT TERM HUP INT QUIT
660 touch "${LOCK_BRITNEY}"
696 TIME="External Updates"
726 lockfile "$LOCK_ACCEPTED"
730 FUNC="process_unchecked"
762 rm -f "$LOCK_ACCEPTED"
767 TIME="make-suite-file-list"
775 TIME="import-keyring"
791 TIME="pkg-file-mapping"
799 TIME="apt-ftparchive"
847 TIME="mirror hardlinks"
862 rm -f "${LOCK_DAILY}"
864 ts "locked part finished"
883 FUNC="transitionsclean"
884 TIME="transitionsclean"
916 TIME="merkel projectb push"
947 FUNC="testingsourcelist"
954 rm -f ${LOCK_BRITNEY}
966 TIME="merkel ddaccessible sync"
982 TIME="apt-ftparchive cleanup"
988 log "Daily cron scripts successful, all done"
990 exec > "$logdir/afterdinstall.log" 2>&1
1001 # Now, at the very (successful) end of dinstall, make sure we remove
1002 # our stage files, so the next dinstall run will do it all again.
1004 touch "${DINSTALLEND}"