2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling merkels QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Create the postgres dump files
47 function pgdump_pre() {
48 log "Creating pre-daily-cron-job backup of $PGDATABASE database..."
49 pg_dump > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
52 function pgdump_post() {
53 log "Creating post-daily-cron-job backup of $PGDATABASE database..."
55 POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
56 pg_dump > $base/backup/dump_$POSTDUMP
57 #pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
58 ln -sf $base/backup/dump_$POSTDUMP current
59 #ln -sf $base/backup/dumpall_$POSTDUMP currentall
62 # Load the dak-dev projectb
64 # Make sure to unset any possible psql variables so we don't drop the wrong
65 # f****** database by accident
75 echo "drop database projectb" | psql -p 5434 template1
76 #cat currentall | psql -p 5433 template1
77 createdb -p 5434 -T template1 projectb
78 fgrep -v '\connect' current | psql -p 5434 projectb
81 # Updating various files
83 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
85 $scriptsdir/update-bugdoctxt
86 $scriptsdir/update-mirrorlists
87 $scriptsdir/update-mailingliststxt
88 $scriptsdir/update-pseudopackages.sh
91 # Process (oldstable)-proposed-updates "NEW" queue
94 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" debian-release@lists.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
98 log "Doing automated p-u-new processing"
99 cd "${queuedir}/p-u-new"
103 log "Doing automated o-p-u-new processing"
104 cd "${queuedir}/o-p-u-new"
108 # The first i18n one, syncing new descriptions
110 log "Synchronizing i18n package descriptions"
111 # First sync their newest data
112 cd ${scriptdir}/i18nsync
113 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
115 # Now check if we still know about the packages for which they created the files
116 # is the timestamp signed by us?
117 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
118 # now read it. As its signed by us we are sure the content is what we expect, no need
119 # to do more here. And we only test -d a directory on it anyway.
120 TSTAMP=$(cat timestamp)
121 # do we have the dir still?
122 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
124 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
125 # Yay, worked, lets copy around
126 for dir in wheezy sid; do
127 if [ -d dists/${dir}/ ]; then
128 cd dists/${dir}/main/i18n
129 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
131 cd ${scriptdir}/i18nsync
134 echo "ARRRR, bad guys, wrong files, ARRR"
135 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
138 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
139 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
142 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
143 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
148 log "Checking for cruft in overrides"
152 function dominate() {
153 log "Removing obsolete source and binary associations"
157 function filelist() {
158 log "Generating file lists for apt-ftparchive"
159 dak generate-filelist
162 function fingerprints() {
163 log "Not updating fingerprints - scripts needs checking"
165 log "Updating fingerprints"
166 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
169 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
171 if [ -s "${OUTFILE}" ]; then
172 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
173 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
174 To: <debian-project@lists.debian.org>
175 Subject: Debian Maintainers Keyring changes
176 Content-Type: text/plain; charset=utf-8
180 The following changes to the debian-maintainers keyring have just been activated:
184 Debian distribution maintenance software,
185 on behalf of the Keyring maintainers
192 function overrides() {
193 log "Writing overrides into text files"
198 rm -f override.sid.all3
199 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
203 log "Generating package / file mapping"
204 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
207 function packages() {
208 log "Generating Packages and Sources files"
210 #apt-ftparchive generate apt.conf
211 dak generate-packages-sources
212 dak contents generate
216 log "Generating pdiff files"
217 dak generate-index-diffs
221 log "Generating Release files"
222 dak generate-releases
225 function dakcleanup() {
226 log "Cleanup old packages/files"
227 dak clean-suites -m 10000
231 function buildd_dir() {
232 # Rebuilt the buildd dir to avoid long times of 403
233 log "Regenerating the buildd incoming dir"
234 STAMP=$(date "+%Y%m%d%H%M")
243 log "Removing any core files ..."
244 find -type f -name core -print0 | xargs -0r rm -v
246 log "Checking permissions on files in the FTP tree ..."
247 find -type f \( \! -perm -444 -o -perm +002 \) -ls
248 find -type d \( \! -perm -555 -o -perm +002 \) -ls
250 log "Checking symlinks ..."
253 log "Creating recursive directory listing ... "
254 rm -f .${FILENAME}.new
255 TZ=UTC ls -lR > .${FILENAME}.new
257 if [ -r ${FILENAME}.gz ] ; then
258 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
259 mv -f .${FILENAME}.new ${FILENAME}
260 rm -f ${FILENAME}.patch.gz
261 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
262 rm -f ${FILENAME}.old.gz
264 mv -f .${FILENAME}.new ${FILENAME}
267 gzip -9cfN ${FILENAME} >${FILENAME}.gz
271 function mkmaintainers() {
272 log 'Creating Maintainers index ... '
275 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
276 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
277 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-maintainers
279 if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then
280 log "installing Maintainers ... "
281 mv -f .new-maintainers Maintainers
282 gzip -9v <Maintainers >.new-maintainers.gz
283 mv -f .new-maintainers.gz Maintainers.gz
285 rm -f .new-maintainers
289 function mkuploaders() {
290 log 'Creating Uploaders index ... '
293 dak make-maintainers -u ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
294 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
295 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-uploaders
297 if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then
298 log "installing Uploaders ... "
299 mv -f .new-uploaders Uploaders
300 gzip -9v <Uploaders >.new-uploaders.gz
301 mv -f .new-uploaders.gz Uploaders.gz
307 function copyoverrides() {
308 log 'Copying override files into public view ...'
310 for ofile in $copyoverrides ; do
312 chmod g+w override.$ofile
316 newofile=override.$ofile.gz
317 rm -f .newover-$ofile.gz
318 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
319 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
320 log " installing new $newofile $pc"
321 mv -f .newover-$ofile.gz $newofile
324 rm -f .newover-$ofile.gz
329 function mkfilesindices() {
331 cd $base/ftp/indices/files/components
335 log "Querying $PGDATABASE..."
336 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
339 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
342 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
345 log "Generating sources list"
347 sed -n 's/|$//p' $ARCHLIST
349 find ./dists -maxdepth 1 \! -type d
350 find ./dists \! -type d | grep "/source/"
351 ) | sort -u | gzip -9 > source.list.gz
353 log "Generating arch lists"
355 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
357 (sed -n "s/|$a$//p" $ARCHLIST
358 sed -n 's/|all$//p' $ARCHLIST
361 find ./dists -maxdepth 1 \! -type d
362 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
363 ) | sort -u | gzip -9 > arch-$a.list.gz
366 log "Generating suite lists"
369 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
371 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
374 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
375 while read id suite; do
376 [ -e $base/ftp/dists/$suite ] || continue
379 distname=$(cd dists; readlink $suite || echo $suite)
380 find ./dists/$distname \! -type d
381 for distdir in ./dists/*; do
382 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
385 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
386 ) | sort -u | gzip -9 > suite-${suite}.list.gz
389 log "Finding everything on the ftp site to generate sundries"
390 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
393 zcat *.list.gz | cat - *.list | sort -u |
394 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
396 log "Generating files list"
399 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
400 cat - sundries.list dists.list project.list docs.list indices.list |
401 sort -u | poolfirst > ../arch-$a.files
405 for dist in sid wheezy; do
406 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
410 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
411 sort -u | poolfirst > ../typical.files
417 function mkchecksums() {
418 dsynclist=$dbdir/dsync.list
419 md5list=$indices/md5sums
421 log -n "Creating md5 / dsync index file ... "
424 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
425 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
426 ${bindir}/dsync-flist -q link-dups $dsynclist || true
430 log "Regenerating \"public\" mirror/ hardlink fun"
431 DATE_SERIAL=$(date +"%Y%m%d01")
432 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
433 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
434 SERIAL="$DATE_SERIAL"
436 SERIAL="$FILESOAPLUS1"
438 date -u > ${TRACEFILE}
439 echo "Using dak v1" >> ${TRACEFILE}
440 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
441 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
443 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
447 log "Expiring old database dumps..."
449 $scriptsdir/expire_dumps -d . -p -f "dump_*"
452 function transitionsclean() {
453 log "Removing out of date transitions..."
455 dak transitions -c -a
459 log "Updating DM html page"
460 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
464 log "Categorizing uncategorized bugs filed against ftp.debian.org"
469 # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
470 log "Trigger merkel/flotows $PGDATABASE sync"
471 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
472 # Also trigger flotow, the ftpmaster test box
473 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
477 # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
478 log "Trigger merkels dd accessible parts sync"
479 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
482 function mirrorpush() {
483 log "Starting the mirrorpush"
484 date -u > /srv/ftp.debian.org/web/mirrorstart
485 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
486 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
487 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
491 log "Exporting package data foo for i18n project"
492 STAMP=$(date "+%Y%m%d%H%M")
493 mkdir -p ${scriptdir}/i18n/${STAMP}
494 cd ${scriptdir}/i18n/${STAMP}
495 dak control-suite -l stable > squeeze
496 dak control-suite -l testing > wheezy
497 dak control-suite -l unstable > sid
498 echo "${STAMP}" > timestamp
499 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
503 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
506 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
510 log "Updating stats data"
512 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
513 R --slave --vanilla < $base/misc/ftpstats.R
514 dak stats arch-space > $webdir/arch-space
515 dak stats pkg-nums > $webdir/pkg-nums
518 function aptftpcleanup() {
519 log "Clean up apt-ftparchive's databases"
521 apt-ftparchive -q clean apt.conf
524 function compress() {
525 log "Compress old psql backups"
527 find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
529 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' \! -name '*.xz' -mmin +720 -print0 \
530 | xargs --no-run-if-empty -0 -P4 -n16 xz -9v
532 find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' \! -name '*.xz' -mmin +720 \
533 | xargs --no-run-if-empty -0 -P4 -n16 xz -9v
534 finddup -l -d $base/backup
537 function logstats() {
538 $masterdir/tools/logs.py "$1"
541 # save timestamp when we start
542 function savetimestamp() {
543 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
544 echo ${NOW} > "${dbdir}/dinstallstart"
547 function maillogfile() {
548 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
551 function renamelogfile() {
552 if [ -f "${dbdir}/dinstallstart" ]; then
553 NOW=$(cat "${dbdir}/dinstallstart")
555 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
556 logstats "$logdir/dinstall_${NOW}.log"
557 bzip2 -9 "$logdir/dinstall_${NOW}.log"
559 error "Problem, I don't know when dinstall started, unable to do log statistics."
560 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
562 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
563 bzip2 -9 "$logdir/dinstall_${NOW}.log"
567 function testingsourcelist() {
568 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
571 # do a last run of process-unchecked before dinstall is on.
572 function process_unchecked() {
573 log "Processing the unchecked queue"
574 UNCHECKED_WITHOUT_LOCK="-p"
579 # do a run of newstage only before dinstall is on.
580 function newstage() {
581 log "Processing the newstage queue"
582 UNCHECKED_WITHOUT_LOCK="-p"
586 # Function to update a "statefile" telling people what we are doing
589 # This should be called with the argument(s)
590 # - Status name we want to show.
593 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
594 cat >"${DINSTALLSTATE}" <<EOF
595 Dinstall start: ${DINSTALLBEGIN}
597 Action start: ${RIGHTNOW}
601 # extract changelogs and stuff
602 function changelogs() {
603 log "Extracting changelogs"
604 dak make-changelog -e
605 mkdir -p ${exportpublic}/changelogs
606 cd ${exportpublic}/changelogs
607 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
608 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &