2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling merkels QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Create the postgres dump files
47 function pgdump_pre() {
48 log "Creating pre-daily-cron-job backup of $PGDATABASE database..."
49 pg_dump > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
52 function pgdump_post() {
53 log "Creating post-daily-cron-job backup of $PGDATABASE database..."
55 POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
56 pg_dump > $base/backup/dump_$POSTDUMP
57 #pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
58 ln -sf $base/backup/dump_$POSTDUMP current
59 #ln -sf $base/backup/dumpall_$POSTDUMP currentall
62 # Updating various files
64 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
66 $scriptsdir/update-bugdoctxt
67 $scriptsdir/update-mirrorlists
68 $scriptsdir/update-mailingliststxt
69 $scriptsdir/update-pseudopackages.sh
72 # Process (oldstable)-proposed-updates "NEW" queue
75 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" debian-release@lists.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
79 log "Doing automated p-u-new processing"
80 cd "${queuedir}/p-u-new"
84 log "Doing automated o-p-u-new processing"
85 cd "${queuedir}/o-p-u-new"
89 # The first i18n one, syncing new descriptions
91 log "Synchronizing i18n package descriptions"
92 # First sync their newest data
93 cd ${scriptdir}/i18nsync
94 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
96 # Now check if we still know about the packages for which they created the files
97 # is the timestamp signed by us?
98 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
99 # now read it. As its signed by us we are sure the content is what we expect, no need
100 # to do more here. And we only test -d a directory on it anyway.
101 TSTAMP=$(cat timestamp)
102 # do we have the dir still?
103 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
105 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
106 # Yay, worked, lets copy around
107 for dir in wheezy sid; do
108 if [ -d dists/${dir}/ ]; then
109 cd dists/${dir}/main/i18n
110 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
112 cd ${scriptdir}/i18nsync
115 echo "ARRRR, bad guys, wrong files, ARRR"
116 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
119 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
120 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
123 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
124 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
129 log "Checking for cruft in overrides"
133 function dominate() {
134 log "Removing obsolete source and binary associations"
138 function filelist() {
139 log "Generating file lists for apt-ftparchive"
140 dak generate-filelist
143 function fingerprints() {
144 log "Not updating fingerprints - scripts needs checking"
146 log "Updating fingerprints"
147 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
150 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
152 if [ -s "${OUTFILE}" ]; then
153 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
154 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
155 To: <debian-project@lists.debian.org>
156 Subject: Debian Maintainers Keyring changes
157 Content-Type: text/plain; charset=utf-8
161 The following changes to the debian-maintainers keyring have just been activated:
165 Debian distribution maintenance software,
166 on behalf of the Keyring maintainers
173 function overrides() {
174 log "Writing overrides into text files"
179 rm -f override.sid.all3
180 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
184 log "Generating package / file mapping"
185 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
188 function packages() {
189 log "Generating Packages and Sources files"
191 #apt-ftparchive generate apt.conf
192 dak generate-packages-sources
193 dak contents generate
197 log "Generating pdiff files"
198 dak generate-index-diffs
202 log "Generating Release files"
203 dak generate-releases
206 function dakcleanup() {
207 log "Cleanup old packages/files"
208 dak clean-suites -m 10000
212 function buildd_dir() {
213 # Rebuilt the buildd dir to avoid long times of 403
214 log "Regenerating the buildd incoming dir"
215 STAMP=$(date "+%Y%m%d%H%M")
224 log "Removing any core files ..."
225 find -type f -name core -print0 | xargs -0r rm -v
227 log "Checking permissions on files in the FTP tree ..."
228 find -type f \( \! -perm -444 -o -perm +002 \) -ls
229 find -type d \( \! -perm -555 -o -perm +002 \) -ls
231 log "Checking symlinks ..."
234 log "Creating recursive directory listing ... "
235 rm -f .${FILENAME}.new
236 TZ=UTC ls -lR > .${FILENAME}.new
238 if [ -r ${FILENAME}.gz ] ; then
239 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
240 mv -f .${FILENAME}.new ${FILENAME}
241 rm -f ${FILENAME}.patch.gz
242 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
243 rm -f ${FILENAME}.old.gz
245 mv -f .${FILENAME}.new ${FILENAME}
248 gzip -9cfN ${FILENAME} >${FILENAME}.gz
252 function mkmaintainers() {
253 log 'Creating Maintainers index ... '
256 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
257 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
258 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-maintainers
260 if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then
261 log "installing Maintainers ... "
262 mv -f .new-maintainers Maintainers
263 gzip -9v <Maintainers >.new-maintainers.gz
264 mv -f .new-maintainers.gz Maintainers.gz
266 rm -f .new-maintainers
270 function mkuploaders() {
271 log 'Creating Uploaders index ... '
274 dak make-maintainers -u ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
275 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
276 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-uploaders
278 if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then
279 log "installing Uploaders ... "
280 mv -f .new-uploaders Uploaders
281 gzip -9v <Uploaders >.new-uploaders.gz
282 mv -f .new-uploaders.gz Uploaders.gz
288 function copyoverrides() {
289 log 'Copying override files into public view ...'
291 for ofile in $copyoverrides ; do
293 chmod g+w override.$ofile
297 newofile=override.$ofile.gz
298 rm -f .newover-$ofile.gz
299 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
300 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
301 log " installing new $newofile $pc"
302 mv -f .newover-$ofile.gz $newofile
305 rm -f .newover-$ofile.gz
310 function mkfilesindices() {
312 cd $base/ftp/indices/files/components
316 log "Querying $PGDATABASE..."
317 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
320 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
323 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
326 log "Generating sources list"
328 sed -n 's/|$//p' $ARCHLIST
330 find ./dists -maxdepth 1 \! -type d
331 find ./dists \! -type d | grep "/source/"
332 ) | sort -u | gzip -9 > source.list.gz
334 log "Generating arch lists"
336 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
338 (sed -n "s/|$a$//p" $ARCHLIST
339 sed -n 's/|all$//p' $ARCHLIST
342 find ./dists -maxdepth 1 \! -type d
343 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
344 ) | sort -u | gzip -9 > arch-$a.list.gz
347 log "Generating suite lists"
350 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
352 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
355 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
356 while read id suite; do
357 [ -e $base/ftp/dists/$suite ] || continue
360 distname=$(cd dists; readlink $suite || echo $suite)
361 find ./dists/$distname \! -type d
362 for distdir in ./dists/*; do
363 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
366 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
367 ) | sort -u | gzip -9 > suite-${suite}.list.gz
370 log "Finding everything on the ftp site to generate sundries"
371 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
374 zcat *.list.gz | cat - *.list | sort -u |
375 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
377 log "Generating files list"
380 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
381 cat - sundries.list dists.list project.list docs.list indices.list |
382 sort -u | poolfirst > ../arch-$a.files
386 for dist in sid wheezy; do
387 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
391 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
392 sort -u | poolfirst > ../typical.files
398 function mkchecksums() {
399 dsynclist=$dbdir/dsync.list
400 md5list=$indices/md5sums
402 log -n "Creating md5 / dsync index file ... "
405 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
406 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
407 ${bindir}/dsync-flist -q link-dups $dsynclist || true
411 log "Regenerating \"public\" mirror/ hardlink fun"
412 DATE_SERIAL=$(date +"%Y%m%d01")
413 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
414 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
415 SERIAL="$DATE_SERIAL"
417 SERIAL="$FILESOAPLUS1"
419 date -u > ${TRACEFILE}
420 echo "Using dak v1" >> ${TRACEFILE}
421 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
422 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
424 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
428 log "Expiring old database dumps..."
430 $scriptsdir/expire_dumps -d . -p -f "dump_*"
433 function transitionsclean() {
434 log "Removing out of date transitions..."
436 dak transitions -c -a
440 log "Updating DM html page"
441 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
445 log "Categorizing uncategorized bugs filed against ftp.debian.org"
450 # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
451 log "Trigger merkel/flotows $PGDATABASE sync"
452 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
453 # Also trigger flotow, the ftpmaster test box
454 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
458 # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
459 log "Trigger merkels dd accessible parts sync"
460 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
463 function mirrorpush() {
464 log "Starting the mirrorpush"
465 date -u > /srv/ftp.debian.org/web/mirrorstart
466 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
467 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
468 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
472 log "Exporting package data foo for i18n project"
473 STAMP=$(date "+%Y%m%d%H%M")
474 mkdir -p ${scriptdir}/i18n/${STAMP}
475 cd ${scriptdir}/i18n/${STAMP}
476 dak control-suite -l stable > squeeze
477 dak control-suite -l testing > wheezy
478 dak control-suite -l unstable > sid
479 echo "${STAMP}" > timestamp
480 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
484 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
487 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
491 log "Updating stats data"
493 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
494 R --slave --vanilla < $base/misc/ftpstats.R
495 dak stats arch-space > $webdir/arch-space
496 dak stats pkg-nums > $webdir/pkg-nums
499 function aptftpcleanup() {
500 log "Clean up apt-ftparchive's databases"
502 apt-ftparchive -q clean apt.conf
505 function cleantransactions() {
506 log "Cleanup transaction ids older than 3 months"
508 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
511 function logstats() {
512 $masterdir/tools/logs.py "$1"
515 # save timestamp when we start
516 function savetimestamp() {
517 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
518 echo ${NOW} > "${dbdir}/dinstallstart"
521 function maillogfile() {
522 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
525 function renamelogfile() {
526 if [ -f "${dbdir}/dinstallstart" ]; then
527 NOW=$(cat "${dbdir}/dinstallstart")
529 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
530 logstats "$logdir/dinstall_${NOW}.log"
531 bzip2 -9 "$logdir/dinstall_${NOW}.log"
533 error "Problem, I don't know when dinstall started, unable to do log statistics."
534 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
536 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
537 bzip2 -9 "$logdir/dinstall_${NOW}.log"
541 function testingsourcelist() {
542 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
545 # do a last run of process-unchecked before dinstall is on.
546 function process_unchecked() {
547 log "Processing the unchecked queue"
548 UNCHECKED_WITHOUT_LOCK="-p"
553 # do a run of newstage only before dinstall is on.
554 function newstage() {
555 log "Processing the newstage queue"
556 UNCHECKED_WITHOUT_LOCK="-p"
560 # Function to update a "statefile" telling people what we are doing
563 # This should be called with the argument(s)
564 # - Status name we want to show.
567 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
568 cat >"${DINSTALLSTATE}" <<EOF
569 Dinstall start: ${DINSTALLBEGIN}
571 Action start: ${RIGHTNOW}
575 # extract changelogs and stuff
576 function changelogs() {
577 log "Extracting changelogs"
578 dak make-changelog -e
579 mkdir -p ${exportpublic}/changelogs
580 cd ${exportpublic}/changelogs
581 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
582 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &