2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
10 rm -f ${LOCK_ACCEPTED}
13 # If we error out this one is called, *FOLLOWED* by cleanup above
15 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
17 subject="ATTENTION ATTENTION!"
18 if [ "${error}" = "false" ]; then
19 subject="${subject} (continued)"
21 subject="${subject} (interrupted)"
23 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
25 cat "${STAGEFILE}.log" | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
28 ########################################################################
29 # the actual dinstall functions follow #
30 ########################################################################
32 # pushing merkels QA user, part one
34 log "Telling merkels QA user that we start dinstall"
35 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
38 # Create the postgres dump files
39 function pgdump_pre() {
40 log "Creating pre-daily-cron-job backup of $PGDATABASE database..."
41 pg_dump > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
44 function pgdump_post() {
45 log "Creating post-daily-cron-job backup of $PGDATABASE database..."
47 POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
48 pg_dump > $base/backup/dump_$POSTDUMP
49 #pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
50 ln -sf $base/backup/dump_$POSTDUMP current
51 #ln -sf $base/backup/dumpall_$POSTDUMP currentall
54 # Load the dak-dev projectb
56 # Make sure to unset any possible psql variables so we don't drop the wrong
57 # f****** database by accident
67 echo "drop database projectb" | psql -p 5434 template1
68 #cat currentall | psql -p 5433 template1
69 createdb -p 5434 -T template1 projectb
70 fgrep -v '\connect' current | psql -p 5434 projectb
73 # Updating various files
75 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
77 $scriptsdir/update-bugdoctxt
78 $scriptsdir/update-mirrorlists
79 $scriptsdir/update-mailingliststxt
80 $scriptsdir/update-pseudopackages.sh
83 # Process (oldstable)-proposed-updates "NEW" queue
86 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" debian-release@lists.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
90 log "Doing automated p-u-new processing"
91 cd "${queuedir}/p-u-new"
95 log "Doing automated o-p-u-new processing"
96 cd "${queuedir}/o-p-u-new"
100 # The first i18n one, syncing new descriptions
102 log "Synchronizing i18n package descriptions"
103 # First sync their newest data
104 cd ${scriptdir}/i18nsync
105 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
107 # Now check if we still know about the packages for which they created the files
108 # is the timestamp signed by us?
109 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
110 # now read it. As its signed by us we are sure the content is what we expect, no need
111 # to do more here. And we only test -d a directory on it anyway.
112 TSTAMP=$(cat timestamp)
113 # do we have the dir still?
114 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
116 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
117 # Yay, worked, lets copy around
118 for dir in squeeze sid; do
119 if [ -d dists/${dir}/ ]; then
120 cd dists/${dir}/main/i18n
121 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
123 cd ${scriptdir}/i18nsync
126 echo "ARRRR, bad guys, wrong files, ARRR"
127 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
130 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
131 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
134 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
135 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
140 log "Checking for cruft in overrides"
144 function dominate() {
145 log "Removing obsolete source and binary associations"
149 function filelist() {
150 log "Generating file lists for apt-ftparchive"
151 dak generate-filelist
154 function fingerprints() {
155 log "Not updating fingerprints - scripts needs checking"
157 log "Updating fingerprints"
158 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
161 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
163 if [ -s "${OUTFILE}" ]; then
164 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
165 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
166 To: <debian-project@lists.debian.org>
167 Subject: Debian Maintainers Keyring changes
168 Content-Type: text/plain; charset=utf-8
172 The following changes to the debian-maintainers keyring have just been activated:
176 Debian distribution maintenance software,
177 on behalf of the Keyring maintainers
184 function overrides() {
185 log "Writing overrides into text files"
190 rm -f override.sid.all3
191 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
195 log "Generating package / file mapping"
196 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
199 function packages() {
200 log "Generating Packages and Sources files"
202 GZIP='--rsyncable' ; export GZIP
203 #apt-ftparchive generate apt.conf
204 dak generate-packages-sources
208 log "Generating pdiff files"
209 dak generate-index-diffs
213 log "Generating Release files"
214 dak generate-releases
217 function dakcleanup() {
218 log "Cleanup old packages/files"
219 dak clean-suites -m 10000
223 function buildd_dir() {
224 # Rebuilt the buildd dir to avoid long times of 403
225 log "Regenerating the buildd incoming dir"
226 STAMP=$(date "+%Y%m%d%H%M")
235 log "Removing any core files ..."
236 find -type f -name core -print0 | xargs -0r rm -v
238 log "Checking permissions on files in the FTP tree ..."
239 find -type f \( \! -perm -444 -o -perm +002 \) -ls
240 find -type d \( \! -perm -555 -o -perm +002 \) -ls
242 log "Checking symlinks ..."
245 log "Creating recursive directory listing ... "
246 rm -f .${FILENAME}.new
247 TZ=UTC ls -lR > .${FILENAME}.new
249 if [ -r ${FILENAME}.gz ] ; then
250 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
251 mv -f .${FILENAME}.new ${FILENAME}
252 rm -f ${FILENAME}.patch.gz
253 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip --rsyncable -9cfn - >${FILENAME}.patch.gz
254 rm -f ${FILENAME}.old.gz
256 mv -f .${FILENAME}.new ${FILENAME}
259 gzip --rsyncable -9cfN ${FILENAME} >${FILENAME}.gz
263 function mkmaintainers() {
264 log 'Creating Maintainers index ... '
267 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
268 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
269 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-maintainers
271 if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then
272 log "installing Maintainers ... "
273 mv -f .new-maintainers Maintainers
274 gzip --rsyncable -9v <Maintainers >.new-maintainers.gz
275 mv -f .new-maintainers.gz Maintainers.gz
277 rm -f .new-maintainers
281 function mkuploaders() {
282 log 'Creating Uploaders index ... '
285 dak make-maintainers -u ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
286 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
287 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-uploaders
289 if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then
290 log "installing Uploaders ... "
291 mv -f .new-uploaders Uploaders
292 gzip --rsyncable -9v <Uploaders >.new-uploaders.gz
293 mv -f .new-uploaders.gz Uploaders.gz
299 function copyoverrides() {
300 log 'Copying override files into public view ...'
302 for ofile in $copyoverrides ; do
304 chmod g+w override.$ofile
308 newofile=override.$ofile.gz
309 rm -f .newover-$ofile.gz
310 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
311 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
312 log " installing new $newofile $pc"
313 mv -f .newover-$ofile.gz $newofile
316 rm -f .newover-$ofile.gz
321 function mkfilesindices() {
323 cd $base/ftp/indices/files/components
327 log "Querying $PGDATABASE..."
328 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
331 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
334 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
337 log "Generating sources list"
339 sed -n 's/|$//p' $ARCHLIST
341 find ./dists -maxdepth 1 \! -type d
342 find ./dists \! -type d | grep "/source/"
343 ) | sort -u | gzip --rsyncable -9 > source.list.gz
345 log "Generating arch lists"
347 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
349 (sed -n "s/|$a$//p" $ARCHLIST
350 sed -n 's/|all$//p' $ARCHLIST
353 find ./dists -maxdepth 1 \! -type d
354 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
355 ) | sort -u | gzip --rsyncable -9 > arch-$a.list.gz
358 log "Generating suite lists"
361 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
363 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
366 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
367 while read id suite; do
368 [ -e $base/ftp/dists/$suite ] || continue
371 distname=$(cd dists; readlink $suite || echo $suite)
372 find ./dists/$distname \! -type d
373 for distdir in ./dists/*; do
374 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
377 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
378 ) | sort -u | gzip --rsyncable -9 > suite-${suite}.list.gz
381 log "Finding everything on the ftp site to generate sundries"
382 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
385 zcat *.list.gz | cat - *.list | sort -u |
386 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
388 log "Generating files list"
391 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
392 cat - sundries.list dists.list project.list docs.list indices.list |
393 sort -u | poolfirst > ../arch-$a.files
397 for dist in sid squeeze; do
398 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip --rsyncable -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
402 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-squeeze.list.gz) |
403 sort -u | poolfirst > ../typical.files
409 function mkchecksums() {
410 dsynclist=$dbdir/dsync.list
411 md5list=$indices/md5sums
413 log -n "Creating md5 / dsync index file ... "
416 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
417 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n --rsyncable > ${md5list}.gz
418 ${bindir}/dsync-flist -q link-dups $dsynclist || true
422 log "Regenerating \"public\" mirror/ hardlink fun"
423 DATE_SERIAL=$(date +"%Y%m%d01")
424 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
425 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
426 SERIAL="$DATE_SERIAL"
428 SERIAL="$FILESOAPLUS1"
430 date -u > ${TRACEFILE}
431 echo "Using dak v1" >> ${TRACEFILE}
432 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
433 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
435 rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
439 log "Expiring old database dumps..."
441 $scriptsdir/expire_dumps -d . -p -f "dump_*"
444 function transitionsclean() {
445 log "Removing out of date transitions..."
447 dak transitions -c -a
451 log "Updating DM html page"
452 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
456 log "Categorizing uncategorized bugs filed against ftp.debian.org"
461 # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
462 log "Trigger merkel/flotows $PGDATABASE sync"
463 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
464 # Also trigger flotow, the ftpmaster test box
465 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
469 # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
470 log "Trigger merkels dd accessible parts sync"
471 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
474 function mirrorpush() {
475 log "Starting the mirrorpush"
476 date -u > /srv/ftp.debian.org/web/mirrorstart
477 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
478 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
479 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
483 log "Exporting package data foo for i18n project"
484 STAMP=$(date "+%Y%m%d%H%M")
485 mkdir -p ${scriptdir}/i18n/${STAMP}
486 cd ${scriptdir}/i18n/${STAMP}
487 dak control-suite -l stable > lenny
488 dak control-suite -l testing > squeeze
489 dak control-suite -l unstable > sid
490 echo "${STAMP}" > timestamp
491 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
495 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
498 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
502 log "Updating stats data"
504 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
505 R --slave --vanilla < $base/misc/ftpstats.R
506 dak stats arch-space > $webdir/arch-space
507 dak stats pkg-nums > $webdir/pkg-nums
510 function aptftpcleanup() {
511 log "Clean up apt-ftparchive's databases"
513 apt-ftparchive -q clean apt.conf
516 function compress() {
517 log "Compress old psql backups"
519 find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
521 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
522 while read dumpname; do
523 echo "Compressing $dumpname"
524 bzip2 -9fv "$dumpname"
526 find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
527 while read dumpname; do
528 echo "Compressing $dumpname"
529 bzip2 -9fv "$dumpname"
531 finddup -l -d $base/backup
534 function logstats() {
535 $masterdir/tools/logs.py "$1"
538 # save timestamp when we start
539 function savetimestamp() {
540 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
541 echo ${NOW} > "${dbdir}/dinstallstart"
544 function maillogfile() {
545 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
548 function renamelogfile() {
549 if [ -f "${dbdir}/dinstallstart" ]; then
550 NOW=$(cat "${dbdir}/dinstallstart")
552 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
553 logstats "$logdir/dinstall_${NOW}.log"
554 bzip2 -9 "$logdir/dinstall_${NOW}.log"
556 error "Problem, I don't know when dinstall started, unable to do log statistics."
557 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
559 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
560 bzip2 -9 "$logdir/dinstall_${NOW}.log"
564 function testingsourcelist() {
565 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
568 # do a last run of process-unchecked before dinstall is on.
569 function process_unchecked() {
570 log "Processing the unchecked queue"
571 UNCHECKED_WITHOUT_LOCK="-p"
576 # do a run of newstage only before dinstall is on.
577 function newstage() {
578 log "Processing the newstage queue"
579 UNCHECKED_WITHOUT_LOCK="-p"
583 # Function to update a "statefile" telling people what we are doing
586 # This should be called with the argument(s)
587 # - Status name we want to show.
590 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
591 cat >"${DINSTALLSTATE}" <<EOF
592 Dinstall start: ${DINSTALLBEGIN}
594 Action start: ${RIGHTNOW}
598 # extract changelogs and stuff
599 function changelogs() {
600 log "Extracting changelogs"
601 dak make-changelog -e
602 mkdir -p ${exportpublic}/changelogs
603 cd ${exportpublic}/changelogs
604 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
605 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &