1 # Timestamp. Used for dinstall stat graphs
3 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
12 # If we error out this one is called, *FOLLOWED* by cleanup above
14 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
16 subject="ATTENTION ATTENTION!"
17 if [ "${error}" = "false" ]; then
18 subject="${subject} (continued)"
20 subject="${subject} (interrupted)"
22 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
24 cat "${STAGEFILE}.log" | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org
27 ########################################################################
28 # the actual dinstall functions follow #
29 ########################################################################
31 # pushing merkels QA user, part one
33 log "Telling merkels QA user that we start dinstall"
34 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1
37 # Create the postgres dump files
38 function pgdump_pre() {
39 log "Creating pre-daily-cron-job backup of projectb database..."
40 pg_dump projectb > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
43 function pgdump_post() {
44 log "Creating post-daily-cron-job backup of projectb database..."
46 POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
47 pg_dump projectb > $base/backup/dump_$POSTDUMP
48 pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
49 ln -sf $base/backup/dump_$POSTDUMP current
50 ln -sf $base/backup/dumpall_$POSTDUMP currentall
53 # Load the dak-dev projectb
56 echo "drop database projectb" | psql -p 5433 template1
57 cat currentall | psql -p 5433 template1
58 createdb -p 5433 -T template0 projectb
59 fgrep -v '\connect' current | psql -p 5433 projectb
62 # Updating various files
64 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
66 $scriptsdir/update-bugdoctxt
67 $scriptsdir/update-mirrorlists
68 $scriptsdir/update-mailingliststxt
69 $scriptsdir/update-pseudopackages.sh
72 # Process (oldstable)-proposed-updates "NEW" queue
76 dak process-new -a -C COMMENTS >> REPORT || true
80 log "Doing automated p-u-new processing"
84 log "Doing automated o-p-u-new processing"
88 # The first i18n one, syncing new descriptions
90 log "Synchronizing i18n package descriptions"
91 # First sync their newest data
92 cd ${scriptdir}/i18nsync
93 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
95 # Now check if we still know about the packages for which they created the files
96 # is the timestamp signed by us?
97 if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
98 # now read it. As its signed by us we are sure the content is what we expect, no need
99 # to do more here. And we only test -d a directory on it anyway.
100 TSTAMP=$(cat timestamp)
101 # do we have the dir still?
102 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
104 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
105 # Yay, worked, lets copy around
106 for dir in squeeze sid; do
107 if [ -d dists/${dir}/ ]; then
108 cd dists/${dir}/main/i18n
109 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
111 cd ${scriptdir}/i18nsync
114 echo "ARRRR, bad guys, wrong files, ARRR"
115 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
118 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
119 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
122 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
123 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
128 log "Checking for cruft in overrides"
133 log "Generating suite file lists for apt-ftparchive"
134 dak make-suite-file-list
137 function filelist() {
138 log "Generating file lists for apt-ftparchive"
139 dak generate-filelist
142 function fingerprints() {
143 log "Not updating fingerprints - scripts needs checking"
145 log "Updating fingerprints"
146 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
149 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
151 if [ -s "${OUTFILE}" ]; then
152 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
153 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
154 To: <debian-project@lists.debian.org>
155 Subject: Debian Maintainers Keyring changes
156 Content-Type: text/plain; charset=utf-8
159 The following changes to the debian-maintainers keyring have just been activated:
163 Debian distribution maintenance software,
164 on behalf of the Keyring maintainers
171 function overrides() {
172 log "Writing overrides into text files"
177 rm -f override.sid.all3
178 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
182 log "Generating package / file mapping"
183 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
186 function packages() {
187 log "Generating Packages and Sources files"
189 GZIP='--rsyncable' ; export GZIP
190 apt-ftparchive generate apt.conf
194 log "Generating pdiff files"
195 dak generate-index-diffs
199 log "Generating Release files"
200 dak generate-releases
203 function dakcleanup() {
204 log "Cleanup old packages/files"
205 # TODO: Fix up clean-suites
206 #dak clean-suites -m 10000
210 function buildd_dir() {
211 # Rebuilt the buildd dir to avoid long times of 403
212 log "Regenerating the buildd incoming dir"
213 STAMP=$(date "+%Y%m%d%H%M")
222 log "Removing any core files ..."
223 find -type f -name core -print0 | xargs -0r rm -v
225 log "Checking permissions on files in the FTP tree ..."
226 find -type f \( \! -perm -444 -o -perm +002 \) -ls
227 find -type d \( \! -perm -555 -o -perm +002 \) -ls
229 log "Checking symlinks ..."
232 log "Creating recursive directory listing ... "
233 rm -f .${FILENAME}.new
234 TZ=UTC ls -lR > .${FILENAME}.new
236 if [ -r ${FILENAME}.gz ] ; then
237 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
238 mv -f .${FILENAME}.new ${FILENAME}
239 rm -f ${FILENAME}.patch.gz
240 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip --rsyncable -9cfn - >${FILENAME}.patch.gz
241 rm -f ${FILENAME}.old.gz
243 mv -f .${FILENAME}.new ${FILENAME}
246 gzip --rsyncable -9cfN ${FILENAME} >${FILENAME}.gz
250 function mkmaintainers() {
251 log 'Creating Maintainers index ... '
254 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
255 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
256 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-maintainers
258 if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then
259 log "installing Maintainers ... "
260 mv -f .new-maintainers Maintainers
261 gzip --rsyncable -9v <Maintainers >.new-maintainers.gz
262 mv -f .new-maintainers.gz Maintainers.gz
264 rm -f .new-maintainers
268 function copyoverrides() {
269 log 'Copying override files into public view ...'
271 for ofile in $copyoverrides ; do
273 chmod g+w override.$f
277 newofile=override.$ofile.gz
278 rm -f .newover-$ofile.gz
279 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
280 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
281 log " installing new $newofile $pc"
282 mv -f .newover-$ofile.gz $newofile
285 rm -f .newover-$ofile.gz
290 function mkfilesindices() {
292 cd $base/ftp/indices/files/components
296 log "Querying projectb..."
297 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql projectb -At | sed 's/|//;s,^/srv/ftp.debian.org/ftp,.,' | sort >$ARCHLIST
300 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
303 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
306 log "Generating sources list"
308 sed -n 's/|$//p' $ARCHLIST
310 find ./dists -maxdepth 1 \! -type d
311 find ./dists \! -type d | grep "/source/"
312 ) | sort -u | gzip --rsyncable -9 > source.list.gz
314 log "Generating arch lists"
316 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
318 (sed -n "s/|$a$//p" $ARCHLIST
319 sed -n 's/|all$//p' $ARCHLIST
322 find ./dists -maxdepth 1 \! -type d
323 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
324 ) | sort -u | gzip --rsyncable -9 > arch-$a.list.gz
327 log "Generating suite lists"
330 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t projectb
332 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t projectb
335 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At projectb |
336 while read id suite; do
337 [ -e $base/ftp/dists/$suite ] || continue
340 distname=$(cd dists; readlink $suite || echo $suite)
341 find ./dists/$distname \! -type d
342 for distdir in ./dists/*; do
343 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
346 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp.debian.org/ftp,.,'
347 ) | sort -u | gzip --rsyncable -9 > suite-${suite}.list.gz
350 log "Finding everything on the ftp site to generate sundries"
351 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
354 zcat *.list.gz | cat - *.list | sort -u |
355 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
357 log "Generating files list"
360 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
361 cat - sundries.list dists.list project.list docs.list indices.list |
362 sort -u | poolfirst > ../arch-$a.files
366 for dist in sid squeeze; do
367 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip --rsyncable -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
371 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-oldstable.list.gz suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-squeeze.list.gz) |
372 sort -u | poolfirst > ../typical.files
378 function mkchecksums() {
379 dsynclist=$dbdir/dsync.list
380 md5list=$indices/md5sums
382 log -n "Creating md5 / dsync index file ... "
385 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
386 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n --rsyncable > ${md5list}.gz
387 ${bindir}/dsync-flist -q link-dups $dsynclist || true
391 log "Regenerating \"public\" mirror/ hardlink fun"
393 rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
397 log "Trigger daily wanna-build run"
402 log "Expiring old database dumps..."
404 $scriptsdir/expire_dumps -d . -p -f "dump_*"
407 function transitionsclean() {
408 log "Removing out of date transitions..."
410 dak transitions -c -a
414 # Send a report on NEW/BYHAND packages
415 log "Nagging ftpteam about NEW/BYHAND packages"
416 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
417 # and one on crufty packages
418 log "Sending information about crufty packages"
419 dak cruft-report > $webdir/cruft-report-daily.txt
420 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
421 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
425 log "Updating DM html page"
426 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
430 log "Categorizing uncategorized bugs filed against ftp.debian.org"
435 # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
436 log "Trigger merkel/flotows projectb sync"
437 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
438 # Also trigger flotow, the ftpmaster test box
439 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
443 # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
444 log "Trigger merkels dd accessible parts sync"
445 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
448 function mirrorpush() {
449 log "Starting the mirrorpush"
450 date -u > /srv/ftp.debian.org/web/mirrorstart
451 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
452 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
453 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
457 log "Exporting package data foo for i18n project"
458 STAMP=$(date "+%Y%m%d%H%M")
459 mkdir -p ${scriptdir}/i18n/${STAMP}
460 cd ${scriptdir}/i18n/${STAMP}
461 dak control-suite -l stable > lenny
462 dak control-suite -l testing > squeeze
463 dak control-suite -l unstable > sid
464 echo "${STAMP}" > timestamp
465 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
469 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
472 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
476 log "Updating stats data"
478 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
479 R --slave --vanilla < $base/misc/ftpstats.R
480 dak stats arch-space > $webdir/arch-space
481 dak stats pkg-nums > $webdir/pkg-nums
484 function aptftpcleanup() {
485 log "Clean up apt-ftparchive's databases"
487 apt-ftparchive -q clean apt.conf
490 function compress() {
491 log "Compress old psql backups"
493 find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
495 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
496 while read dumpname; do
497 echo "Compressing $dumpname"
498 bzip2 -9fv "$dumpname"
500 find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
501 while read dumpname; do
502 echo "Compressing $dumpname"
503 bzip2 -9fv "$dumpname"
505 finddup -l -d $base/backup
508 function logstats() {
509 $masterdir/tools/logs.py "$1"
512 # save timestamp when we start
513 function savetimestamp() {
514 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
515 echo ${NOW} > "${dbdir}/dinstallstart"
518 function maillogfile() {
519 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
522 function renamelogfile() {
523 if [ -f "${dbdir}/dinstallstart" ]; then
524 NOW=$(cat "${dbdir}/dinstallstart")
526 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
527 logstats "$logdir/dinstall_${NOW}.log"
528 bzip2 -9 "$logdir/dinstall_${NOW}.log"
530 error "Problem, I don't know when dinstall started, unable to do log statistics."
531 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
533 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
534 bzip2 -9 "$logdir/dinstall_${NOW}.log"
538 function testingsourcelist() {
539 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
542 # do a last run of process-unchecked before dinstall is on.
543 function process_unchecked() {
544 log "Processing the unchecked queue"
545 UNCHECKED_WITHOUT_LOCK="-p"