2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Updating various files
48 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
50 $scriptsdir/update-bugdoctxt
51 $scriptsdir/update-mirrorlists
52 $scriptsdir/update-mailingliststxt
53 $scriptsdir/update-pseudopackages.sh
56 # Process (oldstable)-proposed-updates "NEW" queue
59 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" debian-release@lists.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
63 log "Doing automated p-u-new processing"
64 cd "${queuedir}/p-u-new"
68 log "Doing automated o-p-u-new processing"
69 cd "${queuedir}/o-p-u-new"
73 # The first i18n one, syncing new descriptions
75 log "Synchronizing i18n package descriptions"
76 # First sync their newest data
77 cd ${scriptdir}/i18nsync
78 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
80 # Now check if we still know about the packages for which they created the files
81 # is the timestamp signed by us?
82 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
83 # now read it. As its signed by us we are sure the content is what we expect, no need
84 # to do more here. And we only test -d a directory on it anyway.
85 TSTAMP=$(cat timestamp)
86 # do we have the dir still?
87 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
89 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
90 # Yay, worked, lets copy around
91 for dir in wheezy sid; do
92 if [ -d dists/${dir}/ ]; then
93 cd dists/${dir}/main/i18n
94 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
96 cd ${scriptdir}/i18nsync
99 echo "ARRRR, bad guys, wrong files, ARRR"
100 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
103 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
104 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
107 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
108 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
113 log "Checking for cruft in overrides"
117 function dominate() {
118 log "Removing obsolete source and binary associations"
122 function filelist() {
123 log "Generating file lists for apt-ftparchive"
124 dak generate-filelist
127 function fingerprints() {
128 log "Updating fingerprints"
129 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
132 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
134 if [ -s "${OUTFILE}" ]; then
135 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
136 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
137 To: <debian-project@lists.debian.org>
138 Subject: Debian Maintainers Keyring changes
139 Content-Type: text/plain; charset=utf-8
143 The following changes to the debian-maintainers keyring have just been activated:
147 Debian distribution maintenance software,
148 on behalf of the Keyring maintainers
155 function overrides() {
156 log "Writing overrides into text files"
161 rm -f override.sid.all3
162 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
166 log "Generating package / file mapping"
167 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
170 function packages() {
171 log "Generating Packages and Sources files"
173 #apt-ftparchive generate apt.conf
174 dak generate-packages-sources
175 dak contents generate
179 log "Generating pdiff files"
180 dak generate-index-diffs
184 log "Generating Release files"
185 dak generate-releases
188 function dakcleanup() {
189 log "Cleanup old packages/files"
190 dak clean-suites -m 10000
194 function buildd_dir() {
195 # Rebuilt the buildd dir to avoid long times of 403
196 log "Regenerating the buildd incoming dir"
197 STAMP=$(date "+%Y%m%d%H%M")
206 log "Removing any core files ..."
207 find -type f -name core -print0 | xargs -0r rm -v
209 log "Checking permissions on files in the FTP tree ..."
210 find -type f \( \! -perm -444 -o -perm +002 \) -ls
211 find -type d \( \! -perm -555 -o -perm +002 \) -ls
213 log "Checking symlinks ..."
216 log "Creating recursive directory listing ... "
217 rm -f .${FILENAME}.new
218 TZ=UTC ls -lR > .${FILENAME}.new
220 if [ -r ${FILENAME}.gz ] ; then
221 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
222 mv -f .${FILENAME}.new ${FILENAME}
223 rm -f ${FILENAME}.patch.gz
224 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
225 rm -f ${FILENAME}.old.gz
227 mv -f .${FILENAME}.new ${FILENAME}
230 gzip -9cfN ${FILENAME} >${FILENAME}.gz
234 function mkmaintainers() {
235 log 'Creating Maintainers index ... '
238 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
239 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
240 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-maintainers
242 if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then
243 log "installing Maintainers ... "
244 mv -f .new-maintainers Maintainers
245 gzip -9v <Maintainers >.new-maintainers.gz
246 mv -f .new-maintainers.gz Maintainers.gz
248 rm -f .new-maintainers
252 function mkuploaders() {
253 log 'Creating Uploaders index ... '
256 dak make-maintainers -u ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
257 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
258 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-uploaders
260 if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then
261 log "installing Uploaders ... "
262 mv -f .new-uploaders Uploaders
263 gzip -9v <Uploaders >.new-uploaders.gz
264 mv -f .new-uploaders.gz Uploaders.gz
270 function copyoverrides() {
271 log 'Copying override files into public view ...'
273 for ofile in $copyoverrides ; do
275 chmod g+w override.$ofile
279 newofile=override.$ofile.gz
280 rm -f .newover-$ofile.gz
281 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
282 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
283 log " installing new $newofile $pc"
284 mv -f .newover-$ofile.gz $newofile
287 rm -f .newover-$ofile.gz
292 function mkfilesindices() {
294 cd $base/ftp/indices/files/components
298 log "Querying $PGDATABASE..."
299 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
302 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
305 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
308 log "Generating sources list"
310 sed -n 's/|$//p' $ARCHLIST
312 find ./dists -maxdepth 1 \! -type d
313 find ./dists \! -type d | grep "/source/"
314 ) | sort -u | gzip -9 > source.list.gz
316 log "Generating arch lists"
318 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
320 (sed -n "s/|$a$//p" $ARCHLIST
321 sed -n 's/|all$//p' $ARCHLIST
324 find ./dists -maxdepth 1 \! -type d
325 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
326 ) | sort -u | gzip -9 > arch-$a.list.gz
329 log "Generating suite lists"
332 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
334 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
337 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
338 while read id suite; do
339 [ -e $base/ftp/dists/$suite ] || continue
342 distname=$(cd dists; readlink $suite || echo $suite)
343 find ./dists/$distname \! -type d
344 for distdir in ./dists/*; do
345 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
348 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
349 ) | sort -u | gzip -9 > suite-${suite}.list.gz
352 log "Finding everything on the ftp site to generate sundries"
353 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
356 zcat *.list.gz | cat - *.list | sort -u |
357 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
359 log "Generating files list"
362 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
363 cat - sundries.list dists.list project.list docs.list indices.list |
364 sort -u | poolfirst > ../arch-$a.files
368 for dist in sid wheezy; do
369 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
373 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
374 sort -u | poolfirst > ../typical.files
380 function mkchecksums() {
381 dsynclist=$dbdir/dsync.list
382 md5list=$indices/md5sums
384 log -n "Creating md5 / dsync index file ... "
387 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
388 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
389 ${bindir}/dsync-flist -q link-dups $dsynclist || true
393 log "Regenerating \"public\" mirror/ hardlink fun"
394 DATE_SERIAL=$(date +"%Y%m%d01")
395 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
396 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
397 SERIAL="$DATE_SERIAL"
399 SERIAL="$FILESOAPLUS1"
401 date -u > ${TRACEFILE}
402 echo "Using dak v1" >> ${TRACEFILE}
403 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
404 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
406 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
410 log "Expiring old database dumps..."
412 $scriptsdir/expire_dumps -d . -p -f "dump_*"
415 function transitionsclean() {
416 log "Removing out of date transitions..."
418 dak transitions -c -a
422 log "Updating DM html page"
423 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
427 log "Categorizing uncategorized bugs filed against ftp.debian.org"
431 function ddaccess() {
432 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
433 log "Trigger dd accessible parts sync including ftp dir"
434 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
437 function mirrorpush() {
438 log "Starting the mirrorpush"
439 date -u > /srv/ftp.debian.org/web/mirrorstart
440 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
441 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
442 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
446 log "Exporting package data foo for i18n project"
447 STAMP=$(date "+%Y%m%d%H%M")
448 mkdir -p ${scriptdir}/i18n/${STAMP}
449 cd ${scriptdir}/i18n/${STAMP}
450 dak control-suite -l stable > squeeze
451 dak control-suite -l testing > wheezy
452 dak control-suite -l unstable > sid
453 echo "${STAMP}" > timestamp
454 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
458 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
461 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
465 log "Updating stats data"
467 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
468 R --slave --vanilla < $base/misc/ftpstats.R
469 dak stats arch-space > $webdir/arch-space
470 dak stats pkg-nums > $webdir/pkg-nums
473 function aptftpcleanup() {
474 log "Clean up apt-ftparchive's databases"
476 apt-ftparchive -q clean apt.conf
479 function cleantransactions() {
480 log "Cleanup transaction ids older than 3 months"
482 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
485 function logstats() {
486 $masterdir/tools/logs.py "$1"
489 # save timestamp when we start
490 function savetimestamp() {
491 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
492 echo ${NOW} > "${dbdir}/dinstallstart"
495 function maillogfile() {
496 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
499 function renamelogfile() {
500 if [ -f "${dbdir}/dinstallstart" ]; then
501 NOW=$(cat "${dbdir}/dinstallstart")
503 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
504 logstats "$logdir/dinstall_${NOW}.log"
505 bzip2 -9 "$logdir/dinstall_${NOW}.log"
507 error "Problem, I don't know when dinstall started, unable to do log statistics."
508 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
510 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
511 bzip2 -9 "$logdir/dinstall_${NOW}.log"
515 function testingsourcelist() {
516 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
519 # do a last run of process-unchecked before dinstall is on.
520 function process_unchecked() {
521 log "Processing the unchecked queue"
522 UNCHECKED_WITHOUT_LOCK="-p"
527 # do a run of newstage only before dinstall is on.
528 function newstage() {
529 log "Processing the newstage queue"
530 UNCHECKED_WITHOUT_LOCK="-p"
534 # Function to update a "statefile" telling people what we are doing
537 # This should be called with the argument(s)
538 # - Status name we want to show.
541 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
542 cat >"${DINSTALLSTATE}" <<EOF
543 Dinstall start: ${DINSTALLBEGIN}
545 Action start: ${RIGHTNOW}
549 # extract changelogs and stuff
550 function changelogs() {
551 log "Extracting changelogs"
552 dak make-changelog -e
553 mkdir -p ${exportpublic}/changelogs
554 cd ${exportpublic}/changelogs
555 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
556 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &