2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Updating various files
48 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
50 $scriptsdir/update-bugdoctxt
51 $scriptsdir/update-mirrorlists
52 $scriptsdir/update-mailingliststxt
53 $scriptsdir/update-pseudopackages.sh
56 # The first i18n one, syncing new descriptions
58 log "Synchronizing i18n package descriptions"
59 # First sync their newest data
60 cd ${scriptdir}/i18nsync
61 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
63 # Now check if we still know about the packages for which they created the files
64 # is the timestamp signed by us?
65 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
66 # now read it. As its signed by us we are sure the content is what we expect, no need
67 # to do more here. And we only test -d a directory on it anyway.
68 TSTAMP=$(cat timestamp)
69 # do we have the dir still?
70 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
72 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
73 # Yay, worked, lets copy around
74 for dir in wheezy sid; do
75 if [ -d dists/${dir}/ ]; then
76 cd dists/${dir}/main/i18n
77 rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/.
79 cd ${scriptdir}/i18nsync
82 echo "ARRRR, bad guys, wrong files, ARRR"
83 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
86 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
87 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
90 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
91 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
96 log "Checking for cruft in overrides"
100 function dominate() {
101 log "Removing obsolete source and binary associations"
105 function filelist() {
106 log "Generating file lists for apt-ftparchive"
107 dak generate-filelist
110 function fingerprints() {
111 log "Updating fingerprints"
112 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
115 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
117 if [ -s "${OUTFILE}" ]; then
118 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
119 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
120 To: <debian-project@lists.debian.org>
121 Subject: Debian Maintainers Keyring changes
122 Content-Type: text/plain; charset=utf-8
126 The following changes to the debian-maintainers keyring have just been activated:
130 Debian distribution maintenance software,
131 on behalf of the Keyring maintainers
138 function overrides() {
139 log "Writing overrides into text files"
144 rm -f override.sid.all3
145 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
149 log "Generating package / file mapping"
150 dak make-pkg-file-mapping ftp-master | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
153 function packages() {
154 log "Generating Packages and Sources files"
155 dak generate-packages-sources2 -a ftp-master
156 dak contents generate -a ftp-master
160 log "Generating pdiff files"
161 dak generate-index-diffs
165 # XXX: disable once we can remove i18n/Index (#649314)
166 log "Generating i18n/Index"
169 for dist in testing unstable experimental proposed-updates testing-proposed-updates; do
170 $scriptsdir/generate-i18n-Index $dist;
173 log "Generating Release files"
174 dak generate-releases -a ftp-master
177 function dakcleanup() {
178 log "Cleanup old packages/files"
179 dak clean-suites -m 10000
180 # XXX: reactivate once clean-queues is fixed
184 function buildd_dir() {
185 # Rebuilt the buildd dir to avoid long times of 403
186 log "Regenerating the buildd incoming dir"
187 STAMP=$(date "+%Y%m%d%H%M")
196 log "Removing any core files ..."
197 find -type f -name core -print -delete
199 log "Checking symlinks ..."
202 log "Creating recursive directory listing ... "
204 TZ=UTC ls -lR | gzip -9c --rsyncable > ${FILENAME}.gz
207 function mkmaintainers() {
208 log 'Creating Maintainers index ... '
211 dak make-maintainers -a ftp-master ${scriptdir}/masterfiles/pseudo-packages.maintainers
212 gzip -9v --rsyncable <Maintainers >Maintainers.gz
213 gzip -9v --rsyncable <Uploaders >Uploaders.gz
216 function copyoverrides() {
217 log 'Copying override files into public view ...'
219 for ofile in ${overridedir}/override.{squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do
221 gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
222 chmod g+w ${indices}/${bname}.gz
226 function mkfilesindices() {
229 cd $base/ftp/indices/files/components
233 log "Querying postgres"
235 SELECT './pool/' || c.name || '/' || f.filename AS path, a.arch_string AS arch_string
237 JOIN files_archive_map af ON f.id = af.file_id
238 JOIN component c ON af.component_id = c.id
239 JOIN archive ON af.archive_id = archive.id
242 JOIN architecture a ON b.architecture = a.id)
244 WHERE archive.name = 'ftp-master'
245 ORDER BY path, arch_string
247 psql -At -c "$query" >$ARCHLIST
250 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
253 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
256 log "Generating sources list"
258 sed -n 's/|$//p' $ARCHLIST
260 find ./dists -maxdepth 1 \! -type d
261 find ./dists \! -type d | grep "/source/"
262 ) | sort -u | gzip -9 > source.list.gz
264 log "Generating arch lists"
266 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
268 (sed -n "s/|$a$//p" $ARCHLIST
269 sed -n 's/|all$//p' $ARCHLIST
272 find ./dists -maxdepth 1 \! -type d
273 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
274 ) | sort -u | gzip -9 > arch-$a.list.gz
277 log "Generating suite lists"
280 local suite_id="$(printf %d $1)"
283 SELECT DISTINCT './pool/' || c.name || '/' || f.filename
285 (SELECT sa.source AS source
286 FROM src_associations sa
287 WHERE sa.suite = $suite_id
290 FROM extra_src_references esr
291 JOIN bin_associations ba ON esr.bin_id = ba.bin
292 WHERE ba.suite = $suite_id
294 SELECT b.source AS source
295 FROM bin_associations ba
296 JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s
297 JOIN dsc_files df ON s.source = df.source
298 JOIN files f ON df.file = f.id
299 JOIN files_archive_map af ON f.id = af.file_id
300 JOIN component c ON af.component_id = c.id
301 JOIN archive ON af.archive_id = archive.id
302 WHERE archive.name = 'ftp-master'
304 psql -F' ' -A -t -c "$query"
307 SELECT './pool/' || c.name || '/' || f.filename
308 FROM bin_associations ba
309 JOIN binaries b ON ba.bin = b.id
310 JOIN files f ON b.file = f.id
311 JOIN files_archive_map af ON f.id = af.file_id
312 JOIN component c ON af.component_id = c.id
313 JOIN archive ON af.archive_id = archive.id
314 WHERE ba.suite = $suite_id AND archive.name = 'ftp-master'
316 psql -F' ' -A -t -c "$query"
319 psql -F' ' -At -c "SELECT id, suite_name FROM suite" |
320 while read id suite; do
321 [ -e $base/ftp/dists/$suite ] || continue
324 distname=$(cd dists; readlink $suite || echo $suite)
325 find ./dists/$distname \! -type d
326 for distdir in ./dists/*; do
327 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
331 ) | sort -u | gzip -9 > suite-${suite}.list.gz
334 log "Finding everything on the ftp site to generate sundries"
335 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
338 zcat *.list.gz | cat - *.list | sort -u |
339 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
341 log "Generating files list"
344 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
345 cat - sundries.list dists.list project.list docs.list indices.list |
346 sort -u | poolfirst > ../arch-$a.files
350 for dist in sid wheezy; do
351 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
355 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
356 sort -u | poolfirst > ../typical.files
363 function mkchecksums() {
364 dsynclist=$dbdir/dsync.list
365 md5list=$indices/md5sums
367 log -n "Creating md5 / dsync index file ... "
370 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
371 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
372 ${bindir}/dsync-flist -q link-dups $dsynclist || true
376 log "Regenerating \"public\" mirror/ hardlink fun"
377 DATE_SERIAL=$(date +"%Y%m%d01")
378 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
379 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
380 SERIAL="$DATE_SERIAL"
382 SERIAL="$FILESOAPLUS1"
384 date -u > ${TRACEFILE}
385 echo "Using dak v1" >> ${TRACEFILE}
386 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
387 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
389 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
393 log "Expiring old database dumps..."
395 $scriptsdir/expire_dumps -d . -p -f "dump_*"
398 function transitionsclean() {
399 log "Removing out of date transitions..."
401 dak transitions -c -a
405 log "Updating DM html page"
406 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
410 log "Categorizing uncategorized bugs filed against ftp.debian.org"
414 function ddaccess() {
415 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
416 log "Trigger dd accessible parts sync including ftp dir"
417 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
420 function mirrorpush() {
421 log "Checking the public archive copy"
422 cd ${mirrordir}/dists
425 for release in $(find . -name "InRelease"); do
426 echo "Processing: ${release}"
427 subdir=${release%/InRelease}
428 while read SHASUM SIZE NAME; do
429 if ! [ -f "${subdir}/${NAME}" ]; then
430 bname=$(basename ${NAME})
431 if [[ "${bname}" =~ ^(Packages|Sources|Translation-[a-zA-Z_]+)$ ]]; then
432 # We don't keep unpacked files, don't check for their existance.
433 # We might want to go and check their unpacked shasum, but right now
434 # I don't care. I believe it should be enough if all the packed shasums
438 broken=$(( broken + 1 ))
439 echo "File ${subdir}/${NAME} is missing"
443 # We do have symlinks in the tree (see the contents files currently).
444 # So we use "readlink -f" to check the size of the target, as thats basically
445 # what gen-releases does
446 fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
447 if [ ${fsize} -ne ${SIZE} ]; then
448 broken=$(( broken + 1 ))
449 echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
453 fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
454 fshasum=${fshasum%% *}
455 if [ "${fshasum}" != "${SHASUM}" ]; then
456 broken=$(( broken + 1 ))
457 echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
460 done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
463 if [ $broken -gt 0 ]; then
464 log_error "Trouble with the public mirror, found ${broken} errors"
468 log "Starting the mirrorpush"
469 date -u > /srv/ftp.debian.org/web/mirrorstart
470 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
471 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
472 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
476 log "Exporting package data foo for i18n project"
477 STAMP=$(date "+%Y%m%d%H%M")
478 mkdir -p ${scriptdir}/i18n/${STAMP}
479 cd ${scriptdir}/i18n/${STAMP}
480 for suite in stable testing unstable; do
481 codename=$(dak admin s show ${suite}|grep '^Codename')
482 codename=${codename##* }
483 echo "Codename is ${codename}"
484 dak control-suite -l ${suite} >${codename}
486 echo "${STAMP}" > timestamp
487 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
491 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
494 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
498 log "Updating stats data"
500 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
501 R --slave --vanilla < $base/misc/ftpstats.R
502 dak stats arch-space > $webdir/arch-space
503 dak stats pkg-nums > $webdir/pkg-nums
506 function aptftpcleanup() {
507 log "Clean up apt-ftparchive's databases"
509 apt-ftparchive -q clean apt.conf
512 function cleantransactions() {
513 log "Cleanup transaction ids older than 3 months"
515 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
518 function logstats() {
519 $masterdir/tools/logs.py "$1"
522 # save timestamp when we start
523 function savetimestamp() {
524 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
525 echo ${NOW} > "${dbdir}/dinstallstart"
528 function maillogfile() {
529 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
532 function renamelogfile() {
533 if [ -f "${dbdir}/dinstallstart" ]; then
534 NOW=$(cat "${dbdir}/dinstallstart")
536 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
537 logstats "$logdir/dinstall_${NOW}.log"
538 bzip2 -9 "$logdir/dinstall_${NOW}.log"
540 error "Problem, I don't know when dinstall started, unable to do log statistics."
541 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
543 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
544 bzip2 -9 "$logdir/dinstall_${NOW}.log"
548 function testingsourcelist() {
549 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
552 # do a last run of process-unchecked before dinstall is on.
553 function process_unchecked() {
554 log "Processing the unchecked queue"
555 UNCHECKED_WITHOUT_LOCK="-p"
560 # Function to update a "statefile" telling people what we are doing
563 # This should be called with the argument(s)
564 # - Status name we want to show.
567 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
568 cat >"${DINSTALLSTATE}" <<EOF
569 Dinstall start: ${DINSTALLBEGIN}
571 Action start: ${RIGHTNOW}
575 # extract changelogs and stuff
576 function changelogs() {
577 log "Extracting changelogs"
578 dak make-changelog -e
579 mkdir -p ${exportpublic}/changelogs
580 cd ${exportpublic}/changelogs
581 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
582 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &
585 function gitpdiff() {
586 # Might be that we want to change this to have more than one git repository.
587 # Advantage of one is that we do not need much space in terms of storage in git itself,
588 # git gc is pretty good on our input.
589 # But it might be faster. Well, lets test.
590 log "Adjusting the git tree for pdiffs"
593 # The regex needs the architectures seperated with \|
594 garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q')
596 # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we
598 # Also, we only want contents, packages and sources.
599 for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do
608 # Second, add all there is into git
611 # Maybe we want to make this the same for tag and commit? But well, shouldn't matter
613 TAGD=$(date +%Y-%m-%d-%H-%M)
614 git commit -m "Commit of ${COMD}"