2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Updating various files
48 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
50 $scriptsdir/update-bugdoctxt
51 $scriptsdir/update-mirrorlists
52 $scriptsdir/update-mailingliststxt
53 $scriptsdir/update-pseudopackages.sh
56 # The first i18n one, syncing new descriptions
58 log "Synchronizing i18n package descriptions"
59 # First sync their newest data
60 cd ${scriptdir}/i18nsync
61 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
63 # Now check if we still know about the packages for which they created the files
64 # is the timestamp signed by us?
65 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
66 # now read it. As its signed by us we are sure the content is what we expect, no need
67 # to do more here. And we only test -d a directory on it anyway.
68 TSTAMP=$(cat timestamp)
69 # do we have the dir still?
70 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
72 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
73 # Yay, worked, lets copy around
74 for dir in wheezy sid; do
75 if [ -d dists/${dir}/ ]; then
76 cd dists/${dir}/main/i18n
77 rsync -aq --delete --delete-after --exclude Translation-en.bz2 . ${ftpdir}/dists/${dir}/main/i18n/.
79 cd ${scriptdir}/i18nsync
82 echo "ARRRR, bad guys, wrong files, ARRR"
83 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
86 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
87 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
90 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
91 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
96 log "Checking for cruft in overrides"
100 function dominate() {
101 log "Removing obsolete source and binary associations"
105 function filelist() {
106 log "Generating file lists for apt-ftparchive"
107 dak generate-filelist
110 function fingerprints() {
111 log "Updating fingerprints"
112 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
115 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
117 if [ -s "${OUTFILE}" ]; then
118 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
119 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
120 To: <debian-project@lists.debian.org>
121 Subject: Debian Maintainers Keyring changes
122 Content-Type: text/plain; charset=utf-8
126 The following changes to the debian-maintainers keyring have just been activated:
130 Debian distribution maintenance software,
131 on behalf of the Keyring maintainers
138 function overrides() {
139 log "Writing overrides into text files"
144 rm -f override.sid.all3
145 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
149 log "Generating package / file mapping"
150 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
153 function packages() {
154 log "Generating Packages and Sources files"
156 #dak generate-packages-sources
157 dak generate-packages-sources2
158 dak contents generate
162 log "Generating pdiff files"
163 dak generate-index-diffs
167 # XXX: disable once we can remove i18n/Index (#649314)
168 log "Generating i18n/Index"
171 for dist in testing unstable experimental proposed-updates testing-proposed-updates; do
172 $scriptsdir/generate-i18n-Index $dist;
175 log "Generating Release files"
176 dak generate-releases
179 function dakcleanup() {
180 log "Cleanup old packages/files"
181 dak clean-suites -m 10000
185 function buildd_dir() {
186 # Rebuilt the buildd dir to avoid long times of 403
187 log "Regenerating the buildd incoming dir"
188 STAMP=$(date "+%Y%m%d%H%M")
197 log "Removing any core files ..."
198 find -type f -name core -print0 | xargs -0r rm -v
200 log "Checking permissions on files in the FTP tree ..."
201 find -type f \( \! -perm -444 -o -perm +002 \) -ls
202 find -type d \( \! -perm -555 -o -perm +002 \) -ls
204 log "Checking symlinks ..."
207 log "Creating recursive directory listing ... "
208 rm -f .${FILENAME}.new
209 TZ=UTC ls -lR > .${FILENAME}.new
211 if [ -r ${FILENAME}.gz ] ; then
212 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
213 mv -f .${FILENAME}.new ${FILENAME}
214 rm -f ${FILENAME}.patch.gz
215 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
216 rm -f ${FILENAME}.old.gz
218 mv -f .${FILENAME}.new ${FILENAME}
221 gzip -9cfN ${FILENAME} >${FILENAME}.gz
225 function mkmaintainers() {
226 log 'Creating Maintainers index ... '
229 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
230 gzip -9v --rsyncable <Maintainers >Maintainers.gz
231 gzip -9v --rsyncable <Uploaders >Uploaders.gz
234 function copyoverrides() {
235 log 'Copying override files into public view ...'
237 for ofile in ${overridedir}/override.{lenny,squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do
239 gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
240 chmod g+w ${indices}/${bname}.gz
244 function mkfilesindices() {
246 cd $base/ftp/indices/files/components
250 log "Querying postgres"
251 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
254 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
257 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
260 log "Generating sources list"
262 sed -n 's/|$//p' $ARCHLIST
264 find ./dists -maxdepth 1 \! -type d
265 find ./dists \! -type d | grep "/source/"
266 ) | sort -u | gzip -9 > source.list.gz
268 log "Generating arch lists"
270 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
272 (sed -n "s/|$a$//p" $ARCHLIST
273 sed -n 's/|all$//p' $ARCHLIST
276 find ./dists -maxdepth 1 \! -type d
277 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
278 ) | sort -u | gzip -9 > arch-$a.list.gz
281 log "Generating suite lists"
284 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
286 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
289 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
290 while read id suite; do
291 [ -e $base/ftp/dists/$suite ] || continue
294 distname=$(cd dists; readlink $suite || echo $suite)
295 find ./dists/$distname \! -type d
296 for distdir in ./dists/*; do
297 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
300 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
301 ) | sort -u | gzip -9 > suite-${suite}.list.gz
304 log "Finding everything on the ftp site to generate sundries"
305 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
308 zcat *.list.gz | cat - *.list | sort -u |
309 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
311 log "Generating files list"
314 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
315 cat - sundries.list dists.list project.list docs.list indices.list |
316 sort -u | poolfirst > ../arch-$a.files
320 for dist in sid wheezy; do
321 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
325 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
326 sort -u | poolfirst > ../typical.files
332 function mkchecksums() {
333 dsynclist=$dbdir/dsync.list
334 md5list=$indices/md5sums
336 log -n "Creating md5 / dsync index file ... "
339 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
340 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
341 ${bindir}/dsync-flist -q link-dups $dsynclist || true
345 log "Regenerating \"public\" mirror/ hardlink fun"
346 DATE_SERIAL=$(date +"%Y%m%d01")
347 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
348 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
349 SERIAL="$DATE_SERIAL"
351 SERIAL="$FILESOAPLUS1"
353 date -u > ${TRACEFILE}
354 echo "Using dak v1" >> ${TRACEFILE}
355 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
356 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
358 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
362 log "Expiring old database dumps..."
364 $scriptsdir/expire_dumps -d . -p -f "dump_*"
367 function transitionsclean() {
368 log "Removing out of date transitions..."
370 dak transitions -c -a
374 log "Updating DM html page"
375 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
379 log "Categorizing uncategorized bugs filed against ftp.debian.org"
383 function ddaccess() {
384 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
385 log "Trigger dd accessible parts sync including ftp dir"
386 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
389 function mirrorpush() {
390 log "Checking the public archive copy"
391 cd ${mirrordir}/dists
394 for release in $(find . -name "InRelease"); do
395 echo "Processing: ${release}"
396 subdir=${release%/InRelease}
397 while read SHASUM SIZE NAME; do
398 if ! [ -f "${subdir}/${NAME}" ]; then
399 bname=$(basename ${NAME})
400 if [[ "${bname}" =~ ^(Packages|Sources|Translation-[a-zA-Z_]+)$ ]]; then
401 # We don't keep unpacked files, don't check for their existance.
402 # We might want to go and check their unpacked shasum, but right now
403 # I don't care. I believe it should be enough if all the packed shasums
407 broken=$(( broken + 1 ))
408 echo "File ${subdir}/${NAME} is missing"
412 # We do have symlinks in the tree (see the contents files currently).
413 # So we use "readlink -f" to check the size of the target, as thats basically
414 # what gen-releases does
415 fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
416 if [ ${fsize} -ne ${SIZE} ]; then
417 broken=$(( broken + 1 ))
418 echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
422 fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
423 fshasum=${fshasum%% *}
424 if [ "${fshasum}" != "${SHASUM}" ]; then
425 broken=$(( broken + 1 ))
426 echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
429 done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
432 if [ $broken -gt 0 ]; then
433 log_error "Trouble with the public mirror, found ${broken} errors"
437 log "Starting the mirrorpush"
438 date -u > /srv/ftp.debian.org/web/mirrorstart
439 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
440 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
441 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
445 log "Exporting package data foo for i18n project"
446 STAMP=$(date "+%Y%m%d%H%M")
447 mkdir -p ${scriptdir}/i18n/${STAMP}
448 cd ${scriptdir}/i18n/${STAMP}
449 dak control-suite -l stable > squeeze
450 dak control-suite -l testing > wheezy
451 dak control-suite -l unstable > sid
452 echo "${STAMP}" > timestamp
453 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
457 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
460 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
464 log "Updating stats data"
466 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
467 R --slave --vanilla < $base/misc/ftpstats.R
468 dak stats arch-space > $webdir/arch-space
469 dak stats pkg-nums > $webdir/pkg-nums
472 function aptftpcleanup() {
473 log "Clean up apt-ftparchive's databases"
475 apt-ftparchive -q clean apt.conf
478 function cleantransactions() {
479 log "Cleanup transaction ids older than 3 months"
481 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
484 function logstats() {
485 $masterdir/tools/logs.py "$1"
488 # save timestamp when we start
489 function savetimestamp() {
490 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
491 echo ${NOW} > "${dbdir}/dinstallstart"
494 function maillogfile() {
495 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
498 function renamelogfile() {
499 if [ -f "${dbdir}/dinstallstart" ]; then
500 NOW=$(cat "${dbdir}/dinstallstart")
502 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
503 logstats "$logdir/dinstall_${NOW}.log"
504 bzip2 -9 "$logdir/dinstall_${NOW}.log"
506 error "Problem, I don't know when dinstall started, unable to do log statistics."
507 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
509 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
510 bzip2 -9 "$logdir/dinstall_${NOW}.log"
514 function testingsourcelist() {
515 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
518 # do a last run of process-unchecked before dinstall is on.
519 function process_unchecked() {
520 log "Processing the unchecked queue"
521 UNCHECKED_WITHOUT_LOCK="-p"
526 # do a run of newstage only before dinstall is on.
527 function newstage() {
528 log "Processing the newstage queue"
529 UNCHECKED_WITHOUT_LOCK="-p"
533 # Function to update a "statefile" telling people what we are doing
536 # This should be called with the argument(s)
537 # - Status name we want to show.
540 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
541 cat >"${DINSTALLSTATE}" <<EOF
542 Dinstall start: ${DINSTALLBEGIN}
544 Action start: ${RIGHTNOW}
548 # extract changelogs and stuff
549 function changelogs() {
550 log "Extracting changelogs"
551 dak make-changelog -e
552 mkdir -p ${exportpublic}/changelogs
553 cd ${exportpublic}/changelogs
554 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
555 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &
558 function gitpdiff() {
559 # Might be that we want to change this to have more than one git repository.
560 # Advantage of one is that we do not need much space in terms of storage in git itself,
561 # git gc is pretty good on our input.
562 # But it might be faster. Well, lets test.
563 log "Adjusting the git tree for pdiffs"
566 # The regex needs the architectures seperated with \|
567 garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q')
569 # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we
571 # Also, we only want contents, packages and sources.
572 for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do
581 # Second, add all there is into git
584 # Maybe we want to make this the same for tag and commit? But well, shouldn't matter
586 TAGD=$(date +%Y-%m-%d-%H-%M)
587 git commit -m "Commit of ${COMD}"