2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Updating various files
48 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
50 $scriptsdir/update-bugdoctxt
51 $scriptsdir/update-mirrorlists
52 $scriptsdir/update-mailingliststxt
53 $scriptsdir/update-pseudopackages.sh
56 # The first i18n one, syncing new descriptions
58 log "Synchronizing i18n package descriptions"
59 # First sync their newest data
60 cd ${scriptdir}/i18nsync
61 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
63 # Now check if we still know about the packages for which they created the files
64 # is the timestamp signed by us?
65 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
66 # now read it. As its signed by us we are sure the content is what we expect, no need
67 # to do more here. And we only test -d a directory on it anyway.
68 TSTAMP=$(cat timestamp)
69 # do we have the dir still?
70 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
72 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
73 # Yay, worked, lets copy around
74 for dir in wheezy sid; do
75 if [ -d dists/${dir}/ ]; then
76 cd dists/${dir}/main/i18n
77 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
79 cd ${scriptdir}/i18nsync
82 echo "ARRRR, bad guys, wrong files, ARRR"
83 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
86 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
87 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
90 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
91 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
96 log "Checking for cruft in overrides"
100 function dominate() {
101 log "Removing obsolete source and binary associations"
105 function filelist() {
106 log "Generating file lists for apt-ftparchive"
107 dak generate-filelist
110 function fingerprints() {
111 log "Updating fingerprints"
112 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
115 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
117 if [ -s "${OUTFILE}" ]; then
118 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
119 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
120 To: <debian-project@lists.debian.org>
121 Subject: Debian Maintainers Keyring changes
122 Content-Type: text/plain; charset=utf-8
126 The following changes to the debian-maintainers keyring have just been activated:
130 Debian distribution maintenance software,
131 on behalf of the Keyring maintainers
138 function overrides() {
139 log "Writing overrides into text files"
144 rm -f override.sid.all3
145 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
149 log "Generating package / file mapping"
150 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
153 function packages() {
154 log "Generating Packages and Sources files"
156 #dak generate-packages-sources
157 dak generate-packages-sources2
158 dak contents generate
162 log "Generating pdiff files"
163 dak generate-index-diffs
167 # XXX: disable once we can remove i18n/Index (#649314)
168 log "Generating i18n/Index"
169 (cd "$ftpdir/dists"; for dist in testing unstable; do $scriptsdir/generate-i18n-Index $dist; done)
170 log "Generating Release files"
171 dak generate-releases
174 function dakcleanup() {
175 log "Cleanup old packages/files"
176 dak clean-suites -m 10000
180 function buildd_dir() {
181 # Rebuilt the buildd dir to avoid long times of 403
182 log "Regenerating the buildd incoming dir"
183 STAMP=$(date "+%Y%m%d%H%M")
192 log "Removing any core files ..."
193 find -type f -name core -print0 | xargs -0r rm -v
195 log "Checking permissions on files in the FTP tree ..."
196 find -type f \( \! -perm -444 -o -perm +002 \) -ls
197 find -type d \( \! -perm -555 -o -perm +002 \) -ls
199 log "Checking symlinks ..."
202 log "Creating recursive directory listing ... "
203 rm -f .${FILENAME}.new
204 TZ=UTC ls -lR > .${FILENAME}.new
206 if [ -r ${FILENAME}.gz ] ; then
207 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
208 mv -f .${FILENAME}.new ${FILENAME}
209 rm -f ${FILENAME}.patch.gz
210 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
211 rm -f ${FILENAME}.old.gz
213 mv -f .${FILENAME}.new ${FILENAME}
216 gzip -9cfN ${FILENAME} >${FILENAME}.gz
220 function mkmaintainers() {
221 log 'Creating Maintainers index ... '
224 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
225 gzip -9v --rsyncable <Maintainers >Maintainers.gz
226 gzip -9v --rsyncable <Uploaders >Uploaders.gz
229 function copyoverrides() {
230 log 'Copying override files into public view ...'
232 for ofile in ${overridedir}/override.{lenny,squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do
234 gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
235 chmod g+w ${indices}/${bname}.gz
239 function mkfilesindices() {
241 cd $base/ftp/indices/files/components
245 log "Querying postgres"
246 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
249 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
252 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
255 log "Generating sources list"
257 sed -n 's/|$//p' $ARCHLIST
259 find ./dists -maxdepth 1 \! -type d
260 find ./dists \! -type d | grep "/source/"
261 ) | sort -u | gzip -9 > source.list.gz
263 log "Generating arch lists"
265 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
267 (sed -n "s/|$a$//p" $ARCHLIST
268 sed -n 's/|all$//p' $ARCHLIST
271 find ./dists -maxdepth 1 \! -type d
272 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
273 ) | sort -u | gzip -9 > arch-$a.list.gz
276 log "Generating suite lists"
279 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
281 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
284 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
285 while read id suite; do
286 [ -e $base/ftp/dists/$suite ] || continue
289 distname=$(cd dists; readlink $suite || echo $suite)
290 find ./dists/$distname \! -type d
291 for distdir in ./dists/*; do
292 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
295 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
296 ) | sort -u | gzip -9 > suite-${suite}.list.gz
299 log "Finding everything on the ftp site to generate sundries"
300 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
303 zcat *.list.gz | cat - *.list | sort -u |
304 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
306 log "Generating files list"
309 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
310 cat - sundries.list dists.list project.list docs.list indices.list |
311 sort -u | poolfirst > ../arch-$a.files
315 for dist in sid wheezy; do
316 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
320 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
321 sort -u | poolfirst > ../typical.files
327 function mkchecksums() {
328 dsynclist=$dbdir/dsync.list
329 md5list=$indices/md5sums
331 log -n "Creating md5 / dsync index file ... "
334 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
335 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
336 ${bindir}/dsync-flist -q link-dups $dsynclist || true
340 log "Regenerating \"public\" mirror/ hardlink fun"
341 DATE_SERIAL=$(date +"%Y%m%d01")
342 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
343 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
344 SERIAL="$DATE_SERIAL"
346 SERIAL="$FILESOAPLUS1"
348 date -u > ${TRACEFILE}
349 echo "Using dak v1" >> ${TRACEFILE}
350 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
351 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
353 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
357 log "Expiring old database dumps..."
359 $scriptsdir/expire_dumps -d . -p -f "dump_*"
362 function transitionsclean() {
363 log "Removing out of date transitions..."
365 dak transitions -c -a
369 log "Updating DM html page"
370 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
374 log "Categorizing uncategorized bugs filed against ftp.debian.org"
378 function ddaccess() {
379 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
380 log "Trigger dd accessible parts sync including ftp dir"
381 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
384 function mirrorpush() {
385 log "Checking the public archive copy"
386 cd ${mirrordir}/dists
389 for release in $(find . -name "InRelease"); do
390 echo "Processing: ${release}"
391 subdir=${release%/InRelease}
392 while read SHASUM SIZE NAME; do
393 if ! [ -f "${subdir}/${NAME}" ]; then
394 bname=$(basename ${NAME})
395 if [[ "${bname}" =~ ^(Packages|Sources|Translation-[a-zA-Z_]+)$ ]]; then
396 # We don't keep unpacked files, don't check for their existance.
397 # We might want to go and check their unpacked shasum, but right now
398 # I don't care. I believe it should be enough if all the packed shasums
402 broken=$(( broken + 1 ))
403 echo "File ${subdir}/${NAME} is missing"
407 # We do have symlinks in the tree (see the contents files currently).
408 # So we use "readlink -f" to check the size of the target, as thats basically
409 # what gen-releases does
410 fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
411 if [ ${fsize} -ne ${SIZE} ]; then
412 broken=$(( broken + 1 ))
413 echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
417 fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
418 fshasum=${fshasum%% *}
419 if [ "${fshasum}" != "${SHASUM}" ]; then
420 broken=$(( broken + 1 ))
421 echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
424 done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
427 if [ $broken -gt 0 ]; then
428 log_error "Trouble with the public mirror, found ${broken} errors"
432 log "Starting the mirrorpush"
433 date -u > /srv/ftp.debian.org/web/mirrorstart
434 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
435 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
436 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
440 log "Exporting package data foo for i18n project"
441 STAMP=$(date "+%Y%m%d%H%M")
442 mkdir -p ${scriptdir}/i18n/${STAMP}
443 cd ${scriptdir}/i18n/${STAMP}
444 dak control-suite -l stable > squeeze
445 dak control-suite -l testing > wheezy
446 dak control-suite -l unstable > sid
447 echo "${STAMP}" > timestamp
448 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
452 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
455 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
459 log "Updating stats data"
461 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
462 R --slave --vanilla < $base/misc/ftpstats.R
463 dak stats arch-space > $webdir/arch-space
464 dak stats pkg-nums > $webdir/pkg-nums
467 function aptftpcleanup() {
468 log "Clean up apt-ftparchive's databases"
470 apt-ftparchive -q clean apt.conf
473 function cleantransactions() {
474 log "Cleanup transaction ids older than 3 months"
476 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
479 function logstats() {
480 $masterdir/tools/logs.py "$1"
483 # save timestamp when we start
484 function savetimestamp() {
485 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
486 echo ${NOW} > "${dbdir}/dinstallstart"
489 function maillogfile() {
490 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
493 function renamelogfile() {
494 if [ -f "${dbdir}/dinstallstart" ]; then
495 NOW=$(cat "${dbdir}/dinstallstart")
497 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
498 logstats "$logdir/dinstall_${NOW}.log"
499 bzip2 -9 "$logdir/dinstall_${NOW}.log"
501 error "Problem, I don't know when dinstall started, unable to do log statistics."
502 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
504 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
505 bzip2 -9 "$logdir/dinstall_${NOW}.log"
509 function testingsourcelist() {
510 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
513 # do a last run of process-unchecked before dinstall is on.
514 function process_unchecked() {
515 log "Processing the unchecked queue"
516 UNCHECKED_WITHOUT_LOCK="-p"
521 # do a run of newstage only before dinstall is on.
522 function newstage() {
523 log "Processing the newstage queue"
524 UNCHECKED_WITHOUT_LOCK="-p"
528 # Function to update a "statefile" telling people what we are doing
531 # This should be called with the argument(s)
532 # - Status name we want to show.
535 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
536 cat >"${DINSTALLSTATE}" <<EOF
537 Dinstall start: ${DINSTALLBEGIN}
539 Action start: ${RIGHTNOW}
543 # extract changelogs and stuff
544 function changelogs() {
545 log "Extracting changelogs"
546 dak make-changelog -e
547 mkdir -p ${exportpublic}/changelogs
548 cd ${exportpublic}/changelogs
549 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
550 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &
553 function gitpdiff() {
554 # Might be that we want to change this to have more than one git repository.
555 # Advantage of one is that we do not need much space in terms of storage in git itself,
556 # git gc is pretty good on our input.
557 # But it might be faster. Well, lets test.
558 log "Adjusting the git tree for pdiffs"
561 # The regex needs the architectures seperated with \|
562 garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q')
564 # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we
566 # Also, we only want contents, packages and sources.
567 for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do
576 # Second, add all there is into git
579 # Maybe we want to make this the same for tag and commit? But well, shouldn't matter
581 TAGD=$(date +%Y-%m-%d-%H-%M)
582 git commit -m "Commit of ${COMD}"