2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Updating various files
48 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
50 $scriptsdir/update-bugdoctxt
51 $scriptsdir/update-mirrorlists
52 $scriptsdir/update-mailingliststxt
53 $scriptsdir/update-pseudopackages.sh
56 # Process (oldstable)-proposed-updates "NEW" queue
59 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" debian-release@lists.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
63 log "Doing automated p-u-new processing"
64 cd "${queuedir}/p-u-new"
68 log "Doing automated o-p-u-new processing"
69 cd "${queuedir}/o-p-u-new"
73 # The first i18n one, syncing new descriptions
75 log "Synchronizing i18n package descriptions"
76 # First sync their newest data
77 cd ${scriptdir}/i18nsync
78 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
80 # Now check if we still know about the packages for which they created the files
81 # is the timestamp signed by us?
82 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
83 # now read it. As its signed by us we are sure the content is what we expect, no need
84 # to do more here. And we only test -d a directory on it anyway.
85 TSTAMP=$(cat timestamp)
86 # do we have the dir still?
87 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
89 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
90 # Yay, worked, lets copy around
91 for dir in wheezy sid; do
92 if [ -d dists/${dir}/ ]; then
93 cd dists/${dir}/main/i18n
94 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
96 cd ${scriptdir}/i18nsync
99 echo "ARRRR, bad guys, wrong files, ARRR"
100 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
103 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
104 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
107 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
108 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
113 log "Checking for cruft in overrides"
117 function dominate() {
118 log "Removing obsolete source and binary associations"
122 function filelist() {
123 log "Generating file lists for apt-ftparchive"
124 dak generate-filelist
127 function fingerprints() {
128 log "Updating fingerprints"
129 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
132 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
134 if [ -s "${OUTFILE}" ]; then
135 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
136 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
137 To: <debian-project@lists.debian.org>
138 Subject: Debian Maintainers Keyring changes
139 Content-Type: text/plain; charset=utf-8
143 The following changes to the debian-maintainers keyring have just been activated:
147 Debian distribution maintenance software,
148 on behalf of the Keyring maintainers
155 function overrides() {
156 log "Writing overrides into text files"
161 rm -f override.sid.all3
162 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
166 log "Generating package / file mapping"
167 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
170 function packages() {
171 log "Generating Packages and Sources files"
173 #dak generate-packages-sources
174 # XXX: disable again later
175 dak generate-packages-sources2 --description-md5 -o Dir::Root=$webdir/newdists/
176 dak generate-packages-sources2
177 dak contents generate
181 log "Generating pdiff files"
182 dak generate-index-diffs
186 log "Generating Release files"
187 dak generate-releases
188 # XXX: disable again later
189 dak generate-releases -o Dir::Root=$webdir/newdists/
192 function dakcleanup() {
193 log "Cleanup old packages/files"
194 dak clean-suites -m 10000
198 function buildd_dir() {
199 # Rebuilt the buildd dir to avoid long times of 403
200 log "Regenerating the buildd incoming dir"
201 STAMP=$(date "+%Y%m%d%H%M")
210 log "Removing any core files ..."
211 find -type f -name core -print0 | xargs -0r rm -v
213 log "Checking permissions on files in the FTP tree ..."
214 find -type f \( \! -perm -444 -o -perm +002 \) -ls
215 find -type d \( \! -perm -555 -o -perm +002 \) -ls
217 log "Checking symlinks ..."
220 log "Creating recursive directory listing ... "
221 rm -f .${FILENAME}.new
222 TZ=UTC ls -lR > .${FILENAME}.new
224 if [ -r ${FILENAME}.gz ] ; then
225 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
226 mv -f .${FILENAME}.new ${FILENAME}
227 rm -f ${FILENAME}.patch.gz
228 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
229 rm -f ${FILENAME}.old.gz
231 mv -f .${FILENAME}.new ${FILENAME}
234 gzip -9cfN ${FILENAME} >${FILENAME}.gz
238 function mkmaintainers() {
239 log 'Creating Maintainers index ... '
242 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
243 gzip -9v --rsyncable <Maintainers >Maintainers.gz
244 gzip -9v --rsyncable <Uploaders >Uploaders.gz
247 function copyoverrides() {
248 log 'Copying override files into public view ...'
250 for ofile in ${overridedir}/override.{lenny,squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do
252 gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
253 chmod g+w ${indices}/${bname}.gz
257 function mkfilesindices() {
259 cd $base/ftp/indices/files/components
263 log "Querying postgres"
264 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
267 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
270 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
273 log "Generating sources list"
275 sed -n 's/|$//p' $ARCHLIST
277 find ./dists -maxdepth 1 \! -type d
278 find ./dists \! -type d | grep "/source/"
279 ) | sort -u | gzip -9 > source.list.gz
281 log "Generating arch lists"
283 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
285 (sed -n "s/|$a$//p" $ARCHLIST
286 sed -n 's/|all$//p' $ARCHLIST
289 find ./dists -maxdepth 1 \! -type d
290 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
291 ) | sort -u | gzip -9 > arch-$a.list.gz
294 log "Generating suite lists"
297 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
299 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
302 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
303 while read id suite; do
304 [ -e $base/ftp/dists/$suite ] || continue
307 distname=$(cd dists; readlink $suite || echo $suite)
308 find ./dists/$distname \! -type d
309 for distdir in ./dists/*; do
310 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
313 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
314 ) | sort -u | gzip -9 > suite-${suite}.list.gz
317 log "Finding everything on the ftp site to generate sundries"
318 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
321 zcat *.list.gz | cat - *.list | sort -u |
322 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
324 log "Generating files list"
327 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
328 cat - sundries.list dists.list project.list docs.list indices.list |
329 sort -u | poolfirst > ../arch-$a.files
333 for dist in sid wheezy; do
334 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
338 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
339 sort -u | poolfirst > ../typical.files
345 function mkchecksums() {
346 dsynclist=$dbdir/dsync.list
347 md5list=$indices/md5sums
349 log -n "Creating md5 / dsync index file ... "
352 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
353 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
354 ${bindir}/dsync-flist -q link-dups $dsynclist || true
358 log "Regenerating \"public\" mirror/ hardlink fun"
359 DATE_SERIAL=$(date +"%Y%m%d01")
360 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
361 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
362 SERIAL="$DATE_SERIAL"
364 SERIAL="$FILESOAPLUS1"
366 date -u > ${TRACEFILE}
367 echo "Using dak v1" >> ${TRACEFILE}
368 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
369 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
371 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
375 log "Expiring old database dumps..."
377 $scriptsdir/expire_dumps -d . -p -f "dump_*"
380 function transitionsclean() {
381 log "Removing out of date transitions..."
383 dak transitions -c -a
387 log "Updating DM html page"
388 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
392 log "Categorizing uncategorized bugs filed against ftp.debian.org"
396 function ddaccess() {
397 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
398 log "Trigger dd accessible parts sync including ftp dir"
399 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
402 function mirrorpush() {
403 log "Checking the public archive copy"
404 cd ${mirrordir}/dists
407 for release in $(find . -name "InRelease"); do
408 echo "Processing: ${release}"
409 subdir=${release%/InRelease}
410 while read SHASUM SIZE NAME; do
411 if ! [ -f "${subdir}/${NAME}" ]; then
412 bname=$(basename ${NAME})
413 if [ "${bname}" = "Packages" ] || [ "${bname}" = "Sources" ]; then
414 # We don't keep unpacked files, don't check for their existance.
415 # We might want to go and check their unpacked shasum, but right now
416 # I don't care. I believe it should be enough if all the packed shasums
420 broken=$(( broken + 1 ))
421 echo "File ${subdir}/${NAME} is missing"
425 # We do have symlinks in the tree (see the contents files currently).
426 # So we use "readlink -f" to check the size of the target, as thats basically
427 # what gen-releases does
428 fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
429 if [ ${fsize} -ne ${SIZE} ]; then
430 broken=$(( broken + 1 ))
431 echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
435 fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
436 fshasum=${fshasum%% *}
437 if [ "${fshasum}" != "${SHASUM}" ]; then
438 broken=$(( broken + 1 ))
439 echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
442 done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
445 if [ $broken -gt 0 ]; then
446 log_error "Trouble with the public mirror, found ${broken} errors"
450 log "Starting the mirrorpush"
451 date -u > /srv/ftp.debian.org/web/mirrorstart
452 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
453 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
454 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
458 log "Exporting package data foo for i18n project"
459 STAMP=$(date "+%Y%m%d%H%M")
460 mkdir -p ${scriptdir}/i18n/${STAMP}
461 cd ${scriptdir}/i18n/${STAMP}
462 dak control-suite -l stable > squeeze
463 dak control-suite -l testing > wheezy
464 dak control-suite -l unstable > sid
465 echo "${STAMP}" > timestamp
466 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
470 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
473 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
477 log "Updating stats data"
479 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
480 R --slave --vanilla < $base/misc/ftpstats.R
481 dak stats arch-space > $webdir/arch-space
482 dak stats pkg-nums > $webdir/pkg-nums
485 function aptftpcleanup() {
486 log "Clean up apt-ftparchive's databases"
488 apt-ftparchive -q clean apt.conf
491 function cleantransactions() {
492 log "Cleanup transaction ids older than 3 months"
494 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
497 function logstats() {
498 $masterdir/tools/logs.py "$1"
501 # save timestamp when we start
502 function savetimestamp() {
503 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
504 echo ${NOW} > "${dbdir}/dinstallstart"
507 function maillogfile() {
508 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
511 function renamelogfile() {
512 if [ -f "${dbdir}/dinstallstart" ]; then
513 NOW=$(cat "${dbdir}/dinstallstart")
515 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
516 logstats "$logdir/dinstall_${NOW}.log"
517 bzip2 -9 "$logdir/dinstall_${NOW}.log"
519 error "Problem, I don't know when dinstall started, unable to do log statistics."
520 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
522 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
523 bzip2 -9 "$logdir/dinstall_${NOW}.log"
527 function testingsourcelist() {
528 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
531 # do a last run of process-unchecked before dinstall is on.
532 function process_unchecked() {
533 log "Processing the unchecked queue"
534 UNCHECKED_WITHOUT_LOCK="-p"
539 # do a run of newstage only before dinstall is on.
540 function newstage() {
541 log "Processing the newstage queue"
542 UNCHECKED_WITHOUT_LOCK="-p"
546 # Function to update a "statefile" telling people what we are doing
549 # This should be called with the argument(s)
550 # - Status name we want to show.
553 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
554 cat >"${DINSTALLSTATE}" <<EOF
555 Dinstall start: ${DINSTALLBEGIN}
557 Action start: ${RIGHTNOW}
561 # extract changelogs and stuff
562 function changelogs() {
563 log "Extracting changelogs"
564 dak make-changelog -e
565 mkdir -p ${exportpublic}/changelogs
566 cd ${exportpublic}/changelogs
567 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
568 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &
571 function gitpdiff() {
572 # Might be that we want to change this to have more than one git repository.
573 # Advantage of one is that we do not need much space in terms of storage in git itself,
574 # git gc is pretty good on our input.
575 # But it might be faster. Well, lets test.
576 log "Adjusting the git tree for pdiffs"
579 # The regex needs the architectures seperated with \|
580 garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q')
582 # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we
584 # Also, we only want contents, packages and sources.
585 for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do
594 # Second, add all there is into git
597 # Maybe we want to make this the same for tag and commit? But well, shouldn't matter
599 TAGD=$(date +%Y-%m-%d-%H-%M)
600 git commit -m "Commit of ${COMD}"