2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Updating various files
48 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
50 $scriptsdir/update-bugdoctxt
51 $scriptsdir/update-mirrorlists
52 $scriptsdir/update-mailingliststxt
53 $scriptsdir/update-pseudopackages.sh
56 # Process (oldstable)-proposed-updates "NEW" queue
59 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" debian-release@lists.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
63 log "Doing automated p-u-new processing"
64 cd "${queuedir}/p-u-new"
68 log "Doing automated o-p-u-new processing"
69 cd "${queuedir}/o-p-u-new"
73 # The first i18n one, syncing new descriptions
75 log "Synchronizing i18n package descriptions"
76 # First sync their newest data
77 cd ${scriptdir}/i18nsync
78 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
80 # Now check if we still know about the packages for which they created the files
81 # is the timestamp signed by us?
82 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
83 # now read it. As its signed by us we are sure the content is what we expect, no need
84 # to do more here. And we only test -d a directory on it anyway.
85 TSTAMP=$(cat timestamp)
86 # do we have the dir still?
87 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
89 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
90 # Yay, worked, lets copy around
91 for dir in wheezy sid; do
92 if [ -d dists/${dir}/ ]; then
93 cd dists/${dir}/main/i18n
94 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
96 cd ${scriptdir}/i18nsync
99 echo "ARRRR, bad guys, wrong files, ARRR"
100 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
103 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
104 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
107 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
108 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
113 log "Checking for cruft in overrides"
117 function dominate() {
118 log "Removing obsolete source and binary associations"
122 function filelist() {
123 log "Generating file lists for apt-ftparchive"
124 dak generate-filelist
127 function fingerprints() {
128 log "Updating fingerprints"
129 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
132 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
134 if [ -s "${OUTFILE}" ]; then
135 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
136 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
137 To: <debian-project@lists.debian.org>
138 Subject: Debian Maintainers Keyring changes
139 Content-Type: text/plain; charset=utf-8
143 The following changes to the debian-maintainers keyring have just been activated:
147 Debian distribution maintenance software,
148 on behalf of the Keyring maintainers
155 function overrides() {
156 log "Writing overrides into text files"
161 rm -f override.sid.all3
162 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
166 log "Generating package / file mapping"
167 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
170 function packages() {
171 log "Generating Packages and Sources files"
173 #dak generate-packages-sources
174 dak generate-packages-sources2
175 dak contents generate
179 log "Generating pdiff files"
180 dak generate-index-diffs
184 log "Generating Release files"
185 dak generate-releases
188 function dakcleanup() {
189 log "Cleanup old packages/files"
190 dak clean-suites -m 10000
194 function buildd_dir() {
195 # Rebuilt the buildd dir to avoid long times of 403
196 log "Regenerating the buildd incoming dir"
197 STAMP=$(date "+%Y%m%d%H%M")
206 log "Removing any core files ..."
207 find -type f -name core -print0 | xargs -0r rm -v
209 log "Checking permissions on files in the FTP tree ..."
210 find -type f \( \! -perm -444 -o -perm +002 \) -ls
211 find -type d \( \! -perm -555 -o -perm +002 \) -ls
213 log "Checking symlinks ..."
216 log "Creating recursive directory listing ... "
217 rm -f .${FILENAME}.new
218 TZ=UTC ls -lR > .${FILENAME}.new
220 if [ -r ${FILENAME}.gz ] ; then
221 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
222 mv -f .${FILENAME}.new ${FILENAME}
223 rm -f ${FILENAME}.patch.gz
224 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
225 rm -f ${FILENAME}.old.gz
227 mv -f .${FILENAME}.new ${FILENAME}
230 gzip -9cfN ${FILENAME} >${FILENAME}.gz
234 function mkmaintainers() {
235 log 'Creating Maintainers index ... '
238 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
239 gzip -9v --rsyncable <Maintainers >Maintainers.gz
240 gzip -9v --rsyncable <Uploaders >Uploaders.gz
243 function copyoverrides() {
244 log 'Copying override files into public view ...'
246 for ofile in $copyoverrides ; do
248 chmod g+w override.$ofile
252 newofile=override.$ofile.gz
253 rm -f .newover-$ofile.gz
254 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
255 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
256 log " installing new $newofile $pc"
257 mv -f .newover-$ofile.gz $newofile
260 rm -f .newover-$ofile.gz
265 function mkfilesindices() {
267 cd $base/ftp/indices/files/components
271 log "Querying postgres"
272 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
275 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
278 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
281 log "Generating sources list"
283 sed -n 's/|$//p' $ARCHLIST
285 find ./dists -maxdepth 1 \! -type d
286 find ./dists \! -type d | grep "/source/"
287 ) | sort -u | gzip -9 > source.list.gz
289 log "Generating arch lists"
291 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
293 (sed -n "s/|$a$//p" $ARCHLIST
294 sed -n 's/|all$//p' $ARCHLIST
297 find ./dists -maxdepth 1 \! -type d
298 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
299 ) | sort -u | gzip -9 > arch-$a.list.gz
302 log "Generating suite lists"
305 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
307 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
310 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
311 while read id suite; do
312 [ -e $base/ftp/dists/$suite ] || continue
315 distname=$(cd dists; readlink $suite || echo $suite)
316 find ./dists/$distname \! -type d
317 for distdir in ./dists/*; do
318 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
321 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
322 ) | sort -u | gzip -9 > suite-${suite}.list.gz
325 log "Finding everything on the ftp site to generate sundries"
326 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
329 zcat *.list.gz | cat - *.list | sort -u |
330 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
332 log "Generating files list"
335 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
336 cat - sundries.list dists.list project.list docs.list indices.list |
337 sort -u | poolfirst > ../arch-$a.files
341 for dist in sid wheezy; do
342 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
346 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
347 sort -u | poolfirst > ../typical.files
353 function mkchecksums() {
354 dsynclist=$dbdir/dsync.list
355 md5list=$indices/md5sums
357 log -n "Creating md5 / dsync index file ... "
360 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
361 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
362 ${bindir}/dsync-flist -q link-dups $dsynclist || true
366 log "Regenerating \"public\" mirror/ hardlink fun"
367 DATE_SERIAL=$(date +"%Y%m%d01")
368 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
369 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
370 SERIAL="$DATE_SERIAL"
372 SERIAL="$FILESOAPLUS1"
374 date -u > ${TRACEFILE}
375 echo "Using dak v1" >> ${TRACEFILE}
376 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
377 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
379 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
383 log "Expiring old database dumps..."
385 $scriptsdir/expire_dumps -d . -p -f "dump_*"
388 function transitionsclean() {
389 log "Removing out of date transitions..."
391 dak transitions -c -a
395 log "Updating DM html page"
396 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
400 log "Categorizing uncategorized bugs filed against ftp.debian.org"
404 function ddaccess() {
405 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
406 log "Trigger dd accessible parts sync including ftp dir"
407 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
410 function mirrorpush() {
411 log "Checking the public archive copy"
412 cd ${mirrordir}/dists
415 for release in $(find . -name "InRelease"); do
416 echo "Processing: ${release}"
417 subdir=${release%/InRelease}
418 while read SHASUM SIZE NAME; do
419 if ! [ -f "${subdir}/${NAME}" ]; then
420 bname=$(basename ${NAME})
421 if [ "${bname}" = "Packages" ] || [ "${bname}" = "Sources" ]; then
422 # We don't keep unpacked files, don't check for their existance.
423 # We might want to go and check their unpacked shasum, but right now
424 # I don't care. I believe it should be enough if all the packed shasums
428 broken=$(( broken + 1 ))
429 echo "File ${subdir}/${NAME} is missing"
433 # We do have symlinks in the tree (see the contents files currently).
434 # So we use "readlink -f" to check the size of the target, as thats basically
435 # what gen-releases does
436 fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
437 if [ ${fsize} -ne ${SIZE} ]; then
438 broken=$(( broken + 1 ))
439 echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
443 fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
444 fshasum=${fshasum%% *}
445 if [ "${fshasum}" != "${SHASUM}" ]; then
446 broken=$(( broken + 1 ))
447 echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
450 done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
453 if [ $broken -gt 0 ]; then
454 log_error "Trouble with the public mirror, found ${broken} errors"
458 log "Starting the mirrorpush"
459 date -u > /srv/ftp.debian.org/web/mirrorstart
460 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
461 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
462 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
466 log "Exporting package data foo for i18n project"
467 STAMP=$(date "+%Y%m%d%H%M")
468 mkdir -p ${scriptdir}/i18n/${STAMP}
469 cd ${scriptdir}/i18n/${STAMP}
470 dak control-suite -l stable > squeeze
471 dak control-suite -l testing > wheezy
472 dak control-suite -l unstable > sid
473 echo "${STAMP}" > timestamp
474 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
478 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
481 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
485 log "Updating stats data"
487 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
488 R --slave --vanilla < $base/misc/ftpstats.R
489 dak stats arch-space > $webdir/arch-space
490 dak stats pkg-nums > $webdir/pkg-nums
493 function aptftpcleanup() {
494 log "Clean up apt-ftparchive's databases"
496 apt-ftparchive -q clean apt.conf
499 function cleantransactions() {
500 log "Cleanup transaction ids older than 3 months"
502 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
505 function logstats() {
506 $masterdir/tools/logs.py "$1"
509 # save timestamp when we start
510 function savetimestamp() {
511 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
512 echo ${NOW} > "${dbdir}/dinstallstart"
515 function maillogfile() {
516 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
519 function renamelogfile() {
520 if [ -f "${dbdir}/dinstallstart" ]; then
521 NOW=$(cat "${dbdir}/dinstallstart")
523 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
524 logstats "$logdir/dinstall_${NOW}.log"
525 bzip2 -9 "$logdir/dinstall_${NOW}.log"
527 error "Problem, I don't know when dinstall started, unable to do log statistics."
528 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
530 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
531 bzip2 -9 "$logdir/dinstall_${NOW}.log"
535 function testingsourcelist() {
536 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
539 # do a last run of process-unchecked before dinstall is on.
540 function process_unchecked() {
541 log "Processing the unchecked queue"
542 UNCHECKED_WITHOUT_LOCK="-p"
547 # do a run of newstage only before dinstall is on.
548 function newstage() {
549 log "Processing the newstage queue"
550 UNCHECKED_WITHOUT_LOCK="-p"
554 # Function to update a "statefile" telling people what we are doing
557 # This should be called with the argument(s)
558 # - Status name we want to show.
561 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
562 cat >"${DINSTALLSTATE}" <<EOF
563 Dinstall start: ${DINSTALLBEGIN}
565 Action start: ${RIGHTNOW}
569 # extract changelogs and stuff
570 function changelogs() {
571 log "Extracting changelogs"
572 dak make-changelog -e
573 mkdir -p ${exportpublic}/changelogs
574 cd ${exportpublic}/changelogs
575 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
576 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &
579 function gitpdiff() {
580 # Might be that we want to change this to have more than one git repository.
581 # Advantage of one is that we do not need much space in terms of storage in git itself,
582 # git gc is pretty good on our input.
583 # But it might be faster. Well, lets test.
584 log "Adjusting the git tree for pdiffs"
587 # The regex needs the architectures seperated with \|
588 garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q')
590 # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we
592 # Also, we only want contents, packages and sources.
593 for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do
602 # Second, add all there is into git
605 # Maybe we want to make this the same for tag and commit? But well, shouldn't matter
607 TAGD=$(date +%Y-%m-%d-%H-%M)
608 git commit -m "Commit of ${COMD}"