2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Updating various files
48 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
50 $scriptsdir/update-bugdoctxt
51 $scriptsdir/update-mirrorlists
52 $scriptsdir/update-mailingliststxt
53 $scriptsdir/update-pseudopackages.sh
56 # Process (oldstable)-proposed-updates "NEW" queue
59 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" debian-release@lists.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
63 log "Doing automated p-u-new processing"
64 cd "${queuedir}/p-u-new"
68 log "Doing automated o-p-u-new processing"
69 cd "${queuedir}/o-p-u-new"
73 # The first i18n one, syncing new descriptions
75 log "Synchronizing i18n package descriptions"
76 # First sync their newest data
77 cd ${scriptdir}/i18nsync
78 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
80 # Now check if we still know about the packages for which they created the files
81 # is the timestamp signed by us?
82 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
83 # now read it. As its signed by us we are sure the content is what we expect, no need
84 # to do more here. And we only test -d a directory on it anyway.
85 TSTAMP=$(cat timestamp)
86 # do we have the dir still?
87 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
89 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
90 # Yay, worked, lets copy around
91 for dir in wheezy sid; do
92 if [ -d dists/${dir}/ ]; then
93 cd dists/${dir}/main/i18n
94 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
96 cd ${scriptdir}/i18nsync
99 echo "ARRRR, bad guys, wrong files, ARRR"
100 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
103 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
104 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
107 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
108 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
113 log "Checking for cruft in overrides"
117 function dominate() {
118 log "Removing obsolete source and binary associations"
122 function filelist() {
123 log "Generating file lists for apt-ftparchive"
124 dak generate-filelist
127 function fingerprints() {
128 log "Updating fingerprints"
129 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
132 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
134 if [ -s "${OUTFILE}" ]; then
135 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
136 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
137 To: <debian-project@lists.debian.org>
138 Subject: Debian Maintainers Keyring changes
139 Content-Type: text/plain; charset=utf-8
143 The following changes to the debian-maintainers keyring have just been activated:
147 Debian distribution maintenance software,
148 on behalf of the Keyring maintainers
155 function overrides() {
156 log "Writing overrides into text files"
161 rm -f override.sid.all3
162 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
166 log "Generating package / file mapping"
167 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
170 function packages() {
171 log "Generating Packages and Sources files"
173 #dak generate-packages-sources
174 # XXX: disable again later
175 dak generate-packages-sources2 --description-md5 -o Dir::Root=$webdir/newdists/
176 dak generate-packages-sources2
177 dak contents generate
181 log "Generating pdiff files"
182 dak generate-index-diffs
186 # XXX: disable once we can remove i18n/Index (#649314)
187 log "Generating i18n/Index"
188 (cd "$ftpdir/dists"; for dist in testing unstable; do $scriptsdir/generate-i18n-Index $dist; done)
189 (cd "$webdir/newdists/dists"; for dist in testing unstable; do $scriptsdir/generate-i18n-Index $dist; done)
190 log "Generating Release files"
191 dak generate-releases
192 # XXX: disable again later
193 dak generate-releases -o Dir::Root=$webdir/newdists/
196 function dakcleanup() {
197 log "Cleanup old packages/files"
198 dak clean-suites -m 10000
202 function buildd_dir() {
203 # Rebuilt the buildd dir to avoid long times of 403
204 log "Regenerating the buildd incoming dir"
205 STAMP=$(date "+%Y%m%d%H%M")
214 log "Removing any core files ..."
215 find -type f -name core -print0 | xargs -0r rm -v
217 log "Checking permissions on files in the FTP tree ..."
218 find -type f \( \! -perm -444 -o -perm +002 \) -ls
219 find -type d \( \! -perm -555 -o -perm +002 \) -ls
221 log "Checking symlinks ..."
224 log "Creating recursive directory listing ... "
225 rm -f .${FILENAME}.new
226 TZ=UTC ls -lR > .${FILENAME}.new
228 if [ -r ${FILENAME}.gz ] ; then
229 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
230 mv -f .${FILENAME}.new ${FILENAME}
231 rm -f ${FILENAME}.patch.gz
232 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
233 rm -f ${FILENAME}.old.gz
235 mv -f .${FILENAME}.new ${FILENAME}
238 gzip -9cfN ${FILENAME} >${FILENAME}.gz
242 function mkmaintainers() {
243 log 'Creating Maintainers index ... '
246 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
247 gzip -9v --rsyncable <Maintainers >Maintainers.gz
248 gzip -9v --rsyncable <Uploaders >Uploaders.gz
251 function copyoverrides() {
252 log 'Copying override files into public view ...'
254 for ofile in ${overridedir}/override.{lenny,squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do
256 gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
257 chmod g+w ${indices}/${bname}.gz
261 function mkfilesindices() {
263 cd $base/ftp/indices/files/components
267 log "Querying postgres"
268 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
271 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
274 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
277 log "Generating sources list"
279 sed -n 's/|$//p' $ARCHLIST
281 find ./dists -maxdepth 1 \! -type d
282 find ./dists \! -type d | grep "/source/"
283 ) | sort -u | gzip -9 > source.list.gz
285 log "Generating arch lists"
287 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
289 (sed -n "s/|$a$//p" $ARCHLIST
290 sed -n 's/|all$//p' $ARCHLIST
293 find ./dists -maxdepth 1 \! -type d
294 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
295 ) | sort -u | gzip -9 > arch-$a.list.gz
298 log "Generating suite lists"
301 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
303 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
306 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
307 while read id suite; do
308 [ -e $base/ftp/dists/$suite ] || continue
311 distname=$(cd dists; readlink $suite || echo $suite)
312 find ./dists/$distname \! -type d
313 for distdir in ./dists/*; do
314 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
317 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
318 ) | sort -u | gzip -9 > suite-${suite}.list.gz
321 log "Finding everything on the ftp site to generate sundries"
322 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
325 zcat *.list.gz | cat - *.list | sort -u |
326 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
328 log "Generating files list"
331 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
332 cat - sundries.list dists.list project.list docs.list indices.list |
333 sort -u | poolfirst > ../arch-$a.files
337 for dist in sid wheezy; do
338 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
342 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
343 sort -u | poolfirst > ../typical.files
349 function mkchecksums() {
350 dsynclist=$dbdir/dsync.list
351 md5list=$indices/md5sums
353 log -n "Creating md5 / dsync index file ... "
356 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
357 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
358 ${bindir}/dsync-flist -q link-dups $dsynclist || true
362 log "Regenerating \"public\" mirror/ hardlink fun"
363 DATE_SERIAL=$(date +"%Y%m%d01")
364 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
365 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
366 SERIAL="$DATE_SERIAL"
368 SERIAL="$FILESOAPLUS1"
370 date -u > ${TRACEFILE}
371 echo "Using dak v1" >> ${TRACEFILE}
372 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
373 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
375 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
379 log "Expiring old database dumps..."
381 $scriptsdir/expire_dumps -d . -p -f "dump_*"
384 function transitionsclean() {
385 log "Removing out of date transitions..."
387 dak transitions -c -a
391 log "Updating DM html page"
392 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
396 log "Categorizing uncategorized bugs filed against ftp.debian.org"
400 function ddaccess() {
401 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
402 log "Trigger dd accessible parts sync including ftp dir"
403 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
406 function mirrorpush() {
407 log "Checking the public archive copy"
408 cd ${mirrordir}/dists
411 for release in $(find . -name "InRelease"); do
412 echo "Processing: ${release}"
413 subdir=${release%/InRelease}
414 while read SHASUM SIZE NAME; do
415 if ! [ -f "${subdir}/${NAME}" ]; then
416 bname=$(basename ${NAME})
417 if [[ "${bname}" =~ ^(Packages|Sources|Translation-[a-zA-Z_]+)$ ]]; then
418 # We don't keep unpacked files, don't check for their existance.
419 # We might want to go and check their unpacked shasum, but right now
420 # I don't care. I believe it should be enough if all the packed shasums
424 broken=$(( broken + 1 ))
425 echo "File ${subdir}/${NAME} is missing"
429 # We do have symlinks in the tree (see the contents files currently).
430 # So we use "readlink -f" to check the size of the target, as thats basically
431 # what gen-releases does
432 fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
433 if [ ${fsize} -ne ${SIZE} ]; then
434 broken=$(( broken + 1 ))
435 echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
439 fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
440 fshasum=${fshasum%% *}
441 if [ "${fshasum}" != "${SHASUM}" ]; then
442 broken=$(( broken + 1 ))
443 echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
446 done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
449 if [ $broken -gt 0 ]; then
450 log_error "Trouble with the public mirror, found ${broken} errors"
454 log "Starting the mirrorpush"
455 date -u > /srv/ftp.debian.org/web/mirrorstart
456 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
457 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
458 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
462 log "Exporting package data foo for i18n project"
463 STAMP=$(date "+%Y%m%d%H%M")
464 mkdir -p ${scriptdir}/i18n/${STAMP}
465 cd ${scriptdir}/i18n/${STAMP}
466 dak control-suite -l stable > squeeze
467 dak control-suite -l testing > wheezy
468 dak control-suite -l unstable > sid
469 echo "${STAMP}" > timestamp
470 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
474 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
477 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
481 log "Updating stats data"
483 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
484 R --slave --vanilla < $base/misc/ftpstats.R
485 dak stats arch-space > $webdir/arch-space
486 dak stats pkg-nums > $webdir/pkg-nums
489 function aptftpcleanup() {
490 log "Clean up apt-ftparchive's databases"
492 apt-ftparchive -q clean apt.conf
495 function cleantransactions() {
496 log "Cleanup transaction ids older than 3 months"
498 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
501 function logstats() {
502 $masterdir/tools/logs.py "$1"
505 # save timestamp when we start
506 function savetimestamp() {
507 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
508 echo ${NOW} > "${dbdir}/dinstallstart"
511 function maillogfile() {
512 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
515 function renamelogfile() {
516 if [ -f "${dbdir}/dinstallstart" ]; then
517 NOW=$(cat "${dbdir}/dinstallstart")
519 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
520 logstats "$logdir/dinstall_${NOW}.log"
521 bzip2 -9 "$logdir/dinstall_${NOW}.log"
523 error "Problem, I don't know when dinstall started, unable to do log statistics."
524 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
526 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
527 bzip2 -9 "$logdir/dinstall_${NOW}.log"
531 function testingsourcelist() {
532 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
535 # do a last run of process-unchecked before dinstall is on.
536 function process_unchecked() {
537 log "Processing the unchecked queue"
538 UNCHECKED_WITHOUT_LOCK="-p"
543 # do a run of newstage only before dinstall is on.
544 function newstage() {
545 log "Processing the newstage queue"
546 UNCHECKED_WITHOUT_LOCK="-p"
550 # Function to update a "statefile" telling people what we are doing
553 # This should be called with the argument(s)
554 # - Status name we want to show.
557 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
558 cat >"${DINSTALLSTATE}" <<EOF
559 Dinstall start: ${DINSTALLBEGIN}
561 Action start: ${RIGHTNOW}
565 # extract changelogs and stuff
566 function changelogs() {
567 log "Extracting changelogs"
568 dak make-changelog -e
569 mkdir -p ${exportpublic}/changelogs
570 cd ${exportpublic}/changelogs
571 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
572 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &
575 function gitpdiff() {
576 # Might be that we want to change this to have more than one git repository.
577 # Advantage of one is that we do not need much space in terms of storage in git itself,
578 # git gc is pretty good on our input.
579 # But it might be faster. Well, lets test.
580 log "Adjusting the git tree for pdiffs"
583 # The regex needs the architectures seperated with \|
584 garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q')
586 # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we
588 # Also, we only want contents, packages and sources.
589 for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do
598 # Second, add all there is into git
601 # Maybe we want to make this the same for tag and commit? But well, shouldn't matter
603 TAGD=$(date +%Y-%m-%d-%H-%M)
604 git commit -m "Commit of ${COMD}"