2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
10 rm -f ${LOCK_ACCEPTED}
13 # If we error out this one is called, *FOLLOWED* by cleanup above
15 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
17 subject="ATTENTION ATTENTION!"
18 if [ "${error}" = "false" ]; then
19 subject="${subject} (continued)"
21 subject="${subject} (interrupted)"
23 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
25 if [ -r "${STAGEFILE}.log" ]; then
26 cat "${STAGEFILE}.log"
28 echo "file ${STAGEFILE}.log does not exist, sorry"
29 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
32 ########################################################################
33 # the actual dinstall functions follow #
34 ########################################################################
37 log "Telling QA user that we start dinstall"
38 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
41 # Updating various files
43 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
45 $scriptsdir/update-bugdoctxt
46 $scriptsdir/update-mirrorlists
47 $scriptsdir/update-mailingliststxt
48 $scriptsdir/update-pseudopackages.sh
51 # Process (oldstable)-proposed-updates "NEW" queue
54 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" team@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
58 log "Doing automated p-u-new processing"
59 cd "${queuedir}/p-u-new"
63 log "Doing automated o-p-u-new processing"
64 cd "${queuedir}/o-p-u-new"
68 # The first i18n one, syncing new descriptions
70 log "Synchronizing i18n package descriptions"
71 # First sync their newest data
72 cd ${scriptdir}/i18nsync
73 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
75 # Now check if we still know about the packages for which they created the files
76 # is the timestamp signed by us?
77 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
78 # now read it. As its signed by us we are sure the content is what we expect, no need
79 # to do more here. And we only test -d a directory on it anyway.
80 TSTAMP=$(cat timestamp)
81 # do we have the dir still?
82 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
84 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
85 # Yay, worked, lets copy around
86 for dir in wheezy sid; do
87 if [ -d dists/${dir}/ ]; then
88 cd dists/${dir}/main/i18n
89 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
91 cd ${scriptdir}/i18nsync
94 echo "ARRRR, bad guys, wrong files, ARRR"
95 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
98 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
99 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
102 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
103 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
108 log "Checking for cruft in overrides"
112 function dominate() {
113 log "Removing obsolete source and binary associations"
117 function filelist() {
118 log "Generating file lists for apt-ftparchive"
119 dak generate-filelist
122 function fingerprints() {
123 log "Updating fingerprints"
124 dak import-keyring -L /srv/backports-master.debian.org/keyrings/keyring.gpg
127 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
129 if [ -s "${OUTFILE}" ]; then
130 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
131 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
132 To: <debian-backports@lists.debian.org>
133 Subject: Debian Maintainers Keyring changes
134 Content-Type: text/plain; charset=utf-8
138 The following changes to the debian-maintainers keyring have just been activated:
142 Debian distribution maintenance software,
143 on behalf of the Keyring maintainers
150 function overrides() {
151 log "Writing overrides into text files"
156 rm -f override.sid.all3
157 for i in main contrib non-free main.debian-installer; do cat override.squeeze-backports.$i >> override.sid.all3; done
161 log "Generating package / file mapping"
162 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
165 function packages() {
166 log "Generating Packages and Sources files"
168 apt-ftparchive generate apt.conf
169 #dak generate-packages-sources
173 log "Generating pdiff files"
174 dak generate-index-diffs
178 log "Generating Release files"
179 dak generate-releases
182 function dakcleanup() {
183 log "Cleanup old packages/files"
184 dak clean-suites -m 10000
188 function buildd_dir() {
189 # Rebuilt the buildd dir to avoid long times of 403
190 log "Regenerating the buildd incoming dir"
191 STAMP=$(date "+%Y%m%d%H%M")
200 log "Removing any core files ..."
201 find -type f -name core -print0 | xargs -0r rm -v
203 log "Checking permissions on files in the FTP tree ..."
204 find -type f \( \! -perm -444 -o -perm +002 \) -ls
205 find -type d \( \! -perm -555 -o -perm +002 \) -ls
207 log "Checking symlinks ..."
210 log "Creating recursive directory listing ... "
211 rm -f .${FILENAME}.new
212 TZ=UTC ls -lR > .${FILENAME}.new
214 if [ -r ${FILENAME}.gz ] ; then
215 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
216 mv -f .${FILENAME}.new ${FILENAME}
217 rm -f ${FILENAME}.patch.gz
218 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
219 rm -f ${FILENAME}.old.gz
221 mv -f .${FILENAME}.new ${FILENAME}
224 gzip -9cfN ${FILENAME} >${FILENAME}.gz
228 function mkmaintainers() {
229 log 'Creating Maintainers index ... '
232 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
233 gzip -9v --rsyncable <Maintainers >Maintainers.gz
234 gzip -9v --rsyncable <Uploaders >Uploaders.gz
237 function copyoverrides() {
238 log 'Copying override files into public view ...'
240 for ofile in $copyoverrides ; do
242 chmod g+w override.$ofile
246 newofile=override.$ofile.gz
247 rm -f .newover-$ofile.gz
248 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
249 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
250 log " installing new $newofile $pc"
251 mv -f .newover-$ofile.gz $newofile
254 rm -f .newover-$ofile.gz
259 function mkfilesindices() {
261 cd $base/ftp/indices/files/components
265 log "Querying postgres"
266 local query='SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)'
267 psql -At -c "$query" | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
270 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
273 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
276 log "Generating sources list"
278 sed -n 's/|$//p' $ARCHLIST
280 find ./dists -maxdepth 1 \! -type d
281 find ./dists \! -type d | grep "/source/"
282 ) | sort -u | gzip -9 > source.list.gz
284 log "Generating arch lists"
286 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
288 (sed -n "s/|$a$//p" $ARCHLIST
289 sed -n 's/|all$//p' $ARCHLIST
292 find ./dists -maxdepth 1 \! -type d
293 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
294 ) | sort -u | gzip -9 > arch-$a.list.gz
297 log "Generating suite lists"
301 query="$(printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)' $1 $1)"
302 psql -F' ' -A -t -c "$query"
304 query="$(printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d' $1)"
305 psql -F' ' -A -t -c "$query"
308 psql -F' ' -At -c 'SELECT id, suite_name FROM suite' |
309 while read id suite; do
310 [ -e $base/ftp/dists/$suite ] || continue
313 distname=$(cd dists; readlink $suite || echo $suite)
314 find ./dists/$distname \! -type d
315 for distdir in ./dists/*; do
316 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
319 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
320 ) | sort -u | gzip -9 > suite-${suite}.list.gz
323 log "Finding everything on the ftp site to generate sundries"
324 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
327 zcat *.list.gz | cat - *.list | sort -u |
328 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
330 log "Generating files list"
333 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
334 cat - sundries.list dists.list project.list docs.list indices.list |
335 sort -u | poolfirst > ../arch-$a.files
339 for dist in sid wheezy; do
340 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
344 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
345 sort -u | poolfirst > ../typical.files
351 function mkchecksums() {
352 dsynclist=$dbdir/dsync.list
353 md5list=$indices/md5sums
355 log -n "Creating md5 / dsync index file ... "
358 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
359 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
360 ${bindir}/dsync-flist -q link-dups $dsynclist || true
364 log "Regenerating \"public\" mirror/ hardlink fun"
365 DATE_SERIAL=$(date +"%Y%m%d01")
366 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
367 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
368 SERIAL="$DATE_SERIAL"
370 SERIAL="$FILESOAPLUS1"
372 date -u > ${TRACEFILE}
373 echo "Using dak v1" >> ${TRACEFILE}
374 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
375 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
377 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
381 log "Expiring old database dumps..."
383 $scriptsdir/expire_dumps -d . -p -f "dump_*"
386 function transitionsclean() {
387 log "Removing out of date transitions..."
389 dak transitions -c -a
393 log "Updating DM html page"
394 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
398 log "Categorizing uncategorized bugs filed against ftp.debian.org"
402 function ddaccess() {
403 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
404 log "Trigger dd accessible parts sync including ftp dir"
405 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
408 function mirrorpush() {
409 log "Starting the mirrorpush"
410 date -u > /srv/backports-web.debian.org/underlay/mirrorstart
411 echo "Using dak v1" >> /srv/backports-web.debian.org/underlay/mirrorstart
412 echo "Running on host $(hostname -f)" >> /srv/backports-web.debian.org/underlay/mirrorstart
413 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
417 log "Exporting package data foo for i18n project"
418 STAMP=$(date "+%Y%m%d%H%M")
419 mkdir -p ${scriptdir}/i18n/${STAMP}
420 cd ${scriptdir}/i18n/${STAMP}
421 dak control-suite -l stable > squeeze
422 dak control-suite -l testing > wheezy
423 dak control-suite -l unstable > sid
424 echo "${STAMP}" > timestamp
425 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
429 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
432 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
436 log "Updating stats data"
438 #$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
439 #R --slave --vanilla < $base/misc/ftpstats.R
440 dak stats arch-space > $webdir/arch-space
441 dak stats pkg-nums > $webdir/pkg-nums
444 function aptftpcleanup() {
445 log "Clean up apt-ftparchive's databases"
447 apt-ftparchive -q clean apt.conf
450 function cleantransactions() {
451 log "Cleanup transaction ids older than 3 months"
453 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
456 function logstats() {
457 $masterdir/tools/logs.py "$1"
460 # save timestamp when we start
461 function savetimestamp() {
462 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
463 echo ${NOW} > "${dbdir}/dinstallstart"
466 function maillogfile() {
467 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
470 function renamelogfile() {
471 if [ -f "${dbdir}/dinstallstart" ]; then
472 NOW=$(cat "${dbdir}/dinstallstart")
474 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
475 # logstats "$logdir/dinstall_${NOW}.log"
476 bzip2 -9 "$logdir/dinstall_${NOW}.log"
478 error "Problem, I don't know when dinstall started, unable to do log statistics."
479 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
481 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
482 bzip2 -9 "$logdir/dinstall_${NOW}.log"
486 function testingsourcelist() {
487 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
490 # do a last run of process-unchecked before dinstall is on.
491 function process_unchecked() {
492 log "Processing the unchecked queue"
493 UNCHECKED_WITHOUT_LOCK="-p"
498 # do a run of newstage only before dinstall is on.
499 function newstage() {
500 log "Processing the newstage queue"
501 UNCHECKED_WITHOUT_LOCK="-p"
505 # Function to update a "statefile" telling people what we are doing
508 # This should be called with the argument(s)
509 # - Status name we want to show.
512 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
513 cat >"${DINSTALLSTATE}" <<EOF
514 Dinstall start: ${DINSTALLBEGIN}
516 Action start: ${RIGHTNOW}
520 # extract changelogs and stuff
521 function changelogs() {
522 log "Extracting changelogs"
523 dak make-changelog -e
524 mkdir -p ${exportpublic}/changelogs
525 cd ${exportpublic}/changelogs
526 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
527 sudo -H -u archvsync /home/archvsync/runmirrors metabdo > ~dak/runmirrors-metadata.log 2>&1 &