2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
10 rm -f ${LOCK_ACCEPTED}
13 # If we error out this one is called, *FOLLOWED* by cleanup above
15 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
17 subject="ATTENTION ATTENTION!"
18 if [ "${error}" = "false" ]; then
19 subject="${subject} (continued)"
21 subject="${subject} (interrupted)"
23 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
25 if [ -r "${STAGEFILE}.log" ]; then
26 cat "${STAGEFILE}.log"
28 echo "file ${STAGEFILE}.log does not exist, sorry"
29 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
32 ########################################################################
33 # the actual dinstall functions follow #
34 ########################################################################
37 log "Telling QA user that we start dinstall"
38 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
41 # Updating various files
43 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
45 $scriptsdir/update-bugdoctxt
46 $scriptsdir/update-mirrorlists
47 $scriptsdir/update-mailingliststxt
48 $scriptsdir/update-pseudopackages.sh
51 # Process (oldstable)-proposed-updates "NEW" queue
54 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" team@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
58 log "Doing automated p-u-new processing"
59 cd "${queuedir}/p-u-new"
63 log "Doing automated o-p-u-new processing"
64 cd "${queuedir}/o-p-u-new"
68 # The first i18n one, syncing new descriptions
70 log "Synchronizing i18n package descriptions"
71 # First sync their newest data
72 cd ${scriptdir}/i18nsync
73 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
75 # Now check if we still know about the packages for which they created the files
76 # is the timestamp signed by us?
77 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
78 # now read it. As its signed by us we are sure the content is what we expect, no need
79 # to do more here. And we only test -d a directory on it anyway.
80 TSTAMP=$(cat timestamp)
81 # do we have the dir still?
82 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
84 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
85 # Yay, worked, lets copy around
86 for dir in wheezy sid; do
87 if [ -d dists/${dir}/ ]; then
88 cd dists/${dir}/main/i18n
89 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
91 cd ${scriptdir}/i18nsync
94 echo "ARRRR, bad guys, wrong files, ARRR"
95 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
98 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
99 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
102 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
103 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
108 log "Checking for cruft in overrides"
112 function dominate() {
113 log "Removing obsolete source and binary associations"
117 function filelist() {
118 log "Generating file lists for apt-ftparchive"
119 dak generate-filelist
122 function fingerprints() {
123 log "Updating fingerprints"
124 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
127 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
129 if [ -s "${OUTFILE}" ]; then
130 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
131 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
132 To: <debian-project@lists.debian.org>
133 Subject: Debian Maintainers Keyring changes
134 Content-Type: text/plain; charset=utf-8
138 The following changes to the debian-maintainers keyring have just been activated:
142 Debian distribution maintenance software,
143 on behalf of the Keyring maintainers
150 function overrides() {
151 log "Writing overrides into text files"
156 rm -f override.sid.all3
157 for i in main contrib non-free main.debian-installer; do cat override.lenny-backports.$i >> override.sid.all3; done
161 log "Generating package / file mapping"
162 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
165 function packages() {
166 log "Generating Packages and Sources files"
168 apt-ftparchive generate apt.conf
169 #dak generate-packages-sources
173 log "Generating pdiff files"
174 dak generate-index-diffs
178 log "Generating Release files"
179 dak generate-releases
182 function dakcleanup() {
183 log "Cleanup old packages/files"
184 dak clean-suites -m 10000
188 function buildd_dir() {
189 # Rebuilt the buildd dir to avoid long times of 403
190 log "Regenerating the buildd incoming dir"
191 STAMP=$(date "+%Y%m%d%H%M")
200 log "Removing any core files ..."
201 find -type f -name core -print0 | xargs -0r rm -v
203 log "Checking permissions on files in the FTP tree ..."
204 find -type f \( \! -perm -444 -o -perm +002 \) -ls
205 find -type d \( \! -perm -555 -o -perm +002 \) -ls
207 log "Checking symlinks ..."
210 log "Creating recursive directory listing ... "
211 rm -f .${FILENAME}.new
212 TZ=UTC ls -lR > .${FILENAME}.new
214 if [ -r ${FILENAME}.gz ] ; then
215 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
216 mv -f .${FILENAME}.new ${FILENAME}
217 rm -f ${FILENAME}.patch.gz
218 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
219 rm -f ${FILENAME}.old.gz
221 mv -f .${FILENAME}.new ${FILENAME}
224 gzip -9cfN ${FILENAME} >${FILENAME}.gz
228 function mkmaintainers() {
229 log 'Creating Maintainers index ... '
232 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
233 gzip -9v --rsyncable <Maintainers >Maintainers.gz
234 gzip -9v --rsyncable <Uploaders >Uploaders.gz
237 function copyoverrides() {
238 log 'Copying override files into public view ...'
240 for ofile in $copyoverrides ; do
242 chmod g+w override.$ofile
246 newofile=override.$ofile.gz
247 rm -f .newover-$ofile.gz
248 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
249 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
250 log " installing new $newofile $pc"
251 mv -f .newover-$ofile.gz $newofile
254 rm -f .newover-$ofile.gz
259 function mkfilesindices() {
261 cd $base/ftp/indices/files/components
265 log "Querying postgres"
266 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
269 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
272 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
275 log "Generating sources list"
277 sed -n 's/|$//p' $ARCHLIST
279 find ./dists -maxdepth 1 \! -type d
280 find ./dists \! -type d | grep "/source/"
281 ) | sort -u | gzip -9 > source.list.gz
283 log "Generating arch lists"
285 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
287 (sed -n "s/|$a$//p" $ARCHLIST
288 sed -n 's/|all$//p' $ARCHLIST
291 find ./dists -maxdepth 1 \! -type d
292 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
293 ) | sort -u | gzip -9 > arch-$a.list.gz
296 log "Generating suite lists"
299 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
301 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
304 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
305 while read id suite; do
306 [ -e $base/ftp/dists/$suite ] || continue
309 distname=$(cd dists; readlink $suite || echo $suite)
310 find ./dists/$distname \! -type d
311 for distdir in ./dists/*; do
312 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
315 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
316 ) | sort -u | gzip -9 > suite-${suite}.list.gz
319 log "Finding everything on the ftp site to generate sundries"
320 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
323 zcat *.list.gz | cat - *.list | sort -u |
324 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
326 log "Generating files list"
329 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
330 cat - sundries.list dists.list project.list docs.list indices.list |
331 sort -u | poolfirst > ../arch-$a.files
335 for dist in sid wheezy; do
336 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
340 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
341 sort -u | poolfirst > ../typical.files
347 function mkchecksums() {
348 dsynclist=$dbdir/dsync.list
349 md5list=$indices/md5sums
351 log -n "Creating md5 / dsync index file ... "
354 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
355 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
356 ${bindir}/dsync-flist -q link-dups $dsynclist || true
360 log "Regenerating \"public\" mirror/ hardlink fun"
361 DATE_SERIAL=$(date +"%Y%m%d01")
362 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
363 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
364 SERIAL="$DATE_SERIAL"
366 SERIAL="$FILESOAPLUS1"
368 date -u > ${TRACEFILE}
369 echo "Using dak v1" >> ${TRACEFILE}
370 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
371 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
373 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
377 log "Expiring old database dumps..."
379 $scriptsdir/expire_dumps -d . -p -f "dump_*"
382 function transitionsclean() {
383 log "Removing out of date transitions..."
385 dak transitions -c -a
389 log "Updating DM html page"
390 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
394 log "Categorizing uncategorized bugs filed against ftp.debian.org"
398 function ddaccess() {
399 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
400 log "Trigger dd accessible parts sync including ftp dir"
401 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
404 function mirrorpush() {
405 log "Starting the mirrorpush"
406 date -u > /srv/backports-web.debian.org/underlay/mirrorstart
407 echo "Using dak v1" >> /srv/backports-web.debian.org/underlay/mirrorstart
408 echo "Running on host $(hostname -f)" >> /srv/backports-web.debian.org/underlay/mirrorstart
409 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
413 log "Exporting package data foo for i18n project"
414 STAMP=$(date "+%Y%m%d%H%M")
415 mkdir -p ${scriptdir}/i18n/${STAMP}
416 cd ${scriptdir}/i18n/${STAMP}
417 dak control-suite -l stable > squeeze
418 dak control-suite -l testing > wheezy
419 dak control-suite -l unstable > sid
420 echo "${STAMP}" > timestamp
421 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
425 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
428 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
432 log "Updating stats data"
434 #$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
435 #R --slave --vanilla < $base/misc/ftpstats.R
436 dak stats arch-space > $webdir/arch-space
437 dak stats pkg-nums > $webdir/pkg-nums
440 function aptftpcleanup() {
441 log "Clean up apt-ftparchive's databases"
443 apt-ftparchive -q clean apt.conf
446 function cleantransactions() {
447 log "Cleanup transaction ids older than 3 months"
449 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
452 function logstats() {
453 $masterdir/tools/logs.py "$1"
456 # save timestamp when we start
457 function savetimestamp() {
458 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
459 echo ${NOW} > "${dbdir}/dinstallstart"
462 function maillogfile() {
463 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
466 function renamelogfile() {
467 if [ -f "${dbdir}/dinstallstart" ]; then
468 NOW=$(cat "${dbdir}/dinstallstart")
470 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
471 # logstats "$logdir/dinstall_${NOW}.log"
472 bzip2 -9 "$logdir/dinstall_${NOW}.log"
474 error "Problem, I don't know when dinstall started, unable to do log statistics."
475 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
477 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
478 bzip2 -9 "$logdir/dinstall_${NOW}.log"
482 function testingsourcelist() {
483 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
486 # do a last run of process-unchecked before dinstall is on.
487 function process_unchecked() {
488 log "Processing the unchecked queue"
489 UNCHECKED_WITHOUT_LOCK="-p"
494 # do a run of newstage only before dinstall is on.
495 function newstage() {
496 log "Processing the newstage queue"
497 UNCHECKED_WITHOUT_LOCK="-p"
501 # Function to update a "statefile" telling people what we are doing
504 # This should be called with the argument(s)
505 # - Status name we want to show.
508 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
509 cat >"${DINSTALLSTATE}" <<EOF
510 Dinstall start: ${DINSTALLBEGIN}
512 Action start: ${RIGHTNOW}
516 # extract changelogs and stuff
517 function changelogs() {
518 log "Extracting changelogs"
519 dak make-changelog -e
520 mkdir -p ${exportpublic}/changelogs
521 cd ${exportpublic}/changelogs
522 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
523 sudo -H -u archvsync /home/archvsync/runmirrors metabdo > ~dak/runmirrors-metadata.log 2>&1 &