2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
10 rm -f ${LOCK_ACCEPTED}
13 # If we error out this one is called, *FOLLOWED* by cleanup above
15 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
17 subject="ATTENTION ATTENTION!"
18 if [ "${error}" = "false" ]; then
19 subject="${subject} (continued)"
21 subject="${subject} (interrupted)"
23 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
25 if [ -r "${STAGEFILE}.log" ]; then
26 cat "${STAGEFILE}.log"
28 echo "file ${STAGEFILE}.log does not exist, sorry"
29 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
32 ########################################################################
33 # the actual dinstall functions follow #
34 ########################################################################
37 log "Telling QA user that we start dinstall"
38 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
41 # Updating various files
43 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
45 $scriptsdir/update-bugdoctxt
46 $scriptsdir/update-mirrorlists
47 $scriptsdir/update-mailingliststxt
48 $scriptsdir/update-pseudopackages.sh
51 # Process (oldstable)-proposed-updates "NEW" queue
54 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" team@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
58 log "Doing automated p-u-new processing"
59 cd "${queuedir}/p-u-new"
63 log "Doing automated o-p-u-new processing"
64 cd "${queuedir}/o-p-u-new"
68 # The first i18n one, syncing new descriptions
70 log "Synchronizing i18n package descriptions"
71 # First sync their newest data
72 cd ${scriptdir}/i18nsync
73 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
75 # Now check if we still know about the packages for which they created the files
76 # is the timestamp signed by us?
77 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
78 # now read it. As its signed by us we are sure the content is what we expect, no need
79 # to do more here. And we only test -d a directory on it anyway.
80 TSTAMP=$(cat timestamp)
81 # do we have the dir still?
82 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
84 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
85 # Yay, worked, lets copy around
86 for dir in wheezy sid; do
87 if [ -d dists/${dir}/ ]; then
88 cd dists/${dir}/main/i18n
89 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
91 cd ${scriptdir}/i18nsync
94 echo "ARRRR, bad guys, wrong files, ARRR"
95 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
98 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
99 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
102 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
103 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
108 log "Checking for cruft in overrides"
112 function dominate() {
113 log "Removing obsolete source and binary associations"
117 function filelist() {
118 log "Generating file lists for apt-ftparchive"
119 dak generate-filelist
122 function fingerprints() {
123 log "Updating fingerprints"
124 dak import-keyring -L /srv/backports-master.debian.org/keyrings/keyring.gpg
127 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
129 if [ -s "${OUTFILE}" ]; then
130 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
131 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
132 To: <debian-backports@lists.debian.org>
133 Subject: Debian Maintainers Keyring changes
134 Content-Type: text/plain; charset=utf-8
138 The following changes to the debian-maintainers keyring have just been activated:
142 Debian distribution maintenance software,
143 on behalf of the Keyring maintainers
150 function overrides() {
151 log "Writing overrides into text files"
156 rm -f override.sid.all3
157 for i in main contrib non-free main.debian-installer; do cat override.squeeze-backports.$i >> override.sid.all3; done
161 log "Generating package / file mapping"
162 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
165 function packages() {
166 log "Generating Packages and Sources files"
168 apt-ftparchive generate apt.conf
169 #dak generate-packages-sources
173 log "Generating pdiff files"
174 dak generate-index-diffs
178 log "Generating Release files"
179 dak generate-releases
182 function dakcleanup() {
183 log "Cleanup old packages/files"
184 dak clean-suites -m 10000
188 function buildd_dir() {
189 # Rebuilt the buildd dir to avoid long times of 403
190 log "Regenerating the buildd incoming dir"
191 STAMP=$(date "+%Y%m%d%H%M")
200 log "Removing any core files ..."
201 find -type f -name core -print0 | xargs -0r rm -v
203 log "Checking permissions on files in the FTP tree ..."
204 find -type f \( \! -perm -444 -o -perm +002 \) -ls
205 find -type d \( \! -perm -555 -o -perm +002 \) -ls
207 log "Checking symlinks ..."
210 log "Creating recursive directory listing ... "
211 rm -f .${FILENAME}.new
212 TZ=UTC ls -lR > .${FILENAME}.new
214 if [ -r ${FILENAME}.gz ] ; then
215 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
216 mv -f .${FILENAME}.new ${FILENAME}
217 rm -f ${FILENAME}.patch.gz
218 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
219 rm -f ${FILENAME}.old.gz
221 mv -f .${FILENAME}.new ${FILENAME}
224 gzip -9cfN ${FILENAME} >${FILENAME}.gz
228 function mkmaintainers() {
229 log 'Creating Maintainers index ... '
232 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
233 gzip -9v --rsyncable <Maintainers >Maintainers.gz
234 gzip -9v --rsyncable <Uploaders >Uploaders.gz
237 function copyoverrides() {
238 log 'Copying override files into public view ...'
240 for ofile in $copyoverrides ; do
242 chmod g+w override.$ofile
246 newofile=override.$ofile.gz
247 rm -f .newover-$ofile.gz
248 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
249 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
250 log " installing new $newofile $pc"
251 mv -f .newover-$ofile.gz $newofile
254 rm -f .newover-$ofile.gz
259 function mkfilesindices() {
261 cd $base/ftp/indices/files/components
265 log "Querying postgres"
267 SELECT CONCAT('./pool/', c.name, '/', f.filename) AS path, a.arch_string AS arch_string
269 JOIN files_archive_map af ON f.id = af.file_id
270 JOIN component c ON af.component_id = c.id
271 JOIN archive ON af.archive_id = archive.id
274 JOIN architecture a ON b.architecture = a.id)
276 WHERE archive.name = 'ftp-master'
277 ORDER BY path, arch_string
279 psql -At -c "$query" >$ARCHLIST
282 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
285 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
288 log "Generating sources list"
290 sed -n 's/|$//p' $ARCHLIST
292 find ./dists -maxdepth 1 \! -type d
293 find ./dists \! -type d | grep "/source/"
294 ) | sort -u | gzip -9 > source.list.gz
296 log "Generating arch lists"
298 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
300 (sed -n "s/|$a$//p" $ARCHLIST
301 sed -n 's/|all$//p' $ARCHLIST
304 find ./dists -maxdepth 1 \! -type d
305 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
306 ) | sort -u | gzip -9 > arch-$a.list.gz
309 log "Generating suite lists"
312 local suite_id="$(printf %d $1)"
315 SELECT DISTINCT CONCAT('./pool/', c.name, '/', f.filename)
317 (SELECT sa.source AS source
318 FROM src_associations sa
319 WHERE sa.suite = $suite_id
322 FROM extra_src_references esr
323 JOIN bin_associations ba ON esr.bin_id = ba.bin
324 WHERE ba.suite = $suite_id
326 SELECT b.source AS source
327 FROM bin_associations ba
328 JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s
329 JOIN dsc_files df ON s.source = df.source
330 JOIN files f ON df.file = f.id
331 JOIN files_archive_map af ON f.id = af.file_id
332 JOIN component c ON af.component_id = c.id
333 JOIN archive ON af.archive_id = archive.id
334 WHERE archive.name = 'ftp-master'
336 psql -F' ' -A -t -c "$query"
339 SELECT CONCAT('./pool/', c.name, '/', f.filename)
340 FROM bin_associations ba
341 JOIN binaries b ON ba.bin = b.id
342 JOIN files f ON b.file = f.id
343 JOIN files_archive_map af ON f.id = af.file_id
344 JOIN component c ON af.component_id = c.id
345 JOIN archive ON af.archive_id = archive.id
346 WHERE ba.suite = $suite_id AND archive.name = 'ftp-master'
348 psql -F' ' -A -t -c "$query"
351 psql -F' ' -At -c 'SELECT id, suite_name FROM suite' |
352 while read id suite; do
353 [ -e $base/ftp/dists/$suite ] || continue
356 distname=$(cd dists; readlink $suite || echo $suite)
357 find ./dists/$distname \! -type d
358 for distdir in ./dists/*; do
359 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
363 ) | sort -u | gzip -9 > suite-${suite}.list.gz
366 log "Finding everything on the ftp site to generate sundries"
367 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
370 zcat *.list.gz | cat - *.list | sort -u |
371 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
373 log "Generating files list"
376 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
377 cat - sundries.list dists.list project.list docs.list indices.list |
378 sort -u | poolfirst > ../arch-$a.files
382 for dist in sid wheezy; do
383 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
387 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
388 sort -u | poolfirst > ../typical.files
394 function mkchecksums() {
395 dsynclist=$dbdir/dsync.list
396 md5list=$indices/md5sums
398 log -n "Creating md5 / dsync index file ... "
401 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
402 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
403 ${bindir}/dsync-flist -q link-dups $dsynclist || true
407 log "Regenerating \"public\" mirror/ hardlink fun"
408 DATE_SERIAL=$(date +"%Y%m%d01")
409 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
410 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
411 SERIAL="$DATE_SERIAL"
413 SERIAL="$FILESOAPLUS1"
415 date -u > ${TRACEFILE}
416 echo "Using dak v1" >> ${TRACEFILE}
417 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
418 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
420 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
424 log "Expiring old database dumps..."
426 $scriptsdir/expire_dumps -d . -p -f "dump_*"
429 function transitionsclean() {
430 log "Removing out of date transitions..."
432 dak transitions -c -a
436 log "Updating DM html page"
437 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
441 log "Categorizing uncategorized bugs filed against ftp.debian.org"
445 function ddaccess() {
446 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
447 log "Trigger dd accessible parts sync including ftp dir"
448 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
451 function mirrorpush() {
452 log "Starting the mirrorpush"
453 date -u > /srv/backports-web.debian.org/underlay/mirrorstart
454 echo "Using dak v1" >> /srv/backports-web.debian.org/underlay/mirrorstart
455 echo "Running on host $(hostname -f)" >> /srv/backports-web.debian.org/underlay/mirrorstart
456 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
460 log "Exporting package data foo for i18n project"
461 STAMP=$(date "+%Y%m%d%H%M")
462 mkdir -p ${scriptdir}/i18n/${STAMP}
463 cd ${scriptdir}/i18n/${STAMP}
464 dak control-suite -l stable > squeeze
465 dak control-suite -l testing > wheezy
466 dak control-suite -l unstable > sid
467 echo "${STAMP}" > timestamp
468 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
472 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
475 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
479 log "Updating stats data"
481 #$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
482 #R --slave --vanilla < $base/misc/ftpstats.R
483 dak stats arch-space > $webdir/arch-space
484 dak stats pkg-nums > $webdir/pkg-nums
487 function aptftpcleanup() {
488 log "Clean up apt-ftparchive's databases"
490 apt-ftparchive -q clean apt.conf
493 function cleantransactions() {
494 log "Cleanup transaction ids older than 3 months"
496 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
499 function logstats() {
500 $masterdir/tools/logs.py "$1"
503 # save timestamp when we start
504 function savetimestamp() {
505 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
506 echo ${NOW} > "${dbdir}/dinstallstart"
509 function maillogfile() {
510 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
513 function renamelogfile() {
514 if [ -f "${dbdir}/dinstallstart" ]; then
515 NOW=$(cat "${dbdir}/dinstallstart")
517 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
518 # logstats "$logdir/dinstall_${NOW}.log"
519 bzip2 -9 "$logdir/dinstall_${NOW}.log"
521 error "Problem, I don't know when dinstall started, unable to do log statistics."
522 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
524 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
525 bzip2 -9 "$logdir/dinstall_${NOW}.log"
529 function testingsourcelist() {
530 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
533 # do a last run of process-unchecked before dinstall is on.
534 function process_unchecked() {
535 log "Processing the unchecked queue"
536 UNCHECKED_WITHOUT_LOCK="-p"
541 # do a run of newstage only before dinstall is on.
542 function newstage() {
543 log "Processing the newstage queue"
544 UNCHECKED_WITHOUT_LOCK="-p"
548 # Function to update a "statefile" telling people what we are doing
551 # This should be called with the argument(s)
552 # - Status name we want to show.
555 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
556 cat >"${DINSTALLSTATE}" <<EOF
557 Dinstall start: ${DINSTALLBEGIN}
559 Action start: ${RIGHTNOW}
563 # extract changelogs and stuff
564 function changelogs() {
565 log "Extracting changelogs"
566 dak make-changelog -e
567 mkdir -p ${exportpublic}/changelogs
568 cd ${exportpublic}/changelogs
569 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
570 sudo -H -u archvsync /home/archvsync/runmirrors metabdo > ~dak/runmirrors-metadata.log 2>&1 &