3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
12 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
13 LOGFILE="$logdir/dinstall_${NOW}.log"
14 exec > "$LOGFILE" 2>&1
18 echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)"
24 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
25 LOCKCU="$lockdir/daily.lock"
26 LOCKAC="$lockdir/unchecked.lock"
27 BRITNEYLOCK="$lockdir/britney.lock"
33 if [ "$lockac" -eq "1" ]; then
38 lockfile -l 3600 $LOCKCU
41 # This file is simply used to indicate to britney whether or not
42 # the Packages file updates completed sucessfully. It's not a lock
43 # from our point of view
48 Packages are currently being installed and indices rebuilt.
49 Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
50 ending about an hour later. This file is then removed.
52 You should not mirror the archive during this period.
55 # Push merkels qa user, so the qa pages can show "dinstall is running" information
56 echo "Telling merkels QA user that we start dinstall"
57 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
60 ################################################################################
62 echo "Creating pre-daily-cron-job backup of projectb database..."
63 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
66 ################################################################################
68 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
70 $scriptsdir/update-bugdoctxt
71 $scriptsdir/update-mirrorlists
72 $scriptsdir/update-mailingliststxt
73 $scriptsdir/update-pseudopackages.sh
76 ################################################################################
78 echo "Doing automated p-u-new processing"
81 dak process-new -a -C COMMENTS >> REPORT || true
85 echo "Doing automated o-p-u-new processing"
86 cd $queuedir/o-p-u-new
88 dak process-new -a -C COMMENTS >> REPORT || true
92 ################################################################################
95 echo "Synchronizing i18n package descriptions"
96 # First sync their newest data
97 cd ${scriptdir}/i18nsync
98 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
100 # Now check if we still know about the packages for which they created the files
101 # is the timestamp signed by us?
102 if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
103 # now read it. As its signed by us we are sure the content is what we expect, no need
104 # to do more here. And we only test -d a directory on it anyway.
105 TSTAMP=$(cat timestamp)
106 # do we have the dir still?
107 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
109 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
110 # Yay, worked, lets copy around
111 for dir in lenny sid; do
112 if [ -d dists/${dir}/ ]; then
113 cd dists/${dir}/main/i18n
114 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
116 cd ${scriptdir}/i18nsync
119 echo "ARRRR, bad guys, wrong files, ARRR"
120 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
123 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
124 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
127 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
128 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
132 ################################################################################
136 echo "Processing queue/accepted"
139 dak process-accepted -pa *.changes | tee REPORT | \
140 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
141 chgrp debadmin REPORT
145 echo "Checking for cruft in overrides"
150 echo "Fixing symlinks in $ftpdir"
151 symlinks -d -r $ftpdir
154 echo "Generating suite file lists for apt-ftparchive"
155 dak make-suite-file-list
156 ts "make-suite-file-list"
158 echo "Updating fingerprints"
159 # Update fingerprints
160 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
163 # Generate override files
164 echo "Writing overrides into text files"
169 rm -f override.sid.all3
170 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
174 # Generate Packages and Sources files
175 echo "Generating Packages and Sources files"
177 apt-ftparchive generate apt.conf
180 # Generate *.diff/ incremental updates
181 echo "Generating pdiff files"
182 dak generate-index-diffs
185 # Generate Release files
186 echo "Generating Release files"
187 dak generate-releases
190 # Clean out old packages
191 echo "Cleanup old packages/files"
196 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
197 # update this before wanna-build is updated.
198 echo "Regenerating wanna-build/buildd information"
199 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
200 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
201 apt-ftparchive generate apt.conf.buildd
204 echo "Running various scripts from $scriptsdir"
213 # (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
214 echo "Regenerating mirror/ hardlink fun"
216 rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
217 ts "mirror hardlinks"
219 echo "Trigger daily wanna-build run"
220 ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
226 ts "locked part finished"
228 ################################################################################
230 echo "Creating post-daily-cron-job backup of projectb database..."
231 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
232 pg_dump projectb > $POSTDUMP
233 (cd $base/backup; ln -sf $POSTDUMP current)
236 ################################################################################
239 echo "Expiring old database dumps..."
240 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
243 ################################################################################
246 # Send a report on NEW/BYHAND packages
247 echo "Nagging ftpteam about NEW/BYHAND packages"
248 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
249 # and one on crufty packages
250 echo "Sending information about crufty packages"
251 dak cruft-report > $webdir/cruft-report-daily.txt
252 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
253 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
256 echo "Updating DM html page"
257 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
259 ################################################################################
261 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
262 echo "Trigger merkels projectb sync"
263 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
264 ts "merkel projectb push"
266 ################################################################################
269 ulimit -m 90000 -d 90000 -s 10000 -v 200000
271 echo "Using run-parts to run scripts in $base/scripts/distmnt"
272 run-parts --report $base/scripts/distmnt
275 echo "Exporting package data foo for i18n project"
276 STAMP=$(date "+%Y%m%d%H%M")
277 mkdir -p ${scriptdir}/i18n/${STAMP}
278 cd ${scriptdir}/i18n/${STAMP}
279 dak control-suite -l stable > etch
280 dak control-suite -l testing > lenny
281 dak control-suite -l unstable > sid
282 echo "${STAMP}" > timestamp
283 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
287 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
290 find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
293 echo "Daily cron scripts successful."
296 echo "Updating stats data"
298 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
299 R --slave --vanilla < $base/misc/ftpstats.R
302 # Remove the britney lock
305 # Clean up apt-ftparchive's databases
306 echo "Clean up apt-ftparchive's databases"
308 apt-ftparchive -q clean apt.conf
309 ts "apt-ftparchive cleanup"
311 # Compress psql backups
312 echo "Compress old psql backups"
314 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 |
315 while read dumpname; do
316 echo "Compressing $dumpname"
322 echo "Removing old dinstall logfiles"
324 find -maxdepth 1 -mindepth 1 -type f -name 'dinstall_*' -mtime +60 |
325 while read dumpname; do
326 echo "Removing $dumpname"
330 find -maxdepth 1 -mindepth 1 -type f -name 'weekly_*' -mtime +60 |
331 while read dumpname; do
332 echo "Removing $dumpname"
338 echo "Finally, all is done, sending mail and compressing logfile"
339 exec > /dev/null 2>&1
341 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
344 ################################################################################