3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
12 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
13 LOGFILE="$logdir/dinstall_${NOW}.log"
14 exec > "$LOGFILE" 2>&1
18 echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)"
24 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
25 LOCKCU="$lockdir/daily.lock"
26 LOCKAC="$lockdir/unchecked.lock"
27 BRITNEYLOCK="$lockdir/britney.lock"
33 if [ "$lockac" -eq "1" ]; then
38 lockfile -l 3600 $LOCKCU
41 # This file is simply used to indicate to britney whether or not
42 # the Packages file updates completed sucessfully. It's not a lock
43 # from our point of view
48 Packages are currently being installed and indices rebuilt.
49 Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
50 ending about an hour later. This file is then removed.
52 You should not mirror the archive during this period.
55 # Push merkels qa user, so the qa pages can show "dinstall is running" information
56 echo "Telling merkels QA user that we start dinstall"
57 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
60 ################################################################################
62 echo "Creating pre-daily-cron-job backup of projectb database..."
63 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
66 ################################################################################
68 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
70 $scriptsdir/update-bugdoctxt
71 $scriptsdir/update-mirrorlists
72 $scriptsdir/update-mailingliststxt
73 $scriptsdir/update-pseudopackages.sh
76 ################################################################################
78 echo "Doing automated p-u-new processing"
81 dak process-new -a -C COMMENTS >> REPORT || true
85 echo "Doing automated o-p-u-new processing"
86 cd $queuedir/o-p-u-new
88 dak process-new -a -C COMMENTS >> REPORT || true
92 ################################################################################
94 echo "Synchronizing i18n package descriptions"
95 # First sync their newest data
96 cd ${scriptdir}/i18nsync
97 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
99 # Now check if we still know about the packages for which they created the files
100 # is the timestamp signed by us?
101 if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
102 # now read it. As its signed by us we are sure the content is what we expect, no need
103 # to do more here. And we only test -d a directory on it anyway.
104 TSTAMP=$(cat timestamp)
105 # do we have the dir still?
106 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
108 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
109 # Yay, worked, lets copy around
110 for dir in squeeze sid; do
111 if [ -d dists/${dir}/ ]; then
112 cd dists/${dir}/main/i18n
113 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
115 cd ${scriptdir}/i18nsync
118 echo "ARRRR, bad guys, wrong files, ARRR"
119 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
122 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
123 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
126 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
127 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
131 ################################################################################
135 echo "Processing queue/accepted"
138 dak process-accepted -pa *.changes | tee REPORT | \
139 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
140 chgrp debadmin REPORT
144 echo "Checking for cruft in overrides"
149 echo "Fixing symlinks in $ftpdir"
150 symlinks -d -r $ftpdir
153 echo "Generating suite file lists for apt-ftparchive"
154 dak make-suite-file-list
155 ts "make-suite-file-list"
157 echo "Updating fingerprints"
158 # Update fingerprints
159 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
162 # Generate override files
163 echo "Writing overrides into text files"
168 rm -f override.sid.all3
169 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
172 # Generate package / file mapping
173 echo "Generating package / file mapping"
174 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
175 ts "pkg-file-mapping"
177 # Generate Packages and Sources files
178 echo "Generating Packages and Sources files"
180 apt-ftparchive generate apt.conf
183 # Generate *.diff/ incremental updates
184 echo "Generating pdiff files"
185 dak generate-index-diffs
188 # Generate Release files
189 echo "Generating Release files"
190 dak generate-releases
193 # Clean out old packages
194 echo "Cleanup old packages/files"
199 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
200 # update this before wanna-build is updated.
201 echo "Regenerating wanna-build/buildd information"
202 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
203 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
204 apt-ftparchive generate apt.conf.buildd
207 echo "Running various scripts from $scriptsdir"
216 # (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
217 echo "Regenerating mirror/ hardlink fun"
219 rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
220 ts "mirror hardlinks"
222 echo "Trigger daily wanna-build run"
223 ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
229 ts "locked part finished"
231 ################################################################################
233 echo "Creating post-daily-cron-job backup of projectb database..."
234 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
235 pg_dump projectb > $POSTDUMP
236 (cd $base/backup; ln -sf $POSTDUMP current)
239 ################################################################################
242 echo "Expiring old database dumps..."
243 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
246 ################################################################################
249 # Send a report on NEW/BYHAND packages
250 echo "Nagging ftpteam about NEW/BYHAND packages"
251 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
252 # and one on crufty packages
253 echo "Sending information about crufty packages"
254 dak cruft-report > $webdir/cruft-report-daily.txt
255 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
256 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
259 echo "Updating DM html page"
260 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
262 echo "Categorizing uncategorized bugs filed against ftp.debian.org"
265 ################################################################################
267 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
268 echo "Trigger merkels projectb sync"
269 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
270 ts "merkel projectb push"
272 ################################################################################
275 ulimit -m 90000 -d 90000 -s 10000 -v 200000
277 echo "Using run-parts to run scripts in $base/scripts/distmnt"
278 run-parts --report $base/scripts/distmnt
281 echo "Exporting package data foo for i18n project"
282 STAMP=$(date "+%Y%m%d%H%M")
283 mkdir -p ${scriptdir}/i18n/${STAMP}
284 cd ${scriptdir}/i18n/${STAMP}
285 dak control-suite -l stable > lenny
286 dak control-suite -l testing > squeeze
287 dak control-suite -l unstable > sid
288 echo "${STAMP}" > timestamp
289 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
293 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
296 find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
299 echo "Daily cron scripts successful."
302 echo "Updating stats data"
304 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
305 R --slave --vanilla < $base/misc/ftpstats.R
308 # Remove the britney lock
311 # Clean up apt-ftparchive's databases
312 echo "Clean up apt-ftparchive's databases"
314 apt-ftparchive -q clean apt.conf
315 ts "apt-ftparchive cleanup"
317 # Compress psql backups
318 echo "Compress old psql backups"
320 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 |
321 while read dumpname; do
322 echo "Compressing $dumpname"
328 echo "Removing old dinstall logfiles"
330 find -maxdepth 1 -mindepth 1 -type f -name 'dinstall_*' -mtime +60 |
331 while read dumpname; do
332 echo "Removing $dumpname"
336 find -maxdepth 1 -mindepth 1 -type f -name 'weekly_*' -mtime +60 |
337 while read dumpname; do
338 echo "Removing $dumpname"
344 echo "Finally, all is done, sending mail and compressing logfile"
345 exec > /dev/null 2>&1
347 $masterdir/tools/logs.py "$LOGFILE"
349 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
352 ################################################################################