3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
12 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
13 LOGFILE="$logdir/dinstall_${NOW}.log"
14 exec > "$LOGFILE" 2>&1
18 echo Archive maintenance timestamp $TS: $(date +%H:%M:%S)
21 echo Archive maintenance started at $(date +%H:%M:%S)
24 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
25 LOCKCU="$lockdir/daily.lock"
26 LOCKAC="$lockdir/unchecked.lock"
27 BRITNEYLOCK="$lockdir/britney.lock"
33 if [ "$lockac" -eq "1" ]; then
38 lockfile -l 3600 $LOCKCU
41 # This file is simply used to indicate to britney whether or not
42 # the Packages file updates completed sucessfully. It's not a lock
43 # from our point of view
48 Packages are currently being installed and indices rebuilt.
49 Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
50 ending about an hour later. This file is then removed.
52 You should not mirror the archive during this period.
55 # Push merkels qa user, so the qa pages can show "dinstall is running" information
56 echo "Telling merkels QA user that we start dinstall"
57 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
60 ################################################################################
62 echo "Creating pre-daily-cron-job backup of projectb database..."
63 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
65 ################################################################################
68 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
70 $scriptsdir/update-bugdoctxt
71 $scriptsdir/update-mirrorlists
72 $scriptsdir/update-mailingliststxt
74 ################################################################################
77 echo "Doing automated p-u-new processing"
80 dak process-new -a -C COMMENTS >> REPORT || true
84 echo "Doing automated o-p-u-new processing"
85 cd $queuedir/o-p-u-new
87 dak process-new -a -C COMMENTS >> REPORT || true
90 ################################################################################
94 echo "Synchronizing i18n package descriptions"
95 # First sync their newest data
96 cd ${scriptdir}/i18nsync
97 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
99 # Now check if we still know about the packages for which they created the files
100 # is the timestamp signed by us?
101 if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
102 # now read it. As its signed by us we are sure the content is what we expect, no need
103 # to do more here. And we only test -d a directory on it anyway.
104 TSTAMP=$(cat timestamp)
105 # do we have the dir still?
106 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
108 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
109 # Yay, worked, lets copy around
110 for dir in lenny sid; do
111 if [ -d dists/${dir}/ ]; then
112 cd dists/${dir}/main/i18n
113 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
115 cd ${scriptdir}/i18nsync
118 echo "ARRRR, bad guys, wrong files, ARRR"
119 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
122 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
123 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
126 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
127 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
130 ################################################################################
135 echo "Processing queue/accepted"
138 dak process-accepted -pa *.changes | tee REPORT | \
139 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
140 chgrp debadmin REPORT
144 echo "Checking for cruft in overrides"
149 echo "Fixing symlinks in $ftpdir"
150 symlinks -d -r $ftpdir
154 echo "Generating suite file lists for apt-ftparchive"
155 dak make-suite-file-list
159 echo "Updating fingerprints"
160 # Update fingerprints
161 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
165 # Generate override files
166 echo "Writing overrides into text files"
173 rm -f override.sid.all3
174 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
178 # Generate Packages and Sources files
179 echo "Generating Packages and Sources files"
181 apt-ftparchive generate apt.conf
185 # Generate *.diff/ incremental updates
186 echo "Generating pdiff files"
187 dak generate-index-diffs
191 # Generate Release files
192 echo "Generating Release files"
193 dak generate-releases
197 # Clean out old packages
198 echo "Cleanup old packages/files"
204 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
205 # update this before wanna-build is updated.
206 echo "Regenerating wanna-build/buildd information"
207 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
208 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
209 apt-ftparchive generate apt.conf.buildd
213 echo "Running various scripts from $scriptsdir"
223 # (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
224 echo "Regenerating mirror/ hardlink fun"
226 rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
231 echo "Trigger daily wanna-build run"
232 ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 buildd@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
236 echo Archive maintenance finished at $(date +%H:%M:%S)
238 ################################################################################
240 echo "Creating post-daily-cron-job backup of projectb database..."
241 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
242 pg_dump projectb > $POSTDUMP
243 (cd $base/backup; ln -sf $POSTDUMP current)
245 ################################################################################
249 echo "Expiring old database dumps..."
250 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
252 ################################################################################
256 # Send a report on NEW/BYHAND packages
257 echo "Nagging ftpteam about NEW/BYHAND packages"
258 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
259 # and one on crufty packages
260 echo "Sending information about crufty packages"
261 dak cruft-report > $webdir/cruft-report-daily.txt
262 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
263 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
267 echo "Updating DM html page"
268 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
270 ################################################################################
272 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
273 echo "Trigger merkels projectb sync"
274 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
276 ################################################################################
280 ulimit -m 90000 -d 90000 -s 10000 -v 200000
282 echo "Using run-parts to run scripts in $base/scripts/distmnt"
283 run-parts --report $base/scripts/distmnt
287 echo "Exporting package data foo for i18n project"
288 STAMP=$(date "+%Y%m%d%H%M")
289 mkdir -p ${scriptdir}/i18n/${STAMP}
290 cd ${scriptdir}/i18n/${STAMP}
291 dak control-suite -l stable > etch
292 dak control-suite -l testing > lenny
293 dak control-suite -l unstable > sid
294 echo "${STAMP}" > timestamp
295 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
299 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
302 find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
306 echo "Daily cron scripts successful."
311 echo "Updating stats data"
313 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
314 R --slave --vanilla < $base/misc/ftpstats.R
318 # Remove the britney lock
321 # Clean up apt-ftparchive's databases
322 echo "Clean up apt-ftparchive's databases"
324 apt-ftparchive -q clean apt.conf
328 # Compress psql backups
329 echo "Compress old psql backups"
331 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 |
332 while read dumpname; do
333 echo "Compressing $dumpname"
340 echo "Finally, all is done, sending mail and compressing logfile"
341 exec > /dev/null 2>&1
343 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
346 ################################################################################