3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
12 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
13 LOGFILE="$logdir/dinstall_${NOW}.log"
14 exec > "$LOGFILE" 2>&1
18 echo Archive maintenance timestamp $TS: $(date +%H:%M:%S)
21 echo Archive maintenance started at $(date +%H:%M:%S)
24 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
25 LOCKCU="$lockdir/daily.lock"
26 LOCKAC="$lockdir/unchecked.lock"
27 BRITNEYLOCK="$lockdir/britney.lock"
33 if [ "$lockac" -eq "1" ]; then
38 lockfile -l 3600 $LOCKCU
41 # This file is simply used to indicate to britney whether or not
42 # the Packages file updates completed sucessfully. It's not a lock
43 # from our point of view
48 Packages are currently being installed and indices rebuilt.
49 Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
50 ending about an hour later. This file is then removed.
52 You should not mirror the archive during this period.
55 # Push merkels qa user, so the qa pages can show "dinstall is running" information
56 echo "Telling merkels QA user that we start dinstall"
57 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
60 ################################################################################
62 echo "Creating pre-daily-cron-job backup of projectb database..."
63 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
65 ################################################################################
68 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
70 $scriptsdir/update-bugdoctxt
71 $scriptsdir/update-mirrorlists
72 $scriptsdir/update-mailingliststxt
74 ################################################################################
77 echo "Doing automated p-u-new processing"
80 dak process-new -a -C COMMENTS >> REPORT || true
84 echo "Doing automated o-p-u-new processing"
85 cd $queuedir/o-p-u-new
87 dak process-new -a -C COMMENTS >> REPORT || true
90 ################################################################################
95 echo "Processing queue/accepted"
98 dak process-accepted -pa *.changes | tee REPORT | \
99 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
100 chgrp debadmin REPORT
104 echo "Checking for cruft in overrides"
109 echo "Fixing symlinks in $ftpdir"
110 symlinks -d -r $ftpdir
114 echo "Generating suite file lists for apt-ftparchive"
115 dak make-suite-file-list
119 echo "Updating fingerprints"
120 # Update fingerprints
121 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
125 # Generate override files
126 echo "Writing overrides into text files"
133 rm -f override.sid.all3
134 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
138 # Generate Packages and Sources files
139 echo "Generating Packages and Sources files"
141 apt-ftparchive generate apt.conf
145 # Generate *.diff/ incremental updates
146 echo "Generating pdiff files"
147 dak generate-index-diffs
151 # Generate Release files
152 echo "Generating Release files"
153 dak generate-releases
157 # Clean out old packages
158 echo "Cleanup old packages/files"
164 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
165 # update this before wanna-build is updated.
166 echo "Regenerating wanna-build/buildd information"
167 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
168 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
169 apt-ftparchive generate apt.conf.buildd
173 echo "Running various scripts from $scriptsdir"
183 # (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
184 echo "Regenerating mirror/ hardlink fun"
186 rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
191 echo "Trigger daily wanna-build run"
192 ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 buildd@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
196 echo Archive maintenance finished at $(date +%H:%M:%S)
198 ################################################################################
200 echo "Creating post-daily-cron-job backup of projectb database..."
201 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
202 pg_dump projectb > $POSTDUMP
203 (cd $base/backup; ln -sf $POSTDUMP current)
205 ################################################################################
209 echo "Expiring old database dumps..."
210 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
212 ################################################################################
216 # Send a report on NEW/BYHAND packages
217 echo "Nagging ftpteam about NEW/BYHAND packages"
218 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
219 # and one on crufty packages
220 echo "Sending information about crufty packages"
221 dak cruft-report > $webdir/cruft-report-daily.txt
222 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
223 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
227 echo "Updating DM html page"
228 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
230 ################################################################################
232 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
233 echo "Trigger merkels projectb sync"
234 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
236 ################################################################################
240 ulimit -m 90000 -d 90000 -s 10000 -v 200000
242 echo "Using run-parts to run scripts in $base/scripts/distmnt"
243 run-parts --report $base/scripts/distmnt
247 echo "Exporting package data foo for i18n project"
249 dak control-suite -l stable > etch
250 dak control-suite -l testing > lenny
251 dak control-suite -l unstable > sid
257 echo "Daily cron scripts successful."
262 echo "Updating stats data"
264 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
265 R --slave --vanilla < $base/misc/ftpstats.R
269 # Remove the britney lock
272 # Clean up apt-ftparchive's databases
273 echo "Clean up apt-ftparchive's databases"
275 apt-ftparchive -q clean apt.conf
279 # Compress psql backups
280 echo "Compress old psql backups"
282 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 |
283 while read dumpname; do
284 echo "Compressing $dumpname"
291 echo "Finally, all is done, sending mail and compressing logfile"
292 exec > /dev/null 2>&1
294 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
297 ################################################################################