3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
12 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
13 LOGFILE="$logdir/dinstall_${NOW}.log"
14 exec > "$LOGFILE" 2>&1
18 echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)"
24 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
25 LOCKCU="$lockdir/daily.lock"
26 LOCKAC="$lockdir/unchecked.lock"
27 BRITNEYLOCK="$lockdir/britney.lock"
33 if [ "$lockac" -eq "1" ]; then
38 lockfile -l 3600 $LOCKCU
41 # This file is simply used to indicate to britney whether or not
42 # the Packages file updates completed sucessfully. It's not a lock
43 # from our point of view
48 Packages are currently being installed and indices rebuilt.
49 Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
50 ending about an hour later. This file is then removed.
52 You should not mirror the archive during this period.
55 # Push merkels qa user, so the qa pages can show "dinstall is running" information
56 echo "Telling merkels QA user that we start dinstall"
57 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
60 ################################################################################
62 echo "Creating pre-daily-cron-job backup of projectb database..."
63 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
66 ################################################################################
68 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
70 $scriptsdir/update-bugdoctxt
71 $scriptsdir/update-mirrorlists
72 $scriptsdir/update-mailingliststxt
73 $scriptsdir/update-pseudopackages.sh
76 ################################################################################
78 echo "Doing automated p-u-new processing"
81 dak process-new -a -C COMMENTS >> REPORT || true
85 echo "Doing automated o-p-u-new processing"
86 cd $queuedir/o-p-u-new
88 dak process-new -a -C COMMENTS >> REPORT || true
92 ################################################################################
94 # Temporary until lenny is there
95 dak control-suite -s lenny-r0 /srv/ftp.debian.org/misc/lenny-r0.txt
97 echo "Synchronizing i18n package descriptions"
98 # First sync their newest data
99 cd ${scriptdir}/i18nsync
100 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
102 # Now check if we still know about the packages for which they created the files
103 # is the timestamp signed by us?
104 if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
105 # now read it. As its signed by us we are sure the content is what we expect, no need
106 # to do more here. And we only test -d a directory on it anyway.
107 TSTAMP=$(cat timestamp)
108 # do we have the dir still?
109 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
111 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
112 # Yay, worked, lets copy around
113 for dir in lenny sid; do
114 if [ -d dists/${dir}/ ]; then
115 cd dists/${dir}/main/i18n
116 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
118 cd ${scriptdir}/i18nsync
121 echo "ARRRR, bad guys, wrong files, ARRR"
122 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
125 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
126 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
129 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
130 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
134 ################################################################################
138 echo "Processing queue/accepted"
141 dak process-accepted -pa *.changes | tee REPORT | \
142 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
143 chgrp debadmin REPORT
147 echo "Checking for cruft in overrides"
152 echo "Fixing symlinks in $ftpdir"
153 symlinks -d -r $ftpdir
156 echo "Generating suite file lists for apt-ftparchive"
157 dak make-suite-file-list
158 ts "make-suite-file-list"
160 echo "Updating fingerprints"
161 # Update fingerprints
162 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
165 # Generate override files
166 echo "Writing overrides into text files"
171 rm -f override.sid.all3
172 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
175 # Generate package / file mapping
176 echo "Generating package / file mapping"
177 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
178 ts "pkg-file-mapping"
180 # Generate Packages and Sources files
181 echo "Generating Packages and Sources files"
183 apt-ftparchive generate apt.conf
186 # Generate *.diff/ incremental updates
187 echo "Generating pdiff files"
188 dak generate-index-diffs
191 # Generate Release files
192 echo "Generating Release files"
193 dak generate-releases
196 # Clean out old packages
197 echo "Cleanup old packages/files"
202 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
203 # update this before wanna-build is updated.
204 echo "Regenerating wanna-build/buildd information"
205 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
206 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
207 apt-ftparchive generate apt.conf.buildd
210 echo "Running various scripts from $scriptsdir"
219 # (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
220 echo "Regenerating mirror/ hardlink fun"
222 rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
223 ts "mirror hardlinks"
225 echo "Trigger daily wanna-build run"
226 ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
232 ts "locked part finished"
234 ################################################################################
236 echo "Creating post-daily-cron-job backup of projectb database..."
237 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
238 pg_dump projectb > $POSTDUMP
239 (cd $base/backup; ln -sf $POSTDUMP current)
242 ################################################################################
245 echo "Expiring old database dumps..."
246 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
249 ################################################################################
252 # Send a report on NEW/BYHAND packages
253 echo "Nagging ftpteam about NEW/BYHAND packages"
254 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
255 # and one on crufty packages
256 echo "Sending information about crufty packages"
257 dak cruft-report > $webdir/cruft-report-daily.txt
258 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
259 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
262 echo "Updating DM html page"
263 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
265 echo "Categorizing uncategorized bugs filed against ftp.debian.org"
268 ################################################################################
270 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
271 echo "Trigger merkels projectb sync"
272 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
273 ts "merkel projectb push"
275 ################################################################################
278 ulimit -m 90000 -d 90000 -s 10000 -v 200000
280 echo "Using run-parts to run scripts in $base/scripts/distmnt"
281 run-parts --report $base/scripts/distmnt
284 echo "Exporting package data foo for i18n project"
285 STAMP=$(date "+%Y%m%d%H%M")
286 mkdir -p ${scriptdir}/i18n/${STAMP}
287 cd ${scriptdir}/i18n/${STAMP}
288 dak control-suite -l stable > etch
289 dak control-suite -l testing > lenny
290 dak control-suite -l unstable > sid
291 echo "${STAMP}" > timestamp
292 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
296 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
299 find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
302 echo "Daily cron scripts successful."
305 echo "Updating stats data"
307 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
308 R --slave --vanilla < $base/misc/ftpstats.R
311 # Remove the britney lock
314 # Clean up apt-ftparchive's databases
315 echo "Clean up apt-ftparchive's databases"
317 apt-ftparchive -q clean apt.conf
318 ts "apt-ftparchive cleanup"
320 # Compress psql backups
321 echo "Compress old psql backups"
323 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 |
324 while read dumpname; do
325 echo "Compressing $dumpname"
331 echo "Removing old dinstall logfiles"
333 find -maxdepth 1 -mindepth 1 -type f -name 'dinstall_*' -mtime +60 |
334 while read dumpname; do
335 echo "Removing $dumpname"
339 find -maxdepth 1 -mindepth 1 -type f -name 'weekly_*' -mtime +60 |
340 while read dumpname; do
341 echo "Removing $dumpname"
347 echo "Finally, all is done, sending mail and compressing logfile"
348 exec > /dev/null 2>&1
350 $masterdir/tools/logs.py "$LOGFILE"
352 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
355 ################################################################################