3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
12 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
13 LOGFILE="$logdir/dinstall_${NOW}.log"
14 exec > "$LOGFILE" 2>&1
18 echo Archive maintenance timestamp $TS: $(date +%H:%M:%S)
21 echo Archive maintenance started at $(date +%H:%M:%S)
24 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
25 LOCKCU="$lockdir/daily.lock"
26 LOCKAC="$lockdir/unchecked.lock"
32 if [ "$lockac" -eq "1" ]; then
37 lockfile -l 3600 $LOCKCU
42 Packages are currently being installed and indices rebuilt.
43 Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
44 ending about an hour later. This file is then removed.
46 You should not mirror the archive during this period.
49 # Push merkels qa user, so the qa pages can show "dinstall is running" information
50 echo "Telling merkels QA user that we start dinstall"
51 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
54 ################################################################################
56 echo "Creating pre-daily-cron-job backup of projectb database..."
57 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
59 ################################################################################
62 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
64 $scriptsdir/update-bugdoctxt
65 $scriptsdir/update-mirrorlists
66 $scriptsdir/update-mailingliststxt
68 ################################################################################
71 echo "Doing automated p-u-new processing"
74 dak process-new -a -C COMMENTS >> REPORT || true
78 echo "Doing automated o-p-u-new processing"
79 cd $queuedir/o-p-u-new
81 dak process-new -a -C COMMENTS >> REPORT || true
84 ################################################################################
89 echo "Processing queue/accepted"
92 dak process-accepted -pa *.changes | tee REPORT | \
93 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
98 echo "Checking for cruft in overrides"
103 echo "Fixing symlinks in $ftpdir"
104 symlinks -d -r $ftpdir
108 echo "Generating suite file lists for apt-ftparchive"
109 dak make-suite-file-list
113 echo "Updating fingerprints"
114 # Update fingerprints
115 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
119 # Generate override files
120 echo "Writing overrides into text files"
127 rm -f override.sid.all3
128 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
132 # Generate Packages and Sources files
133 echo "Generating Packages and Sources files"
135 apt-ftparchive generate apt.conf
139 # Generate *.diff/ incremental updates
140 echo "Generating pdiff files"
141 dak generate-index-diffs
145 # Generate Release files
146 echo "Generating Release files"
147 dak generate-releases
151 # Clean out old packages
152 echo "Cleanup old packages/files"
158 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
159 # update this before wanna-build is updated.
160 echo "Regenerating wanna-build/buildd information"
161 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
162 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
163 apt-ftparchive generate apt.conf.buildd
167 echo "Running various scripts from $scriptsdir"
177 echo "Trigger daily wanna-build run"
178 ssh buildd@buildd /org/wanna-build/trigger.daily
182 echo Archive maintenance finished at $(date +%H:%M:%S)
184 ################################################################################
186 echo "Creating post-daily-cron-job backup of projectb database..."
187 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
188 pg_dump projectb > $POSTDUMP
189 (cd $base/backup; ln -sf $POSTDUMP current)
191 ################################################################################
195 echo "Expiring old database dumps..."
196 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
198 ################################################################################
202 # Send a report on NEW/BYHAND packages
203 echo "Nagging ftpteam about NEW/BYHAND packages"
204 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
205 # and one on crufty packages
206 echo "Sending information about crufty packages"
207 dak cruft-report > $webdir/cruft-report-daily.txt
208 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
209 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
213 echo "Updating DM html page"
214 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
216 ################################################################################
218 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
219 echo "Trigger merkels projectb sync"
220 ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
222 ################################################################################
226 ulimit -m 90000 -d 90000 -s 10000 -v 200000
228 echo "Using run-parts to run scripts in $base/scripts/distmnt"
229 run-parts --report $base/scripts/distmnt
231 echo "Daily cron scripts successful."
236 echo "Updating stats data"
238 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
239 R --slave --vanilla < $base/misc/ftpstats.R
243 # Clean up apt-ftparchive's databases
244 echo "Clean up apt-ftparchive's databases"
246 apt-ftparchive -q clean apt.conf
250 # Compress psql backups older than a week, but no more than 20 of them
251 echo "Compress old psql backups"
253 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 |
254 sort | head -n20 | while read dumpname; do
255 echo "Compressing $dumpname"
262 echo "Finally, all is done, sending mail and compressing logfile"
263 exec > /dev/null 2>&1
265 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
268 ################################################################################