3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
12 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
13 LOGFILE="$logdir/dinstall_${NOW}.log"
14 exec > "$LOGFILE" 2>&1
16 echo Archive maintenance started at $(date +%X)
19 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
20 LOCKCU="$lockdir/daily.lock"
21 LOCKAC="$lockdir/unchecked.lock"
27 if [ "$lockac" -eq "1" ]; then
33 lockfile -l 3600 $LOCKCU
38 Packages are currently being installed and indices rebuilt.
39 Maintenance is automatic, starting at 13:52 US Central time, and
40 ending at about 15:30. This file is then removed.
42 You should not mirror the archive during this period.
45 # Push merkels qa user, so the qa pages can show "dinstall is running" information
46 echo "Telling merkels QA user that we start dinstall"
47 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
49 ################################################################################
51 echo "Creating pre-daily-cron-job backup of projectb database..."
52 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
54 ################################################################################
56 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
57 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
59 $scriptsdir/update-bugdoctxt
60 $scriptsdir/update-mirrorlists
61 $scriptsdir/update-mailingliststxt
63 ################################################################################
65 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
66 echo "Doing automated p-u-new processing"
69 dak process-new -a -C COMMENTS >> REPORT
72 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
73 echo "Doing automated o-p-u-new processing"
74 cd $queuedir/o-p-u-new
76 dak process-new -a -C COMMENTS >> REPORT
79 ################################################################################
81 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
84 echo "Processing queue/accepted"
87 dak process-accepted -pa *.changes | tee REPORT | \
88 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
92 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
93 echo "Checking for cruft in overrides"
98 echo "Fixing symlinks in $ftpdir"
99 symlinks -d -r $ftpdir
101 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
103 echo "Generating suite file lists for apt-ftparchive"
104 dak make-suite-file-list
106 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
108 echo "Updating fingerprints"
109 # Update fingerprints
110 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
112 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
114 # Generate override files
115 echo "Writing overrides into text files"
119 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
122 rm -f override.sid.all3
123 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
125 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
127 # Generate Packages and Sources files
128 echo "Generating Packages and Sources files"
130 apt-ftparchive generate apt.conf
131 # Generate *.diff/ incremental updates
133 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
135 echo "Generating pdiff files"
136 dak generate-index-diffs
138 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
140 # Generate Release files
141 echo "Generating Release files"
142 dak generate-releases
144 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
146 # Clean out old packages
147 echo "Cleanup old packages/files"
151 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
153 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
154 # update this before wanna-build is updated.
155 echo "Regenerating wanna-build/buildd information"
156 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
157 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
158 apt-ftparchive generate apt.conf.buildd
160 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
162 echo "Running various scripts from $scriptsdir"
172 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
174 echo "Trigger daily wanna-build run"
175 ssh buildd@buildd /org/wanna-build/trigger.daily
178 echo Archive maintenance finished at $(date +%X)
180 ################################################################################
182 echo "Creating post-daily-cron-job backup of projectb database..."
183 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
184 pg_dump projectb > $POSTDUMP
185 (cd $base/backup; ln -sf $POSTDUMP current)
187 ################################################################################
189 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
191 echo "Expiring old database dumps..."
192 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
194 ################################################################################
196 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
198 # Send a report on NEW/BYHAND packages
199 echo "Nagging ftpteam about NEW/BYHAND packages"
200 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
201 # and one on crufty packages
202 echo "Sending information about crufty packages"
203 dak cruft-report > $webdir/cruft-report-daily.txt
204 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
205 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
207 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
209 echo "Updating DM html page"
210 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
212 ################################################################################
214 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
215 echo "Trigger merkels projectb sync"
216 ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
218 ################################################################################
220 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
222 ulimit -m 90000 -d 90000 -s 10000 -v 200000
224 echo "Using run-parts to run scripts in $base/scripts/distmnt"
225 run-parts --report $base/scripts/distmnt
227 echo "Daily cron scripts successful."
229 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
232 echo "Updating stats data"
234 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
235 R --slave --vanilla < $base/misc/ftpstats.R
237 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
239 # Clean up apt-ftparchive's databases
240 echo "Clean up apt-ftparchive's databases"
242 apt-ftparchive -q clean apt.conf
244 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
246 # Compress psql backups older than a week, but no more than 20 of them
247 echo "Compress old psql backups"
249 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 |
250 sort | head -n20 | while read dumpname; do
251 echo "Compressing $dumpname"
256 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
258 echo "Finally, all is done, sending mail and compressing logfile"
259 exec > /dev/null 2>&1
261 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
264 ################################################################################