3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
12 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
13 LOGFILE="$logdir/dinstall_${NOW}.log"
14 exec > "$LOGFILE" 2>&1
16 echo Archive maintenance started at $(date +%X)
19 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
20 LOCKCU="$lockdir/daily.lock"
21 LOCKAC="$lockdir/unchecked.lock"
27 if [ "$lockac" -eq "1" ]; then
32 lockfile -l 3600 $LOCKCU
37 Packages are currently being installed and indices rebuilt.
38 Maintenance is automatic, starting at 13:52 US Central time, and
39 ending at about 15:30. This file is then removed.
41 You should not mirror the archive during this period.
44 # Push merkels qa user, so the qa pages can show "dinstall is running" information
45 echo "Telling merkels QA user that we start dinstall"
46 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
48 ################################################################################
50 echo "Creating pre-daily-cron-job backup of projectb database..."
51 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
53 ################################################################################
55 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
56 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
58 $scriptsdir/update-bugdoctxt
59 $scriptsdir/update-mirrorlists
60 $scriptsdir/update-mailingliststxt
62 ################################################################################
64 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
65 echo "Doing automated p-u-new processing"
68 dak process-new -a -C COMMENTS >> REPORT || true
71 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
72 echo "Doing automated o-p-u-new processing"
73 cd $queuedir/o-p-u-new
75 dak process-new -a -C COMMENTS >> REPORT || true
78 ################################################################################
80 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
83 echo "Processing queue/accepted"
86 dak process-accepted -pa *.changes | tee REPORT | \
87 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
91 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
92 echo "Checking for cruft in overrides"
97 echo "Fixing symlinks in $ftpdir"
98 symlinks -d -r $ftpdir
100 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
102 echo "Generating suite file lists for apt-ftparchive"
103 dak make-suite-file-list
105 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
107 echo "Updating fingerprints"
108 # Update fingerprints
109 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
111 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
113 # Generate override files
114 echo "Writing overrides into text files"
118 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
121 rm -f override.sid.all3
122 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
124 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
126 # Generate Packages and Sources files
127 echo "Generating Packages and Sources files"
129 apt-ftparchive generate apt.conf
130 # Generate *.diff/ incremental updates
132 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
134 echo "Generating pdiff files"
135 dak generate-index-diffs
137 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
139 # Generate Release files
140 echo "Generating Release files"
141 dak generate-releases
143 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
145 # Clean out old packages
146 echo "Cleanup old packages/files"
150 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
152 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
153 # update this before wanna-build is updated.
154 echo "Regenerating wanna-build/buildd information"
155 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
156 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
157 apt-ftparchive generate apt.conf.buildd
159 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
161 echo "Running various scripts from $scriptsdir"
171 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
173 echo "Trigger daily wanna-build run"
174 ssh buildd@buildd /org/wanna-build/trigger.daily
177 echo Archive maintenance finished at $(date +%X)
179 ################################################################################
181 echo "Creating post-daily-cron-job backup of projectb database..."
182 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
183 pg_dump projectb > $POSTDUMP
184 (cd $base/backup; ln -sf $POSTDUMP current)
186 ################################################################################
188 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
190 echo "Expiring old database dumps..."
191 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
193 ################################################################################
195 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
197 # Send a report on NEW/BYHAND packages
198 echo "Nagging ftpteam about NEW/BYHAND packages"
199 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
200 # and one on crufty packages
201 echo "Sending information about crufty packages"
202 dak cruft-report > $webdir/cruft-report-daily.txt
203 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
204 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
206 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
208 echo "Updating DM html page"
209 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
211 ################################################################################
213 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
214 echo "Trigger merkels projectb sync"
215 ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
217 ################################################################################
219 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
221 ulimit -m 90000 -d 90000 -s 10000 -v 200000
223 echo "Using run-parts to run scripts in $base/scripts/distmnt"
224 run-parts --report $base/scripts/distmnt
226 echo "Daily cron scripts successful."
228 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
231 echo "Updating stats data"
233 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
234 R --slave --vanilla < $base/misc/ftpstats.R
236 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
238 # Clean up apt-ftparchive's databases
239 echo "Clean up apt-ftparchive's databases"
241 apt-ftparchive -q clean apt.conf
243 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
245 # Compress psql backups older than a week, but no more than 20 of them
246 echo "Compress old psql backups"
248 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 |
249 sort | head -n20 | while read dumpname; do
250 echo "Compressing $dumpname"
255 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
257 echo "Finally, all is done, sending mail and compressing logfile"
258 exec > /dev/null 2>&1
260 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
263 ################################################################################