3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
12 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
13 LOGFILE="$logdir/daily_${NOW}.log"
14 exec > "$LOGFILE" 2>&1
16 echo Archive maintenance started at $(date +%X)
19 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
20 LOCKCU="$lockdir/daily.lock"
21 LOCKAC="$lockdir/unchecked.lock"
27 if [ "$lockac" -eq "1" ]; then
33 lockfile -l 3600 $LOCKCU
38 Packages are currently being installed and indices rebuilt.
39 Maintenance is automatic, starting at 13:52 US Central time, and
40 ending at about 15:30. This file is then removed.
42 You should not mirror the archive during this period.
45 # Push merkels qa user, so the qa pages can show "dinstall is running" information
46 echo "Telling merkels QA user that we start dinstall"
47 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
49 ################################################################################
51 echo "Creating pre-daily-cron-job backup of projectb database..."
52 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
54 ################################################################################
56 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
57 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
59 $scriptsdir/update-bugdoctxt
60 $scriptsdir/update-mirrorlists
61 $scriptsdir/update-mailingliststxt
63 ################################################################################
65 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
66 echo "Doing automated p-u-new processing"
69 dak process-new -a -C COMMENTS >> REPORT
72 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
73 echo "Doing automated o-p-u-new processing"
74 cd $queuedir/o-p-u-new
76 dak process-new -a -C COMMENTS >> REPORT
79 ################################################################################
81 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
84 echo "Processing queue/accepted"
87 dak process-accepted -pa *.changes | tee REPORT | \
88 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
92 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
93 echo "Checking for cruft in overrides"
98 echo "Fixing symlinks in $ftpdir"
99 symlinks -d -r $ftpdir
101 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
103 echo "Generating suite file lists for apt-ftparchive"
104 dak make-suite-file-list
106 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
108 echo "Updating fingerprints"
109 # Update fingerprints
110 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
112 # Generate override files
113 echo "Writing overrides into text files"
118 rm -f override.sid.all3
119 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
121 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
123 # Generate Packages and Sources files
124 echo "Generating Packages and Sources files"
126 apt-ftparchive generate apt.conf
127 # Generate *.diff/ incremental updates
128 echo "Generating pdiff files"
129 dak generate-index-diffs
130 # Generate Release files
131 echo "Generating Release files"
132 dak generate-releases
134 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
136 # Clean out old packages
137 echo "Cleanup old packages/files"
141 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
143 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
144 # update this before wanna-build is updated.
145 echo "Regenerating wanna-build/buildd information"
146 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
147 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
148 apt-ftparchive generate apt.conf.buildd
150 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
152 echo "Running various scripts from $scriptsdir"
161 echo "Trigger daily wanna-build run"
162 ssh buildd@buildd /org/wanna-build/trigger.daily
165 echo Archive maintenance finished at $(date +%X)
167 ################################################################################
169 echo "Creating post-daily-cron-job backup of projectb database..."
170 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
171 pg_dump projectb > $POSTDUMP
172 (cd $base/backup; ln -sf $POSTDUMP current)
174 ################################################################################
176 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
178 echo "Expiring old database dumps..."
179 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
181 ################################################################################
183 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
185 # Send a report on NEW/BYHAND packages
186 echo "Nagging ftpteam about NEW/BYHAND packages"
187 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
188 # and one on crufty packages
189 echo "Sending information about crufty packages"
190 dak cruft-report > $webdir/cruft-report-daily.txt
191 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
192 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
194 echo "Updating DM html page"
195 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
197 ################################################################################
199 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
200 echo "Trigger merkels projectb sync"
201 ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
203 ################################################################################
205 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
207 ulimit -m 90000 -d 90000 -s 10000 -v 200000
209 echo "Using run-parts to run scripts in $base/scripts/distmnt"
210 run-parts --report $base/scripts/distmnt
212 echo "Daily cron scripts successful."
214 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
217 echo "Updating stats data"
219 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
220 R --slave --vanilla < $base/misc/ftpstats.R
222 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
224 # Clean up apt-ftparchive's databases
225 echo "Clean up apt-ftparchive's databases"
227 apt-ftparchive -q clean apt.conf
229 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
231 # Compress psql backups older than a week, but no more than 20 of them
232 echo "Compress old psql backups"
234 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 |
235 sort | head -n20 | while read dumpname; do
236 echo "Compressing $dumpname"
241 echo "Finally, all is done, sending mail and compressing logfile"
242 exec > /dev/null 2>&1
244 cat "$LOGFILE" | mail -s "Log for cron.daily run of $(date +%Y.%m.%d)" cron@ftp-master.debian.org
247 ################################################################################