3 # Executed daily via cron, out of dak's crontab.
6 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
9 ################################################################################
11 echo Archive maintenance started at $(date +%X)
14 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
15 LOCKCU="$lockdir/daily.lock"
16 LOCKAC="$lockdir/unchecked.lock"
22 if [ "$lockac" -eq "1" ]; then
26 lockfile -l 3600 $LOCKCU
31 Packages are currently being installed and indices rebuilt.
32 Maintenance is automatic, starting at 13:52 US Central time, and
33 ending at about 15:30. This file is then removed.
35 You should not mirror the archive during this period.
38 # Push merkels qa user, so the qa pages can show "dinstall is running" information
39 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
41 ################################################################################
43 echo "Creating pre-daily-cron-job backup of projectb database..."
44 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
46 ################################################################################
48 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
50 $scriptsdir/update-bugdoctxt
51 $scriptsdir/update-mirrorlists
52 $scriptsdir/update-mailingliststxt
54 ################################################################################
56 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
59 dak process-new -a -C COMMENTS >> REPORT
62 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
63 cd $queuedir/o-p-u-new
65 dak process-new -a -C COMMENTS >> REPORT
68 ################################################################################
70 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
75 dak process-accepted -pa *.changes | tee REPORT | \
76 mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
80 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
86 symlinks -d -r $ftpdir
88 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
90 dak make-suite-file-list
92 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
95 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
97 # Generate override files
101 # Update task overrides for testing and unstable
102 # [JT 2004-02-04 disabled; copying in by hand for now]
103 #cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sarge.extra.main
104 #cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sid.extra.main
107 rm -f override.sid.all3
108 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
110 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
112 # Generate Packages and Sources files
114 apt-ftparchive generate apt.conf
115 # Generate *.diff/ incremental updates
116 dak generate-index-diffs
117 # Generate Release files
118 dak generate-releases
120 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
122 # Clean out old packages
126 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
128 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
129 # update this before wanna-build is updated.
130 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
131 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
132 apt-ftparchive generate apt.conf.buildd
134 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
143 # Fetch bugs information before unchecked processing is allowed again.
144 $base/testing/britney allowdaklock bugs || true
146 ssh buildd@buildd /org/wanna-build/trigger.daily
149 echo Archive maintenance finished at $(date +%X)
151 ################################################################################
153 echo "Creating post-daily-cron-job backup of projectb database..."
154 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
155 pg_dump projectb > $POSTDUMP
156 (cd $base/backup; ln -sf $POSTDUMP current)
158 ################################################################################
160 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
162 # Vacuum the database
163 # (JJ, 20-04-2008) disabled, as we have autovacuum set to on in postgres.
164 # refer to http://www.postgresql.org/docs/current/static/routine-vacuuming.html#AUTOVACUUM
165 # which says "Beginning in PostgreSQL 8.1, there is an optional feature called autovacuum,
166 # whose purpose is to automate the execution of VACUUM and ANALYZE commands."
167 # echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$"
169 echo "Expiring old database dumps..."
170 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
172 ################################################################################
174 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
176 # Send a report on NEW/BYHAND packages
177 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
178 # and one on crufty packages
179 dak cruft-report > $webdir/cruft-report-daily.txt
180 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
181 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
183 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
185 ################################################################################
187 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
188 ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
192 #time dak mirror-split
194 ################################################################################
196 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
198 ulimit -m 90000 -d 90000 -s 10000 -v 200000
200 run-parts --report $base/scripts/distmnt
202 echo Daily cron scripts successful.
204 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
208 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
209 R --slave --vanilla < $base/misc/ftpstats.R
211 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
213 # Clean up apt-ftparchive's databases
216 apt-ftparchive -q clean apt.conf
218 TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
220 # Compress psql backups older than a week, but no more than 20 of them
223 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 |
224 sort | head -n20 | while read dumpname; do
225 echo "Compressing $dumpname"
230 ################################################################################