#! /bin/sh
#
# Executed daily via cron, out of dak's crontab.

set -e
export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
. $SCRIPTVARS

################################################################################

echo Archive maintenance started at $(date +%X)
TS=0

NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
LOCKCU="$lockdir/daily.lock"
LOCKAC="$lockdir/unchecked.lock"
lockac=0

cleanup() {
  rm -f "$NOTICE"
  rm -f "$LOCKCU"
  if [ "$lockac" -eq "1" ]; then
    rm -f "$LOCKAC"
  fi
}
lockfile -l 3600 $LOCKCU
trap cleanup 0

rm -f "$NOTICE"
cat > "$NOTICE" <<EOF
Packages are currently being installed and indices rebuilt.
Maintenance is automatic, starting at 13:52 US Central time, and
ending at about 15:30.  This file is then removed.

You should not mirror the archive during this period.
EOF

################################################################################

echo "Creating pre-daily-cron-job backup of projectb database..."
pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)

################################################################################

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
cd $configdir
$scriptsdir/update-bugdoctxt
$scriptsdir/update-mirrorlists
$scriptsdir/update-mailingliststxt

################################################################################

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
cd $queuedir/p-u-new
date -u -R >> REPORT
dak process-new -a -C COMMENTS >> REPORT
echo >> REPORT

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
cd $queuedir/o-p-u-new
date -u -R >> REPORT
dak process-new -a -C COMMENTS >> REPORT
echo >> REPORT

################################################################################

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
lockfile $LOCKAC
lockac=1
cd $accepted
rm -f REPORT
dak process-accepted -pa *.changes | tee REPORT | \
     mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
chgrp debadmin REPORT
chmod 664 REPORT

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

dak check-overrides
rm -f $LOCKAC
lockac=0

symlinks -d -r $ftpdir

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

dak make-suite-file-list

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

# Update fingerprints
dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg

# Generate override files
cd $overridedir
dak make-overrides

# Update task overrides for testing and unstable
# [JT 2004-02-04 disabled; copying in by hand for now]
#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sarge.extra.main
#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sid.extra.main

# FIXME
rm -f override.sid.all3
for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

# Generate Packages and Sources files
cd $configdir
apt-ftparchive generate apt.conf
# Generate *.diff/ incremental updates
dak generate-index-diffs
# Generate Release files
dak generate-releases

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

# Clean out old packages
dak clean-suites
dak clean-queues

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

# Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
# update this before wanna-build is updated.
psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
symlinks -d /srv/incoming.debian.org/buildd > /dev/null
apt-ftparchive generate apt.conf.buildd

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

cd $scriptsdir
./mkmaintainers
./copyoverrides
./mklslar
./mkfilesindices
./mkchecksums
#
# Fetch bugs information before unchecked processing is allowed again.
$base/testing/britney allowdaklock bugs || true
rm -f $NOTICE
ssh buildd@buildd /org/wanna-build/trigger.daily

rm -f $LOCKCU
echo Archive maintenance finished at $(date +%X)

################################################################################

echo "Creating post-daily-cron-job backup of projectb database..."
POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
pg_dump projectb > $POSTDUMP
(cd $base/backup; ln -sf $POSTDUMP current)

################################################################################

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

# Vacuum the database
# (JJ, 20-04-2008) disabled, as we have autovacuum set to on in postgres.
# refer to http://www.postgresql.org/docs/current/static/routine-vacuuming.html#AUTOVACUUM
# which says "Beginning in PostgreSQL 8.1, there is an optional feature called autovacuum,
# whose purpose is to automate the execution of VACUUM and ANALYZE  commands."
# echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE:  Skipping.*only table owner can VACUUM it$"

echo "Expiring old database dumps..."
(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")

################################################################################

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

# Send a report on NEW/BYHAND packages
dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
# and one on crufty packages
dak cruft-report | tee $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org

$scriptsdir/dm-monitor >$webdir/dm-uploaders.html

################################################################################

# Run mirror-split

#time dak mirror-split

################################################################################

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

ulimit -m 90000 -d 90000 -s 10000 -v 200000

run-parts --report $base/scripts/distmnt

echo Daily cron scripts successful.

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

# Stats pr0n
cd $configdir
$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
R --slave --vanilla < $base/misc/ftpstats.R

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

# Clean up apt-ftparchive's databases

cd $configdir
apt-ftparchive -q clean apt.conf

TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

# Compress psql backups older than a week, but no more than 20 of them

(cd $base/backup/
 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | 
   sort | head -n20 | while read dumpname; do
     echo "Compressing $dumpname"
     bzip2 -9 "$dumpname"
   done
)

################################################################################