#! /bin/sh
#
# Executed daily via cron, out of dak's crontab.

set -e
export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
. $SCRIPTVARS

################################################################################

# Start logging
NOW=`date "+%Y.%m.%d-%H:%M:%S"`
LOGFILE="$logdir/dinstall_${NOW}.log"
exec > "$LOGFILE" 2>&1

ts() {
	TS=$(($TS+1));
	echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)"
}

TS=-1
ts "startup"

NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
LOCKCU="$lockdir/daily.lock"
LOCKAC="$lockdir/unchecked.lock"
BRITNEYLOCK="$lockdir/britney.lock"
lockac=0

cleanup() {
  rm -f "$NOTICE"
  rm -f "$LOCKCU"
  if [ "$lockac" -eq "1" ]; then
    rm -f "$LOCKAC"
  fi
  echo "Cleanup"
}
lockfile -l 3600 $LOCKCU
trap cleanup 0

# This file is simply used to indicate to britney whether or not
# the Packages file updates completed sucessfully.  It's not a lock
# from our point of view
touch ${BRITNEYLOCK}

rm -f "$NOTICE"
cat > "$NOTICE" <<EOF
Packages are currently being installed and indices rebuilt.
Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
ending about an hour later.  This file is then removed.

You should not mirror the archive during this period.
EOF

# Push merkels qa user, so the qa pages can show "dinstall is running" information
echo "Telling merkels QA user that we start dinstall"
ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
ts "init"

################################################################################

echo "Creating pre-daily-cron-job backup of projectb database..."
pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
ts "pg_dump1"

################################################################################

echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
cd $configdir
$scriptsdir/update-bugdoctxt
$scriptsdir/update-mirrorlists
$scriptsdir/update-mailingliststxt
$scriptsdir/update-pseudopackages.sh
ts "External Updates"

################################################################################

echo "Doing automated p-u-new processing"
cd $queuedir/p-u-new
date -u -R >> REPORT
dak process-new -a -C COMMENTS >> REPORT || true
echo >> REPORT
ts "p-u-new"

echo "Doing automated o-p-u-new processing"
cd $queuedir/o-p-u-new
date -u -R >> REPORT
dak process-new -a -C COMMENTS >> REPORT || true
echo >> REPORT
ts "o-p-u-new"

################################################################################

echo "Synchronizing i18n package descriptions"
# First sync their newest data
cd ${scriptdir}/i18nsync
rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true

# Now check if we still know about the packages for which they created the files
# is the timestamp signed by us?
if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
    # now read it. As its signed by us we are sure the content is what we expect, no need
	# to do more here. And we only test -d a directory on it anyway.
    TSTAMP=$(cat timestamp)
    # do we have the dir still?
    if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
        # Lets check!
        if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
			# Yay, worked, lets copy around
			for dir in squeeze sid; do
				if [ -d dists/${dir}/ ]; then
					cd dists/${dir}/main/i18n
					rsync -aq --delete --delete-after  . ${ftpdir}/dists/${dir}/main/i18n/.
				fi
				cd ${scriptdir}/i18nsync
			done
		else
			echo "ARRRR, bad guys, wrong files, ARRR"
			echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
		fi
    else
		echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
		echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
	fi
else
    echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
	echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
fi
ts "i18n 1"

################################################################################

lockfile $LOCKAC
lockac=1
echo "Processing queue/accepted"
cd $accepted
rm -f REPORT
dak process-accepted -pa *.changes | tee REPORT | \
     mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
chgrp debadmin REPORT
chmod 664 REPORT
ts "accepted"

echo "Checking for cruft in overrides"
dak check-overrides
rm -f $LOCKAC
lockac=0

echo "Fixing symlinks in $ftpdir"
symlinks -d -r $ftpdir
ts "cruft"

echo "Generating suite file lists for apt-ftparchive"
dak make-suite-file-list
ts "make-suite-file-list"

echo "Updating fingerprints"
# Update fingerprints
dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
ts "import-keyring"

# Generate override files
echo "Writing overrides into text files"
cd $overridedir
dak make-overrides

# FIXME
rm -f override.sid.all3
for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
ts "overrides"

# Generate package / file mapping
echo "Generating package / file mapping"
dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
ts "pkg-file-mapping"

# Generate Packages and Sources files
echo "Generating Packages and Sources files"
cd $configdir
apt-ftparchive generate apt.conf
ts "apt-ftparchive"

# Generate *.diff/ incremental updates
echo "Generating pdiff files"
dak generate-index-diffs
ts "pdiff"

# Generate Release files
echo "Generating Release files"
dak generate-releases
ts "release files"

# Clean out old packages
echo "Cleanup old packages/files"
dak clean-suites
dak clean-queues
ts "cleanup"

# Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
# update this before wanna-build is updated.
echo "Regenerating wanna-build/buildd information"
psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
symlinks -d /srv/incoming.debian.org/buildd > /dev/null
apt-ftparchive generate apt.conf.buildd
ts "buildd"

echo "Running various scripts from $scriptsdir"
cd $scriptsdir
./mkmaintainers
./copyoverrides
./mklslar
./mkfilesindices
./mkchecksums
ts "scripts"

# (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
echo "Regenerating mirror/ hardlink fun"
cd ${mirrordir}
rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
ts "mirror hardlinks"

echo "Trigger daily wanna-build run"
ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
ts "w-b"

rm -f $NOTICE
rm -f $LOCKCU

ts "locked part finished"

################################################################################

echo "Creating post-daily-cron-job backup of projectb database..."
POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
pg_dump projectb > $POSTDUMP
(cd $base/backup; ln -sf $POSTDUMP current)
ts "pg_dump2"

################################################################################


echo "Expiring old database dumps..."
(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
ts "expire_dumps"

################################################################################


# Send a report on NEW/BYHAND packages
echo "Nagging ftpteam about NEW/BYHAND packages"
dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
# and one on crufty packages
echo "Sending information about crufty packages"
dak cruft-report > $webdir/cruft-report-daily.txt
dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
ts "reports"

echo "Updating DM html page"
$scriptsdir/dm-monitor >$webdir/dm-uploaders.html

echo "Categorizing uncategorized bugs filed against ftp.debian.org"
dak bts-categorize

################################################################################

# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
echo "Trigger merkels projectb sync"
ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
ts "merkel projectb push"

################################################################################


ulimit -m 90000 -d 90000 -s 10000 -v 200000

echo "Using run-parts to run scripts in $base/scripts/distmnt"
run-parts --report $base/scripts/distmnt
ts "run-parts"

echo "Exporting package data foo for i18n project"
STAMP=$(date "+%Y%m%d%H%M")
mkdir -p ${scriptdir}/i18n/${STAMP}
cd ${scriptdir}/i18n/${STAMP}
dak control-suite -l stable > lenny
dak control-suite -l testing > squeeze
dak control-suite -l unstable > sid
echo "${STAMP}" > timestamp
gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
rm -f md5sum
md5sum * > md5sum
cd ${webdir}/
ln -sfT ${scriptdir}/i18n/${STAMP} i18n

cd ${scriptdir}
find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
ts "i18n 2"

echo "Daily cron scripts successful."

# Stats pr0n
echo "Updating stats data"
cd $configdir
$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
R --slave --vanilla < $base/misc/ftpstats.R
ts "stats"

# Remove the britney lock
rm -f ${BRITNEYLOCK}

# Clean up apt-ftparchive's databases
echo "Clean up apt-ftparchive's databases"
cd $configdir
apt-ftparchive -q clean apt.conf
ts "apt-ftparchive cleanup"

# Compress psql backups
echo "Compress old psql backups"
(cd $base/backup/
	find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 | 
	while read dumpname; do
		echo "Compressing $dumpname"
		bzip2 -9 "$dumpname"
	done
)
ts "compress"

echo "Removing old dinstall logfiles"
(cd $logdir
	find -maxdepth 1 -mindepth 1 -type f -name 'dinstall_*' -mtime +60 | 
	while read dumpname; do
		echo "Removing $dumpname"
		rm -f "$dumpname"
	done

	find -maxdepth 1 -mindepth 1 -type f -name 'weekly_*' -mtime +60 | 
	while read dumpname; do
		echo "Removing $dumpname"
		rm -f "$dumpname"
	done
)
ts "logremove"

echo "Finally, all is done, sending mail and compressing logfile"
exec > /dev/null 2>&1

$masterdir/tools/logs.py "$LOGFILE"

cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
bzip2 -9 "$LOGFILE"

################################################################################