#! /bin/sh
#
# Executed daily via cron, out of dak's crontab.

set -e
export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
. $SCRIPTVARS

################################################################################

# Start logging
NOW=`date "+%Y.%m.%d-%H:%M:%S"`
LOGFILE="$logdir/dinstall_${NOW}.log"
exec > "$LOGFILE" 2>&1

ts() {
	TS=$(($TS+1));
	echo Archive maintenance timestamp $TS: $(date +%H:%M:%S)
}

echo Archive maintenance started at $(date +%H:%M:%S)
TS=0

NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
LOCKCU="$lockdir/daily.lock"
LOCKAC="$lockdir/unchecked.lock"
BRITNEYLOCK="$lockdir/britney.lock"
lockac=0

cleanup() {
  rm -f "$NOTICE"
  rm -f "$LOCKCU"
  if [ "$lockac" -eq "1" ]; then
    rm -f "$LOCKAC"
  fi
  echo "Cleanup"
}
lockfile -l 3600 $LOCKCU
trap cleanup 0

# This file is simply used to indicate to britney whether or not
# the Packages file updates completed sucessfully.  It's not a lock
# from our point of view
touch ${BRITNEYLOCK}

rm -f "$NOTICE"
cat > "$NOTICE" <<EOF
Packages are currently being installed and indices rebuilt.
Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
ending about an hour later.  This file is then removed.

You should not mirror the archive during this period.
EOF

# Push merkels qa user, so the qa pages can show "dinstall is running" information
echo "Telling merkels QA user that we start dinstall"
ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
ts

################################################################################

echo "Creating pre-daily-cron-job backup of projectb database..."
pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)

################################################################################

ts
echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
cd $configdir
$scriptsdir/update-bugdoctxt
$scriptsdir/update-mirrorlists
$scriptsdir/update-mailingliststxt
$scriptsdir/update-pseudopackages.sh

################################################################################

ts
echo "Doing automated p-u-new processing"
cd $queuedir/p-u-new
date -u -R >> REPORT
dak process-new -a -C COMMENTS >> REPORT || true
echo >> REPORT

ts
echo "Doing automated o-p-u-new processing"
cd $queuedir/o-p-u-new
date -u -R >> REPORT
dak process-new -a -C COMMENTS >> REPORT || true
echo >> REPORT

################################################################################

ts

echo "Synchronizing i18n package descriptions"
# First sync their newest data
cd ${scriptdir}/i18nsync
rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true

# Now check if we still know about the packages for which they created the files
# is the timestamp signed by us?
if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
    # now read it. As its signed by us we are sure the content is what we expect, no need
	# to do more here. And we only test -d a directory on it anyway.
    TSTAMP=$(cat timestamp)
    # do we have the dir still?
    if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
        # Lets check!
        if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
			# Yay, worked, lets copy around
			for dir in lenny sid; do
				if [ -d dists/${dir}/ ]; then
					cd dists/${dir}/main/i18n
					rsync -aq --delete --delete-after  . ${ftpdir}/dists/${dir}/main/i18n/.
				fi
				cd ${scriptdir}/i18nsync
			done
		else
			echo "ARRRR, bad guys, wrong files, ARRR"
			echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
		fi
    else
		echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
		echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
	fi
else
    echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
	echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
fi

################################################################################

ts
lockfile $LOCKAC
lockac=1
echo "Processing queue/accepted"
cd $accepted
rm -f REPORT
dak process-accepted -pa *.changes | tee REPORT | \
     mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
chgrp debadmin REPORT
chmod 664 REPORT

ts
echo "Checking for cruft in overrides"
dak check-overrides
rm -f $LOCKAC
lockac=0

echo "Fixing symlinks in $ftpdir"
symlinks -d -r $ftpdir

ts

echo "Generating suite file lists for apt-ftparchive"
dak make-suite-file-list

ts

echo "Updating fingerprints"
# Update fingerprints
dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true

ts

# Generate override files
echo "Writing overrides into text files"
cd $overridedir
dak make-overrides

ts

# FIXME
rm -f override.sid.all3
for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done

ts

# Generate Packages and Sources files
echo "Generating Packages and Sources files"
cd $configdir
apt-ftparchive generate apt.conf

ts

# Generate *.diff/ incremental updates
echo "Generating pdiff files"
dak generate-index-diffs

ts

# Generate Release files
echo "Generating Release files"
dak generate-releases

ts

# Clean out old packages
echo "Cleanup old packages/files"
dak clean-suites
dak clean-queues

ts

# Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
# update this before wanna-build is updated.
echo "Regenerating wanna-build/buildd information"
psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
symlinks -d /srv/incoming.debian.org/buildd > /dev/null
apt-ftparchive generate apt.conf.buildd

ts

echo "Running various scripts from $scriptsdir"
cd $scriptsdir
./mkmaintainers
./copyoverrides
./mklslar
./mkfilesindices
./mkchecksums

ts

# (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
echo "Regenerating mirror/ hardlink fun"
cd ${mirrordir}
rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .


ts

echo "Trigger daily wanna-build run"
ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 buildd@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org

rm -f $NOTICE
rm -f $LOCKCU
echo Archive maintenance finished at $(date +%H:%M:%S)

################################################################################

echo "Creating post-daily-cron-job backup of projectb database..."
POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
pg_dump projectb > $POSTDUMP
(cd $base/backup; ln -sf $POSTDUMP current)

################################################################################

ts

echo "Expiring old database dumps..."
(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")

################################################################################

ts

# Send a report on NEW/BYHAND packages
echo "Nagging ftpteam about NEW/BYHAND packages"
dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
# and one on crufty packages
echo "Sending information about crufty packages"
dak cruft-report > $webdir/cruft-report-daily.txt
dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org

ts

echo "Updating DM html page"
$scriptsdir/dm-monitor >$webdir/dm-uploaders.html

################################################################################

# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
echo "Trigger merkels projectb sync"
ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true

################################################################################

ts

ulimit -m 90000 -d 90000 -s 10000 -v 200000

echo "Using run-parts to run scripts in $base/scripts/distmnt"
run-parts --report $base/scripts/distmnt

ts

echo "Exporting package data foo for i18n project"
STAMP=$(date "+%Y%m%d%H%M")
mkdir -p ${scriptdir}/i18n/${STAMP}
cd ${scriptdir}/i18n/${STAMP}
dak control-suite -l stable > etch
dak control-suite -l testing > lenny
dak control-suite -l unstable > sid
echo "${STAMP}" > timestamp
gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
rm -f md5sum
md5sum * > md5sum
cd ${webdir}/
ln -sfT ${scriptdir}/i18n/${STAMP} i18n

cd ${scriptdir}
find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf

ts

echo "Daily cron scripts successful."

ts

# Stats pr0n
echo "Updating stats data"
cd $configdir
$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
R --slave --vanilla < $base/misc/ftpstats.R

ts

# Remove the britney lock
rm -f ${BRITNEYLOCK}

# Clean up apt-ftparchive's databases
echo "Clean up apt-ftparchive's databases"
cd $configdir
apt-ftparchive -q clean apt.conf

ts

# Compress psql backups
echo "Compress old psql backups"
(cd $base/backup/
	find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 | 
	while read dumpname; do
		echo "Compressing $dumpname"
		bzip2 -9 "$dumpname"
	done
)

ts

echo "Removing old dinstall logfiles"
(cd $logdir
	find -maxdepth 1 -mindepth 1 -type f -name 'dinstall_*' -mtime +60 | 
	while read dumpname; do
		echo "Removing $dumpname"
		rm -f "$dumpname"
	done

	find -maxdepth 1 -mindepth 1 -type f -name 'weekly_*' -mtime +60 | 
	while read dumpname; do
		echo "Removing $dumpname"
		rm -f "$dumpname"
	done
)

echo "Finally, all is done, sending mail and compressing logfile"
exec > /dev/null 2>&1

cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
bzip2 -9 "$LOGFILE"

################################################################################