# -*- mode:sh -*-
# Timestamp. Used for dinstall stat graphs
function ts() {
        echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
}

# Cleanup actions
function cleanup() {
	rm -f ${LOCK_DAILY}
	rm -f ${LOCK_ACCEPTED}
}

# If we error out this one is called, *FOLLOWED* by cleanup above
function onerror() {
    ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")

    subject="ATTENTION ATTENTION!"
    if [ "${error}" = "false" ]; then
        subject="${subject} (continued)"
    else
        subject="${subject} (interrupted)"
    fi
    subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"

    if [ -r "${STAGEFILE}.log" ]; then
        cat "${STAGEFILE}.log"
    else
        echo  "file ${STAGEFILE}.log does not exist, sorry"
    fi | mail -s "${subject}" -a "X-Debian: DAK" cron@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
}

########################################################################
# the actual dinstall functions follow                                 #
########################################################################

function qa1() {
    log "Telling QA user that we start dinstall"
    ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
 }

# Updating various files
function updates() {
    log "Updating Bugs docu, Mirror list and mailing-lists.txt"
    cd $configdir
    $scriptsdir/update-bugdoctxt
    $scriptsdir/update-mirrorlists
    $scriptsdir/update-mailingliststxt
    $scriptsdir/update-pseudopackages.sh
}

# Process (oldstable)-proposed-updates "NEW" queue
function punew_do() {
    date -u -R >> REPORT
    dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" team@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
    echo >> REPORT
}
function punew() {
    log "Doing automated p-u-new processing"
    cd "${queuedir}/p-u-new"
    punew_do "$1"
}
function opunew() {
    log "Doing automated o-p-u-new processing"
    cd "${queuedir}/o-p-u-new"
    punew_do "$1"
}

# The first i18n one, syncing new descriptions
function i18n1() {
    log "Synchronizing i18n package descriptions"
    # First sync their newest data
    cd ${scriptdir}/i18nsync
    rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true

    # Now check if we still know about the packages for which they created the files
    # is the timestamp signed by us?
    if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
        # now read it. As its signed by us we are sure the content is what we expect, no need
        # to do more here. And we only test -d a directory on it anyway.
        TSTAMP=$(cat timestamp)
        # do we have the dir still?
        if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
            # Lets check!
            if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
                # Yay, worked, lets copy around
                for dir in wheezy sid; do
                    if [ -d dists/${dir}/ ]; then
                        cd dists/${dir}/main/i18n
                        rsync -aq --delete --delete-after  . ${ftpdir}/dists/${dir}/main/i18n/.
                    fi
                    cd ${scriptdir}/i18nsync
                done
            else
                echo "ARRRR, bad guys, wrong files, ARRR"
                echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
            fi
        else
            echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
            echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
        fi
    else
        echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
        echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
    fi
}

function cruft() {
    log "Checking for cruft in overrides"
    dak check-overrides
}

function dominate() {
    log "Removing obsolete source and binary associations"
    dak dominate
}

function filelist() {
    log "Generating file lists for apt-ftparchive"
    dak generate-filelist
}

function fingerprints() {
    log "Updating fingerprints"
    dak import-keyring -L /srv/backports-master.debian.org/keyrings/keyring.gpg

    OUTFILE=$(mktemp)
    dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"

    if [ -s "${OUTFILE}" ]; then
        /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
To: <debian-backports@lists.debian.org>
Subject: Debian Maintainers Keyring changes
Content-Type: text/plain; charset=utf-8
X-Debian: DAK
MIME-Version: 1.0

The following changes to the debian-maintainers keyring have just been activated:

$(cat $OUTFILE)

Debian distribution maintenance software,
on behalf of the Keyring maintainers

EOF
    fi
    rm -f "$OUTFILE"
}

function overrides() {
    log "Writing overrides into text files"
    cd $overridedir
    dak make-overrides

    # FIXME
    rm -f override.sid.all3
    for i in main contrib non-free main.debian-installer; do cat override.squeeze-backports.$i >> override.sid.all3; done
}

function mpfm() {
    log "Generating package / file mapping"
    dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
}

function packages() {
    log "Generating Packages and Sources files"
    cd $configdir
    apt-ftparchive generate apt.conf
    #dak generate-packages-sources
}

function pdiff() {
    log "Generating pdiff files"
    dak generate-index-diffs
}

function release() {
    log "Generating Release files"
    dak generate-releases
}

function dakcleanup() {
    log "Cleanup old packages/files"
    dak clean-suites -m 10000
    dak clean-queues
}

function buildd_dir() {
    # Rebuilt the buildd dir to avoid long times of 403
    log "Regenerating the buildd incoming dir"
    STAMP=$(date "+%Y%m%d%H%M")
    make_buildd_dir
}

function mklslar() {
    cd $ftpdir

    FILENAME=ls-lR

    log "Removing any core files ..."
    find -type f -name core -print0 | xargs -0r rm -v

    log "Checking permissions on files in the FTP tree ..."
    find -type f \( \! -perm -444 -o -perm +002 \) -ls
    find -type d \( \! -perm -555 -o -perm +002 \) -ls

    log "Checking symlinks ..."
    symlinks -rd .

    log "Creating recursive directory listing ... "
    rm -f .${FILENAME}.new
    TZ=UTC ls -lR > .${FILENAME}.new

    if [ -r ${FILENAME}.gz ] ; then
        mv -f ${FILENAME}.gz ${FILENAME}.old.gz
        mv -f .${FILENAME}.new ${FILENAME}
        rm -f ${FILENAME}.patch.gz
        zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
        rm -f ${FILENAME}.old.gz
    else
        mv -f .${FILENAME}.new ${FILENAME}
    fi

    gzip -9cfN ${FILENAME} >${FILENAME}.gz
    rm -f ${FILENAME}
}

function mkmaintainers() {
    log 'Creating Maintainers index ... '

    cd $indices
    dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
    gzip -9v --rsyncable <Maintainers >Maintainers.gz
    gzip -9v --rsyncable <Uploaders >Uploaders.gz
}

function copyoverrides() {
    log 'Copying override files into public view ...'

    for ofile in $copyoverrides ; do
	    cd $overridedir
	    chmod g+w override.$ofile

	    cd $indices

	    newofile=override.$ofile.gz
	    rm -f .newover-$ofile.gz
	    pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
        if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
		    log "   installing new $newofile $pc"
		    mv -f .newover-$ofile.gz $newofile
		    chmod g+w $newofile
        else
		    rm -f .newover-$ofile.gz
	    fi
    done
}

function mkfilesindices() {
    umask 002
    cd $base/ftp/indices/files/components

    ARCHLIST=$(tempfile)

    log "Querying postgres"
    local query="
      SELECT CONCAT('./pool/', c.name, '/', f.filename) AS path, a.arch_string AS arch_string
      FROM files f
      JOIN files_archive_map af ON f.id = af.file_id
      JOIN component c ON af.component_id = c.id
      JOIN archive ON af.archive_id = archive.id
      LEFT OUTER JOIN
        (binaries b
         JOIN architecture a ON b.architecture = a.id)
        ON f.id = b.file
      WHERE archive.name = 'ftp-master'
      ORDER BY path, arch_string
    "
    psql -At -c "$query" >$ARCHLIST

    includedirs () {
        perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
    }
    poolfirst () {
        perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
    }

    log "Generating sources list"
    (
        sed -n 's/|$//p' $ARCHLIST
        cd $base/ftp
        find ./dists -maxdepth 1 \! -type d
        find ./dists \! -type d | grep "/source/"
    ) | sort -u | gzip -9 > source.list.gz

    log "Generating arch lists"

    ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
    for a in $ARCHES; do
        (sed -n "s/|$a$//p" $ARCHLIST
            sed -n 's/|all$//p' $ARCHLIST

            cd $base/ftp
            find ./dists -maxdepth 1 \! -type d
            find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
        ) | sort -u | gzip -9 > arch-$a.list.gz
    done

    log "Generating suite lists"

    suite_list () {
	local suite_id="$(printf %d $1)"
	local query
	query="
          SELECT DISTINCT CONCAT('./pool/', c.name, '/', f.filename)
          FROM
            (SELECT sa.source AS source
               FROM src_associations sa
              WHERE sa.suite = $suite_id
             UNION
             SELECT esr.src_id
               FROM extra_src_references esr
               JOIN bin_associations ba ON esr.bin_id = ba.bin
               WHERE ba.suite = $suite_id
             UNION
             SELECT b.source AS source
               FROM bin_associations ba
               JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s
            JOIN dsc_files df ON s.source = df.source
            JOIN files f ON df.file = f.id
            JOIN files_archive_map af ON f.id = af.file_id
            JOIN component c ON af.component_id = c.id
            JOIN archive ON af.archive_id = archive.id
            WHERE archive.name = 'ftp-master'
        "
	psql -F' ' -A -t -c "$query"

	query="
          SELECT CONCAT('./pool/', c.name, '/', f.filename)
          FROM bin_associations ba
          JOIN binaries b ON ba.bin = b.id
          JOIN files f ON b.file = f.id
          JOIN files_archive_map af ON f.id = af.file_id
          JOIN component c ON af.component_id = c.id
          JOIN archive ON af.archive_id = archive.id
          WHERE ba.suite = $suite_id AND archive.name = 'ftp-master'
        "
	psql -F' ' -A -t -c "$query"
    }

    psql -F' ' -At -c 'SELECT id, suite_name FROM suite' |
    while read id suite; do
        [ -e $base/ftp/dists/$suite ] || continue
        (
            (cd $base/ftp
                distname=$(cd dists; readlink $suite || echo $suite)
                find ./dists/$distname \! -type d
                for distdir in ./dists/*; do
                    [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
                done
            )
            suite_list $id
        ) | sort -u | gzip -9 > suite-${suite}.list.gz
    done

    log "Finding everything on the ftp site to generate sundries"
    (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST

    rm -f sundries.list
    zcat *.list.gz | cat - *.list | sort -u |
    diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list

    log "Generating files list"

    for a in $ARCHES; do
        (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
        cat - sundries.list dists.list project.list docs.list indices.list |
        sort -u | poolfirst > ../arch-$a.files
    done

    (cd $base/ftp/
	    for dist in sid wheezy; do
		    find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
	    done
    )

    (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
    sort -u | poolfirst > ../typical.files

    rm -f $ARCHLIST
    log "Done!"
}

function mkchecksums() {
    dsynclist=$dbdir/dsync.list
    md5list=$indices/md5sums

    log -n "Creating md5 / dsync index file ... "

    cd "$ftpdir"
    ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
    ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
    ${bindir}/dsync-flist -q link-dups $dsynclist || true
}

function mirror() {
    log "Regenerating \"public\" mirror/ hardlink fun"
    DATE_SERIAL=$(date +"%Y%m%d01")
    FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
    if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
        SERIAL="$DATE_SERIAL"
    else
        SERIAL="$FILESOAPLUS1"
    fi
    date -u > ${TRACEFILE}
    echo "Using dak v1" >> ${TRACEFILE}
    echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
    echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
    cd ${mirrordir}
    rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new  --ignore-errors ${ftpdir}/. .
}

function expire() {
    log "Expiring old database dumps..."
    cd $base/backup
    $scriptsdir/expire_dumps -d . -p -f "dump_*"
}

function transitionsclean() {
    log "Removing out of date transitions..."
    cd $base
    dak transitions -c -a
}

function dm() {
    log "Updating DM html page"
    #$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
}

function bts() {
    log "Categorizing uncategorized bugs filed against ftp.debian.org"
    dak bts-categorize
}

function ddaccess() {
    # Tell our dd accessible mirror to sync itself up. Including ftp dir.
    log "Trigger dd accessible parts sync including ftp dir"
    ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
}

function mirrorpush() {
    log "Starting the mirrorpush"
    date -u > /srv/backports-web.debian.org/underlay/mirrorstart
    echo "Using dak v1" >> /srv/backports-web.debian.org/underlay/mirrorstart
    echo "Running on host $(hostname -f)" >> /srv/backports-web.debian.org/underlay/mirrorstart
    sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
}

function i18n2() {
    log "Exporting package data foo for i18n project"
    STAMP=$(date "+%Y%m%d%H%M")
    mkdir -p ${scriptdir}/i18n/${STAMP}
    cd ${scriptdir}/i18n/${STAMP}
    dak control-suite -l stable > squeeze
    dak control-suite -l testing > wheezy
    dak control-suite -l unstable > sid
    echo "${STAMP}" > timestamp
    gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --detach-sign -o timestamp.gpg timestamp
    rm -f md5sum
    md5sum * > md5sum
    cd ${webdir}/
    ln -sfT ${scriptdir}/i18n/${STAMP} i18n

    cd ${scriptdir}
    find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
}

function stats() {
    log "Updating stats data"
    cd $configdir
    #$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
    #R --slave --vanilla < $base/misc/ftpstats.R
    dak stats arch-space > $webdir/arch-space
    dak stats pkg-nums > $webdir/pkg-nums
}

function aptftpcleanup() {
    log "Clean up apt-ftparchive's databases"
    cd $configdir
    apt-ftparchive -q clean apt.conf
}

function cleantransactions() {
    log "Cleanup transaction ids older than 3 months"
    cd $base/backup/
    find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
}

function logstats() {
    $masterdir/tools/logs.py "$1"
}

# save timestamp when we start
function savetimestamp() {
	NOW=`date "+%Y.%m.%d-%H:%M:%S"`
	echo ${NOW} > "${dbdir}/dinstallstart"
}

function maillogfile() {
    cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
}

function renamelogfile() {
    if [ -f "${dbdir}/dinstallstart" ]; then
        NOW=$(cat "${dbdir}/dinstallstart")
#        maillogfile
        mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
#        logstats "$logdir/dinstall_${NOW}.log"
        bzip2 -9 "$logdir/dinstall_${NOW}.log"
    else
        error "Problem, I don't know when dinstall started, unable to do log statistics."
        NOW=`date "+%Y.%m.%d-%H:%M:%S"`
#        maillogfile
        mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
        bzip2 -9 "$logdir/dinstall_${NOW}.log"
    fi
}

function testingsourcelist() {
    dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
}

# do a last run of process-unchecked before dinstall is on.
function process_unchecked() {
    log "Processing the unchecked queue"
    UNCHECKED_WITHOUT_LOCK="-p"
    do_unchecked
    sync_debbugs
}

# do a run of newstage only before dinstall is on.
function newstage() {
    log "Processing the newstage queue"
    UNCHECKED_WITHOUT_LOCK="-p"
    do_newstage
}

# Function to update a "statefile" telling people what we are doing
# (more or less).
#
# This should be called with the argument(s)
#  - Status name we want to show.
#
function state() {
    RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
    cat >"${DINSTALLSTATE}" <<EOF
Dinstall start: ${DINSTALLBEGIN}
Current action: ${1}
Action start: ${RIGHTNOW}
EOF
}

# extract changelogs and stuff
function changelogs() {
    log "Extracting changelogs"
    dak make-changelog -e
    mkdir -p ${exportpublic}/changelogs
    cd ${exportpublic}/changelogs
    rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
    sudo -H -u archvsync /home/archvsync/runmirrors metabdo > ~dak/runmirrors-metadata.log 2>&1 &
}