dsync

[dak.git] / config / debian / cron.dinstall

diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index d8a69d62de4478af48045c1ad2c4b0fa368993ec..15096657235cc09b43b70157f34d1db174b4466e 100755 (executable)
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -13,12 +13,18 @@ NOW=`date "+%Y.%m.%d-%H:%M:%S"`
 LOGFILE="$logdir/dinstall_${NOW}.log"
 exec > "$LOGFILE" 2>&1
 
 LOGFILE="$logdir/dinstall_${NOW}.log"
 exec > "$LOGFILE" 2>&1
 

-echo Archive maintenance started at $(date +%X)
-TS=0
+ts() {
+       TS=$(($TS+1));
+       echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)"
+}
+
+TS=-1
+ts "startup"

 
 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
 LOCKCU="$lockdir/daily.lock"
 LOCKAC="$lockdir/unchecked.lock"
 
 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
 LOCKCU="$lockdir/daily.lock"
 LOCKAC="$lockdir/unchecked.lock"

+BRITNEYLOCK="$lockdir/britney.lock"

 lockac=0
 
 cleanup() {
 lockac=0
 
 cleanup() {


@@ -32,52 +38,101 @@ cleanup() {
 lockfile -l 3600 $LOCKCU
 trap cleanup 0
 
 lockfile -l 3600 $LOCKCU
 trap cleanup 0
 

+# This file is simply used to indicate to britney whether or not
+# the Packages file updates completed sucessfully.  It's not a lock
+# from our point of view
+touch ${BRITNEYLOCK}
+

 rm -f "$NOTICE"
 cat > "$NOTICE" <<EOF
 Packages are currently being installed and indices rebuilt.
 rm -f "$NOTICE"
 cat > "$NOTICE" <<EOF
 Packages are currently being installed and indices rebuilt.

-Maintenance is automatic, starting at 13:52 US Central time, and
-ending at about 15:30.  This file is then removed.
+Maintenance is automatic, starting at 07:52 and 19:52 UTC, and
+ending about an hour later.  This file is then removed.

 
 You should not mirror the archive during this period.
 EOF
 
 # Push merkels qa user, so the qa pages can show "dinstall is running" information
 echo "Telling merkels QA user that we start dinstall"
 
 You should not mirror the archive during this period.
 EOF
 
 # Push merkels qa user, so the qa pages can show "dinstall is running" information
 echo "Telling merkels QA user that we start dinstall"

-ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1
+ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
+ts "init"

 
 ################################################################################
 
 echo "Creating pre-daily-cron-job backup of projectb database..."
 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
 
 ################################################################################
 
 echo "Creating pre-daily-cron-job backup of projectb database..."
 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)

+ts "pg_dump1"

 
 ################################################################################
 
 
 ################################################################################
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
 cd $configdir
 $scriptsdir/update-bugdoctxt
 $scriptsdir/update-mirrorlists
 $scriptsdir/update-mailingliststxt
 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
 cd $configdir
 $scriptsdir/update-bugdoctxt
 $scriptsdir/update-mirrorlists
 $scriptsdir/update-mailingliststxt

+$scriptsdir/update-pseudopackages.sh
+ts "External Updates"

 
 ################################################################################
 
 
 ################################################################################
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

 echo "Doing automated p-u-new processing"
 cd $queuedir/p-u-new
 date -u -R >> REPORT
 echo "Doing automated p-u-new processing"
 cd $queuedir/p-u-new
 date -u -R >> REPORT

-dak process-new -a -C COMMENTS >> REPORT
+dak process-new -a -C COMMENTS >> REPORT || true

 echo >> REPORT
 echo >> REPORT

+ts "p-u-new"

 
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

 echo "Doing automated o-p-u-new processing"
 cd $queuedir/o-p-u-new
 date -u -R >> REPORT
 echo "Doing automated o-p-u-new processing"
 cd $queuedir/o-p-u-new
 date -u -R >> REPORT

-dak process-new -a -C COMMENTS >> REPORT
+dak process-new -a -C COMMENTS >> REPORT || true

 echo >> REPORT
 echo >> REPORT

+ts "o-p-u-new"
+
+################################################################################
+
+# Temporary until lenny is there
+dak control-suite -s lenny-r0 /srv/ftp.debian.org/misc/lenny-r0.txt
+
+echo "Synchronizing i18n package descriptions"
+# First sync their newest data
+cd ${scriptdir}/i18nsync
+rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
+
+# Now check if we still know about the packages for which they created the files
+# is the timestamp signed by us?
+if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
+    # now read it. As its signed by us we are sure the content is what we expect, no need
+       # to do more here. And we only test -d a directory on it anyway.
+    TSTAMP=$(cat timestamp)
+    # do we have the dir still?
+    if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
+        # Lets check!
+        if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
+                       # Yay, worked, lets copy around
+                       for dir in lenny sid; do
+                               if [ -d dists/${dir}/ ]; then
+                                       cd dists/${dir}/main/i18n
+                                       rsync -aq --delete --delete-after  . ${ftpdir}/dists/${dir}/main/i18n/.
+                               fi
+                               cd ${scriptdir}/i18nsync
+                       done
+               else
+                       echo "ARRRR, bad guys, wrong files, ARRR"
+                       echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
+               fi
+    else
+               echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
+               echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
+       fi
+else
+    echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
+       echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
+fi
+ts "i18n 1"

 
 ################################################################################
 
 
 ################################################################################
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

 lockfile $LOCKAC
 lockac=1
 echo "Processing queue/accepted"
 lockfile $LOCKAC
 lockac=1
 echo "Processing queue/accepted"


@@ -87,8 +142,8 @@ dak process-accepted -pa *.changes | tee REPORT | \
      mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
 chgrp debadmin REPORT
 chmod 664 REPORT
      mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
 chgrp debadmin REPORT
 chmod 664 REPORT

+ts "accepted"

 
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

 echo "Checking for cruft in overrides"
 dak check-overrides
 rm -f $LOCKAC
 echo "Checking for cruft in overrides"
 dak check-overrides
 rm -f $LOCKAC


@@ -96,58 +151,53 @@ lockac=0
 
 echo "Fixing symlinks in $ftpdir"
 symlinks -d -r $ftpdir
 
 echo "Fixing symlinks in $ftpdir"
 symlinks -d -r $ftpdir

-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts "cruft"

 
 echo "Generating suite file lists for apt-ftparchive"
 dak make-suite-file-list
 
 echo "Generating suite file lists for apt-ftparchive"
 dak make-suite-file-list

-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts "make-suite-file-list"

 
 echo "Updating fingerprints"
 # Update fingerprints
 
 echo "Updating fingerprints"
 # Update fingerprints

-dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
+ts "import-keyring"

 
 # Generate override files
 echo "Writing overrides into text files"
 cd $overridedir
 dak make-overrides
 
 
 # Generate override files
 echo "Writing overrides into text files"
 cd $overridedir
 dak make-overrides
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-

 # FIXME
 rm -f override.sid.all3
 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
 # FIXME
 rm -f override.sid.all3
 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done

+ts "overrides"

 
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+# Generate package / file mapping
+echo "Generating package / file mapping"
+dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
+ts "pkg-file-mapping"

 
 # Generate Packages and Sources files
 echo "Generating Packages and Sources files"
 cd $configdir
 apt-ftparchive generate apt.conf
 
 # Generate Packages and Sources files
 echo "Generating Packages and Sources files"
 cd $configdir
 apt-ftparchive generate apt.conf

-# Generate *.diff/ incremental updates
-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts "apt-ftparchive"

 
 

+# Generate *.diff/ incremental updates

 echo "Generating pdiff files"
 dak generate-index-diffs
 echo "Generating pdiff files"
 dak generate-index-diffs

-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts "pdiff"

 
 # Generate Release files
 echo "Generating Release files"
 dak generate-releases
 
 # Generate Release files
 echo "Generating Release files"
 dak generate-releases

-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts "release files"

 
 # Clean out old packages
 echo "Cleanup old packages/files"
 dak clean-suites
 dak clean-queues
 
 # Clean out old packages
 echo "Cleanup old packages/files"
 dak clean-suites
 dak clean-queues

-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts "cleanup"

 
 # Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
 # update this before wanna-build is updated.
 
 # Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
 # update this before wanna-build is updated.


@@ -155,8 +205,7 @@ echo "Regenerating wanna-build/buildd information"
 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
 apt-ftparchive generate apt.conf.buildd
 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
 apt-ftparchive generate apt.conf.buildd

-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts "buildd"

 
 echo "Running various scripts from $scriptsdir"
 cd $scriptsdir
 
 echo "Running various scripts from $scriptsdir"
 cd $scriptsdir


@@ -165,16 +214,22 @@ cd $scriptsdir
 ./mklslar
 ./mkfilesindices
 ./mkchecksums
 ./mklslar
 ./mkfilesindices
 ./mkchecksums

-#
-rm -f $NOTICE
+ts "scripts"

 
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+# (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
+echo "Regenerating mirror/ hardlink fun"
+cd ${mirrordir}
+rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
+ts "mirror hardlinks"

 
 echo "Trigger daily wanna-build run"
 
 echo "Trigger daily wanna-build run"

-ssh buildd@buildd /org/wanna-build/trigger.daily
+ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
+ts "w-b"

 
 

+rm -f $NOTICE

 rm -f $LOCKCU
 rm -f $LOCKCU

-echo Archive maintenance finished at $(date +%X)
+
+ts "locked part finished"

 
 ################################################################################
 
 
 ################################################################################
 


@@ -182,17 +237,17 @@ echo "Creating post-daily-cron-job backup of projectb database..."
 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
 pg_dump projectb > $POSTDUMP
 (cd $base/backup; ln -sf $POSTDUMP current)
 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
 pg_dump projectb > $POSTDUMP
 (cd $base/backup; ln -sf $POSTDUMP current)

+ts "pg_dump2"

 
 ################################################################################
 
 
 ################################################################################
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

 
 echo "Expiring old database dumps..."
 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
 
 echo "Expiring old database dumps..."
 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")

+ts "expire_dumps"

 
 ################################################################################
 
 
 ################################################################################
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

 
 # Send a report on NEW/BYHAND packages
 echo "Nagging ftpteam about NEW/BYHAND packages"
 
 # Send a report on NEW/BYHAND packages
 echo "Nagging ftpteam about NEW/BYHAND packages"


@@ -202,61 +257,98 @@ echo "Sending information about crufty packages"
 dak cruft-report > $webdir/cruft-report-daily.txt
 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
 dak cruft-report > $webdir/cruft-report-daily.txt
 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org

-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts "reports"

 
 echo "Updating DM html page"
 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
 
 
 echo "Updating DM html page"
 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
 

+echo "Categorizing uncategorized bugs filed against ftp.debian.org"
+dak bts-categorize
+

 ################################################################################
 
 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
 echo "Trigger merkels projectb sync"
 ################################################################################
 
 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
 echo "Trigger merkels projectb sync"

-ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
+ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
+ts "merkel projectb push"

 
 ################################################################################
 
 
 ################################################################################
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)

 
 ulimit -m 90000 -d 90000 -s 10000 -v 200000
 
 echo "Using run-parts to run scripts in $base/scripts/distmnt"
 run-parts --report $base/scripts/distmnt
 
 ulimit -m 90000 -d 90000 -s 10000 -v 200000
 
 echo "Using run-parts to run scripts in $base/scripts/distmnt"
 run-parts --report $base/scripts/distmnt

+ts "run-parts"
+
+echo "Exporting package data foo for i18n project"
+STAMP=$(date "+%Y%m%d%H%M")
+mkdir -p ${scriptdir}/i18n/${STAMP}
+cd ${scriptdir}/i18n/${STAMP}
+dak control-suite -l stable > etch
+dak control-suite -l testing > lenny
+dak control-suite -l unstable > sid
+echo "${STAMP}" > timestamp
+gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
+rm -f md5sum
+md5sum * > md5sum
+cd ${webdir}/
+ln -sfT ${scriptdir}/i18n/${STAMP} i18n
+
+cd ${scriptdir}
+find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
+ts "i18n 2"

 
 echo "Daily cron scripts successful."
 
 
 echo "Daily cron scripts successful."
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-

 # Stats pr0n
 echo "Updating stats data"
 cd $configdir
 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
 R --slave --vanilla < $base/misc/ftpstats.R
 # Stats pr0n
 echo "Updating stats data"
 cd $configdir
 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
 R --slave --vanilla < $base/misc/ftpstats.R

+ts "stats"

 
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+# Remove the britney lock
+rm -f ${BRITNEYLOCK}

 
 # Clean up apt-ftparchive's databases
 echo "Clean up apt-ftparchive's databases"
 cd $configdir
 apt-ftparchive -q clean apt.conf
 
 # Clean up apt-ftparchive's databases
 echo "Clean up apt-ftparchive's databases"
 cd $configdir
 apt-ftparchive -q clean apt.conf

+ts "apt-ftparchive cleanup"

 
 

-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-
-# Compress psql backups older than a week, but no more than 20 of them
+# Compress psql backups

 echo "Compress old psql backups"
 (cd $base/backup/
 echo "Compress old psql backups"
 (cd $base/backup/

- find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | 
-   sort | head -n20 | while read dumpname; do
-     echo "Compressing $dumpname"
-     bzip2 -9 "$dumpname"
-   done
+       find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 | 
+       while read dumpname; do
+               echo "Compressing $dumpname"
+               bzip2 -9 "$dumpname"
+       done

 )
 )

-
-TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
+ts "compress"
+
+echo "Removing old dinstall logfiles"
+(cd $logdir
+       find -maxdepth 1 -mindepth 1 -type f -name 'dinstall_*' -mtime +60 | 
+       while read dumpname; do
+               echo "Removing $dumpname"
+               rm -f "$dumpname"
+       done
+
+       find -maxdepth 1 -mindepth 1 -type f -name 'weekly_*' -mtime +60 | 
+       while read dumpname; do
+               echo "Removing $dumpname"
+               rm -f "$dumpname"
+       done
+)
+ts "logremove"

 
 echo "Finally, all is done, sending mail and compressing logfile"
 exec > /dev/null 2>&1
 
 
 echo "Finally, all is done, sending mail and compressing logfile"
 exec > /dev/null 2>&1
 

+$masterdir/tools/logs.py "$LOGFILE"
+

 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
 bzip2 -9 "$LOGFILE"
 
 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
 bzip2 -9 "$LOGFILE"