Merge branch 'master' into bpo

[dak.git] / config / debian / cron.dinstall

diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index 6df0bc69a77c56575cf646fa2f18ec0a3bb06514..0629a102ae59545c78df18f258be489c6784e8d0 100755 (executable)
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -15,11 +15,11 @@ exec > "$LOGFILE" 2>&1
 
 ts() {
        TS=$(($TS+1));
 
 ts() {
        TS=$(($TS+1));

-       echo Archive maintenance timestamp $TS: $(date +%H:%M:%S)
+       echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)"

 }
 
 }
 

-echo Archive maintenance started at $(date +%H:%M:%S)
-TS=0
+TS=-1
+ts "startup"

 
 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
 LOCKCU="$lockdir/daily.lock"
 
 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
 LOCKCU="$lockdir/daily.lock"


@@ -55,42 +55,42 @@ EOF
 # Push merkels qa user, so the qa pages can show "dinstall is running" information
 echo "Telling merkels QA user that we start dinstall"
 ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
 # Push merkels qa user, so the qa pages can show "dinstall is running" information
 echo "Telling merkels QA user that we start dinstall"
 ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true

-ts
+ts "init"

 
 ################################################################################
 
 echo "Creating pre-daily-cron-job backup of projectb database..."
 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
 
 ################################################################################
 
 echo "Creating pre-daily-cron-job backup of projectb database..."
 pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)

+ts "pg_dump1"

 
 ################################################################################
 
 
 ################################################################################
 

-ts

 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
 cd $configdir
 $scriptsdir/update-bugdoctxt
 $scriptsdir/update-mirrorlists
 $scriptsdir/update-mailingliststxt
 echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
 cd $configdir
 $scriptsdir/update-bugdoctxt
 $scriptsdir/update-mirrorlists
 $scriptsdir/update-mailingliststxt

+$scriptsdir/update-pseudopackages.sh
+ts "External Updates"

 
 ################################################################################
 
 
 ################################################################################
 

-ts

 echo "Doing automated p-u-new processing"
 cd $queuedir/p-u-new
 date -u -R >> REPORT
 dak process-new -a -C COMMENTS >> REPORT || true
 echo >> REPORT
 echo "Doing automated p-u-new processing"
 cd $queuedir/p-u-new
 date -u -R >> REPORT
 dak process-new -a -C COMMENTS >> REPORT || true
 echo >> REPORT

+ts "p-u-new"

 
 

-ts

 echo "Doing automated o-p-u-new processing"
 cd $queuedir/o-p-u-new
 date -u -R >> REPORT
 dak process-new -a -C COMMENTS >> REPORT || true
 echo >> REPORT
 echo "Doing automated o-p-u-new processing"
 cd $queuedir/o-p-u-new
 date -u -R >> REPORT
 dak process-new -a -C COMMENTS >> REPORT || true
 echo >> REPORT

+ts "o-p-u-new"

 
 ################################################################################
 
 
 ################################################################################
 

-ts
-

 echo "Synchronizing i18n package descriptions"
 # First sync their newest data
 cd ${scriptdir}/i18nsync
 echo "Synchronizing i18n package descriptions"
 # First sync their newest data
 cd ${scriptdir}/i18nsync


@@ -107,7 +107,7 @@ if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.g
         # Lets check!
         if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
                        # Yay, worked, lets copy around
         # Lets check!
         if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
                        # Yay, worked, lets copy around

-                       for dir in lenny sid; do
+                       for dir in squeeze sid; do

                                if [ -d dists/${dir}/ ]; then
                                        cd dists/${dir}/main/i18n
                                        rsync -aq --delete --delete-after  . ${ftpdir}/dists/${dir}/main/i18n/.
                                if [ -d dists/${dir}/ ]; then
                                        cd dists/${dir}/main/i18n
                                        rsync -aq --delete --delete-after  . ${ftpdir}/dists/${dir}/main/i18n/.


@@ -116,20 +116,20 @@ if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.g
                        done
                else
                        echo "ARRRR, bad guys, wrong files, ARRR"
                        done
                else
                        echo "ARRRR, bad guys, wrong files, ARRR"

-                       echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
+                       echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org

                fi
     else
                echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
                fi
     else
                echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"

-               echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
+               echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org

        fi
 else
     echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
        fi
 else
     echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."

-       echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
+       echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org

 fi
 fi

+ts "i18n 1"

 
 ################################################################################
 
 
 ################################################################################
 

-ts

 lockfile $LOCKAC
 lockac=1
 echo "Processing queue/accepted"
 lockfile $LOCKAC
 lockac=1
 echo "Processing queue/accepted"


@@ -139,8 +139,8 @@ dak process-accepted -pa *.changes | tee REPORT | \
      mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
 chgrp debadmin REPORT
 chmod 664 REPORT
      mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
 chgrp debadmin REPORT
 chmod 664 REPORT

+ts "accepted"

 
 

-ts

 echo "Checking for cruft in overrides"
 dak check-overrides
 rm -f $LOCKAC
 echo "Checking for cruft in overrides"
 dak check-overrides
 rm -f $LOCKAC


@@ -148,58 +148,53 @@ lockac=0
 
 echo "Fixing symlinks in $ftpdir"
 symlinks -d -r $ftpdir
 
 echo "Fixing symlinks in $ftpdir"
 symlinks -d -r $ftpdir

-
-ts
+ts "cruft"

 
 echo "Generating suite file lists for apt-ftparchive"
 dak make-suite-file-list
 
 echo "Generating suite file lists for apt-ftparchive"
 dak make-suite-file-list

-
-ts
+ts "make-suite-file-list"

 
 echo "Updating fingerprints"
 # Update fingerprints
 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
 
 echo "Updating fingerprints"
 # Update fingerprints
 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true

-
-ts
+ts "import-keyring"

 
 # Generate override files
 echo "Writing overrides into text files"
 cd $overridedir
 dak make-overrides
 
 
 # Generate override files
 echo "Writing overrides into text files"
 cd $overridedir
 dak make-overrides
 

-ts
-

 # FIXME
 rm -f override.sid.all3
 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
 # FIXME
 rm -f override.sid.all3
 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done

+ts "overrides"

 
 

-ts
+# Generate package / file mapping
+echo "Generating package / file mapping"
+dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
+ts "pkg-file-mapping"

 
 # Generate Packages and Sources files
 echo "Generating Packages and Sources files"
 cd $configdir
 apt-ftparchive generate apt.conf
 
 # Generate Packages and Sources files
 echo "Generating Packages and Sources files"
 cd $configdir
 apt-ftparchive generate apt.conf

-
-ts
+ts "apt-ftparchive"

 
 # Generate *.diff/ incremental updates
 echo "Generating pdiff files"
 dak generate-index-diffs
 
 # Generate *.diff/ incremental updates
 echo "Generating pdiff files"
 dak generate-index-diffs

-
-ts
+ts "pdiff"

 
 # Generate Release files
 echo "Generating Release files"
 dak generate-releases
 
 # Generate Release files
 echo "Generating Release files"
 dak generate-releases

-
-ts
+ts "release files"

 
 # Clean out old packages
 echo "Cleanup old packages/files"
 dak clean-suites
 dak clean-queues
 
 # Clean out old packages
 echo "Cleanup old packages/files"
 dak clean-suites
 dak clean-queues

-
-ts
+ts "cleanup"

 
 # Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
 # update this before wanna-build is updated.
 
 # Needs to be rebuilt, as files have moved.  Due to unaccepts, we need to
 # update this before wanna-build is updated.


@@ -207,8 +202,7 @@ echo "Regenerating wanna-build/buildd information"
 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
 apt-ftparchive generate apt.conf.buildd
 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
 apt-ftparchive generate apt.conf.buildd

-
-ts
+ts "buildd"

 
 echo "Running various scripts from $scriptsdir"
 cd $scriptsdir
 
 echo "Running various scripts from $scriptsdir"
 cd $scriptsdir


@@ -217,23 +211,22 @@ cd $scriptsdir
 ./mklslar
 ./mkfilesindices
 ./mkchecksums
 ./mklslar
 ./mkfilesindices
 ./mkchecksums

-
-ts
+ts "scripts"

 
 # (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
 echo "Regenerating mirror/ hardlink fun"
 cd ${mirrordir}
 rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
 
 # (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
 echo "Regenerating mirror/ hardlink fun"
 cd ${mirrordir}
 rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .

-
-
-ts
+ts "mirror hardlinks"

 
 echo "Trigger daily wanna-build run"
 
 echo "Trigger daily wanna-build run"

-ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 buildd@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
+ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
+ts "w-b"

 
 rm -f $NOTICE
 rm -f $LOCKCU
 
 rm -f $NOTICE
 rm -f $LOCKCU

-echo Archive maintenance finished at $(date +%H:%M:%S)
+
+ts "locked part finished"

 
 ################################################################################
 
 
 ################################################################################
 


@@ -241,17 +234,17 @@ echo "Creating post-daily-cron-job backup of projectb database..."
 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
 pg_dump projectb > $POSTDUMP
 (cd $base/backup; ln -sf $POSTDUMP current)
 POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
 pg_dump projectb > $POSTDUMP
 (cd $base/backup; ln -sf $POSTDUMP current)

+ts "pg_dump2"

 
 ################################################################################
 
 
 ################################################################################
 

-ts

 
 echo "Expiring old database dumps..."
 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
 
 echo "Expiring old database dumps..."
 (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")

+ts "expire_dumps"

 
 ################################################################################
 
 
 ################################################################################
 

-ts

 
 # Send a report on NEW/BYHAND packages
 echo "Nagging ftpteam about NEW/BYHAND packages"
 
 # Send a report on NEW/BYHAND packages
 echo "Nagging ftpteam about NEW/BYHAND packages"


@@ -261,35 +254,36 @@ echo "Sending information about crufty packages"
 dak cruft-report > $webdir/cruft-report-daily.txt
 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
 dak cruft-report > $webdir/cruft-report-daily.txt
 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org

-
-ts
+ts "reports"

 
 echo "Updating DM html page"
 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
 
 
 echo "Updating DM html page"
 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
 

+echo "Categorizing uncategorized bugs filed against ftp.debian.org"
+dak bts-categorize
+

 ################################################################################
 
 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
 echo "Trigger merkels projectb sync"
 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
 ################################################################################
 
 # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
 echo "Trigger merkels projectb sync"
 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true

+ts "merkel projectb push"

 
 ################################################################################
 
 
 ################################################################################
 

-ts

 
 ulimit -m 90000 -d 90000 -s 10000 -v 200000
 
 echo "Using run-parts to run scripts in $base/scripts/distmnt"
 run-parts --report $base/scripts/distmnt
 
 ulimit -m 90000 -d 90000 -s 10000 -v 200000
 
 echo "Using run-parts to run scripts in $base/scripts/distmnt"
 run-parts --report $base/scripts/distmnt

-
-ts
+ts "run-parts"

 
 echo "Exporting package data foo for i18n project"
 STAMP=$(date "+%Y%m%d%H%M")
 mkdir -p ${scriptdir}/i18n/${STAMP}
 cd ${scriptdir}/i18n/${STAMP}
 
 echo "Exporting package data foo for i18n project"
 STAMP=$(date "+%Y%m%d%H%M")
 mkdir -p ${scriptdir}/i18n/${STAMP}
 cd ${scriptdir}/i18n/${STAMP}

-dak control-suite -l stable > etch
-dak control-suite -l testing > lenny
+dak control-suite -l stable > lenny
+dak control-suite -l testing > squeeze

 dak control-suite -l unstable > sid
 echo "${STAMP}" > timestamp
 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
 dak control-suite -l unstable > sid
 echo "${STAMP}" > timestamp
 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp


@@ -300,20 +294,16 @@ ln -sfT ${scriptdir}/i18n/${STAMP} i18n
 
 cd ${scriptdir}
 find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
 
 cd ${scriptdir}
 find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf

-
-ts
+ts "i18n 2"

 
 echo "Daily cron scripts successful."
 
 
 echo "Daily cron scripts successful."
 

-ts
-

 # Stats pr0n
 echo "Updating stats data"
 cd $configdir
 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
 R --slave --vanilla < $base/misc/ftpstats.R
 # Stats pr0n
 echo "Updating stats data"
 cd $configdir
 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
 R --slave --vanilla < $base/misc/ftpstats.R

-
-ts
+ts "stats"

 
 # Remove the britney lock
 rm -f ${BRITNEYLOCK}
 
 # Remove the britney lock
 rm -f ${BRITNEYLOCK}


@@ -322,8 +312,7 @@ rm -f ${BRITNEYLOCK}
 echo "Clean up apt-ftparchive's databases"
 cd $configdir
 apt-ftparchive -q clean apt.conf
 echo "Clean up apt-ftparchive's databases"
 cd $configdir
 apt-ftparchive -q clean apt.conf

-
-ts
+ts "apt-ftparchive cleanup"

 
 # Compress psql backups
 echo "Compress old psql backups"
 
 # Compress psql backups
 echo "Compress old psql backups"


@@ -334,12 +323,29 @@ echo "Compress old psql backups"
                bzip2 -9 "$dumpname"
        done
 )
                bzip2 -9 "$dumpname"
        done
 )

+ts "compress"

 
 

-ts
+echo "Removing old dinstall logfiles"
+(cd $logdir
+       find -maxdepth 1 -mindepth 1 -type f -name 'dinstall_*' -mtime +60 | 
+       while read dumpname; do
+               echo "Removing $dumpname"
+               rm -f "$dumpname"
+       done
+
+       find -maxdepth 1 -mindepth 1 -type f -name 'weekly_*' -mtime +60 | 
+       while read dumpname; do
+               echo "Removing $dumpname"
+               rm -f "$dumpname"
+       done
+)
+ts "logremove"

 
 echo "Finally, all is done, sending mail and compressing logfile"
 exec > /dev/null 2>&1
 
 
 echo "Finally, all is done, sending mail and compressing logfile"
 exec > /dev/null 2>&1
 

+$masterdir/tools/logs.py "$LOGFILE"
+

 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
 bzip2 -9 "$LOGFILE"
 
 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
 bzip2 -9 "$LOGFILE"