From: Mike O'Connor Date: Wed, 25 Feb 2009 19:43:24 +0000 (-0500) Subject: Merge branch 'master' into content_generation X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=f96dff8bac6b5d13eceb2bf1bbe59d5998a27cce;hp=7bd3a212dc79eadd396f101e43983450f1dca53a;p=dak.git Merge branch 'master' into content_generation --- diff --git a/config/debian/common b/config/debian/common new file mode 100644 index 00000000..ad10ea6c --- /dev/null +++ b/config/debian/common @@ -0,0 +1,26 @@ +# log something (basically echo it together with a timestamp) +# +# Set $PROGRAM to a string to have it added to the output. +function log () { + if [ -z "${PROGRAM}" ]; then + echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@" + else + echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@" + fi +} + +# log the message using log() but then also send a mail +# to the address configured in MAILTO (if non-empty) +function log_error () { + log "$@" + if [ -z "${MAILTO}" ]; then + echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO} + fi +} + +# debug log, only output when DEBUG=1 +function debug () { + if [ $DEBUG -eq 1 ]; then + log "$*" + fi +} diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall index babef86a..383ba334 100755 --- a/config/debian/cron.dinstall +++ b/config/debian/cron.dinstall @@ -28,32 +28,8 @@ export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars ######################################################################## # Functions # ######################################################################## -# log something (basically echo it together with a timestamp) -# -# Set $PROGRAM to a string to have it added to the output. -function log () { - if [ -z "${PROGRAM}" ]; then - echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@" - else - echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@" - fi -} - -# log the message using log() but then also send a mail -# to the address configured in MAILTO (if non-empty) -function log_error () { - log "$@" - if [ -z "${MAILTO}" ]; then - echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO} - fi -} - -# debug log, only output when DEBUG=1 -function debug () { - if [ $DEBUG -eq 1 ]; then - log "$*" - fi -} +# common functions are "outsourced" +. "${configdir}/common" # Timestamp. Used for dinstall stat graphs function ts() { @@ -426,14 +402,14 @@ function stage() { touch "${stagedir}/${FUNC}" + if [ -n "${TIME}" ]; then + ts "${TIME}" + fi + if [ -f "${LOCK_STOP}" ]; then log "${LOCK_STOP} exists, exiting immediately" exit 42 fi - - if [ -n "${TIME}" ]; then - ts "${TIME}" - fi } ######################################################################## @@ -461,8 +437,14 @@ fi # How many logfiles to keep LOGROTATE=${LOGROTATE:-400} +# Marker for dinstall start +DINSTALLSTART="${lockdir}/dinstallstart" +# Marker for dinstall end +DINSTALLEND="${lockdir}/dinstallend" + # Timestamps start at -1. so first gets 0 TS=-1 +touch "${DINSTALLSTART}" ts "startup" # Tell everyone we are doing some work @@ -767,7 +749,7 @@ stage $GO log "Daily cron scripts successful, all done" -exec > /dev/null 2>&1 +exec > "$logdir/afterdinstall.log" 2>&1 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org @@ -782,4 +764,5 @@ stage $GO # Now, at the very (successful) end of dinstall, make sure we remove # our stage files, so the next dinstall run will do it all again. -rm -f "${stagedir}/*" +rm -f ${stagedir}/* +touch "${DINSTALLEND}" diff --git a/config/debian/cron.reboot b/config/debian/cron.reboot new file mode 100755 index 00000000..b3c0b09a --- /dev/null +++ b/config/debian/cron.reboot @@ -0,0 +1,72 @@ +#!/bin/bash +# No way I try to deal with a crippled sh just for POSIX foo. + +# Copyright (C) 2009 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +# exit on errors +set -e +# make sure to only use defined variables +set -u + +# import the general variable set. +export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars +. $SCRIPTVARS + +# common functions are "outsourced" +. "${configdir}/common" + +# usually we are not using debug logs. Set to 1 if you want them. +DEBUG=0 + +# our name +PROGRAM="dinstall_reboot" + +# where do we want mails to go? For example log entries made with error() +if [ "x$(hostname -s)x" != "xriesx" ]; then + # Not our ftpmaster host + MAILTO=${MAILTO:-"root"} +else + # Yay, ftpmaster + MAILTO=${MAILTO:-"ftpmaster@debian.org"} +fi + +# Marker for dinstall start +DINSTALLSTART="${lockdir}/dinstallstart" +# Marker for dinstall end +DINSTALLEND="${lockdir}/dinstallend" + +set +e +starttime=$(/usr/bin/stat -c %Z "${DINSTALLSTART}") +endtime=$(/usr/bin/stat -c %Z "${DINSTALLEND}") +set -e + +if [ ${endtime} -gt ${starttime} ]; then + # Great, last dinstall run did seem to end without trouble, no need to rerun + log "Last dinstall run did end without trouble, not rerunning" + exit 0 +else + # Hrm, it looks like we did not successfully end the last run. + # This either means dinstall did abort due to an error, or we had a reboot + # No way to tell, so lets restart and see what happens. + + # Make sure we are not fooled by some random touching of the files, only + # really restart if we have the first stage stampfile there, indicating that + # dinstall got started + if [ -f "${stagedir}/savetimestamp" ]; then + log "Seems we have to restart a dinstall run after reboot" + ${configdir}/cron.dinstall + fi +fi diff --git a/dak/clean_suites.py b/dak/clean_suites.py index 5523d633..7537eb12 100755 --- a/dak/clean_suites.py +++ b/dak/clean_suites.py @@ -39,6 +39,7 @@ Cnf = None Options = None now_date = None; # mark newly "deleted" things as deleted "now" delete_date = None; # delete things marked "deleted" earler than this +max_delete = None ################################################################################ @@ -47,7 +48,8 @@ def usage (exit_code=0): Clean old packages from suites. -n, --no-action don't do anything - -h, --help show this help and exit""" + -h, --help show this help and exit + -m, --maximum maximum number of files to remove""" sys.exit(exit_code) ################################################################################ @@ -183,7 +185,7 @@ def clean_binaries(): ######################################## def clean(): - global delete_date, now_date + global delete_date, now_date, max_delete count = 0 size = 0 @@ -204,7 +206,12 @@ def clean(): sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before))) # Delete files from the pool - q = projectB.query("SELECT l.path, f.filename FROM location l, files f WHERE f.last_used <= '%s' AND l.id = f.location" % (delete_date)) + query = "SELECT l.path, f.filename FROM location l, files f WHERE f.last_used <= '%s' AND l.id = f.location" % (delete_date) + if max_delete is not None: + query += " LIMIT %d" % maximum + sys.stdout.write("Limiting removals to %d" % Cnf["Clean-Suites::Options::Maximum"]) + + q=projectB.query(query) for i in q.getresult(): filename = i[0] + i[1] if not os.path.exists(filename): @@ -321,19 +328,31 @@ def clean_queue_build(): ################################################################################ def main(): - global Cnf, Options, projectB, delete_date, now_date + global Cnf, Options, projectB, delete_date, now_date, max_delete Cnf = utils.get_conf() - for i in ["Help", "No-Action" ]: + for i in ["Help", "No-Action", "Maximum" ]: if not Cnf.has_key("Clean-Suites::Options::%s" % (i)): Cnf["Clean-Suites::Options::%s" % (i)] = "" Arguments = [('h',"help","Clean-Suites::Options::Help"), - ('n',"no-action","Clean-Suites::Options::No-Action")] + ('n',"no-action","Clean-Suites::Options::No-Action"), + ('m',"maximum","Clean-Suites::Options::Maximum", "HasArg")] apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Clean-Suites::Options") + if Cnf["Clean-Suites::Options::Maximum"] != "": + try: + # Only use Maximum if it's an integer + max_delete = int(Cnf["Clean-Suites::Options::Maximum"]) + if max_delete < 1: + utils.fubar("If given, Maximum must be at least 1") + except ValueError, e: + utils.fubar("If given, Maximum must be an integer") + else: + max_delete = None + if Options["Help"]: usage() diff --git a/dak/generate_releases.py b/dak/generate_releases.py index 1c258bd5..95b32c17 100755 --- a/dak/generate_releases.py +++ b/dak/generate_releases.py @@ -26,6 +26,7 @@ import sys, os, stat, time, pg import gzip, bz2 import apt_pkg from daklib import utils +from daklib import database from daklib.dak_exceptions import * ################################################################################ @@ -149,6 +150,7 @@ def main (): apt_pkg.ReadConfigFileISC(AptCnf, Options["Apt-Conf"]) projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + database.init(Cnf, projectB) if not suites: suites = Cnf.SubTree("Suite").List() @@ -176,6 +178,10 @@ def main (): if qs[0][0] != "-": version = qs[0][0] if qs[0][1]: description = qs[0][1] + architectures = database.get_suite_architectures(suite) + if architectures == None: + architectures = [] + if SuiteBlock.has_key("NotAutomatic"): notautomatic = "yes" else: @@ -221,7 +227,7 @@ def main (): if notautomatic != "": out.write("NotAutomatic: %s\n" % (notautomatic)) - out.write("Architectures: %s\n" % (" ".join(filter(utils.real_arch, SuiteBlock.ValueList("Architectures"))))) + out.write("Architectures: %s\n" % (" ".join(filter(utils.real_arch, architectures)))) if components: out.write("Components: %s\n" % (" ".join(components)))