From: Mike O'Connor Date: Fri, 13 Mar 2009 13:30:41 +0000 (-0400) Subject: merge from master X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=a650ca814d48fbe72cde347828d405e5da780b80;hp=5c4242ac752fcc98787ecf0d84927f907808e36e;p=dak.git merge from master Signed-off-by: Mike O'Connor --- diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall index 383ba334..599a96d4 100755 --- a/config/debian/cron.dinstall +++ b/config/debian/cron.dinstall @@ -43,6 +43,12 @@ function cleanup() { rm -f ${LOCK_ACCEPTED} } +# If we error out this one is called, *FOLLOWED* by cleanup above +function onerror() { + ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S") + cat "$LOGFILE" | mail -s "ATTENTION ATTENTION! dinstall error at ${ERRDATE} (Be quiet, Brain, or I'll stab you with a Q-tip)" cron@ftp-master.debian.org +} + ######################################################################## # the actual dinstall functions follow # ######################################################################## @@ -74,9 +80,20 @@ function pgdump_pre() { function pgdump_post() { log "Creating post-daily-cron-job backup of projectb database..." cd $base/backup - POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) - pg_dump projectb > $POSTDUMP - ln -sf $POSTDUMP current + POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S) + pg_dump projectb > $base/backup/dump_$POSTDUMP + pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP + ln -sf $base/backup/dump_$POSTDUMP current + ln -sf $base/backup/dumpall_$POSTDUMP currentall +} + +# Load the dak-dev projectb +function pgdakdev() { + cd $base/backup + echo "drop database projectb" | psql -p 5433 template1 + cat currentall | psql -p 5433 template1 + createdb -p 5433 -T template0 projectb + fgrep -v '\connect' current | psql -p 5433 projectb } # Updating various files @@ -267,9 +284,9 @@ function bts() { } function merkel2() { - # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached + # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached log "Trigger merkels projectb sync" - ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 + ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1 } function runparts() { @@ -329,20 +346,29 @@ function savetimestamp() { echo ${NOW} > "${dbdir}/dinstallstart" } +function maillogfile() { + cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org +} + function renamelogfile() { - if [ -f "${dbdir}/dinstallstart" ]; then - RENAMETO=$(cat "${dbdir}/dinstallstart") - mv "$LOGFILE" "$logdir/dinstall_${RENAMETO}.log" - logstats "$logdir/dinstall_${RENAMETO}.log" - bzip2 -9 "$logdir/dinstall_${RENAMETO}.log" - else - error "Problem, I don't know when dinstall started, unable to do log statistics." - NOW=`date "+%Y.%m.%d-%H:%M:%S"` - mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" - bzip2 -9 "$logdir/dinstall_${NOW}.log" - fi + if [ -f "${dbdir}/dinstallstart" ]; then + NOW=$(cat "${dbdir}/dinstallstart") + maillogfile + mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" + logstats "$logdir/dinstall_${NOW}.log" + bzip2 -9 "$logdir/dinstall_${NOW}.log" + else + error "Problem, I don't know when dinstall started, unable to do log statistics." + NOW=`date "+%Y.%m.%d-%H:%M:%S"` + maillogfile + mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" + bzip2 -9 "$logdir/dinstall_${NOW}.log" + fi } +function testingsourcelist() { + dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list +} ######################################################################## ######################################################################## @@ -384,10 +410,10 @@ function stage() { # it has to cd first! cd ${configdir} - if [ -f "${LOCK_STOP}" ]; then - log "${LOCK_STOP} exists, exiting immediately" - exit 42 - fi + if [ -f "${LOCK_STOP}" ]; then + log "${LOCK_STOP} exists, exiting immediately" + exit 42 + fi if [ "${ERR}" = "false" ]; then set +e @@ -406,10 +432,10 @@ function stage() { ts "${TIME}" fi - if [ -f "${LOCK_STOP}" ]; then - log "${LOCK_STOP} exists, exiting immediately" - exit 42 - fi + if [ -f "${LOCK_STOP}" ]; then + log "${LOCK_STOP} exists, exiting immediately" + exit 42 + fi } ######################################################################## @@ -466,7 +492,8 @@ LOCK_BRITNEY="$lockdir/britney.lock" LOCK_STOP="$lockdir/archive.stop" lockfile -l 3600 "${LOCK_DAILY}" -trap cleanup EXIT ERR TERM HUP INT QUIT +trap onerror ERR +trap cleanup EXIT TERM HUP INT QUIT touch "${LOCK_BRITNEY}" @@ -663,6 +690,14 @@ GO=( ) stage $GO +GO=( + FUNC="pgdakdev" + TIME="dak-dev db" + ARGS="" + ERR="false" +) +stage $GO + GO=( FUNC="expire" TIME="expire_dumps" @@ -691,7 +726,7 @@ GO=( FUNC="bts" TIME="" ARGS="" - ERR="" + ERR="false" ) stage $GO @@ -729,6 +764,13 @@ GO=( ) stage $GO +GO=( + FUNC="testingsourcelist" + TIME="" + ARGS="" + ERR="false" +) + rm -f ${LOCK_BRITNEY} GO=( @@ -751,8 +793,6 @@ log "Daily cron scripts successful, all done" exec > "$logdir/afterdinstall.log" 2>&1 -cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org - GO=( FUNC="renamelogfile" TIME="" diff --git a/config/debian/dak.conf b/config/debian/dak.conf index 60df7567..b8560c5e 100644 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@ -8,7 +8,7 @@ Dinstall SigningKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg"; SigningPubKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg"; SigningKeyIds "6070D3A1"; - SendmailCommand "/usr/sbin/sendmail -odq -oi -t"; + SendmailCommand "/usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org"; MyEmailAddress "Debian Installer "; MyAdminAddress "ftpmaster@debian.org"; MyHost "debian.org"; // used for generating user@my_host addresses in e.g. manual_reject() @@ -317,12 +317,12 @@ Suite { Stable; }; - MustBeOlderThan - { - Testing; - Unstable; - Experimental; - }; +// MustBeOlderThan +// { +// Testing; +// Unstable; +// Experimental; +// }; Enhances { Stable; @@ -378,7 +378,7 @@ Suite MustBeNewerThan { Stable; - Proposed-Updates; +// Proposed-Updates; Testing; }; MustBeOlderThan @@ -439,7 +439,7 @@ Suite MustBeNewerThan { Stable; - Proposed-Updates; +// Proposed-Updates; Testing; Testing-Proposed-Updates; }; @@ -473,7 +473,7 @@ Suite MustBeNewerThan { Stable; - Proposed-Updates; +// Proposed-Updates; Testing; Testing-Proposed-Updates; Unstable; diff --git a/config/debian/extensions.py b/config/debian/extensions.py index 0bb51bd7..83ae3076 100644 --- a/config/debian/extensions.py +++ b/config/debian/extensions.py @@ -2,7 +2,7 @@ import sys, os, textwrap import apt_pkg import daklib.utils, daklib.database -import syck +import yaml import daklib.extensions from daklib.extensions import replace_dak_function @@ -30,8 +30,8 @@ def check_transition(): sourcefile = file(transpath, 'r') sourcecontent = sourcefile.read() try: - transitions = syck.load(sourcecontent) - except syck.error, msg: + transitions = yaml.load(sourcecontent) + except yaml.YAMLError, msg: # This shouldn't happen, there is a wrapper to edit the file which # checks it, but we prefer to be safe than ending up rejecting # everything. diff --git a/dak/bts_categorize.py b/dak/bts_categorize.py index fd9cd090..1e0349d0 100755 --- a/dak/bts_categorize.py +++ b/dak/bts_categorize.py @@ -122,10 +122,14 @@ class BugClassifier(object): controls = "" bc = BugClassifier() - for bug in bc.unclassified_bugs(): - controls += bc.classify_bug(bug) - - return controls + try: + for bug in bc.unclassified_bugs(): + controls += bc.classify_bug(bug) + + return controls + except: + log.error("couldn't retreive bugs from soap interface: %s" % sys.exc_info()[0]) + return None def send_email(commands, simulate=False): global Cnf diff --git a/dak/generate_index_diffs.py b/dak/generate_index_diffs.py index 2b06146b..774e0467 100755 --- a/dak/generate_index_diffs.py +++ b/dak/generate_index_diffs.py @@ -37,7 +37,9 @@ import tempfile import subprocess import time import apt_pkg +import pg from daklib import utils +from daklib import database ################################################################################ @@ -278,7 +280,7 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 14): def main(): - global Cnf, Options, Logger + global Cnf, Options, Logger, projectB os.umask(0002) @@ -309,6 +311,9 @@ def main(): if Options.has_key("RootDir"): Cnf["Dir::Root"] = Options["RootDir"] + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + database.init(Cnf, projectB) + if not suites: suites = Cnf.SubTree("Suite").List() @@ -322,7 +327,9 @@ def main(): suite = suite.lower() - architectures = SuiteBlock.ValueList("Architectures") + architectures = database.get_suite_architectures(suite) + if architectures == None: + architectures = [] if SuiteBlock.has_key("Components"): components = SuiteBlock.ValueList("Components") diff --git a/dak/generate_releases.py b/dak/generate_releases.py index 95b32c17..983c8573 100755 --- a/dak/generate_releases.py +++ b/dak/generate_releases.py @@ -237,6 +237,13 @@ def main (): files = [] if AptCnf.has_key("tree::%s" % (tree)): + if AptCnf.has_key("tree::%s::Contents" % (tree)): + pass + else: + for x in os.listdir("%s/%s" % (Cnf["Dir::Root"], tree)): + if x.startswith('Contents-'): + files.append(x) + for sec in AptCnf["tree::%s::Sections" % (tree)].split(): for arch in AptCnf["tree::%s::Architectures" % (tree)].split(): if arch == "source": diff --git a/dak/init_db.py b/dak/init_db.py index 31a2a5a7..af0a03b2 100755 --- a/dak/init_db.py +++ b/dak/init_db.py @@ -23,8 +23,8 @@ import psycopg2, sys import apt_pkg from daklib import utils -from daklib.dbconn import DBConn -from daklib.config import Config +from daklib.DBConn import DBConn +from daklib.Config import Config ################################################################################ diff --git a/dak/process_new.py b/dak/process_new.py index 8741d25e..acc4522f 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -1,9 +1,12 @@ #!/usr/bin/env python # vim:set et ts=4 sw=4: -""" Handles NEW and BYHAND packages """ -# Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 James Troup +""" Handles NEW and BYHAND packages +@contact: Debian FTP Master +@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -37,7 +40,13 @@ ################################################################################ -import copy, errno, os, readline, stat, sys, time +import copy +import errno +import os +import readline +import stat +import sys +import time import apt_pkg, apt_inst import examine_package from daklib import database @@ -47,10 +56,10 @@ from daklib import utils from daklib.regexes import re_no_epoch, re_default_answer, re_isanum # Globals -Cnf = None +Cnf = None #: Configuration, apt_pkg.Configuration Options = None Upload = None -projectB = None +projectB = None #: database connection, pgobject Logger = None Priorities = None @@ -727,6 +736,9 @@ def init(): Cnf["Process-New::Options::%s" % (i)] = "" changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) + if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""): + changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"]) + Options = Cnf.SubTree("Process-New::Options") if Options["Help"]: diff --git a/daklib/binary.py b/daklib/binary.py index 8dd9b301..57b0f65d 100755 --- a/daklib/binary.py +++ b/daklib/binary.py @@ -191,7 +191,6 @@ class Binary(object): except: traceback.print_exc() - result = False os.chdir(cwd) return result diff --git a/docs/meta/lenny/README.Debian b/docs/meta/lenny/README.Debian new file mode 100644 index 00000000..49659fcb --- /dev/null +++ b/docs/meta/lenny/README.Debian @@ -0,0 +1,11 @@ +ftpmaster meta package for DSA + + +This is a dummy package that makes Debian's package management +system believe that certain packages needed for ftpmaster have +to be installed. The intention is that DSA can easily see what +needs to be there. + +If you, for whatever reason, need a package added to this meta- +packages dependency list, contact ftpmaster@debian.org, NOT +the Debian admins. diff --git a/docs/meta/lenny/changelog b/docs/meta/lenny/changelog new file mode 100644 index 00000000..f2449b88 --- /dev/null +++ b/docs/meta/lenny/changelog @@ -0,0 +1,6 @@ +ftpmaster-lenny (1.0) unstable; urgency=low + + * New "package", to help DSA + + -- Joerg Jaspert Mon, 09 Mar 2009 14:09:09 +0100 + diff --git a/docs/meta/lenny/copyright b/docs/meta/lenny/copyright new file mode 100644 index 00000000..f778e680 --- /dev/null +++ b/docs/meta/lenny/copyright @@ -0,0 +1,25 @@ +This package was put together by: + + Joerg Jaspert on Mon, 09 Mar 2009 14:07:44 +0100 + +Copyright: + + + +License: + + This package is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2 as + published by the Free Software Foundation. + + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this package; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +On Debian systems, the complete text of the GNU General +Public License can be found in `/usr/share/common-licenses/GPL'. diff --git a/docs/meta/lenny/ftpmaster-lenny b/docs/meta/lenny/ftpmaster-lenny new file mode 100644 index 00000000..14488625 --- /dev/null +++ b/docs/meta/lenny/ftpmaster-lenny @@ -0,0 +1,22 @@ +Section: devel +Priority: optional +Standards-Version: 3.8.1 + +Package: ftpmaster-lenny +Version: 1.0 +Maintainer: Debian FTP Master +Depends: apt-utils, bicyclerepair, binutils-multiarch, build-essential, bzip2, cron, curl, cvs, debian-el, debian-bug, dpkg-dev-el, easypg, devscripts, emacs-goodies-el, emacs22-nox, gnupg, gpgv, graphviz, ikiwiki, irb, libapt-pkg-dev, libdbd-pg-ruby, lintian, mc, mutt, postgresql-plperl-8.3, pychecker, pylint, pymacs, python, python-apt, python-btsutils, python-debian, python-epydoc, python-ldap, python-mode, python-numpy, python-psycopg2, python-pygresql, python-pyrss2gen, python-soappy, python-yaml, r-base, rsync, ruby, ruby-elisp, subversion, git-core, symlinks +Architecture: all +Copyright: copyright +Changelog: changelog +Readme: README.Debian +Description: Meta package for DSA listing ftpmaster needs + This is a small meta package for the Debian System Administrators + so DSA easily knows (and can keep installed) all packages + Ftpmaster needs. + . + If, for whatever reason, you need a package added to this ones + Dependencies, ask ftpmaster, not the Debian admins. + . + This is not only "What DAK needs", but a general "FTPMaster needs this + to do the work" diff --git a/tools/removals.pl b/tools/removals.pl index b07845bf..839fc618 100755 --- a/tools/removals.pl +++ b/tools/removals.pl @@ -71,7 +71,7 @@ for my $removal (@removals ) { $rss->add_item(title => "$reason", link => "http://ftp-master.debian.org/removals.txt?" . $link, - description => qq[<pre>$body</pre>], + description => qq[
$body
], dc => { creator => "$ftpmaster", }