From: Paul Wise Date: Sun, 30 Jan 2011 10:35:19 +0000 (+0800) Subject: Implement NEW/BYHAND/DEFERRED/OSPU/SPU graphs X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=b43634b6faf221ee10fc63116baead379f68f92f;p=dak.git Implement NEW/BYHAND/DEFERRED/OSPU/SPU graphs --- diff --git a/config/backports/cron.hourly b/config/backports/cron.hourly index 5cca477c..5357653e 100755 --- a/config/backports/cron.hourly +++ b/config/backports/cron.hourly @@ -10,8 +10,10 @@ export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars dak import-users-from-passwd dak queue-report -n > $webdir/new.html -dak queue-report -8 -d new,byhand,proposedupdates,oldproposedupdates +dak queue-report -8 -d new,byhand,proposedupdates,oldproposedupdates -r $webdir #dak show-deferred > ${webdir}/deferred.html +dak graph -n new,byhand,proposedupdates,oldproposedupdates -r $webdir -i $webdir -x $configdir/rrd-release-freeze-dates +#dak graph -n new,byhand,proposedupdates,oldproposedupdates,deferred -r $webdir -i $webdir -x $configdir/rrd-release-freeze-dates dak show-new > /dev/null # cd $webdir diff --git a/config/backports/rrd-release-freeze-dates b/config/backports/rrd-release-freeze-dates new file mode 100644 index 00000000..ef70834e --- /dev/null +++ b/config/backports/rrd-release-freeze-dates @@ -0,0 +1,2 @@ +VRULE:1234691928#632a5b:lenny release +VRULE:1281102258#0b19c1:squeeze freeze diff --git a/config/debian/cron.hourly b/config/debian/cron.hourly index ccfbe38c..3446182e 100755 --- a/config/debian/cron.hourly +++ b/config/debian/cron.hourly @@ -10,8 +10,9 @@ export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars dak import-users-from-passwd dak queue-report -n > $webdir/new.html -dak queue-report -8 -d new,byhand,proposedupdates,oldproposedupdates +dak queue-report -8 -d new,byhand,proposedupdates,oldproposedupdates -r $webdir dak show-deferred > ${webdir}/deferred.html +dak graph -n new,byhand,proposedupdates,oldproposedupdates,deferred -r $webdir -i $webdir -x $configdir/rrd-release-freeze-dates # do not run show-new and other stuff in parallel LOCKFILE="$lockdir/unchecked.lock" diff --git a/config/debian/rrd-release-freeze-dates b/config/debian/rrd-release-freeze-dates new file mode 100644 index 00000000..ef70834e --- /dev/null +++ b/config/debian/rrd-release-freeze-dates @@ -0,0 +1,2 @@ +VRULE:1234691928#632a5b:lenny release +VRULE:1281102258#0b19c1:squeeze freeze diff --git a/dak/dak.py b/dak/dak.py index a25afb66..5a659d8c 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -61,6 +61,8 @@ def init(): "Output html for packages in NEW"), ("show-deferred", "Output html and symlinks for packages in DEFERRED"), + ("graph", + "Output graphs of number of packages in various queues"), ("rm", "Remove packages from suites"), diff --git a/dak/graph.py b/dak/graph.py new file mode 100755 index 00000000..06deaef1 --- /dev/null +++ b/dak/graph.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python + +""" Produces a set of graphs of NEW/BYHAND/DEFERRED""" +# Copyright 2011 Paul Wise + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import sys + +import rrdtool +import apt_pkg + +from daklib import utils +from daklib.dak_exceptions import * + +Cnf = None +default_names = ["byhand", "new", "deferred"] + +################################################################################ + +def usage(exit_code=0): + print """Usage: dak queue-graph +Graphs the number of packages in queue directories (usually new and byhand). + + -h, --help show this help and exit. + -r, --rrd=key Directory where rrd files to be updated are stored + -x, --extra-rrd=key File containing extra options to rrdtool graphing + -i, --images=key Directory where image graphs to be updated are stored + -n, --names=key A comma seperated list of rrd files to be scanned + +""" + sys.exit(exit_code) + +################################################################################ + +def graph(rrd_dir, image_dir, name, extra_args, graph, title, start, year_lines=False): + image_file = os.path.join(image_dir, "%s-%s.png" % (name, graph)) + rrd_file = os.path.join(rrd_dir, "%s.rrd" % name) + rrd_args = [image_file, "--start", start] + rrd_args += (""" +--end +now +--width +600 +--height +150 +--vertical-label +packages +--title +Package count: %s +--lower-limit +0 +-E +""" % title).strip().split("\n") + + if year_lines: + rrd_args += ["--x-grid", "MONTH:1:YEAR:1:YEAR:1:31536000:%Y"] + + rrd_args += (""" +DEF:ds1=%s:ds1:AVERAGE +LINE2:ds1#D9382B:Total package count: +VDEF:lds1=ds1,LAST +VDEF:minds1=ds1,MINIMUM +VDEF:maxds1=ds1,MAXIMUM +VDEF:avgds1=ds1,AVERAGE +GPRINT:lds1:%%3.0lf +GPRINT:minds1:\tMin\\: %%3.0lf +GPRINT:maxds1:\tMax\\: %%3.0lf +GPRINT:avgds1:\tAvg\\: %%3.0lf\\j +DEF:ds0=%s:ds0:AVERAGE +VDEF:lds0=ds0,LAST +VDEF:minds0=ds0,MINIMUM +VDEF:maxds0=ds0,MAXIMUM +VDEF:avgds0=ds0,AVERAGE +LINE2:ds0#3069DA:Package count in %s: +GPRINT:lds0:%%3.0lf +GPRINT:minds0:\tMin\\: %%3.0lf +GPRINT:maxds0:\tMax\\: %%3.0lf +GPRINT:avgds0:\tAvg\\: %%3.0lf\\j +""" % (rrd_file, rrd_file, name.upper())).strip().split("\n") + + rrd_args += extra_args + rrdtool.graph(*rrd_args) + +################################################################################ + +def main(): + global Cnf + + Cnf = utils.get_conf() + Arguments = [('h',"help","Graph::Options::Help"), + ('x',"extra-rrd","Graph::Options::Extra-Rrd", "HasArg"), + ('r',"rrd","Graph::Options::Rrd", "HasArg"), + ('i',"images","Graph::Options::Images", "HasArg"), + ('n',"names","Graph::Options::Names", "HasArg")] + for i in [ "help" ]: + if not Cnf.has_key("Graph::Options::%s" % (i)): + Cnf["Graph::Options::%s" % (i)] = "" + + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) + + Options = Cnf.SubTree("Graph::Options") + if Options["Help"]: + usage() + + names = [] + + if Cnf.has_key("Graph::Options::Names"): + for i in Cnf["Graph::Options::Names"].split(","): + names.append(i) + elif Cnf.has_key("Graph::Names"): + names = Cnf.ValueList("Graph::Names") + else: + names = default_names + + extra_rrdtool_args = [] + + if Cnf.has_key("Graph::Options::Extra-Rrd"): + for i in Cnf["Graph::Options::Extra-Rrd"].split(","): + f = open(i) + extra_rrdtool_args.extend(f.read().strip().split("\n")) + f.close() + elif Cnf.has_key("Graph::Extra-Rrd"): + for i in Cnf.ValueList("Graph::Extra-Rrd"): + f = open(i) + extra_rrdtool_args.extend(f.read().strip().split("\n")) + f.close() + + if Cnf.has_key("Graph::Options::Rrd"): + rrd_dir = Cnf["Graph::Options::Rrd"] + elif Cnf.has_key("Dir::Rrd"): + rrd_dir = Cnf["Dir::Rrd"] + else: + print >> sys.stderr, "No directory to read RRD files from\n" + sys.exit(1) + + if Cnf.has_key("Graph::Options::Images"): + image_dir = Cnf["Graph::Options::Images"] + else: + print >> sys.stderr, "No directory to write graph images to\n" + sys.exit(1) + + for name in names: + stdargs = [rrd_dir, image_dir, name, extra_rrdtool_args] + graph(*(stdargs+['day', 'day', 'now-1d'])) + graph(*(stdargs+['week', 'week', 'now-1w'])) + graph(*(stdargs+['month', 'month', 'now-1m'])) + graph(*(stdargs+['year', 'year', 'now-1y'])) + graph(*(stdargs+['5years', '5 years', 'now-5y', True])) + graph(*(stdargs+['10years', '10 years', 'now-10y', True])) + +################################################################################ + +if __name__ == '__main__': + main() diff --git a/dak/queue_report.py b/dak/queue_report.py index e2d8578b..d3777a27 100755 --- a/dak/queue_report.py +++ b/dak/queue_report.py @@ -37,6 +37,10 @@ from copy import copy import glob, os, stat, sys, time import apt_pkg +try: + import rrdtool +except ImportError: + pass from daklib import utils from daklib.queue import Upload @@ -60,6 +64,7 @@ Prints a report of packages in queue directories (usually new and byhand). -s, --sort=key sort output according to key, see below. -a, --age=key if using sort by age, how should time be treated? If not given a default of hours will be used. + -r, --rrd=key Directory where rrd files to be updated are stored -d, --directories=key A comma seperated list of queues to be scanned Sorting Keys: ao=age, oldest first. an=age, newest first. @@ -319,8 +324,41 @@ def table_row(source, version, arch, last_mod, maint, distribution, closes, fing ############################################################ -def process_changes_files(changes_files, type, log): - session = DBConn().session() +def update_graph_database(rrd_dir, type, n_source, n_binary): + if not rrd_dir: + return + + rrd_file = os.path.join(rrd_dir, type.lower()+'.rrd') + update = [rrd_file, "N:%s:%s" % (n_source, n_binary)] + + try: + rrdtool.update(*update) + except rrdtool.error: + create = [rrd_file]+""" +--step +300 +--start +0 +DS:ds0:GAUGE:7200:0:1000 +DS:ds1:GAUGE:7200:0:1000 +RRA:AVERAGE:0.5:1:599 +RRA:AVERAGE:0.5:6:700 +RRA:AVERAGE:0.5:24:775 +RRA:AVERAGE:0.5:288:795 +RRA:MAX:0.5:1:600 +RRA:MAX:0.5:6:700 +RRA:MAX:0.5:24:775 +RRA:MAX:0.5:288:795 +""".strip().split("\n") + rrdtool.create(*create) + rrdtool.update(*update) + except NameError: + pass + +############################################################ + +def process_changes_files(changes_files, type, log, rrd_dir): + #session = DBConn().session() msg = "" cache = {} # Read in all the .changes files @@ -366,6 +404,8 @@ def process_changes_files(changes_files, type, log): per_source_items = per_source.items() per_source_items.sort(sg_compare) + update_graph_database(rrd_dir, type, len(per_source_items), len(changes_files)) + entries = [] max_source_len = 0 max_version_len = 0 @@ -562,6 +602,7 @@ def main(): ('8','822',"Queue-Report::Options::822"), ('s',"sort","Queue-Report::Options::Sort", "HasArg"), ('a',"age","Queue-Report::Options::Age", "HasArg"), + ('r',"rrd","Queue-Report::Options::Rrd", "HasArg"), ('d',"directories","Queue-Report::Options::Directories", "HasArg")] for i in [ "help" ]: if not Cnf.has_key("Queue-Report::Options::%s" % (i)): @@ -589,6 +630,13 @@ def main(): else: directories = [ "byhand", "new" ] + if Cnf.has_key("Queue-Report::Options::Rrd"): + rrd_dir = Cnf["Queue-Report::Options::Rrd"] + elif Cnf.has_key("Dir::Rrd"): + rrd_dir = Cnf["Dir::Rrd"] + else: + rrd_dir = None + f = None if Cnf.has_key("Queue-Report::Options::822"): # Open the report file @@ -596,11 +644,22 @@ def main(): for directory in directories: changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)])) - process_changes_files(changes_files, directory, f) + process_changes_files(changes_files, directory, f, rrd_dir) if Cnf.has_key("Queue-Report::Options::822"): f.close() + if Cnf.has_key("Queue-Report::Options::New"): + for dir in directories: + print """ +

%s, last day

+

%s, last week

+

%s, last month

+

%s, last year

+

%s, last 5 years

+

%s, last 10 years

+""" % ((dir,)*12) + if Cnf.has_key("Queue-Report::Options::New"): footer() diff --git a/dak/show_deferred.py b/dak/show_deferred.py index fac16927..afa12b4b 100755 --- a/dak/show_deferred.py +++ b/dak/show_deferred.py @@ -88,7 +88,15 @@ def header(): """ def footer(): - res = "

Timestamp: %s (UTC)

" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) + res = """ +

deferred, last day

+

deferred, last week

+

deferred, last month

+

deferred, last year

+

deferred, last 5 years

+

deferred, last 10 years

+""" + res += "

Timestamp: %s (UTC)

" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) res += """ Valid HTML 4.01! diff --git a/docs/README.first b/docs/README.first index bcfa1f3f..86cb73cd 100644 --- a/docs/README.first +++ b/docs/README.first @@ -41,6 +41,7 @@ o Information display: * dak ls - shows information about package(s) * dak queue-report - shows information about package(s) in queue/ * dak override - can show you individual override entries + * dak graph - creates some pretty graphs of queue sizes over time Generic and useful, but only for those with existing archives -------------------------------------------------------------