X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fqueue_report.py;h=29d0788124082c672d84eebebca86670d2a8608d;hb=076955c92fab7438ca1076e17ed07a997e9bd4de;hp=d33b19e385f3176e005accc27f71cccc073ce2c7;hpb=59fd5aa2a8be3b76dbc968429c457f096adfa472;p=dak.git diff --git a/dak/queue_report.py b/dak/queue_report.py index d33b19e3..29d07881 100755 --- a/dak/queue_report.py +++ b/dak/queue_report.py @@ -1,8 +1,7 @@ #!/usr/bin/env python # Produces a report on NEW and BYHAND packages -# Copyright (C) 2001, 2002, 2003, 2005 James Troup -# $Id: helena,v 1.6 2005-11-15 09:50:32 ajt Exp $ +# Copyright (C) 2001, 2002, 2003, 2005, 2006 James Troup # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,34 +34,38 @@ ################################################################################ -import copy, glob, os, stat, sys, time; -import apt_pkg; -import katie, utils; -import encodings.utf_8, encodings.latin_1, string; +import copy, glob, os, stat, sys, time +import apt_pkg +import cgi +from daklib import queue +from daklib import utils +from daklib.dak_exceptions import * -Cnf = None; -Katie = None; -direction = []; -row_number = 0; +Cnf = None +Upload = None +direction = [] +row_number = 0 ################################################################################ def usage(exit_code=0): - print """Usage: helena + print """Usage: dak queue-report Prints a report of packages in queue directories (usually new and byhand). -h, --help show this help and exit. + -8, --822 writes 822 formated output to the location set in dak.conf -n, --new produce html-output -s, --sort=key sort output according to key, see below. -a, --age=key if using sort by age, how should time be treated? If not given a default of hours will be used. + -d, --directories=key A comma seperated list of queues to be scanned Sorting Keys: ao=age, oldest first. an=age, newest first. na=name, ascending nd=name, descending nf=notes, first nl=notes, last Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years - + """ sys.exit(exit_code) @@ -70,398 +73,494 @@ Prints a report of packages in queue directories (usually new and byhand). def plural(x): if x > 1: - return "s"; + return "s" else: - return ""; + return "" ################################################################################ def time_pp(x): if x < 60: - unit="second"; + unit="second" elif x < 3600: - x /= 60; - unit="minute"; + x /= 60 + unit="minute" elif x < 86400: - x /= 3600; - unit="hour"; + x /= 3600 + unit="hour" elif x < 604800: - x /= 86400; - unit="day"; + x /= 86400 + unit="day" elif x < 2419200: - x /= 604800; - unit="week"; + x /= 604800 + unit="week" elif x < 29030400: - x /= 2419200; - unit="month"; + x /= 2419200 + unit="month" else: - x /= 29030400; - unit="year"; - x = int(x); - return "%s %s%s" % (x, unit, plural(x)); + x /= 29030400 + unit="year" + x = int(x) + return "%s %s%s" % (x, unit, plural(x)) ################################################################################ def sg_compare (a, b): - a = a[1]; - b = b[1]; + a = a[1] + b = b[1] """Sort by have note, time of oldest upload.""" # Sort by have note - a_note_state = a["note_state"]; - b_note_state = b["note_state"]; + a_note_state = a["note_state"] + b_note_state = b["note_state"] if a_note_state < b_note_state: - return -1; + return -1 elif a_note_state > b_note_state: - return 1; + return 1 # Sort by time of oldest upload - return cmp(a["oldest"], b["oldest"]); + return cmp(a["oldest"], b["oldest"]) ############################################################ def sortfunc(a,b): - for sorting in direction: - (sortkey, way, time) = sorting; - ret = 0 - if time == "m": - x=int(a[sortkey]/60) - y=int(b[sortkey]/60) - elif time == "h": - x=int(a[sortkey]/3600) - y=int(b[sortkey]/3600) - elif time == "d": - x=int(a[sortkey]/86400) - y=int(b[sortkey]/86400) - elif time == "w": - x=int(a[sortkey]/604800) - y=int(b[sortkey]/604800) - elif time == "o": - x=int(a[sortkey]/2419200) - y=int(b[sortkey]/2419200) - elif time == "y": - x=int(a[sortkey]/29030400) - y=int(b[sortkey]/29030400) - else: - x=a[sortkey] - y=b[sortkey] - if x < y: - ret = -1 - elif x > y: - ret = 1 - if ret != 0: - if way < 0: - ret = ret*-1 - return ret - return 0 + for sorting in direction: + (sortkey, way, time) = sorting + ret = 0 + if time == "m": + x=int(a[sortkey]/60) + y=int(b[sortkey]/60) + elif time == "h": + x=int(a[sortkey]/3600) + y=int(b[sortkey]/3600) + elif time == "d": + x=int(a[sortkey]/86400) + y=int(b[sortkey]/86400) + elif time == "w": + x=int(a[sortkey]/604800) + y=int(b[sortkey]/604800) + elif time == "o": + x=int(a[sortkey]/2419200) + y=int(b[sortkey]/2419200) + elif time == "y": + x=int(a[sortkey]/29030400) + y=int(b[sortkey]/29030400) + else: + x=a[sortkey] + y=b[sortkey] + if x < y: + ret = -1 + elif x > y: + ret = 1 + if ret != 0: + if way < 0: + ret = ret*-1 + return ret + return 0 ############################################################ def header(): - print """ - - Debian NEW and BYHAND Packages - - - - -
- - - - Debian Project -
-
- - - - - - - - - - -
- Debian NEW and BYHAND Packages -
- -
- """ + print """ + + + + + + + Debian NEW and BYHAND Packages + + + + +
+ + corner image + corner image + corner image + corner image + + Debian NEW and BYHAND Packages + +
+ """ def footer(): - print "

Timestamp: %s (UTC)

" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) - print "

Hint: Age is the youngest upload of the package, if there is more than one version.

" - print "

You may want to look at the REJECT-FAQ for possible reasons why one of the above packages may get rejected.

" - print """ - Valid HTML 4.01! - - Valid CSS! + print "

Timestamp: %s (UTC)

" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) + + print """ +
+

Hint: Age is the youngest upload of the package, if there is more than + one version.
+ You may want to look at the REJECT-FAQ + for possible reasons why one of the above packages may get rejected.

+

+ Valid XHTML 1.0 Strict + + Valid CSS! + +

+
""" - print "" -def table_header(type): +def table_header(type, source_count, total_count): print "

Summary for: %s

" % (type) - print """
- - - - - - - - - - """ + print """ +
PackageVersionArchDistributionAgeMaintainerCloses
+ + + + + + + + + + + + + + """ -def table_footer(type, source_count, total_count): - print "
+ """ + print "Package count in %s: %s |  Total Package count: %s" % (type, source_count, total_count) + print """ +
PackageVersionArchDistributionAgeUpload infoCloses

\n" - print "

Package count in %s: %s\n" % (type, source_count) - print "
Total Package count: %s

\n" % (total_count) +def table_footer(type): + print "" -def force_to_latin(s): - """Forces a string to Latin-1.""" - latin1_s = unicode(s,'utf-8'); - return latin1_s.encode('iso8859-1', 'replace'); +def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby): -def table_row(source, version, arch, last_mod, maint, distribution, closes): + global row_number - global row_number; + trclass = "sid" + for dist in distribution: + if dist == "experimental": + trclass = "exp" if row_number % 2 != 0: - print "" + print "" % (trclass) else: - print "" + print "" % (trclass) - tdclass = "sid" - for dist in distribution: - if dist == "experimental": - tdclass = "exp"; - print "%s" % (tdclass, source); - print "" % (tdclass) + print "%s" % (source) + print "" for vers in version.split(): - print "%s
" % (vers); - print "%s" % (tdclass, arch, tdclass); + print "%s
" % (source, utils.html_escape(vers), utils.html_escape(vers)) + print "" + print "%s" % (arch) + print "" for dist in distribution: - print "%s
" % (dist); - print "%s" % (tdclass, last_mod); - (name, mail) = maint.split(":"); - name = force_to_latin(name); - - print "%s" % (tdclass, mail, name); - print "" % (tdclass) + print "%s
" % (dist) + print "" + print "%s" % (last_mod) + (name, mail) = maint.split(":") + + print "" + print "Maintainer: %s
" % (utils.html_escape(mail), utils.html_escape(name)) + (name, mail) = changedby.split(":") + print "Changed-By: %s
" % (utils.html_escape(mail), utils.html_escape(name)) + + try: + (login, domain) = sponsor.split("@") + print "Sponsor: %s@debian.org
" % (utils.html_escape(login), utils.html_escape(login)) + except: + pass + + print "Fingerprint: %s" % (fingerprint) + print "" + + print "" for close in closes: - print "#%s
" % (close, close); - print ""; - row_number+=1; - + print "#%s
" % (utils.html_escape(close), utils.html_escape(close)) + print "" + row_number+=1 + ############################################################ -def process_changes_files(changes_files, type): - msg = ""; - cache = {}; +def process_changes_files(changes_files, type, log): + msg = "" + cache = {} # Read in all the .changes files for filename in changes_files: try: - Katie.pkg.changes_file = filename; - Katie.init_vars(); - Katie.update_vars(); - cache[filename] = copy.copy(Katie.pkg.changes); - cache[filename]["filename"] = filename; + Upload.pkg.changes_file = filename + Upload.init_vars() + Upload.update_vars() + cache[filename] = copy.copy(Upload.pkg.changes) + cache[filename]["filename"] = filename except: - break; + break # Divide the .changes into per-source groups - per_source = {}; + per_source = {} for filename in cache.keys(): - source = cache[filename]["source"]; + source = cache[filename]["source"] if not per_source.has_key(source): - per_source[source] = {}; - per_source[source]["list"] = []; - per_source[source]["list"].append(cache[filename]); + per_source[source] = {} + per_source[source]["list"] = [] + per_source[source]["list"].append(cache[filename]) # Determine oldest time and have note status for each source group for source in per_source.keys(): - source_list = per_source[source]["list"]; - first = source_list[0]; - oldest = os.stat(first["filename"])[stat.ST_MTIME]; - have_note = 0; + source_list = per_source[source]["list"] + first = source_list[0] + oldest = os.stat(first["filename"])[stat.ST_MTIME] + have_note = 0 for d in per_source[source]["list"]: - mtime = os.stat(d["filename"])[stat.ST_MTIME]; - if Cnf.has_key("Helena::Options::New"): + mtime = os.stat(d["filename"])[stat.ST_MTIME] + if Cnf.has_key("Queue-Report::Options::New"): if mtime > oldest: - oldest = mtime; + oldest = mtime else: if mtime < oldest: - oldest = mtime; - have_note += (d.has_key("lisa note")); - per_source[source]["oldest"] = oldest; + oldest = mtime + have_note += (d.has_key("process-new note")) + per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none elif have_note < len(source_list): per_source[source]["note_state"] = 1; # some else: per_source[source]["note_state"] = 2; # all - per_source_items = per_source.items(); - per_source_items.sort(sg_compare); - - entries = []; - max_source_len = 0; - max_version_len = 0; - max_arch_len = 0; - maintainer = {}; - maint=""; - distribution=""; - closes=""; - source_exists=""; + per_source_items = per_source.items() + per_source_items.sort(sg_compare) + + entries = [] + max_source_len = 0 + max_version_len = 0 + max_arch_len = 0 for i in per_source_items: - last_modified = time.time()-i[1]["oldest"]; - source = i[1]["list"][0]["source"]; + maintainer = {} + maint="" + distribution="" + closes="" + fingerprint="" + changeby = {} + changedby="" + sponsor="" + filename=i[1]["list"][0]["filename"] + last_modified = time.time()-i[1]["oldest"] + source = i[1]["list"][0]["source"] if len(source) > max_source_len: - max_source_len = len(source); - arches = {}; - versions = {}; + max_source_len = len(source) + arches = {} + versions = {} for j in i[1]["list"]: - if Cnf.has_key("Helena::Options::New"): + if Cnf.has_key("Queue-Report::Options::New") or Cnf.has_key("Queue-Report::Options::822"): try: (maintainer["maintainer822"], maintainer["maintainer2047"], maintainer["maintainername"], maintainer["maintaineremail"]) = \ - utils.fix_maintainer (j["maintainer"]); - except utils.ParseMaintError, msg: - print "Problems while parsing maintainer address\n"; - maintainer["maintainername"] = "Unknown"; - maintainer["maintaineremail"] = "Unknown"; - maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"]); - distribution=j["distribution"].keys(); - closes=j["closes"].keys(); + utils.fix_maintainer (j["maintainer"]) + except ParseMaintError, msg: + print "Problems while parsing maintainer address\n" + maintainer["maintainername"] = "Unknown" + maintainer["maintaineremail"] = "Unknown" + maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"]) + # ...likewise for the Changed-By: field if it exists. + try: + (changeby["changedby822"], changeby["changedby2047"], + changeby["changedbyname"], changeby["changedbyemail"]) = \ + utils.fix_maintainer (j["changed-by"]) + except ParseMaintError, msg: + (changeby["changedby822"], changeby["changedby2047"], + changeby["changedbyname"], changeby["changedbyemail"]) = \ + ("", "", "", "") + changedby="%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"]) + + distribution=j["distribution"].keys() + closes=j["closes"].keys() + fingerprint=j["fingerprint"] + if j.has_key("sponsoremail"): + sponsor=j["sponsoremail"] for arch in j["architecture"].keys(): - arches[arch] = ""; - version = j["version"]; - versions[version] = ""; - arches_list = arches.keys(); - arches_list.sort(utils.arch_compare_sw); - arch_list = " ".join(arches_list); - version_list = " ".join(versions.keys()); + arches[arch] = "" + version = j["version"] + versions[version] = "" + arches_list = arches.keys() + arches_list.sort(utils.arch_compare_sw) + arch_list = " ".join(arches_list) + version_list = " ".join(versions.keys()) if len(version_list) > max_version_len: - max_version_len = len(version_list); + max_version_len = len(version_list) if len(arch_list) > max_arch_len: - max_arch_len = len(arch_list); + max_arch_len = len(arch_list) if i[1]["note_state"]: - note = " | [N]"; + note = " | [N]" else: - note = ""; - entries.append([source, version_list, arch_list, note, last_modified, maint, distribution, closes]); + note = "" + entries.append([source, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename]) # direction entry consists of "Which field, which direction, time-consider" where # time-consider says how we should treat last_modified. Thats all. # Look for the options for sort and then do the sort. age = "h" - if Cnf.has_key("Helena::Options::Age"): - age = Cnf["Helena::Options::Age"] - if Cnf.has_key("Helena::Options::New"): + if Cnf.has_key("Queue-Report::Options::Age"): + age = Cnf["Queue-Report::Options::Age"] + if Cnf.has_key("Queue-Report::Options::New"): # If we produce html we always have oldest first. - direction.append([4,-1,"ao"]); + direction.append([4,-1,"ao"]) else: - if Cnf.has_key("Helena::Options::Sort"): - for i in Cnf["Helena::Options::Sort"].split(","): - if i == "ao": - # Age, oldest first. - direction.append([4,-1,age]); - elif i == "an": - # Age, newest first. - direction.append([4,1,age]); - elif i == "na": - # Name, Ascending. - direction.append([0,1,0]); - elif i == "nd": - # Name, Descending. - direction.append([0,-1,0]); - elif i == "nl": - # Notes last. - direction.append([3,1,0]); - elif i == "nf": - # Notes first. - direction.append([3,-1,0]); + if Cnf.has_key("Queue-Report::Options::Sort"): + for i in Cnf["Queue-Report::Options::Sort"].split(","): + if i == "ao": + # Age, oldest first. + direction.append([4,-1,age]) + elif i == "an": + # Age, newest first. + direction.append([4,1,age]) + elif i == "na": + # Name, Ascending. + direction.append([0,1,0]) + elif i == "nd": + # Name, Descending. + direction.append([0,-1,0]) + elif i == "nl": + # Notes last. + direction.append([3,1,0]) + elif i == "nf": + # Notes first. + direction.append([3,-1,0]) entries.sort(lambda x, y: sortfunc(x, y)) # Yes, in theory you can add several sort options at the commandline with. But my mind is to small # at the moment to come up with a real good sorting function that considers all the sidesteps you # have with it. (If you combine options it will simply take the last one at the moment). # Will be enhanced in the future. - if Cnf.has_key("Helena::Options::New"): - direction.append([4,1,"ao"]); + if Cnf.has_key("Queue-Report::Options::822"): + # print stuff out in 822 format + for entry in entries: + (source, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry + + # We'll always have Source, Version, Arch, Mantainer, and Dist + # For the rest, check to see if we have them, then print them out + log.write("Source: " + source + "\n") + log.write("Version: " + version_list + "\n") + log.write("Architectures: ") + log.write( (", ".join(arch_list.split(" "))) + "\n") + log.write("Age: " + time_pp(last_modified) + "\n") + log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n") + log.write("Queue: " + type + "\n") + + (name, mail) = maint.split(":") + log.write("Maintainer: " + name + " <"+mail+">" + "\n") + if changedby: + (name, mail) = changedby.split(":") + log.write("Changed-By: " + name + " <"+mail+">" + "\n") + if sponsor: + log.write("Sponsored-By: " + sponsor + "\n") + log.write("Distribution:") + for dist in distribution: + log.write(" " + dist) + log.write("\n") + log.write("Fingerprint: " + fingerprint + "\n") + if closes: + bug_string = "" + for bugs in closes: + bug_string += "#"+bugs+", " + log.write("Closes: " + bug_string[:-2] + "\n") + log.write("Changes-File: " + os.path.basename(changes_file) + "\n") + log.write("\n") + + if Cnf.has_key("Queue-Report::Options::New"): + direction.append([4,1,"ao"]) entries.sort(lambda x, y: sortfunc(x, y)) # Output for a html file. First table header. then table_footer. # Any line between them is then a printed from subroutine table_row. if len(entries) > 0: - table_header(type.upper()); + total_count = len(changes_files) + source_count = len(per_source_items) + table_header(type.upper(), source_count, total_count) for entry in entries: - (source, version_list, arch_list, note, last_modified, maint, distribution, closes) = entry; - table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes); - total_count = len(changes_files); - source_count = len(per_source_items); - table_footer(type.upper(), source_count, total_count); - else: + (source, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, undef) = entry + table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes, fingerprint, sponsor, changedby) + table_footer(type.upper()) + elif not Cnf.has_key("Queue-Report::Options::822"): # The "normal" output without any formatting. format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len) - msg = ""; + msg = "" for entry in entries: - (source, version_list, arch_list, note, last_modified, undef, undef, undef) = entry; - msg += format % (source, version_list, arch_list, note, time_pp(last_modified)); + (source, version_list, arch_list, note, last_modified, undef, undef, undef, undef, undef, undef, undef) = entry + msg += format % (source, version_list, arch_list, note, time_pp(last_modified)) if msg: - total_count = len(changes_files); - source_count = len(per_source_items); - print type.upper(); - print "-"*len(type); + total_count = len(changes_files) + source_count = len(per_source_items) + print type.upper() + print "-"*len(type) print - print msg; - print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count)); + print msg + print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count)) print ################################################################################ def main(): - global Cnf, Katie; - - Cnf = utils.get_conf(); - Arguments = [('h',"help","Helena::Options::Help"), - ('n',"new","Helena::Options::New"), - ('s',"sort","Helena::Options::Sort", "HasArg"), - ('a',"age","Helena::Options::Age", "HasArg")]; + global Cnf, Upload + + Cnf = utils.get_conf() + Arguments = [('h',"help","Queue-Report::Options::Help"), + ('n',"new","Queue-Report::Options::New"), + ('8','822',"Queue-Report::Options::822"), + ('s',"sort","Queue-Report::Options::Sort", "HasArg"), + ('a',"age","Queue-Report::Options::Age", "HasArg"), + ('d',"directories","Queue-Report::Options::Directories", "HasArg")] for i in [ "help" ]: - if not Cnf.has_key("Helena::Options::%s" % (i)): - Cnf["Helena::Options::%s" % (i)] = ""; + if not Cnf.has_key("Queue-Report::Options::%s" % (i)): + Cnf["Queue-Report::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) - Options = Cnf.SubTree("Helena::Options") + Options = Cnf.SubTree("Queue-Report::Options") if Options["Help"]: - usage(); + usage() + + Upload = queue.Upload(Cnf) - Katie = katie.Katie(Cnf); + if Cnf.has_key("Queue-Report::Options::New"): + header() - if Cnf.has_key("Helena::Options::New"): - header(); + directories = [ ] - directories = Cnf.ValueList("Helena::Directories"); - if not directories: - directories = [ "byhand", "new" ]; + if Cnf.has_key("Queue-Report::Options::Directories"): + for i in Cnf["Queue-Report::Options::Directories"].split(","): + directories.append(i) + elif Cnf.has_key("Queue-Report::Directories"): + directories = Cnf.ValueList("Queue-Report::Directories") + else: + directories = [ "byhand", "new" ] + + f = None + if Cnf.has_key("Queue-Report::Options::822"): + # Open the report file + f = open(Cnf["Queue-Report::ReportLocations::822Location"], "w") for directory in directories: - changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)])); - process_changes_files(changes_files, directory); + changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)])) + process_changes_files(changes_files, directory, f) + + if Cnf.has_key("Queue-Report::Options::822"): + f.close() - if Cnf.has_key("Helena::Options::New"): - footer(); + if Cnf.has_key("Queue-Report::Options::New"): + footer() ################################################################################ if __name__ == '__main__': - main(); + main()