]> git.decadent.org.uk Git - dak.git/blob - dak/queue_report.py
Merge commit 'ftpmaster/master'
[dak.git] / dak / queue_report.py
1 #!/usr/bin/env python
2
3 """ Produces a report on NEW and BYHAND packages """
4 # Copyright (C) 2001, 2002, 2003, 2005, 2006  James Troup <james@nocrew.org>
5
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19
20 ################################################################################
21
22 # <o-o> XP runs GCC, XFREE86, SSH etc etc,.,, I feel almost like linux....
23 # <o-o> I am very confident that I can replicate any Linux application on XP
24 # <willy> o-o: *boggle*
25 # <o-o> building from source.
26 # <o-o> Viiru: I already run GIMP under XP
27 # <willy> o-o: why do you capitalise the names of all pieces of software?
28 # <o-o> willy: because I want the EMPHASIZE them....
29 # <o-o> grr s/the/to/
30 # <willy> o-o: it makes you look like ZIPPY the PINHEAD
31 # <o-o> willy: no idea what you are talking about.
32 # <willy> o-o: do some research
33 # <o-o> willy: for what reason?
34
35 ################################################################################
36
37 import copy, glob, os, stat, sys, time
38 import apt_pkg
39 import cgi
40 from daklib import queue
41 from daklib import utils
42 from daklib.dak_exceptions import *
43
44 Cnf = None
45 Upload = None
46 direction = []
47 row_number = 0
48
49 ################################################################################
50
51 def usage(exit_code=0):
52     print """Usage: dak queue-report
53 Prints a report of packages in queue directories (usually new and byhand).
54
55   -h, --help                show this help and exit.
56   -8, --822                 writes 822 formated output to the location set in dak.conf
57   -n, --new                 produce html-output
58   -s, --sort=key            sort output according to key, see below.
59   -a, --age=key             if using sort by age, how should time be treated?
60                             If not given a default of hours will be used.
61   -d, --directories=key     A comma seperated list of queues to be scanned
62
63      Sorting Keys: ao=age,   oldest first.   an=age,   newest first.
64                    na=name,  ascending       nd=name,  descending
65                    nf=notes, first           nl=notes, last
66
67      Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years
68
69 """
70     sys.exit(exit_code)
71
72 ################################################################################
73
74 def plural(x):
75     if x > 1:
76         return "s"
77     else:
78         return ""
79
80 ################################################################################
81
82 def time_pp(x):
83     if x < 60:
84         unit="second"
85     elif x < 3600:
86         x /= 60
87         unit="minute"
88     elif x < 86400:
89         x /= 3600
90         unit="hour"
91     elif x < 604800:
92         x /= 86400
93         unit="day"
94     elif x < 2419200:
95         x /= 604800
96         unit="week"
97     elif x < 29030400:
98         x /= 2419200
99         unit="month"
100     else:
101         x /= 29030400
102         unit="year"
103     x = int(x)
104     return "%s %s%s" % (x, unit, plural(x))
105
106 ################################################################################
107
108 def sg_compare (a, b):
109     a = a[1]
110     b = b[1]
111     """Sort by have note, time of oldest upload."""
112     # Sort by have note
113     a_note_state = a["note_state"]
114     b_note_state = b["note_state"]
115     if a_note_state < b_note_state:
116         return -1
117     elif a_note_state > b_note_state:
118         return 1
119
120     # Sort by time of oldest upload
121     return cmp(a["oldest"], b["oldest"])
122
123 ############################################################
124
125 def sortfunc(a,b):
126     for sorting in direction:
127         (sortkey, way, time) = sorting
128         ret = 0
129         if time == "m":
130             x=int(a[sortkey]/60)
131             y=int(b[sortkey]/60)
132         elif time == "h":
133             x=int(a[sortkey]/3600)
134             y=int(b[sortkey]/3600)
135         elif time == "d":
136             x=int(a[sortkey]/86400)
137             y=int(b[sortkey]/86400)
138         elif time == "w":
139             x=int(a[sortkey]/604800)
140             y=int(b[sortkey]/604800)
141         elif time == "o":
142             x=int(a[sortkey]/2419200)
143             y=int(b[sortkey]/2419200)
144         elif time == "y":
145             x=int(a[sortkey]/29030400)
146             y=int(b[sortkey]/29030400)
147         else:
148             x=a[sortkey]
149             y=b[sortkey]
150         if x < y:
151             ret = -1
152         elif x > y:
153             ret = 1
154         if ret != 0:
155             if way < 0:
156                 ret = ret*-1
157             return ret
158     return 0
159
160 ############################################################
161
162 def header():
163     print """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
164 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
165 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="de" lang="de">
166   <head>
167     <meta http-equiv="content-type" content="text/xhtml+xml; charset=utf-8" />
168     <link type="text/css" rel="stylesheet" href="style.css" />
169     <link rel="shortcut icon" href="http://www.debian.org/favicon.ico" />
170     <title>
171       Debian NEW and BYHAND Packages
172     </title>
173   </head>
174   <body id="NEW">
175     <div id="logo">
176       <a href="http://www.debian.org/">
177         <img src="http://www.debian.org/logos/openlogo-nd-50.png"
178         alt="debian logo" /></a>
179       <a href="http://www.debian.org/">
180         <img src="http://www.debian.org/Pics/debian.png"
181         alt="Debian Project" /></a>
182     </div>
183     <div id="titleblock">
184
185       <img src="http://www.debian.org/Pics/red-upperleft.png"
186       id="red-upperleft" alt="corner image"/>
187       <img src="http://www.debian.org/Pics/red-lowerleft.png"
188       id="red-lowerleft" alt="corner image"/>
189       <img src="http://www.debian.org/Pics/red-upperright.png"
190       id="red-upperright" alt="corner image"/>
191       <img src="http://www.debian.org/Pics/red-lowerright.png"
192       id="red-lowerright" alt="corner image"/>
193       <span class="title">
194         Debian NEW and BYHAND Packages
195       </span>
196     </div>
197     """
198
199 def footer():
200     print "<p class=\"timestamp\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
201
202     print """
203     <div class="footer">
204     <p>Hint: Age is the youngest upload of the package, if there is more than
205     one version.<br />
206     You may want to look at <a href="http://ftp-master.debian.org/REJECT-FAQ.html">the REJECT-FAQ</a>
207       for possible reasons why one of the above packages may get rejected.</p>
208       <p>
209       <a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-xhtml10"
210         alt="Valid XHTML 1.0 Strict" height="31" width="88" /></a>
211       <a href="http://jigsaw.w3.org/css-validator/">
212         <img style="border:0;width:88px;height:31px" src="http://jigsaw.w3.org/css-validator/images/vcss"
213         alt="Valid CSS!" />
214       </a>
215       </p>
216     </div> </body> </html>
217     """
218
219 def table_header(type, source_count, total_count):
220     print "<h1>Summary for: %s</h1>" % (type)
221     print """
222     <table class="NEW">
223       <caption>
224     """
225     print "Package count in <strong>%s</strong>: <em>%s</em>&nbsp;|&nbsp; Total Package count: <em>%s</em>" % (type, source_count, total_count)
226     print """
227       </caption>
228       <thead>
229         <tr>
230           <th>Package</th>
231           <th>Version</th>
232           <th>Arch</th>
233           <th>Distribution</th>
234           <th>Age</th>
235           <th>Upload info</th>
236           <th>Closes</th>
237         </tr>
238       </thead>
239       <tbody>
240     """
241
242 def table_footer(type):
243     print "</tbody></table>"
244
245
246 def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
247
248     global row_number
249
250     trclass = "sid"
251     for dist in distribution:
252         if dist == "experimental":
253             trclass = "exp"
254
255     if row_number % 2 != 0:
256         print "<tr class=\"%s even\">" % (trclass)
257     else:
258         print "<tr class=\"%s odd\">" % (trclass)
259
260     print "<td class=\"package\">%s</td>" % (source)
261     print "<td class=\"version\">"
262     for vers in version.split():
263         print "<a href=\"/new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers))
264     print "</td>"
265     print "<td class=\"arch\">%s</td>" % (arch)
266     print "<td class=\"distribution\">"
267     for dist in distribution:
268         print "%s<br/>" % (dist)
269     print "</td>"
270     print "<td class=\"age\">%s</td>" % (last_mod)
271     (name, mail) = maint.split(":", 1)
272
273     print "<td class=\"upload-data\">"
274     print "<span class=\"maintainer\">Maintainer: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
275     (name, mail) = changedby.split(":", 1)
276     print "<span class=\"changed-by\">Changed-By: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
277
278     try:
279         (login, domain) = sponsor.split("@", 1)
280         print "<span class=\"sponsor\">Sponsor: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span>@debian.org<br/>" % (utils.html_escape(login), utils.html_escape(login))
281     except:
282         pass
283
284     print "<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint)
285     print "</td>"
286
287     print "<td class=\"closes\">"
288     for close in closes:
289         print "<a href=\"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close))
290     print "</td></tr>"
291     row_number+=1
292
293 ############################################################
294
295 def process_changes_files(changes_files, type, log):
296     msg = ""
297     cache = {}
298     # Read in all the .changes files
299     for filename in changes_files:
300         try:
301             Upload.pkg.changes_file = filename
302             Upload.init_vars()
303             Upload.update_vars()
304             cache[filename] = copy.copy(Upload.pkg.changes)
305             cache[filename]["filename"] = filename
306         except:
307             break
308     # Divide the .changes into per-source groups
309     per_source = {}
310     for filename in cache.keys():
311         source = cache[filename]["source"]
312         if not per_source.has_key(source):
313             per_source[source] = {}
314             per_source[source]["list"] = []
315         per_source[source]["list"].append(cache[filename])
316     # Determine oldest time and have note status for each source group
317     for source in per_source.keys():
318         source_list = per_source[source]["list"]
319         first = source_list[0]
320         oldest = os.stat(first["filename"])[stat.ST_MTIME]
321         have_note = 0
322         for d in per_source[source]["list"]:
323             mtime = os.stat(d["filename"])[stat.ST_MTIME]
324             if Cnf.has_key("Queue-Report::Options::New"):
325                 if mtime > oldest:
326                     oldest = mtime
327             else:
328                 if mtime < oldest:
329                     oldest = mtime
330             have_note += (d.has_key("process-new note"))
331         per_source[source]["oldest"] = oldest
332         if not have_note:
333             per_source[source]["note_state"] = 0; # none
334         elif have_note < len(source_list):
335             per_source[source]["note_state"] = 1; # some
336         else:
337             per_source[source]["note_state"] = 2; # all
338     per_source_items = per_source.items()
339     per_source_items.sort(sg_compare)
340
341     entries = []
342     max_source_len = 0
343     max_version_len = 0
344     max_arch_len = 0
345     for i in per_source_items:
346         maintainer = {}
347         maint=""
348         distribution=""
349         closes=""
350         fingerprint=""
351         changeby = {}
352         changedby=""
353         sponsor=""
354         filename=i[1]["list"][0]["filename"]
355         last_modified = time.time()-i[1]["oldest"]
356         source = i[1]["list"][0]["source"]
357         if len(source) > max_source_len:
358             max_source_len = len(source)
359         arches = {}
360         versions = {}
361         for j in i[1]["list"]:
362             if Cnf.has_key("Queue-Report::Options::New") or Cnf.has_key("Queue-Report::Options::822"):
363                 try:
364                     (maintainer["maintainer822"], maintainer["maintainer2047"],
365                     maintainer["maintainername"], maintainer["maintaineremail"]) = \
366                     utils.fix_maintainer (j["maintainer"])
367                 except ParseMaintError, msg:
368                     print "Problems while parsing maintainer address\n"
369                     maintainer["maintainername"] = "Unknown"
370                     maintainer["maintaineremail"] = "Unknown"
371                 maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
372                 # ...likewise for the Changed-By: field if it exists.
373                 try:
374                     (changeby["changedby822"], changeby["changedby2047"],
375                      changeby["changedbyname"], changeby["changedbyemail"]) = \
376                      utils.fix_maintainer (j["changed-by"])
377                 except ParseMaintError, msg:
378                     (changeby["changedby822"], changeby["changedby2047"],
379                      changeby["changedbyname"], changeby["changedbyemail"]) = \
380                      ("", "", "", "")
381                 changedby="%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"])
382
383                 distribution=j["distribution"].keys()
384                 closes=j["closes"].keys()
385                 fingerprint=j["fingerprint"]
386                 if j.has_key("sponsoremail"):
387                     sponsor=j["sponsoremail"]
388             for arch in j["architecture"].keys():
389                 arches[arch] = ""
390             version = j["version"]
391             versions[version] = ""
392         arches_list = arches.keys()
393         arches_list.sort(utils.arch_compare_sw)
394         arch_list = " ".join(arches_list)
395         version_list = " ".join(versions.keys())
396         if len(version_list) > max_version_len:
397             max_version_len = len(version_list)
398         if len(arch_list) > max_arch_len:
399             max_arch_len = len(arch_list)
400         if i[1]["note_state"]:
401             note = " | [N]"
402         else:
403             note = ""
404         entries.append([source, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename])
405
406     # direction entry consists of "Which field, which direction, time-consider" where
407     # time-consider says how we should treat last_modified. Thats all.
408
409     # Look for the options for sort and then do the sort.
410     age = "h"
411     if Cnf.has_key("Queue-Report::Options::Age"):
412         age =  Cnf["Queue-Report::Options::Age"]
413     if Cnf.has_key("Queue-Report::Options::New"):
414     # If we produce html we always have oldest first.
415         direction.append([4,-1,"ao"])
416     else:
417         if Cnf.has_key("Queue-Report::Options::Sort"):
418             for i in Cnf["Queue-Report::Options::Sort"].split(","):
419                 if i == "ao":
420                     # Age, oldest first.
421                     direction.append([4,-1,age])
422                 elif i == "an":
423                     # Age, newest first.
424                     direction.append([4,1,age])
425                 elif i == "na":
426                     # Name, Ascending.
427                     direction.append([0,1,0])
428                 elif i == "nd":
429                     # Name, Descending.
430                     direction.append([0,-1,0])
431                 elif i == "nl":
432                     # Notes last.
433                     direction.append([3,1,0])
434                 elif i == "nf":
435                     # Notes first.
436                     direction.append([3,-1,0])
437     entries.sort(lambda x, y: sortfunc(x, y))
438     # Yes, in theory you can add several sort options at the commandline with. But my mind is to small
439     # at the moment to come up with a real good sorting function that considers all the sidesteps you
440     # have with it. (If you combine options it will simply take the last one at the moment).
441     # Will be enhanced in the future.
442
443     if Cnf.has_key("Queue-Report::Options::822"):
444         # print stuff out in 822 format
445         for entry in entries:
446             (source, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry
447
448             # We'll always have Source, Version, Arch, Mantainer, and Dist
449             # For the rest, check to see if we have them, then print them out
450             log.write("Source: " + source + "\n")
451             log.write("Version: " + version_list + "\n")
452             log.write("Architectures: ")
453             log.write( (", ".join(arch_list.split(" "))) + "\n")
454             log.write("Age: " + time_pp(last_modified) + "\n")
455             log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n")
456             log.write("Queue: " + type + "\n")
457
458             (name, mail) = maint.split(":", 1)
459             log.write("Maintainer: " + name + " <"+mail+">" + "\n")
460             if changedby:
461                (name, mail) = changedby.split(":", 1)
462                log.write("Changed-By: " + name + " <"+mail+">" + "\n")
463             if sponsor:
464                log.write("Sponsored-By: " + sponsor + "\n")
465             log.write("Distribution:")
466             for dist in distribution:
467                log.write(" " + dist)
468             log.write("\n")
469             log.write("Fingerprint: " + fingerprint + "\n")
470             if closes:
471                 bug_string = ""
472                 for bugs in closes:
473                     bug_string += "#"+bugs+", "
474                 log.write("Closes: " + bug_string[:-2] + "\n")
475             log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
476             log.write("\n")
477
478     if Cnf.has_key("Queue-Report::Options::New"):
479         direction.append([4,1,"ao"])
480         entries.sort(lambda x, y: sortfunc(x, y))
481     # Output for a html file. First table header. then table_footer.
482     # Any line between them is then a <tr> printed from subroutine table_row.
483         if len(entries) > 0:
484             total_count = len(changes_files)
485             source_count = len(per_source_items)
486             table_header(type.upper(), source_count, total_count)
487             for entry in entries:
488                 (source, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, undef) = entry
489                 table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes, fingerprint, sponsor, changedby)
490             table_footer(type.upper())
491     elif not Cnf.has_key("Queue-Report::Options::822"):
492     # The "normal" output without any formatting.
493         format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
494
495         msg = ""
496         for entry in entries:
497             (source, version_list, arch_list, note, last_modified, undef, undef, undef, undef, undef, undef, undef) = entry
498             msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
499
500         if msg:
501             total_count = len(changes_files)
502             source_count = len(per_source_items)
503             print type.upper()
504             print "-"*len(type)
505             print
506             print msg
507             print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count))
508             print
509
510
511 ################################################################################
512
513 def main():
514     global Cnf, Upload
515
516     Cnf = utils.get_conf()
517     Arguments = [('h',"help","Queue-Report::Options::Help"),
518                  ('n',"new","Queue-Report::Options::New"),
519                  ('8','822',"Queue-Report::Options::822"),
520                  ('s',"sort","Queue-Report::Options::Sort", "HasArg"),
521                  ('a',"age","Queue-Report::Options::Age", "HasArg"),
522                  ('d',"directories","Queue-Report::Options::Directories", "HasArg")]
523     for i in [ "help" ]:
524         if not Cnf.has_key("Queue-Report::Options::%s" % (i)):
525             Cnf["Queue-Report::Options::%s" % (i)] = ""
526
527     apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
528
529     Options = Cnf.SubTree("Queue-Report::Options")
530     if Options["Help"]:
531         usage()
532
533     Upload = queue.Upload(Cnf)
534
535     if Cnf.has_key("Queue-Report::Options::New"):
536         header()
537
538     directories = [ ]
539
540     if Cnf.has_key("Queue-Report::Options::Directories"):
541         for i in Cnf["Queue-Report::Options::Directories"].split(","):
542             directories.append(i)
543     elif Cnf.has_key("Queue-Report::Directories"):
544         directories = Cnf.ValueList("Queue-Report::Directories")
545     else:
546         directories = [ "byhand", "new" ]
547
548     f = None
549     if Cnf.has_key("Queue-Report::Options::822"):
550         # Open the report file
551         f = open(Cnf["Queue-Report::ReportLocations::822Location"], "w")
552
553     for directory in directories:
554         changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)]))
555         process_changes_files(changes_files, directory, f)
556
557     if Cnf.has_key("Queue-Report::Options::822"):
558         f.close()
559
560     if Cnf.has_key("Queue-Report::Options::New"):
561         footer()
562
563 ################################################################################
564
565 if __name__ == '__main__':
566     main()