]> git.decadent.org.uk Git - dak.git/blob - dak/queue_report.py
Merge remote branch 'drkranz/master' into merge
[dak.git] / dak / queue_report.py
1 #!/usr/bin/env python
2
3 """ Produces a report on NEW and BYHAND packages """
4 # Copyright (C) 2001, 2002, 2003, 2005, 2006  James Troup <james@nocrew.org>
5
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19
20 ################################################################################
21
22 # <o-o> XP runs GCC, XFREE86, SSH etc etc,.,, I feel almost like linux....
23 # <o-o> I am very confident that I can replicate any Linux application on XP
24 # <willy> o-o: *boggle*
25 # <o-o> building from source.
26 # <o-o> Viiru: I already run GIMP under XP
27 # <willy> o-o: why do you capitalise the names of all pieces of software?
28 # <o-o> willy: because I want the EMPHASIZE them....
29 # <o-o> grr s/the/to/
30 # <willy> o-o: it makes you look like ZIPPY the PINHEAD
31 # <o-o> willy: no idea what you are talking about.
32 # <willy> o-o: do some research
33 # <o-o> willy: for what reason?
34
35 ################################################################################
36
37 from copy import copy
38 import glob, os, stat, sys, time
39 import apt_pkg
40
41 from daklib import utils
42 from daklib.queue import Upload
43 from daklib.dbconn import DBConn, has_new_comment, DBChange, DBSource, get_uid_from_fingerprint
44 from daklib.textutils import fix_maintainer
45 from daklib.dak_exceptions import *
46
47 Cnf = None
48 direction = []
49 row_number = 0
50
51 ################################################################################
52
53 def usage(exit_code=0):
54     print """Usage: dak queue-report
55 Prints a report of packages in queue directories (usually new and byhand).
56
57   -h, --help                show this help and exit.
58   -8, --822                 writes 822 formated output to the location set in dak.conf
59   -n, --new                 produce html-output
60   -s, --sort=key            sort output according to key, see below.
61   -a, --age=key             if using sort by age, how should time be treated?
62                             If not given a default of hours will be used.
63   -d, --directories=key     A comma seperated list of queues to be scanned
64
65      Sorting Keys: ao=age,   oldest first.   an=age,   newest first.
66                    na=name,  ascending       nd=name,  descending
67                    nf=notes, first           nl=notes, last
68
69      Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years
70
71 """
72     sys.exit(exit_code)
73
74 ################################################################################
75
76 def plural(x):
77     if x > 1:
78         return "s"
79     else:
80         return ""
81
82 ################################################################################
83
84 def time_pp(x):
85     if x < 60:
86         unit="second"
87     elif x < 3600:
88         x /= 60
89         unit="minute"
90     elif x < 86400:
91         x /= 3600
92         unit="hour"
93     elif x < 604800:
94         x /= 86400
95         unit="day"
96     elif x < 2419200:
97         x /= 604800
98         unit="week"
99     elif x < 29030400:
100         x /= 2419200
101         unit="month"
102     else:
103         x /= 29030400
104         unit="year"
105     x = int(x)
106     return "%s %s%s" % (x, unit, plural(x))
107
108 ################################################################################
109
110 def sg_compare (a, b):
111     a = a[1]
112     b = b[1]
113     """Sort by have note, time of oldest upload."""
114     # Sort by have note
115     a_note_state = a["note_state"]
116     b_note_state = b["note_state"]
117     if a_note_state < b_note_state:
118         return -1
119     elif a_note_state > b_note_state:
120         return 1
121
122     # Sort by time of oldest upload
123     return cmp(a["oldest"], b["oldest"])
124
125 ############################################################
126
127 def sortfunc(a,b):
128     for sorting in direction:
129         (sortkey, way, time) = sorting
130         ret = 0
131         if time == "m":
132             x=int(a[sortkey]/60)
133             y=int(b[sortkey]/60)
134         elif time == "h":
135             x=int(a[sortkey]/3600)
136             y=int(b[sortkey]/3600)
137         elif time == "d":
138             x=int(a[sortkey]/86400)
139             y=int(b[sortkey]/86400)
140         elif time == "w":
141             x=int(a[sortkey]/604800)
142             y=int(b[sortkey]/604800)
143         elif time == "o":
144             x=int(a[sortkey]/2419200)
145             y=int(b[sortkey]/2419200)
146         elif time == "y":
147             x=int(a[sortkey]/29030400)
148             y=int(b[sortkey]/29030400)
149         else:
150             x=a[sortkey]
151             y=b[sortkey]
152         if x < y:
153             ret = -1
154         elif x > y:
155             ret = 1
156         if ret != 0:
157             if way < 0:
158                 ret = ret*-1
159             return ret
160     return 0
161
162 ############################################################
163
164 def header():
165     print """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
166 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
167 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="de" lang="de">
168   <head>
169     <meta http-equiv="content-type" content="text/xhtml+xml; charset=utf-8" />
170     <link type="text/css" rel="stylesheet" href="style.css" />
171     <link rel="shortcut icon" href="http://www.debian.org/favicon.ico" />
172     <title>
173       Debian NEW and BYHAND Packages
174     </title>
175     <script type="text/javascript">
176     function togglePkg() {
177         var children = document.getElementsByTagName("*");
178         for (var i = 0; i < children.length; i++) {
179             if(!children[i].hasAttribute("class"))
180                 continue;
181             c = children[i].getAttribute("class").split(" ");
182             for(var j = 0; j < c.length; j++) {
183                 if(c[j] == "binNEW") {
184                     if (children[i].style.display == '')
185                         children[i].style.display = 'none';
186                     else children[i].style.display = '';
187                 }
188             }
189         }
190     }
191     </script>
192   </head>
193   <body id="NEW">
194     <div id="logo">
195       <a href="http://www.debian.org/">
196         <img src="http://www.debian.org/logos/openlogo-nd-50.png"
197         alt="debian logo" /></a>
198       <a href="http://www.debian.org/">
199         <img src="http://www.debian.org/Pics/debian.png"
200         alt="Debian Project" /></a>
201     </div>
202     <div id="titleblock">
203
204       <img src="http://www.debian.org/Pics/red-upperleft.png"
205       id="red-upperleft" alt="corner image"/>
206       <img src="http://www.debian.org/Pics/red-lowerleft.png"
207       id="red-lowerleft" alt="corner image"/>
208       <img src="http://www.debian.org/Pics/red-upperright.png"
209       id="red-upperright" alt="corner image"/>
210       <img src="http://www.debian.org/Pics/red-lowerright.png"
211       id="red-lowerright" alt="corner image"/>
212       <span class="title">
213         Debian NEW and BYHAND Packages
214       </span>
215     </div>
216     """
217
218 def footer():
219     print "<p class=\"timestamp\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
220
221     print """
222     <div class="footer">
223     <p>Hint: Age is the youngest upload of the package, if there is more than
224     one version.<br />
225     You may want to look at <a href="http://ftp-master.debian.org/REJECT-FAQ.html">the REJECT-FAQ</a>
226       for possible reasons why one of the above packages may get rejected.</p>
227       <p>
228       <a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-xhtml10"
229         alt="Valid XHTML 1.0 Strict" height="31" width="88" /></a>
230       <a href="http://jigsaw.w3.org/css-validator/">
231         <img style="border:0;width:88px;height:31px" src="http://jigsaw.w3.org/css-validator/images/vcss"
232         alt="Valid CSS!" />
233       </a>
234       </p>
235     </div> </body> </html>
236     """
237
238 def table_header(type, source_count, total_count):
239     print "<h1 class='binNEW'>Summary for: %s</h1>" % (type)
240     print "<h1 class='binNEW' style='display: none'>Summary for: binary-%s only</h1>" % (type)
241     print """
242     <table class="NEW">
243       <p class="togglepkg" onclick="togglePkg()">Click to toggle all/binary-NEW packages</p>
244       <caption class="binNEW">
245     """
246     print "Package count in <strong>%s</strong>: <em>%s</em>&nbsp;|&nbsp; Total Package count: <em>%s</em>" % (type, source_count, total_count)
247     print """
248       </caption>
249       <thead>
250         <tr>
251           <th>Package</th>
252           <th>Version</th>
253           <th>Arch</th>
254           <th>Distribution</th>
255           <th>Age</th>
256           <th>Upload info</th>
257           <th>Closes</th>
258         </tr>
259       </thead>
260       <tbody>
261     """
262
263 def table_footer(type):
264     print "</tbody></table>"
265
266
267 def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
268
269     global row_number
270
271     trclass = "sid"
272     session = DBConn().session()
273     for dist in distribution:
274         if dist == "experimental":
275             trclass = "exp"
276
277     if not len(session.query(DBSource).filter_by(source = source).all()):
278         trclass += " binNEW"
279     session.commit()
280
281     if row_number % 2 != 0:
282         print "<tr class=\"%s even\">" % (trclass)
283     else:
284         print "<tr class=\"%s odd\">" % (trclass)
285
286     print "<td class=\"package\">%s</td>" % (source)
287     print "<td class=\"version\">"
288     for vers in version.split():
289         print "<a href=\"/new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers))
290     print "</td>"
291     print "<td class=\"arch\">%s</td>" % (arch)
292     print "<td class=\"distribution\">"
293     for dist in distribution:
294         print "%s<br/>" % (dist)
295     print "</td>"
296     print "<td class=\"age\">%s</td>" % (last_mod)
297     (name, mail) = maint.split(":", 1)
298
299     print "<td class=\"upload-data\">"
300     print "<span class=\"maintainer\">Maintainer: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
301     (name, mail) = changedby.split(":", 1)
302     print "<span class=\"changed-by\">Changed-By: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
303
304     if sponsor:
305         try:
306             (login, domain) = sponsor.split("@", 1)
307             print "<span class=\"sponsor\">Sponsor: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span>@debian.org<br/>" % (utils.html_escape(login), utils.html_escape(login))
308         except Exception, e:
309             pass
310
311     print "<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint)
312     print "</td>"
313
314     print "<td class=\"closes\">"
315     for close in closes:
316         print "<a href=\"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close))
317     print "</td></tr>"
318     row_number+=1
319
320 ############################################################
321
322 def process_changes_files(changes_files, type, log):
323     session = DBConn().session()
324     msg = ""
325     cache = {}
326     # Read in all the .changes files
327     for filename in changes_files:
328         try:
329             u = Upload()
330             u.load_changes(filename)
331             cache[filename] = copy(u.pkg.changes)
332             cache[filename]["filename"] = filename
333         except Exception, e:
334             print "WARNING: Exception %s" % e
335             continue
336     # Divide the .changes into per-source groups
337     per_source = {}
338     for filename in cache.keys():
339         source = cache[filename]["source"]
340         if not per_source.has_key(source):
341             per_source[source] = {}
342             per_source[source]["list"] = []
343         per_source[source]["list"].append(cache[filename])
344     # Determine oldest time and have note status for each source group
345     for source in per_source.keys():
346         source_list = per_source[source]["list"]
347         first = source_list[0]
348         oldest = os.stat(first["filename"])[stat.ST_MTIME]
349         have_note = 0
350         for d in per_source[source]["list"]:
351             mtime = os.stat(d["filename"])[stat.ST_MTIME]
352             if Cnf.has_key("Queue-Report::Options::New"):
353                 if mtime > oldest:
354                     oldest = mtime
355             else:
356                 if mtime < oldest:
357                     oldest = mtime
358             have_note += has_new_comment(d["source"], d["version"])
359         per_source[source]["oldest"] = oldest
360         if not have_note:
361             per_source[source]["note_state"] = 0; # none
362         elif have_note < len(source_list):
363             per_source[source]["note_state"] = 1; # some
364         else:
365             per_source[source]["note_state"] = 2; # all
366     per_source_items = per_source.items()
367     per_source_items.sort(sg_compare)
368
369     entries = []
370     max_source_len = 0
371     max_version_len = 0
372     max_arch_len = 0
373     for i in per_source_items:
374         maintainer = {}
375         maint=""
376         distribution=""
377         closes=""
378         fingerprint=""
379         changeby = {}
380         changedby=""
381         sponsor=""
382         filename=i[1]["list"][0]["filename"]
383         last_modified = time.time()-i[1]["oldest"]
384         source = i[1]["list"][0]["source"]
385         if len(source) > max_source_len:
386             max_source_len = len(source)
387         binary_list = i[1]["list"][0]["binary"].keys()
388         binary = ', '.join(binary_list)
389         arches = {}
390         versions = {}
391         for j in i[1]["list"]:
392             changesbase = os.path.basename(j["filename"])
393             try:
394                 dbc = session.query(DBChange).filter_by(changesname=changesbase).one()
395             except Exception, e:
396                 print "Can't find changes file in NEW for %s (%s)" % (changesbase, e)
397                 dbc = None
398
399             if Cnf.has_key("Queue-Report::Options::New") or Cnf.has_key("Queue-Report::Options::822"):
400                 try:
401                     (maintainer["maintainer822"], maintainer["maintainer2047"],
402                     maintainer["maintainername"], maintainer["maintaineremail"]) = \
403                     fix_maintainer (j["maintainer"])
404                 except ParseMaintError, msg:
405                     print "Problems while parsing maintainer address\n"
406                     maintainer["maintainername"] = "Unknown"
407                     maintainer["maintaineremail"] = "Unknown"
408                 maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
409                 # ...likewise for the Changed-By: field if it exists.
410                 try:
411                     (changeby["changedby822"], changeby["changedby2047"],
412                      changeby["changedbyname"], changeby["changedbyemail"]) = \
413                      fix_maintainer (j["changed-by"])
414                 except ParseMaintError, msg:
415                     (changeby["changedby822"], changeby["changedby2047"],
416                      changeby["changedbyname"], changeby["changedbyemail"]) = \
417                      ("", "", "", "")
418                 changedby="%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"])
419
420                 distribution=j["distribution"].keys()
421                 closes=j["closes"].keys()
422                 if dbc:
423                     fingerprint = dbc.fingerprint
424                     sponsor_name = get_uid_from_fingerprint(fingerprint).name
425                     sponsor_email = get_uid_from_fingerprint(fingerprint).uid + "@debian.org"
426                     if sponsor_name != maintainer["maintainername"] and sponsor_name != changeby["changedbyname"] and \
427                     sponsor_email != maintainer["maintaineremail"] and sponsor_name != changeby["changedbyemail"]:
428                         sponsor = sponsor_email
429
430             for arch in j["architecture"].keys():
431                 arches[arch] = ""
432             version = j["version"]
433             versions[version] = ""
434         arches_list = arches.keys()
435         arches_list.sort(utils.arch_compare_sw)
436         arch_list = " ".join(arches_list)
437         version_list = " ".join(versions.keys())
438         if len(version_list) > max_version_len:
439             max_version_len = len(version_list)
440         if len(arch_list) > max_arch_len:
441             max_arch_len = len(arch_list)
442         if i[1]["note_state"]:
443             note = " | [N]"
444         else:
445             note = ""
446         entries.append([source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename])
447
448     # direction entry consists of "Which field, which direction, time-consider" where
449     # time-consider says how we should treat last_modified. Thats all.
450
451     # Look for the options for sort and then do the sort.
452     age = "h"
453     if Cnf.has_key("Queue-Report::Options::Age"):
454         age =  Cnf["Queue-Report::Options::Age"]
455     if Cnf.has_key("Queue-Report::Options::New"):
456     # If we produce html we always have oldest first.
457         direction.append([5,-1,"ao"])
458     else:
459         if Cnf.has_key("Queue-Report::Options::Sort"):
460             for i in Cnf["Queue-Report::Options::Sort"].split(","):
461                 if i == "ao":
462                     # Age, oldest first.
463                     direction.append([5,-1,age])
464                 elif i == "an":
465                     # Age, newest first.
466                     direction.append([5,1,age])
467                 elif i == "na":
468                     # Name, Ascending.
469                     direction.append([0,1,0])
470                 elif i == "nd":
471                     # Name, Descending.
472                     direction.append([0,-1,0])
473                 elif i == "nl":
474                     # Notes last.
475                     direction.append([4,1,0])
476                 elif i == "nf":
477                     # Notes first.
478                     direction.append([4,-1,0])
479     entries.sort(lambda x, y: sortfunc(x, y))
480     # Yes, in theory you can add several sort options at the commandline with. But my mind is to small
481     # at the moment to come up with a real good sorting function that considers all the sidesteps you
482     # have with it. (If you combine options it will simply take the last one at the moment).
483     # Will be enhanced in the future.
484
485     if Cnf.has_key("Queue-Report::Options::822"):
486         # print stuff out in 822 format
487         for entry in entries:
488             (source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry
489
490             # We'll always have Source, Version, Arch, Mantainer, and Dist
491             # For the rest, check to see if we have them, then print them out
492             log.write("Source: " + source + "\n")
493             log.write("Binary: " + binary + "\n")
494             log.write("Version: " + version_list + "\n")
495             log.write("Architectures: ")
496             log.write( (", ".join(arch_list.split(" "))) + "\n")
497             log.write("Age: " + time_pp(last_modified) + "\n")
498             log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n")
499             log.write("Queue: " + type + "\n")
500
501             (name, mail) = maint.split(":", 1)
502             log.write("Maintainer: " + name + " <"+mail+">" + "\n")
503             if changedby:
504                (name, mail) = changedby.split(":", 1)
505                log.write("Changed-By: " + name + " <"+mail+">" + "\n")
506             if sponsor:
507                log.write("Sponsored-By: " + sponsor + "\n")
508             log.write("Distribution:")
509             for dist in distribution:
510                log.write(" " + dist)
511             log.write("\n")
512             log.write("Fingerprint: " + fingerprint + "\n")
513             if closes:
514                 bug_string = ""
515                 for bugs in closes:
516                     bug_string += "#"+bugs+", "
517                 log.write("Closes: " + bug_string[:-2] + "\n")
518             log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
519             log.write("\n")
520
521     if Cnf.has_key("Queue-Report::Options::New"):
522         direction.append([5,1,"ao"])
523         entries.sort(lambda x, y: sortfunc(x, y))
524     # Output for a html file. First table header. then table_footer.
525     # Any line between them is then a <tr> printed from subroutine table_row.
526         if len(entries) > 0:
527             total_count = len(changes_files)
528             source_count = len(per_source_items)
529             table_header(type.upper(), source_count, total_count)
530             for entry in entries:
531                 (source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, undef) = entry
532                 table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes, fingerprint, sponsor, changedby)
533             table_footer(type.upper())
534     elif not Cnf.has_key("Queue-Report::Options::822"):
535     # The "normal" output without any formatting.
536         format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
537
538         msg = ""
539         for entry in entries:
540             (source, binary, version_list, arch_list, note, last_modified, undef, undef, undef, undef, undef, undef, undef) = entry
541             msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
542
543         if msg:
544             total_count = len(changes_files)
545             source_count = len(per_source_items)
546             print type.upper()
547             print "-"*len(type)
548             print
549             print msg
550             print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count))
551             print
552
553
554 ################################################################################
555
556 def main():
557     global Cnf
558
559     Cnf = utils.get_conf()
560     Arguments = [('h',"help","Queue-Report::Options::Help"),
561                  ('n',"new","Queue-Report::Options::New"),
562                  ('8','822',"Queue-Report::Options::822"),
563                  ('s',"sort","Queue-Report::Options::Sort", "HasArg"),
564                  ('a',"age","Queue-Report::Options::Age", "HasArg"),
565                  ('d',"directories","Queue-Report::Options::Directories", "HasArg")]
566     for i in [ "help" ]:
567         if not Cnf.has_key("Queue-Report::Options::%s" % (i)):
568             Cnf["Queue-Report::Options::%s" % (i)] = ""
569
570     apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
571
572     Options = Cnf.SubTree("Queue-Report::Options")
573     if Options["Help"]:
574         usage()
575
576     if Cnf.has_key("Queue-Report::Options::New"):
577         header()
578
579     # Initialize db so we can get the NEW comments
580     dbconn = DBConn()
581
582     directories = [ ]
583
584     if Cnf.has_key("Queue-Report::Options::Directories"):
585         for i in Cnf["Queue-Report::Options::Directories"].split(","):
586             directories.append(i)
587     elif Cnf.has_key("Queue-Report::Directories"):
588         directories = Cnf.ValueList("Queue-Report::Directories")
589     else:
590         directories = [ "byhand", "new" ]
591
592     f = None
593     if Cnf.has_key("Queue-Report::Options::822"):
594         # Open the report file
595         f = open(Cnf["Queue-Report::ReportLocations::822Location"], "w")
596
597     for directory in directories:
598         changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)]))
599         process_changes_files(changes_files, directory, f)
600
601     if Cnf.has_key("Queue-Report::Options::822"):
602         f.close()
603
604     if Cnf.has_key("Queue-Report::Options::New"):
605         footer()
606
607 ################################################################################
608
609 if __name__ == '__main__':
610     main()