dak/queue_report.py

   1 #!/usr/bin/env python
   2
   3 """ Produces a report on NEW and BYHAND packages """
   4 # Copyright (C) 2001, 2002, 2003, 2005, 2006  James Troup <james@nocrew.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 ################################################################################
  21
  22 # <o-o> XP runs GCC, XFREE86, SSH etc etc,.,, I feel almost like linux....
  23 # <o-o> I am very confident that I can replicate any Linux application on XP
  24 # <willy> o-o: *boggle*
  25 # <o-o> building from source.
  26 # <o-o> Viiru: I already run GIMP under XP
  27 # <willy> o-o: why do you capitalise the names of all pieces of software?
  28 # <o-o> willy: because I want the EMPHASIZE them....
  29 # <o-o> grr s/the/to/
  30 # <willy> o-o: it makes you look like ZIPPY the PINHEAD
  31 # <o-o> willy: no idea what you are talking about.
  32 # <willy> o-o: do some research
  33 # <o-o> willy: for what reason?
  34
  35 ################################################################################
  36
  37 from copy import copy
  38 import glob, os, stat, sys, time
  39 import apt_pkg
  40
  41 from daklib import utils
  42 from daklib.queue import Upload
  43 from daklib.dbconn import DBConn, has_new_comment, DBChange, get_uid_from_fingerprint
  44 from daklib.textutils import fix_maintainer
  45 from daklib.dak_exceptions import *
  46
  47 Cnf = None
  48 direction = []
  49 row_number = 0
  50
  51 ################################################################################
  52
  53 def usage(exit_code=0):
  54     print """Usage: dak queue-report
  55 Prints a report of packages in queue directories (usually new and byhand).
  56
  57   -h, --help                show this help and exit.
  58   -8, --822                 writes 822 formated output to the location set in dak.conf
  59   -n, --new                 produce html-output
  60   -s, --sort=key            sort output according to key, see below.
  61   -a, --age=key             if using sort by age, how should time be treated?
  62                             If not given a default of hours will be used.
  63   -d, --directories=key     A comma seperated list of queues to be scanned
  64
  65      Sorting Keys: ao=age,   oldest first.   an=age,   newest first.
  66                    na=name,  ascending       nd=name,  descending
  67                    nf=notes, first           nl=notes, last
  68
  69      Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years
  70
  71 """
  72     sys.exit(exit_code)
  73
  74 ################################################################################
  75
  76 def plural(x):
  77     if x > 1:
  78         return "s"
  79     else:
  80         return ""
  81
  82 ################################################################################
  83
  84 def time_pp(x):
  85     if x < 60:
  86         unit="second"
  87     elif x < 3600:
  88         x /= 60
  89         unit="minute"
  90     elif x < 86400:
  91         x /= 3600
  92         unit="hour"
  93     elif x < 604800:
  94         x /= 86400
  95         unit="day"
  96     elif x < 2419200:
  97         x /= 604800
  98         unit="week"
  99     elif x < 29030400:
 100         x /= 2419200
 101         unit="month"
 102     else:
 103         x /= 29030400
 104         unit="year"
 105     x = int(x)
 106     return "%s %s%s" % (x, unit, plural(x))
 107
 108 ################################################################################
 109
 110 def sg_compare (a, b):
 111     a = a[1]
 112     b = b[1]
 113     """Sort by have note, time of oldest upload."""
 114     # Sort by have note
 115     a_note_state = a["note_state"]
 116     b_note_state = b["note_state"]
 117     if a_note_state < b_note_state:
 118         return -1
 119     elif a_note_state > b_note_state:
 120         return 1
 121
 122     # Sort by time of oldest upload
 123     return cmp(a["oldest"], b["oldest"])
 124
 125 ############################################################
 126
 127 def sortfunc(a,b):
 128     for sorting in direction:
 129         (sortkey, way, time) = sorting
 130         ret = 0
 131         if time == "m":
 132             x=int(a[sortkey]/60)
 133             y=int(b[sortkey]/60)
 134         elif time == "h":
 135             x=int(a[sortkey]/3600)
 136             y=int(b[sortkey]/3600)
 137         elif time == "d":
 138             x=int(a[sortkey]/86400)
 139             y=int(b[sortkey]/86400)
 140         elif time == "w":
 141             x=int(a[sortkey]/604800)
 142             y=int(b[sortkey]/604800)
 143         elif time == "o":
 144             x=int(a[sortkey]/2419200)
 145             y=int(b[sortkey]/2419200)
 146         elif time == "y":
 147             x=int(a[sortkey]/29030400)
 148             y=int(b[sortkey]/29030400)
 149         else:
 150             x=a[sortkey]
 151             y=b[sortkey]
 152         if x < y:
 153             ret = -1
 154         elif x > y:
 155             ret = 1
 156         if ret != 0:
 157             if way < 0:
 158                 ret = ret*-1
 159             return ret
 160     return 0
 161
 162 ############################################################
 163
 164 def header():
 165     print """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 166 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 167 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="de" lang="de">
 168   <head>
 169     <meta http-equiv="content-type" content="text/xhtml+xml; charset=utf-8" />
 170     <link type="text/css" rel="stylesheet" href="style.css" />
 171     <link rel="shortcut icon" href="http://www.debian.org/favicon.ico" />
 172     <title>
 173       Debian NEW and BYHAND Packages
 174     </title>
 175   </head>
 176   <body id="NEW">
 177     <div id="logo">
 178       <a href="http://www.debian.org/">
 179         <img src="http://www.debian.org/logos/openlogo-nd-50.png"
 180         alt="debian logo" /></a>
 181       <a href="http://www.debian.org/">
 182         <img src="http://www.debian.org/Pics/debian.png"
 183         alt="Debian Project" /></a>
 184     </div>
 185     <div id="titleblock">
 186
 187       <img src="http://www.debian.org/Pics/red-upperleft.png"
 188       id="red-upperleft" alt="corner image"/>
 189       <img src="http://www.debian.org/Pics/red-lowerleft.png"
 190       id="red-lowerleft" alt="corner image"/>
 191       <img src="http://www.debian.org/Pics/red-upperright.png"
 192       id="red-upperright" alt="corner image"/>
 193       <img src="http://www.debian.org/Pics/red-lowerright.png"
 194       id="red-lowerright" alt="corner image"/>
 195       <span class="title">
 196         Debian NEW and BYHAND Packages
 197       </span>
 198     </div>
 199     """
 200
 201 def footer():
 202     print "<p class=\"timestamp\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
 203
 204     print """
 205     <div class="footer">
 206     <p>Hint: Age is the youngest upload of the package, if there is more than
 207     one version.<br />
 208     You may want to look at <a href="http://ftp-master.debian.org/REJECT-FAQ.html">the REJECT-FAQ</a>
 209       for possible reasons why one of the above packages may get rejected.</p>
 210       <p>
 211       <a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-xhtml10"
 212         alt="Valid XHTML 1.0 Strict" height="31" width="88" /></a>
 213       <a href="http://jigsaw.w3.org/css-validator/">
 214         <img style="border:0;width:88px;height:31px" src="http://jigsaw.w3.org/css-validator/images/vcss"
 215         alt="Valid CSS!" />
 216       </a>
 217       </p>
 218     </div> </body> </html>
 219     """
 220
 221 def table_header(type, source_count, total_count):
 222     print "<h1>Summary for: %s</h1>" % (type)
 223     print """
 224     <table class="NEW">
 225       <caption>
 226     """
 227     print "Package count in <strong>%s</strong>: <em>%s</em>&nbsp;|&nbsp; Total Package count: <em>%s</em>" % (type, source_count, total_count)
 228     print """
 229       </caption>
 230       <thead>
 231         <tr>
 232           <th>Package</th>
 233           <th>Version</th>
 234           <th>Arch</th>
 235           <th>Distribution</th>
 236           <th>Age</th>
 237           <th>Upload info</th>
 238           <th>Closes</th>
 239         </tr>
 240       </thead>
 241       <tbody>
 242     """
 243
 244 def table_footer(type):
 245     print "</tbody></table>"
 246
 247
 248 def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
 249
 250     global row_number
 251
 252     trclass = "sid"
 253     for dist in distribution:
 254         if dist == "experimental":
 255             trclass = "exp"
 256
 257     if row_number % 2 != 0:
 258         print "<tr class=\"%s even\">" % (trclass)
 259     else:
 260         print "<tr class=\"%s odd\">" % (trclass)
 261
 262     print "<td class=\"package\">%s</td>" % (source)
 263     print "<td class=\"version\">"
 264     for vers in version.split():
 265         print "<a href=\"/new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers))
 266     print "</td>"
 267     print "<td class=\"arch\">%s</td>" % (arch)
 268     print "<td class=\"distribution\">"
 269     for dist in distribution:
 270         print "%s<br/>" % (dist)
 271     print "</td>"
 272     print "<td class=\"age\">%s</td>" % (last_mod)
 273     (name, mail) = maint.split(":", 1)
 274
 275     print "<td class=\"upload-data\">"
 276     print "<span class=\"maintainer\">Maintainer: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
 277     (name, mail) = changedby.split(":", 1)
 278     print "<span class=\"changed-by\">Changed-By: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
 279
 280     if sponsor:
 281         try:
 282             (login, domain) = sponsor.split("@", 1)
 283             print "<span class=\"sponsor\">Sponsor: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span>@debian.org<br/>" % (utils.html_escape(login), utils.html_escape(login))
 284         except Exception, e:
 285             pass
 286
 287     print "<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint)
 288     print "</td>"
 289
 290     print "<td class=\"closes\">"
 291     for close in closes:
 292         print "<a href=\"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close))
 293     print "</td></tr>"
 294     row_number+=1
 295
 296 ############################################################
 297
 298 def process_changes_files(changes_files, type, log):
 299     session = DBConn().session()
 300     msg = ""
 301     cache = {}
 302     # Read in all the .changes files
 303     for filename in changes_files:
 304         try:
 305             u = Upload()
 306             u.load_changes(filename)
 307             cache[filename] = copy(u.pkg.changes)
 308             cache[filename]["filename"] = filename
 309         except Exception, e:
 310             print "WARNING: Exception %s" % e
 311             continue
 312     # Divide the .changes into per-source groups
 313     per_source = {}
 314     for filename in cache.keys():
 315         source = cache[filename]["source"]
 316         if not per_source.has_key(source):
 317             per_source[source] = {}
 318             per_source[source]["list"] = []
 319         per_source[source]["list"].append(cache[filename])
 320     # Determine oldest time and have note status for each source group
 321     for source in per_source.keys():
 322         source_list = per_source[source]["list"]
 323         first = source_list[0]
 324         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 325         have_note = 0
 326         for d in per_source[source]["list"]:
 327             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 328             if Cnf.has_key("Queue-Report::Options::New"):
 329                 if mtime > oldest:
 330                     oldest = mtime
 331             else:
 332                 if mtime < oldest:
 333                     oldest = mtime
 334             have_note += has_new_comment(d["source"], d["version"])
 335         per_source[source]["oldest"] = oldest
 336         if not have_note:
 337             per_source[source]["note_state"] = 0; # none
 338         elif have_note < len(source_list):
 339             per_source[source]["note_state"] = 1; # some
 340         else:
 341             per_source[source]["note_state"] = 2; # all
 342     per_source_items = per_source.items()
 343     per_source_items.sort(sg_compare)
 344
 345     entries = []
 346     max_source_len = 0
 347     max_version_len = 0
 348     max_arch_len = 0
 349     for i in per_source_items:
 350         maintainer = {}
 351         maint=""
 352         distribution=""
 353         closes=""
 354         fingerprint=""
 355         changeby = {}
 356         changedby=""
 357         sponsor=""
 358         filename=i[1]["list"][0]["filename"]
 359         last_modified = time.time()-i[1]["oldest"]
 360         source = i[1]["list"][0]["source"]
 361         if len(source) > max_source_len:
 362             max_source_len = len(source)
 363         binary_list = i[1]["list"][0]["binary"].keys()
 364         binary = ', '.join(binary_list)
 365         arches = {}
 366         versions = {}
 367         for j in i[1]["list"]:
 368             changesbase = os.path.basename(j["filename"])
 369             try:
 370                 dbc = session.query(DBChange).filter_by(changesname=changesbase).one()
 371             except Exception, e:
 372                 print "Can't find changes file in NEW for %s (%s)" % (changesbase, e)
 373                 dbc = None
 374
 375             if Cnf.has_key("Queue-Report::Options::New") or Cnf.has_key("Queue-Report::Options::822"):
 376                 try:
 377                     (maintainer["maintainer822"], maintainer["maintainer2047"],
 378                     maintainer["maintainername"], maintainer["maintaineremail"]) = \
 379                     fix_maintainer (j["maintainer"])
 380                 except ParseMaintError, msg:
 381                     print "Problems while parsing maintainer address\n"
 382                     maintainer["maintainername"] = "Unknown"
 383                     maintainer["maintaineremail"] = "Unknown"
 384                 maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
 385                 # ...likewise for the Changed-By: field if it exists.
 386                 try:
 387                     (changeby["changedby822"], changeby["changedby2047"],
 388                      changeby["changedbyname"], changeby["changedbyemail"]) = \
 389                      fix_maintainer (j["changed-by"])
 390                 except ParseMaintError, msg:
 391                     (changeby["changedby822"], changeby["changedby2047"],
 392                      changeby["changedbyname"], changeby["changedbyemail"]) = \
 393                      ("", "", "", "")
 394                 changedby="%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"])
 395
 396                 distribution=j["distribution"].keys()
 397                 closes=j["closes"].keys()
 398                 if dbc:
 399                     fingerprint = dbc.fingerprint
 400                     sponsor_name = get_uid_from_fingerprint(fingerprint).name
 401                     sponsor_email = get_uid_from_fingerprint(fingerprint).uid + "@debian.org"
 402                     if sponsor_name != maintainer["maintainername"] and sponsor_name != changeby["changedbyname"] and \
 403                     sponsor_email != maintainer["maintaineremail"] and sponsor_name != changeby["changedbyemail"]:
 404                         sponsor = sponsor_email
 405
 406             for arch in j["architecture"].keys():
 407                 arches[arch] = ""
 408             version = j["version"]
 409             versions[version] = ""
 410         arches_list = arches.keys()
 411         arches_list.sort(utils.arch_compare_sw)
 412         arch_list = " ".join(arches_list)
 413         version_list = " ".join(versions.keys())
 414         if len(version_list) > max_version_len:
 415             max_version_len = len(version_list)
 416         if len(arch_list) > max_arch_len:
 417             max_arch_len = len(arch_list)
 418         if i[1]["note_state"]:
 419             note = " | [N]"
 420         else:
 421             note = ""
 422         entries.append([source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename])
 423
 424     # direction entry consists of "Which field, which direction, time-consider" where
 425     # time-consider says how we should treat last_modified. Thats all.
 426
 427     # Look for the options for sort and then do the sort.
 428     age = "h"
 429     if Cnf.has_key("Queue-Report::Options::Age"):
 430         age =  Cnf["Queue-Report::Options::Age"]
 431     if Cnf.has_key("Queue-Report::Options::New"):
 432     # If we produce html we always have oldest first.
 433         direction.append([5,-1,"ao"])
 434     else:
 435         if Cnf.has_key("Queue-Report::Options::Sort"):
 436             for i in Cnf["Queue-Report::Options::Sort"].split(","):
 437                 if i == "ao":
 438                     # Age, oldest first.
 439                     direction.append([5,-1,age])
 440                 elif i == "an":
 441                     # Age, newest first.
 442                     direction.append([5,1,age])
 443                 elif i == "na":
 444                     # Name, Ascending.
 445                     direction.append([0,1,0])
 446                 elif i == "nd":
 447                     # Name, Descending.
 448                     direction.append([0,-1,0])
 449                 elif i == "nl":
 450                     # Notes last.
 451                     direction.append([4,1,0])
 452                 elif i == "nf":
 453                     # Notes first.
 454                     direction.append([4,-1,0])
 455     entries.sort(lambda x, y: sortfunc(x, y))
 456     # Yes, in theory you can add several sort options at the commandline with. But my mind is to small
 457     # at the moment to come up with a real good sorting function that considers all the sidesteps you
 458     # have with it. (If you combine options it will simply take the last one at the moment).
 459     # Will be enhanced in the future.
 460
 461     if Cnf.has_key("Queue-Report::Options::822"):
 462         # print stuff out in 822 format
 463         for entry in entries:
 464             (source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry
 465
 466             # We'll always have Source, Version, Arch, Mantainer, and Dist
 467             # For the rest, check to see if we have them, then print them out
 468             log.write("Source: " + source + "\n")
 469             log.write("Binary: " + binary + "\n")
 470             log.write("Version: " + version_list + "\n")
 471             log.write("Architectures: ")
 472             log.write( (", ".join(arch_list.split(" "))) + "\n")
 473             log.write("Age: " + time_pp(last_modified) + "\n")
 474             log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n")
 475             log.write("Queue: " + type + "\n")
 476
 477             (name, mail) = maint.split(":", 1)
 478             log.write("Maintainer: " + name + " <"+mail+">" + "\n")
 479             if changedby:
 480                (name, mail) = changedby.split(":", 1)
 481                log.write("Changed-By: " + name + " <"+mail+">" + "\n")
 482             if sponsor:
 483                log.write("Sponsored-By: " + sponsor + "\n")
 484             log.write("Distribution:")
 485             for dist in distribution:
 486                log.write(" " + dist)
 487             log.write("\n")
 488             log.write("Fingerprint: " + fingerprint + "\n")
 489             if closes:
 490                 bug_string = ""
 491                 for bugs in closes:
 492                     bug_string += "#"+bugs+", "
 493                 log.write("Closes: " + bug_string[:-2] + "\n")
 494             log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
 495             log.write("\n")
 496
 497     if Cnf.has_key("Queue-Report::Options::New"):
 498         direction.append([5,1,"ao"])
 499         entries.sort(lambda x, y: sortfunc(x, y))
 500     # Output for a html file. First table header. then table_footer.
 501     # Any line between them is then a <tr> printed from subroutine table_row.
 502         if len(entries) > 0:
 503             total_count = len(changes_files)
 504             source_count = len(per_source_items)
 505             table_header(type.upper(), source_count, total_count)
 506             for entry in entries:
 507                 (source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, undef) = entry
 508                 table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes, fingerprint, sponsor, changedby)
 509             table_footer(type.upper())
 510     elif not Cnf.has_key("Queue-Report::Options::822"):
 511     # The "normal" output without any formatting.
 512         format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
 513
 514         msg = ""
 515         for entry in entries:
 516             (source, binary, version_list, arch_list, note, last_modified, undef, undef, undef, undef, undef, undef, undef) = entry
 517             msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
 518
 519         if msg:
 520             total_count = len(changes_files)
 521             source_count = len(per_source_items)
 522             print type.upper()
 523             print "-"*len(type)
 524             print
 525             print msg
 526             print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count))
 527             print
 528
 529
 530 ################################################################################
 531
 532 def main():
 533     global Cnf
 534
 535     Cnf = utils.get_conf()
 536     Arguments = [('h',"help","Queue-Report::Options::Help"),
 537                  ('n',"new","Queue-Report::Options::New"),
 538                  ('8','822',"Queue-Report::Options::822"),
 539                  ('s',"sort","Queue-Report::Options::Sort", "HasArg"),
 540                  ('a',"age","Queue-Report::Options::Age", "HasArg"),
 541                  ('d',"directories","Queue-Report::Options::Directories", "HasArg")]
 542     for i in [ "help" ]:
 543         if not Cnf.has_key("Queue-Report::Options::%s" % (i)):
 544             Cnf["Queue-Report::Options::%s" % (i)] = ""
 545
 546     apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
 547
 548     Options = Cnf.SubTree("Queue-Report::Options")
 549     if Options["Help"]:
 550         usage()
 551
 552     if Cnf.has_key("Queue-Report::Options::New"):
 553         header()
 554
 555     # Initialize db so we can get the NEW comments
 556     dbconn = DBConn()
 557
 558     directories = [ ]
 559
 560     if Cnf.has_key("Queue-Report::Options::Directories"):
 561         for i in Cnf["Queue-Report::Options::Directories"].split(","):
 562             directories.append(i)
 563     elif Cnf.has_key("Queue-Report::Directories"):
 564         directories = Cnf.ValueList("Queue-Report::Directories")
 565     else:
 566         directories = [ "byhand", "new" ]
 567
 568     f = None
 569     if Cnf.has_key("Queue-Report::Options::822"):
 570         # Open the report file
 571         f = open(Cnf["Queue-Report::ReportLocations::822Location"], "w")
 572
 573     for directory in directories:
 574         changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)]))
 575         process_changes_files(changes_files, directory, f)
 576
 577     if Cnf.has_key("Queue-Report::Options::822"):
 578         f.close()
 579
 580     if Cnf.has_key("Queue-Report::Options::New"):
 581         footer()
 582
 583 ################################################################################
 584
 585 if __name__ == '__main__':
 586     main()