dak/queue_report.py

   1 #!/usr/bin/env python
   2
   3 # Produces a report on NEW and BYHAND packages
   4 # Copyright (C) 2001, 2002, 2003, 2005, 2006  James Troup <james@nocrew.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 ################################################################################
  21
  22 # <o-o> XP runs GCC, XFREE86, SSH etc etc,.,, I feel almost like linux....
  23 # <o-o> I am very confident that I can replicate any Linux application on XP
  24 # <willy> o-o: *boggle*
  25 # <o-o> building from source.
  26 # <o-o> Viiru: I already run GIMP under XP
  27 # <willy> o-o: why do you capitalise the names of all pieces of software?
  28 # <o-o> willy: because I want the EMPHASIZE them....
  29 # <o-o> grr s/the/to/
  30 # <willy> o-o: it makes you look like ZIPPY the PINHEAD
  31 # <o-o> willy: no idea what you are talking about.
  32 # <willy> o-o: do some research
  33 # <o-o> willy: for what reason?
  34
  35 ################################################################################
  36
  37 import copy, glob, os, stat, sys, time
  38 import apt_pkg
  39 import dak.lib.queue, dak.lib.utils
  40
  41 Cnf = None
  42 Upload = None
  43 direction = []
  44 row_number = 0
  45
  46 ################################################################################
  47
  48 def usage(exit_code=0):
  49     print """Usage: dak queue-report
  50 Prints a report of packages in queue directories (usually new and byhand).
  51
  52   -h, --help                show this help and exit.
  53   -n, --new                 produce html-output
  54   -s, --sort=key            sort output according to key, see below.
  55   -a, --age=key             if using sort by age, how should time be treated?
  56                             If not given a default of hours will be used.
  57
  58      Sorting Keys: ao=age,   oldest first.   an=age,   newest first.
  59                    na=name,  ascending       nd=name,  descending
  60                    nf=notes, first           nl=notes, last
  61
  62      Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years
  63
  64 """
  65     sys.exit(exit_code)
  66
  67 ################################################################################
  68
  69 def plural(x):
  70     if x > 1:
  71         return "s"
  72     else:
  73         return ""
  74
  75 ################################################################################
  76
  77 def time_pp(x):
  78     if x < 60:
  79         unit="second"
  80     elif x < 3600:
  81         x /= 60
  82         unit="minute"
  83     elif x < 86400:
  84         x /= 3600
  85         unit="hour"
  86     elif x < 604800:
  87         x /= 86400
  88         unit="day"
  89     elif x < 2419200:
  90         x /= 604800
  91         unit="week"
  92     elif x < 29030400:
  93         x /= 2419200
  94         unit="month"
  95     else:
  96         x /= 29030400
  97         unit="year"
  98     x = int(x)
  99     return "%s %s%s" % (x, unit, plural(x))
 100
 101 ################################################################################
 102
 103 def sg_compare (a, b):
 104     a = a[1]
 105     b = b[1]
 106     """Sort by have note, time of oldest upload."""
 107     # Sort by have note
 108     a_note_state = a["note_state"]
 109     b_note_state = b["note_state"]
 110     if a_note_state < b_note_state:
 111         return -1
 112     elif a_note_state > b_note_state:
 113         return 1
 114
 115     # Sort by time of oldest upload
 116     return cmp(a["oldest"], b["oldest"])
 117
 118 ############################################################
 119
 120 def sortfunc(a,b):
 121      for sorting in direction:
 122          (sortkey, way, time) = sorting
 123          ret = 0
 124          if time == "m":
 125              x=int(a[sortkey]/60)
 126              y=int(b[sortkey]/60)
 127          elif time == "h":
 128              x=int(a[sortkey]/3600)
 129              y=int(b[sortkey]/3600)
 130          elif time == "d":
 131              x=int(a[sortkey]/86400)
 132              y=int(b[sortkey]/86400)
 133          elif time == "w":
 134              x=int(a[sortkey]/604800)
 135              y=int(b[sortkey]/604800)
 136          elif time == "o":
 137              x=int(a[sortkey]/2419200)
 138              y=int(b[sortkey]/2419200)
 139          elif time == "y":
 140              x=int(a[sortkey]/29030400)
 141              y=int(b[sortkey]/29030400)
 142          else:
 143              x=a[sortkey]
 144              y=b[sortkey]
 145          if x < y:
 146              ret = -1
 147          elif x > y:
 148              ret = 1
 149          if ret != 0:
 150              if way < 0:
 151                  ret = ret*-1
 152              return ret
 153      return 0
 154
 155 ############################################################
 156
 157 def header():
 158     print """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 159         <html><head><meta http-equiv="Content-Type" content="text/html; charset=iso8859-1">
 160         <title>Debian NEW and BYHAND Packages</title>
 161         <link type="text/css" rel="stylesheet" href="style.css">
 162         <link rel="shortcut icon" href="http://www.debian.org/favicon.ico">
 163         </head>
 164         <body>
 165         <div align="center">
 166         <a href="http://www.debian.org/">
 167      <img src="http://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a>
 168         <a href="http://www.debian.org/">
 169      <img src="http://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a>
 170         </div>
 171         <br />
 172         <table class="reddy" width="100%">
 173         <tr>
 174         <td class="reddy">
 175     <img src="http://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0"
 176      alt="" width="15" height="16"></td>
 177         <td rowspan="2" class="reddy">Debian NEW and BYHAND Packages</td>
 178         <td class="reddy">
 179     <img src="http://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0"
 180      alt="" width="16" height="16"></td>
 181         </tr>
 182         <tr>
 183         <td class="reddy">
 184     <img src="http://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0"
 185      alt="" width="16" height="16"></td>
 186         <td class="reddy">
 187     <img src="http://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0"
 188      alt="" width="15" height="16"></td>
 189         </tr>
 190         </table>
 191         """
 192
 193 def footer():
 194     print "<p class=\"validate\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
 195     print "<hr><p>Hint: Age is the youngest upload of the package, if there is more than one version.</p>"
 196     print "<p>You may want to look at <a href=\"http://ftp-master.debian.org/REJECT-FAQ.html\">the REJECT-FAQ</a> for possible reasons why one of the above packages may get rejected.</p>"
 197     print """<a href="http://validator.w3.org/check?uri=referer">
 198     <img border="0" src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88"></a>
 199         <a href="http://jigsaw.w3.org/css-validator/check/referer">
 200     <img border="0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!"
 201      height="31" width="88"></a>
 202     """
 203     print "</body></html>"
 204
 205 def table_header(type):
 206     print "<h1>Summary for: %s</h1>" % (type)
 207     print """<center><table border="0">
 208         <tr>
 209           <th align="center">Package</th>
 210           <th align="center">Version</th>
 211           <th align="center">Arch</th>
 212           <th align="center">Distribution</th>
 213           <th align="center">Age</th>
 214           <th align="center">Maintainer</th>
 215           <th align="center">Closes</th>
 216         </tr>
 217         """
 218
 219 def table_footer(type, source_count, total_count):
 220     print "</table></center><br>\n"
 221     print "<p class=\"validate\">Package count in <b>%s</b>: <i>%s</i>\n" % (type, source_count)
 222     print "<br>Total Package count: <i>%s</i></p>\n" % (total_count)
 223
 224 def force_to_latin(s):
 225     """Forces a string to Latin-1."""
 226     latin1_s = unicode(s,'utf-8')
 227     return latin1_s.encode('iso8859-1', 'replace')
 228
 229
 230 def table_row(source, version, arch, last_mod, maint, distribution, closes):
 231
 232     global row_number
 233
 234     if row_number % 2 != 0:
 235         print "<tr class=\"even\">"
 236     else:
 237         print "<tr class=\"odd\">"
 238
 239     tdclass = "sid"
 240     for dist in distribution:
 241         if dist == "experimental":
 242             tdclass = "exp"
 243     print "<td valign=\"top\" class=\"%s\">%s</td>" % (tdclass, source)
 244     print "<td valign=\"top\" class=\"%s\">" % (tdclass)
 245     for vers in version.split():
 246         print "%s<br>" % (vers)
 247     print "</td><td valign=\"top\" class=\"%s\">%s</td><td valign=\"top\" class=\"%s\">" % (tdclass, arch, tdclass)
 248     for dist in distribution:
 249         print "%s<br>" % (dist)
 250     print "</td><td valign=\"top\" class=\"%s\">%s</td>" % (tdclass, last_mod)
 251     (name, mail) = maint.split(":")
 252     name = force_to_latin(name)
 253
 254     print "<td valign=\"top\" class=\"%s\"><a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></td>" % (tdclass, mail, name)
 255     print "<td valign=\"top\" class=\"%s\">" % (tdclass)
 256     for close in closes:
 257         print "<a href=\"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br>" % (close, close)
 258     print "</td></tr>"
 259     row_number+=1
 260
 261 ############################################################
 262
 263 def process_changes_files(changes_files, type):
 264     msg = ""
 265     cache = {}
 266     # Read in all the .changes files
 267     for filename in changes_files:
 268         try:
 269             Upload.pkg.changes_file = filename
 270             Upload.init_vars()
 271             Upload.update_vars()
 272             cache[filename] = copy.copy(Upload.pkg.changes)
 273             cache[filename]["filename"] = filename
 274         except:
 275             break
 276     # Divide the .changes into per-source groups
 277     per_source = {}
 278     for filename in cache.keys():
 279         source = cache[filename]["source"]
 280         if not per_source.has_key(source):
 281             per_source[source] = {}
 282             per_source[source]["list"] = []
 283         per_source[source]["list"].append(cache[filename])
 284     # Determine oldest time and have note status for each source group
 285     for source in per_source.keys():
 286         source_list = per_source[source]["list"]
 287         first = source_list[0]
 288         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 289         have_note = 0
 290         for d in per_source[source]["list"]:
 291             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 292             if Cnf.has_key("Queue-Report::Options::New"):
 293                 if mtime > oldest:
 294                     oldest = mtime
 295             else:
 296                 if mtime < oldest:
 297                     oldest = mtime
 298             have_note += (d.has_key("process-new note"))
 299         per_source[source]["oldest"] = oldest
 300         if not have_note:
 301             per_source[source]["note_state"] = 0; # none
 302         elif have_note < len(source_list):
 303             per_source[source]["note_state"] = 1; # some
 304         else:
 305             per_source[source]["note_state"] = 2; # all
 306     per_source_items = per_source.items()
 307     per_source_items.sort(sg_compare)
 308
 309     entries = []
 310     max_source_len = 0
 311     max_version_len = 0
 312     max_arch_len = 0
 313     maintainer = {}
 314     maint=""
 315     distribution=""
 316     closes=""
 317     source_exists=""
 318     for i in per_source_items:
 319         last_modified = time.time()-i[1]["oldest"]
 320         source = i[1]["list"][0]["source"]
 321         if len(source) > max_source_len:
 322             max_source_len = len(source)
 323         arches = {}
 324         versions = {}
 325         for j in i[1]["list"]:
 326             if Cnf.has_key("Queue-Report::Options::New"):
 327                 try:
 328                     (maintainer["maintainer822"], maintainer["maintainer2047"],
 329                     maintainer["maintainername"], maintainer["maintaineremail"]) = \
 330                     dak.lib.utils.fix_maintainer (j["maintainer"])
 331                 except dak.lib.utils.ParseMaintError, msg:
 332                     print "Problems while parsing maintainer address\n"
 333                     maintainer["maintainername"] = "Unknown"
 334                     maintainer["maintaineremail"] = "Unknown"
 335                 maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
 336                 distribution=j["distribution"].keys()
 337                 closes=j["closes"].keys()
 338             for arch in j["architecture"].keys():
 339                 arches[arch] = ""
 340             version = j["version"]
 341             versions[version] = ""
 342         arches_list = arches.keys()
 343         arches_list.sort(dak.lib.utils.arch_compare_sw)
 344         arch_list = " ".join(arches_list)
 345         version_list = " ".join(versions.keys())
 346         if len(version_list) > max_version_len:
 347             max_version_len = len(version_list)
 348         if len(arch_list) > max_arch_len:
 349             max_arch_len = len(arch_list)
 350         if i[1]["note_state"]:
 351             note = " | [N]"
 352         else:
 353             note = ""
 354         entries.append([source, version_list, arch_list, note, last_modified, maint, distribution, closes])
 355
 356     # direction entry consists of "Which field, which direction, time-consider" where
 357     # time-consider says how we should treat last_modified. Thats all.
 358
 359     # Look for the options for sort and then do the sort.
 360     age = "h"
 361     if Cnf.has_key("Queue-Report::Options::Age"):
 362         age =  Cnf["Queue-Report::Options::Age"]
 363     if Cnf.has_key("Queue-Report::Options::New"):
 364     # If we produce html we always have oldest first.
 365         direction.append([4,-1,"ao"])
 366     else:
 367                 if Cnf.has_key("Queue-Report::Options::Sort"):
 368                         for i in Cnf["Queue-Report::Options::Sort"].split(","):
 369                           if i == "ao":
 370                                   # Age, oldest first.
 371                                   direction.append([4,-1,age])
 372                           elif i == "an":
 373                                   # Age, newest first.
 374                                   direction.append([4,1,age])
 375                           elif i == "na":
 376                                   # Name, Ascending.
 377                                   direction.append([0,1,0])
 378                           elif i == "nd":
 379                                   # Name, Descending.
 380                                   direction.append([0,-1,0])
 381                           elif i == "nl":
 382                                   # Notes last.
 383                                   direction.append([3,1,0])
 384                           elif i == "nf":
 385                                   # Notes first.
 386                                   direction.append([3,-1,0])
 387     entries.sort(lambda x, y: sortfunc(x, y))
 388     # Yes, in theory you can add several sort options at the commandline with. But my mind is to small
 389     # at the moment to come up with a real good sorting function that considers all the sidesteps you
 390     # have with it. (If you combine options it will simply take the last one at the moment).
 391     # Will be enhanced in the future.
 392
 393     if Cnf.has_key("Queue-Report::Options::New"):
 394         direction.append([4,1,"ao"])
 395         entries.sort(lambda x, y: sortfunc(x, y))
 396     # Output for a html file. First table header. then table_footer.
 397     # Any line between them is then a <tr> printed from subroutine table_row.
 398         if len(entries) > 0:
 399             table_header(type.upper())
 400             for entry in entries:
 401                 (source, version_list, arch_list, note, last_modified, maint, distribution, closes) = entry
 402                 table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes)
 403             total_count = len(changes_files)
 404             source_count = len(per_source_items)
 405             table_footer(type.upper(), source_count, total_count)
 406     else:
 407     # The "normal" output without any formatting.
 408         format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
 409
 410         msg = ""
 411         for entry in entries:
 412             (source, version_list, arch_list, note, last_modified, undef, undef, undef) = entry
 413             msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
 414
 415         if msg:
 416             total_count = len(changes_files)
 417             source_count = len(per_source_items)
 418             print type.upper()
 419             print "-"*len(type)
 420             print
 421             print msg
 422             print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count))
 423             print
 424
 425
 426 ################################################################################
 427
 428 def main():
 429     global Cnf, Upload
 430
 431     Cnf = dak.lib.utils.get_conf()
 432     Arguments = [('h',"help","Queue-Report::Options::Help"),
 433                  ('n',"new","Queue-Report::Options::New"),
 434                  ('s',"sort","Queue-Report::Options::Sort", "HasArg"),
 435                  ('a',"age","Queue-Report::Options::Age", "HasArg")]
 436     for i in [ "help" ]:
 437         if not Cnf.has_key("Queue-Report::Options::%s" % (i)):
 438             Cnf["Queue-Report::Options::%s" % (i)] = ""
 439
 440     apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
 441
 442     Options = Cnf.SubTree("Queue-Report::Options")
 443     if Options["Help"]:
 444         usage()
 445
 446     Upload = dak.lib.queue.Upload(Cnf)
 447
 448     if Cnf.has_key("Queue-Report::Options::New"):
 449         header()
 450
 451     directories = Cnf.ValueList("Queue-Report::Directories")
 452     if not directories:
 453         directories = [ "byhand", "new" ]
 454
 455     for directory in directories:
 456         changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)]))
 457         process_changes_files(changes_files, directory)
 458
 459     if Cnf.has_key("Queue-Report::Options::New"):
 460         footer()
 461
 462 ################################################################################
 463
 464 if __name__ == '__main__':
 465     main()