dak/queue_report.py

   1 #!/usr/bin/env python
   2
   3 # Produces a report on NEW and BYHAND packages
   4 # Copyright (C) 2001, 2002, 2003, 2005, 2006  James Troup <james@nocrew.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 ################################################################################
  21
  22 # <o-o> XP runs GCC, XFREE86, SSH etc etc,.,, I feel almost like linux....
  23 # <o-o> I am very confident that I can replicate any Linux application on XP
  24 # <willy> o-o: *boggle*
  25 # <o-o> building from source.
  26 # <o-o> Viiru: I already run GIMP under XP
  27 # <willy> o-o: why do you capitalise the names of all pieces of software?
  28 # <o-o> willy: because I want the EMPHASIZE them....
  29 # <o-o> grr s/the/to/
  30 # <willy> o-o: it makes you look like ZIPPY the PINHEAD
  31 # <o-o> willy: no idea what you are talking about.
  32 # <willy> o-o: do some research
  33 # <o-o> willy: for what reason?
  34
  35 ################################################################################
  36
  37 import copy, glob, os, stat, sys, time
  38 import apt_pkg
  39 import cgi
  40 from daklib import queue
  41 from daklib import utils
  42 from daklib.dak_exceptions import *
  43
  44 Cnf = None
  45 Upload = None
  46 direction = []
  47 row_number = 0
  48
  49 ################################################################################
  50
  51 def usage(exit_code=0):
  52     print """Usage: dak queue-report
  53 Prints a report of packages in queue directories (usually new and byhand).
  54
  55   -h, --help                show this help and exit.
  56   -n, --new                 produce html-output
  57   -s, --sort=key            sort output according to key, see below.
  58   -a, --age=key             if using sort by age, how should time be treated?
  59                             If not given a default of hours will be used.
  60
  61      Sorting Keys: ao=age,   oldest first.   an=age,   newest first.
  62                    na=name,  ascending       nd=name,  descending
  63                    nf=notes, first           nl=notes, last
  64
  65      Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years
  66
  67 """
  68     sys.exit(exit_code)
  69
  70 ################################################################################
  71
  72 def plural(x):
  73     if x > 1:
  74         return "s"
  75     else:
  76         return ""
  77
  78 ################################################################################
  79
  80 def time_pp(x):
  81     if x < 60:
  82         unit="second"
  83     elif x < 3600:
  84         x /= 60
  85         unit="minute"
  86     elif x < 86400:
  87         x /= 3600
  88         unit="hour"
  89     elif x < 604800:
  90         x /= 86400
  91         unit="day"
  92     elif x < 2419200:
  93         x /= 604800
  94         unit="week"
  95     elif x < 29030400:
  96         x /= 2419200
  97         unit="month"
  98     else:
  99         x /= 29030400
 100         unit="year"
 101     x = int(x)
 102     return "%s %s%s" % (x, unit, plural(x))
 103
 104 ################################################################################
 105
 106 def sg_compare (a, b):
 107     a = a[1]
 108     b = b[1]
 109     """Sort by have note, time of oldest upload."""
 110     # Sort by have note
 111     a_note_state = a["note_state"]
 112     b_note_state = b["note_state"]
 113     if a_note_state < b_note_state:
 114         return -1
 115     elif a_note_state > b_note_state:
 116         return 1
 117
 118     # Sort by time of oldest upload
 119     return cmp(a["oldest"], b["oldest"])
 120
 121 ############################################################
 122
 123 def sortfunc(a,b):
 124     for sorting in direction:
 125         (sortkey, way, time) = sorting
 126         ret = 0
 127         if time == "m":
 128             x=int(a[sortkey]/60)
 129             y=int(b[sortkey]/60)
 130         elif time == "h":
 131             x=int(a[sortkey]/3600)
 132             y=int(b[sortkey]/3600)
 133         elif time == "d":
 134             x=int(a[sortkey]/86400)
 135             y=int(b[sortkey]/86400)
 136         elif time == "w":
 137             x=int(a[sortkey]/604800)
 138             y=int(b[sortkey]/604800)
 139         elif time == "o":
 140             x=int(a[sortkey]/2419200)
 141             y=int(b[sortkey]/2419200)
 142         elif time == "y":
 143             x=int(a[sortkey]/29030400)
 144             y=int(b[sortkey]/29030400)
 145         else:
 146             x=a[sortkey]
 147             y=b[sortkey]
 148         if x < y:
 149             ret = -1
 150         elif x > y:
 151             ret = 1
 152         if ret != 0:
 153             if way < 0:
 154                 ret = ret*-1
 155             return ret
 156     return 0
 157
 158 ############################################################
 159
 160 def header():
 161     print """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 162 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 163 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="de" lang="de">
 164   <head>
 165     <meta http-equiv="content-type" content="text/xhtml+xml; charset=utf-8" />
 166     <link type="text/css" rel="stylesheet" href="style.css" />
 167     <link rel="shortcut icon" href="http://www.debian.org/favicon.ico" />
 168     <title>
 169       Debian NEW and BYHAND Packages
 170     </title>
 171   </head>
 172   <body id="NEW">
 173     <div id="logo">
 174       <a href="http://www.debian.org/">
 175         <img src="http://www.debian.org/logos/openlogo-nd-50.png"
 176         alt="debian logo" /></a>
 177       <a href="http://www.debian.org/">
 178         <img src="http://www.debian.org/Pics/debian.png"
 179         alt="Debian Project" /></a>
 180     </div>
 181     <div id="titleblock">
 182
 183       <img src="http://www.debian.org/Pics/red-upperleft.png"
 184       id="red-upperleft" alt="corner image"/>
 185       <img src="http://www.debian.org/Pics/red-lowerleft.png"
 186       id="red-lowerleft" alt="corner image"/>
 187       <img src="http://www.debian.org/Pics/red-upperright.png"
 188       id="red-upperright" alt="corner image"/>
 189       <img src="http://www.debian.org/Pics/red-lowerright.png"
 190       id="red-lowerright" alt="corner image"/>
 191       <span class="title">
 192         Debian NEW and BYHAND Packages
 193       </span>
 194     </div>
 195     """
 196
 197 def footer():
 198     print "<p class=\"timestamp\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
 199
 200     print """
 201     <div class="footer">
 202     <p>Hint: Age is the youngest upload of the package, if there is more than
 203     one version.<br />
 204     You may want to look at <a href="http://ftp-master.debian.org/REJECT-FAQ.html">the REJECT-FAQ</a>
 205       for possible reasons why one of the above packages may get rejected.</p>
 206       <p>
 207       <a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-xhtml10"
 208         alt="Valid XHTML 1.0 Strict" height="31" width="88" /></a>
 209       <a href="http://jigsaw.w3.org/css-validator/">
 210         <img style="border:0;width:88px;height:31px" src="http://jigsaw.w3.org/css-validator/images/vcss"
 211         alt="Valid CSS!" />
 212       </a>
 213       </p>
 214     </div> </body> </html>
 215     """
 216
 217 def table_header(type, source_count, total_count):
 218     print "<h1>Summary for: %s</h1>" % (type)
 219     print """
 220     <table class="NEW">
 221       <caption>
 222     """
 223     print "Package count in <strong>%s</strong>: <em>%s</em>&nbsp;|&nbsp; Total Package count: <em>%s</em>" % (type, source_count, total_count)
 224     print """
 225       </caption>
 226       <thead>
 227         <tr>
 228           <th>Package</th>
 229           <th>Version</th>
 230           <th>Arch</th>
 231           <th>Distribution</th>
 232           <th>Age</th>
 233           <th>Upload info</th>
 234           <th>Closes</th>
 235         </tr>
 236       </thead>
 237       <tbody>
 238     """
 239
 240 def table_footer(type):
 241     print "</tbody></table>"
 242
 243
 244 def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
 245
 246     global row_number
 247
 248     trclass = "sid"
 249     for dist in distribution:
 250         if dist == "experimental":
 251             trclass = "exp"
 252
 253     if row_number % 2 != 0:
 254         print "<tr class=\"%s even\">" % (trclass)
 255     else:
 256         print "<tr class=\"%s odd\">" % (trclass)
 257
 258     print "<td class=\"package\">%s</td>" % (source)
 259     print "<td class=\"version\">"
 260     for vers in version.split():
 261         print "<a href=\"/new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers))
 262     print "</td>"
 263     print "<td class=\"arch\">%s</td>" % (arch)
 264     print "<td class=\"distribution\">"
 265     for dist in distribution:
 266         print "%s<br/>" % (dist)
 267     print "</td>"
 268     print "<td class=\"age\">%s</td>" % (last_mod)
 269     (name, mail) = maint.split(":")
 270
 271     print "<td class=\"upload-data\">"
 272     print "<span class=\"maintainer\">Maintainer: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
 273     (name, mail) = changedby.split(":")
 274     print "<span class=\"changed-by\">Changed-By: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
 275
 276     try:
 277         (login, domain) = sponsor.split("@")
 278         print "<span class=\"sponsor\">Sponsor: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span>@debian.org<br/>" % (utils.html_escape(login), utils.html_escape(login))
 279     except:
 280         pass
 281
 282     print "<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint)
 283     print "</td>"
 284
 285     print "<td class=\"closes\">"
 286     for close in closes:
 287         print "<a href=\"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close))
 288     print "</td></tr>"
 289     row_number+=1
 290
 291 ############################################################
 292
 293 def process_changes_files(changes_files, type):
 294     msg = ""
 295     cache = {}
 296     # Read in all the .changes files
 297     for filename in changes_files:
 298         try:
 299             Upload.pkg.changes_file = filename
 300             Upload.init_vars()
 301             Upload.update_vars()
 302             cache[filename] = copy.copy(Upload.pkg.changes)
 303             cache[filename]["filename"] = filename
 304         except:
 305             break
 306     # Divide the .changes into per-source groups
 307     per_source = {}
 308     for filename in cache.keys():
 309         source = cache[filename]["source"]
 310         if not per_source.has_key(source):
 311             per_source[source] = {}
 312             per_source[source]["list"] = []
 313         per_source[source]["list"].append(cache[filename])
 314     # Determine oldest time and have note status for each source group
 315     for source in per_source.keys():
 316         source_list = per_source[source]["list"]
 317         first = source_list[0]
 318         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 319         have_note = 0
 320         for d in per_source[source]["list"]:
 321             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 322             if Cnf.has_key("Queue-Report::Options::New"):
 323                 if mtime > oldest:
 324                     oldest = mtime
 325             else:
 326                 if mtime < oldest:
 327                     oldest = mtime
 328             have_note += (d.has_key("process-new note"))
 329         per_source[source]["oldest"] = oldest
 330         if not have_note:
 331             per_source[source]["note_state"] = 0; # none
 332         elif have_note < len(source_list):
 333             per_source[source]["note_state"] = 1; # some
 334         else:
 335             per_source[source]["note_state"] = 2; # all
 336     per_source_items = per_source.items()
 337     per_source_items.sort(sg_compare)
 338
 339     entries = []
 340     max_source_len = 0
 341     max_version_len = 0
 342     max_arch_len = 0
 343     for i in per_source_items:
 344         maintainer = {}
 345         maint=""
 346         distribution=""
 347         closes=""
 348         fingerprint=""
 349         changeby = {}
 350         changedby=""
 351         sponsor=""
 352         last_modified = time.time()-i[1]["oldest"]
 353         source = i[1]["list"][0]["source"]
 354         if len(source) > max_source_len:
 355             max_source_len = len(source)
 356         arches = {}
 357         versions = {}
 358         for j in i[1]["list"]:
 359             if Cnf.has_key("Queue-Report::Options::New"):
 360                 try:
 361                     (maintainer["maintainer822"], maintainer["maintainer2047"],
 362                     maintainer["maintainername"], maintainer["maintaineremail"]) = \
 363                     utils.fix_maintainer (j["maintainer"])
 364                 except ParseMaintError, msg:
 365                     print "Problems while parsing maintainer address\n"
 366                     maintainer["maintainername"] = "Unknown"
 367                     maintainer["maintaineremail"] = "Unknown"
 368                 maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
 369                 # ...likewise for the Changed-By: field if it exists.
 370                 try:
 371                     (changeby["changedby822"], changeby["changedby2047"],
 372                      changeby["changedbyname"], changeby["changedbyemail"]) = \
 373                      utils.fix_maintainer (j["changed-by"])
 374                 except ParseMaintError, msg:
 375                     (changeby["changedby822"], changeby["changedby2047"],
 376                      changeby["changedbyname"], changeby["changedbyemail"]) = \
 377                      ("", "", "", "")
 378                 changedby="%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"])
 379
 380                 distribution=j["distribution"].keys()
 381                 closes=j["closes"].keys()
 382                 fingerprint=j["fingerprint"]
 383                 if j.has_key("sponsoremail"):
 384                     sponsor=j["sponsoremail"]
 385             for arch in j["architecture"].keys():
 386                 arches[arch] = ""
 387             version = j["version"]
 388             versions[version] = ""
 389         arches_list = arches.keys()
 390         arches_list.sort(utils.arch_compare_sw)
 391         arch_list = " ".join(arches_list)
 392         version_list = " ".join(versions.keys())
 393         if len(version_list) > max_version_len:
 394             max_version_len = len(version_list)
 395         if len(arch_list) > max_arch_len:
 396             max_arch_len = len(arch_list)
 397         if i[1]["note_state"]:
 398             note = " | [N]"
 399         else:
 400             note = ""
 401         entries.append([source, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby])
 402
 403     # direction entry consists of "Which field, which direction, time-consider" where
 404     # time-consider says how we should treat last_modified. Thats all.
 405
 406     # Look for the options for sort and then do the sort.
 407     age = "h"
 408     if Cnf.has_key("Queue-Report::Options::Age"):
 409         age =  Cnf["Queue-Report::Options::Age"]
 410     if Cnf.has_key("Queue-Report::Options::New"):
 411     # If we produce html we always have oldest first.
 412         direction.append([4,-1,"ao"])
 413     else:
 414         if Cnf.has_key("Queue-Report::Options::Sort"):
 415             for i in Cnf["Queue-Report::Options::Sort"].split(","):
 416                 if i == "ao":
 417                     # Age, oldest first.
 418                     direction.append([4,-1,age])
 419                 elif i == "an":
 420                     # Age, newest first.
 421                     direction.append([4,1,age])
 422                 elif i == "na":
 423                     # Name, Ascending.
 424                     direction.append([0,1,0])
 425                 elif i == "nd":
 426                     # Name, Descending.
 427                     direction.append([0,-1,0])
 428                 elif i == "nl":
 429                     # Notes last.
 430                     direction.append([3,1,0])
 431                 elif i == "nf":
 432                     # Notes first.
 433                     direction.append([3,-1,0])
 434     entries.sort(lambda x, y: sortfunc(x, y))
 435     # Yes, in theory you can add several sort options at the commandline with. But my mind is to small
 436     # at the moment to come up with a real good sorting function that considers all the sidesteps you
 437     # have with it. (If you combine options it will simply take the last one at the moment).
 438     # Will be enhanced in the future.
 439
 440     if Cnf.has_key("Queue-Report::Options::New"):
 441         direction.append([4,1,"ao"])
 442         entries.sort(lambda x, y: sortfunc(x, y))
 443     # Output for a html file. First table header. then table_footer.
 444     # Any line between them is then a <tr> printed from subroutine table_row.
 445         if len(entries) > 0:
 446             total_count = len(changes_files)
 447             source_count = len(per_source_items)
 448             table_header(type.upper(), source_count, total_count)
 449             for entry in entries:
 450                 (source, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby) = entry
 451                 table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes, fingerprint, sponsor, changedby)
 452             table_footer(type.upper())
 453     else:
 454     # The "normal" output without any formatting.
 455         format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
 456
 457         msg = ""
 458         for entry in entries:
 459             (source, version_list, arch_list, note, last_modified, undef, undef, undef, undef, undef, undef) = entry
 460             msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
 461
 462         if msg:
 463             total_count = len(changes_files)
 464             source_count = len(per_source_items)
 465             print type.upper()
 466             print "-"*len(type)
 467             print
 468             print msg
 469             print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count))
 470             print
 471
 472
 473 ################################################################################
 474
 475 def main():
 476     global Cnf, Upload
 477
 478     Cnf = utils.get_conf()
 479     Arguments = [('h',"help","Queue-Report::Options::Help"),
 480                  ('n',"new","Queue-Report::Options::New"),
 481                  ('s',"sort","Queue-Report::Options::Sort", "HasArg"),
 482                  ('a',"age","Queue-Report::Options::Age", "HasArg")]
 483     for i in [ "help" ]:
 484         if not Cnf.has_key("Queue-Report::Options::%s" % (i)):
 485             Cnf["Queue-Report::Options::%s" % (i)] = ""
 486
 487     apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
 488
 489     Options = Cnf.SubTree("Queue-Report::Options")
 490     if Options["Help"]:
 491         usage()
 492
 493     Upload = queue.Upload(Cnf)
 494
 495     if Cnf.has_key("Queue-Report::Options::New"):
 496         header()
 497
 498     directories = Cnf.ValueList("Queue-Report::Directories")
 499     if not directories:
 500         directories = [ "byhand", "new" ]
 501
 502     for directory in directories:
 503         changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)]))
 504         process_changes_files(changes_files, directory)
 505
 506     if Cnf.has_key("Queue-Report::Options::New"):
 507         footer()
 508
 509 ################################################################################
 510
 511 if __name__ == '__main__':
 512     main()