dak/show_deferred.py

   1 #!/usr/bin/env python
   2
   3 """ Overview of the DEFERRED queue, based on queue-report """
   4 #    Copyright (C) 2001, 2002, 2003, 2005, 2006  James Troup <james@nocrew.org>
   5 # Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ################################################################################
  22
  23 import sys, os, re, time
  24 import apt_pkg
  25 import rrdtool
  26
  27 from debian import deb822
  28
  29 from daklib.dbconn import *
  30 from daklib.gpg import SignedFile
  31 from daklib import utils
  32 from daklib.regexes import re_html_escaping, html_escaping
  33
  34 ################################################################################
  35 ### work around bug #487902 in debian-python 0.1.10
  36 deb822.Changes._multivalued_fields = {
  37             "files": [ "md5sum", "size", "section", "priority", "name" ],
  38             "checksums-sha1": ["sha1", "size", "name"],
  39             "checksums-sha256": ["sha256", "size", "name"],
  40           }
  41
  42 ################################################################################
  43
  44 row_number = 1
  45
  46 def html_escape(s):
  47     return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)
  48
  49 ################################################################################
  50
  51 def header():
  52   return  """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
  53         <html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
  54         <title>Deferred uploads to Debian</title>
  55         <link type="text/css" rel="stylesheet" href="style.css">
  56         <link rel="shortcut icon" href="https://www.debian.org/favicon.ico">
  57         </head>
  58         <body>
  59         <div align="center">
  60         <a href="https://www.debian.org/">
  61      <img src="https://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a>
  62         <a href="https://www.debian.org/">
  63      <img src="https://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a>
  64         </div>
  65         <br />
  66         <table class="reddy" width="100%">
  67         <tr>
  68         <td class="reddy">
  69     <img src="https://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0"
  70      alt="" width="15" height="16"></td>
  71         <td rowspan="2" class="reddy">Deferred uploads to Debian</td>
  72         <td class="reddy">
  73     <img src="https://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0"
  74      alt="" width="16" height="16"></td>
  75         </tr>
  76         <tr>
  77         <td class="reddy">
  78     <img src="https://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0"
  79      alt="" width="16" height="16"></td>
  80         <td class="reddy">
  81     <img src="https://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0"
  82      alt="" width="15" height="16"></td>
  83         </tr>
  84         </table>
  85         """
  86
  87 def footer():
  88     res = "<p class=\"validate\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
  89     res += "<p class=\"timestamp\">There are <a href=\"/stat.html\">graphs about the queues</a> available.</p>"
  90     res += """<a href="http://validator.w3.org/check?uri=referer">
  91     <img border="0" src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88"></a>
  92         <a href="http://jigsaw.w3.org/css-validator/check/referer">
  93     <img border="0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!"
  94      height="31" width="88"></a>
  95     """
  96     res += "</body></html>"
  97     return res.encode('utf-8')
  98
  99 def table_header():
 100     return """<h1>Deferred uploads</h1>
 101       <center><table border="0">
 102         <tr>
 103           <th align="center">Change</th>
 104           <th align="center">Time remaining</th>
 105           <th align="center">Uploader</th>
 106           <th align="center">Closes</th>
 107         </tr>
 108         """
 109
 110 def table_footer():
 111     return '</table><br/><p>non-NEW uploads are <a href="/deferred/">available</a>, see the <a href="ftp://ftp-master.debian.org/pub/UploadQueue/README">UploadQueue-README</a> for more information.</p></center><br/>\n'
 112
 113 def table_row(changesname, delay, changed_by, closes, fingerprint):
 114     global row_number
 115
 116     res = '<tr class="%s">'%((row_number%2) and 'odd' or 'even')
 117     res += (2*'<td valign="top">%s</td>')%tuple(map(html_escape,(changesname,delay)))
 118     res += '<td valign="top">%s<br><span class=\"deferredfp\">Fingerprint: %s</span></td>' % (html_escape(changed_by), fingerprint)
 119     res += ('<td valign="top">%s</td>' %
 120              ''.join(map(lambda close:  '<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>' % (close, close),closes)))
 121     res += '</tr>\n'
 122     row_number+=1
 123     return res
 124
 125 def update_graph_database(rrd_dir, *counts):
 126     if not rrd_dir:
 127         return
 128
 129     rrd_file = os.path.join(rrd_dir, 'deferred.rrd')
 130     counts = [str(count) for count in counts]
 131     update = [rrd_file, "N:"+":".join(counts)]
 132
 133     try:
 134         rrdtool.update(*update)
 135     except rrdtool.error:
 136         create = [rrd_file]+"""
 137 --step
 138 300
 139 --start
 140 0
 141 DS:day0:GAUGE:7200:0:1000
 142 DS:day1:GAUGE:7200:0:1000
 143 DS:day2:GAUGE:7200:0:1000
 144 DS:day3:GAUGE:7200:0:1000
 145 DS:day4:GAUGE:7200:0:1000
 146 DS:day5:GAUGE:7200:0:1000
 147 DS:day6:GAUGE:7200:0:1000
 148 DS:day7:GAUGE:7200:0:1000
 149 DS:day8:GAUGE:7200:0:1000
 150 DS:day9:GAUGE:7200:0:1000
 151 DS:day10:GAUGE:7200:0:1000
 152 DS:day11:GAUGE:7200:0:1000
 153 DS:day12:GAUGE:7200:0:1000
 154 DS:day13:GAUGE:7200:0:1000
 155 DS:day14:GAUGE:7200:0:1000
 156 DS:day15:GAUGE:7200:0:1000
 157 RRA:AVERAGE:0.5:1:599
 158 RRA:AVERAGE:0.5:6:700
 159 RRA:AVERAGE:0.5:24:775
 160 RRA:AVERAGE:0.5:288:795
 161 RRA:MIN:0.5:1:600
 162 RRA:MIN:0.5:6:700
 163 RRA:MIN:0.5:24:775
 164 RRA:MIN:0.5:288:795
 165 RRA:MAX:0.5:1:600
 166 RRA:MAX:0.5:6:700
 167 RRA:MAX:0.5:24:775
 168 RRA:MAX:0.5:288:795
 169 """.strip().split("\n")
 170         try:
 171             rc = rrdtool.create(*create)
 172             ru = rrdtool.update(*update)
 173         except rrdtool.error as e:
 174             print('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e))
 175     except NameError:
 176         pass
 177
 178 def get_upload_data(changesfn):
 179     achanges = deb822.Changes(file(changesfn))
 180     changesname = os.path.basename(changesfn)
 181     delay = os.path.basename(os.path.dirname(changesfn))
 182     m = re.match(r'([0-9]+)-day', delay)
 183     if m:
 184         delaydays = int(m.group(1))
 185         remainingtime = (delaydays>0)*max(0,24*60*60+os.stat(changesfn).st_mtime-time.time())
 186         delay = "%d days %02d:%02d" %(max(delaydays-1,0), int(remainingtime/3600),int(remainingtime/60)%60)
 187     else:
 188         delaydays = 0
 189         remainingtime = 0
 190
 191     uploader = achanges.get('changed-by')
 192     uploader = re.sub(r'^\s*(\S.*)\s+<.*>',r'\1',uploader)
 193     with utils.open_file(changesfn) as f:
 194         fingerprint = SignedFile(f.read(), keyrings=get_active_keyring_paths()).fingerprint
 195     if Cnf.has_key("Show-Deferred::LinkPath"):
 196         isnew = 0
 197         suites = get_suites_source_in(achanges['source'])
 198         if 'unstable' not in suites and 'experimental' not in suites:
 199             isnew = 1
 200
 201         for b in achanges['binary'].split():
 202             suites = get_suites_binary_in(b)
 203             if 'unstable' not in suites and 'experimental' not in suites:
 204                 isnew = 1
 205
 206         if not isnew:
 207             # we don't link .changes because we don't want other people to
 208             # upload it with the existing signature.
 209             for afn in map(lambda x: x['name'],achanges['files']):
 210                 lfn = os.path.join(Cnf["Show-Deferred::LinkPath"],afn)
 211                 qfn = os.path.join(os.path.dirname(changesfn),afn)
 212                 if os.path.islink(lfn):
 213                     os.unlink(lfn)
 214                 if os.path.exists(qfn):
 215                     os.symlink(qfn,lfn)
 216                     os.chmod(qfn, 0o644)
 217     return (max(delaydays-1,0)*24*60*60+remainingtime, changesname, delay, uploader, achanges.get('closes','').split(), fingerprint, achanges, delaydays)
 218
 219 def list_uploads(filelist, rrd_dir):
 220     uploads = map(get_upload_data, filelist)
 221     uploads.sort()
 222     # print the summary page
 223     print header()
 224     if uploads:
 225         print table_header()
 226         print ''.join(map(lambda x: table_row(*x[1:6]), uploads)).encode('utf-8')
 227         print table_footer()
 228     else:
 229         print '<h1>Currently no deferred uploads to Debian</h1>'
 230     print footer()
 231     # machine readable summary
 232     if Cnf.has_key("Show-Deferred::LinkPath"):
 233         fn = os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp')
 234         f = open(fn,"w")
 235         try:
 236             counts = [0]*16
 237             for u in uploads:
 238                 counts[u[7]] += 1
 239                 print >> f, "Changes-file: %s"%u[1]
 240                 fields = """Location: DEFERRED
 241 Delayed-Until: %s
 242 Delay-Remaining: %s
 243 Fingerprint: %s"""%(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time()+u[0])),u[2], u[5])
 244                 print >> f, fields
 245                 encoded = unicode(u[6]).encode('utf-8')
 246                 print >> f, encoded.rstrip()
 247                 open(os.path.join(Cnf["Show-Deferred::LinkPath"],u[1]),"w").write(encoded+fields+'\n')
 248                 print >> f
 249             f.close()
 250             os.rename(os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp'),
 251                       os.path.join(Cnf["Show-Deferred::LinkPath"],'status'))
 252             update_graph_database(rrd_dir, *counts)
 253         except:
 254             os.unlink(fn)
 255             raise
 256
 257 def usage (exit_code=0):
 258     if exit_code:
 259         f = sys.stderr
 260     else:
 261         f = sys.stdout
 262     print >> f, """Usage: dak show-deferred
 263   -h, --help                    show this help and exit.
 264   -p, --link-path [path]        override output directory.
 265   -d, --deferred-queue [path]   path to the deferred queue
 266   -r, --rrd=key                 Directory where rrd files to be updated are stored
 267   """
 268     sys.exit(exit_code)
 269
 270 def init():
 271     global Cnf, Options
 272     Cnf = utils.get_conf()
 273     Arguments = [('h',"help","Show-Deferred::Options::Help"),
 274                  ("p","link-path","Show-Deferred::LinkPath","HasArg"),
 275                  ("d","deferred-queue","Show-Deferred::DeferredQueue","HasArg"),
 276                  ('r',"rrd","Show-Deferred::Options::Rrd", "HasArg")]
 277     args = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
 278     for i in ["help"]:
 279         if not Cnf.has_key("Show-Deferred::Options::%s" % (i)):
 280             Cnf["Show-Deferred::Options::%s" % (i)] = ""
 281     for i,j in [("DeferredQueue","--deferred-queue")]:
 282         if not Cnf.has_key("Show-Deferred::%s" % (i)):
 283             print >> sys.stderr, """Show-Deferred::%s is mandatory.
 284   set via config file or command-line option %s"""%(i,j)
 285
 286     Options = Cnf.subtree("Show-Deferred::Options")
 287     if Options["help"]:
 288         usage()
 289
 290     # Initialise database connection
 291     DBConn()
 292
 293     return args
 294
 295 def main():
 296     args = init()
 297     if len(args)!=0:
 298         usage(1)
 299
 300     if Cnf.has_key("Show-Deferred::Options::Rrd"):
 301         rrd_dir = Cnf["Show-Deferred::Options::Rrd"]
 302     elif Cnf.has_key("Dir::Rrd"):
 303         rrd_dir = Cnf["Dir::Rrd"]
 304     else:
 305         rrd_dir = None
 306
 307     filelist = []
 308     for r,d,f  in os.walk(Cnf["Show-Deferred::DeferredQueue"]):
 309         filelist += map (lambda x: os.path.join(r,x),
 310                          filter(lambda x: x.endswith('.changes'), f))
 311     list_uploads(filelist, rrd_dir)
 312
 313     available_changes = set(map(os.path.basename,filelist))
 314     if Cnf.has_key("Show-Deferred::LinkPath"):
 315         # remove dead links
 316         for r,d,f in os.walk(Cnf["Show-Deferred::LinkPath"]):
 317             for af in f:
 318                 afp = os.path.join(r,af)
 319                 if (not os.path.exists(afp) or
 320                     (af.endswith('.changes') and af not in available_changes)):
 321                     os.unlink(afp)
 322
 323 if __name__ == '__main__':
 324     main()