]> git.decadent.org.uk Git - dak.git/blob - dak/show_deferred.py
e7dded0f92393f846e88fad9f8aeebac4eabe84b
[dak.git] / dak / show_deferred.py
1 #!/usr/bin/env python
2
3 """ Overview of the DEFERRED queue, based on queue-report """
4 #    Copyright (C) 2001, 2002, 2003, 2005, 2006  James Troup <james@nocrew.org>
5 # Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de>
6
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
11
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20
21 ################################################################################
22
23 import sys, os, re, time
24 import apt_pkg
25 import rrdtool
26
27 from debian import deb822
28
29 from daklib.dbconn import *
30 from daklib import utils
31 from daklib.regexes import re_html_escaping, html_escaping
32
33 ################################################################################
34 ### work around bug #487902 in debian-python 0.1.10
35 deb822.Changes._multivalued_fields = {
36             "files": [ "md5sum", "size", "section", "priority", "name" ],
37             "checksums-sha1": ["sha1", "size", "name"],
38             "checksums-sha256": ["sha256", "size", "name"],
39           }
40
41 ################################################################################
42
43 row_number = 1
44
45 def html_escape(s):
46     return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)
47
48 ################################################################################
49
50 def header():
51   return  """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
52         <html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
53         <title>Deferred uploads to Debian</title>
54         <link type="text/css" rel="stylesheet" href="style.css">
55         <link rel="shortcut icon" href="http://www.debian.org/favicon.ico">
56         </head>
57         <body>
58         <div align="center">
59         <a href="http://www.debian.org/">
60      <img src="http://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a>
61         <a href="http://www.debian.org/">
62      <img src="http://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a>
63         </div>
64         <br />
65         <table class="reddy" width="100%">
66         <tr>
67         <td class="reddy">
68     <img src="http://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0"
69      alt="" width="15" height="16"></td>
70         <td rowspan="2" class="reddy">Deferred uploads to Debian</td>
71         <td class="reddy">
72     <img src="http://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0"
73      alt="" width="16" height="16"></td>
74         </tr>
75         <tr>
76         <td class="reddy">
77     <img src="http://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0"
78      alt="" width="16" height="16"></td>
79         <td class="reddy">
80     <img src="http://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0"
81      alt="" width="15" height="16"></td>
82         </tr>
83         </table>
84         """
85
86 def footer():
87     res = "<p class=\"validate\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
88     res += "<p class=\"timestamp\">There are <a href=\"/stat.html\">graphs about the queues</a> available.</p>"
89     res += """<a href="http://validator.w3.org/check?uri=referer">
90     <img border="0" src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88"></a>
91         <a href="http://jigsaw.w3.org/css-validator/check/referer">
92     <img border="0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!"
93      height="31" width="88"></a>
94     """
95     res += "</body></html>"
96     return res.encode('utf-8')
97
98 def table_header():
99     return """<h1>Deferred uploads</h1>
100       <center><table border="0">
101         <tr>
102           <th align="center">Change</th>
103           <th align="center">Time remaining</th>
104           <th align="center">Uploader</th>
105           <th align="center">Closes</th>
106         </tr>
107         """
108
109 def table_footer():
110     return '</table><br/><p>non-NEW uploads are <a href="/deferred/">available</a>, see the <a href="ftp://ftp-master.debian.org/pub/UploadQueue/README">UploadQueue-README</a> for more information.</p></center><br/>\n'
111
112 def table_row(changesname, delay, changed_by, closes):
113     global row_number
114
115     res = '<tr class="%s">'%((row_number%2) and 'odd' or 'even')
116     res += (3*'<td valign="top">%s</td>')%tuple(map(html_escape,(changesname,delay,changed_by)))
117     res += ('<td valign="top">%s</td>' %
118              ''.join(map(lambda close:  '<a href="http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>' % (close, close),closes)))
119     res += '</tr>\n'
120     row_number+=1
121     return res
122
123 def update_graph_database(rrd_dir, *counts):
124     if not rrd_dir:
125         return
126
127     rrd_file = os.path.join(rrd_dir, 'deferred.rrd')
128     counts = [str(count) for count in counts]
129     update = [rrd_file, "N:"+":".join(counts)]
130
131     try:
132         rrdtool.update(*update)
133     except rrdtool.error:
134         create = [rrd_file]+"""
135 --step
136 300
137 --start
138 0
139 DS:day0:GAUGE:7200:0:1000
140 DS:day1:GAUGE:7200:0:1000
141 DS:day2:GAUGE:7200:0:1000
142 DS:day3:GAUGE:7200:0:1000
143 DS:day4:GAUGE:7200:0:1000
144 DS:day5:GAUGE:7200:0:1000
145 DS:day6:GAUGE:7200:0:1000
146 DS:day7:GAUGE:7200:0:1000
147 DS:day8:GAUGE:7200:0:1000
148 DS:day9:GAUGE:7200:0:1000
149 DS:day10:GAUGE:7200:0:1000
150 DS:day11:GAUGE:7200:0:1000
151 DS:day12:GAUGE:7200:0:1000
152 DS:day13:GAUGE:7200:0:1000
153 DS:day14:GAUGE:7200:0:1000
154 DS:day15:GAUGE:7200:0:1000
155 RRA:AVERAGE:0.5:1:599
156 RRA:AVERAGE:0.5:6:700
157 RRA:AVERAGE:0.5:24:775
158 RRA:AVERAGE:0.5:288:795
159 RRA:MIN:0.5:1:600
160 RRA:MIN:0.5:6:700
161 RRA:MIN:0.5:24:775
162 RRA:MIN:0.5:288:795
163 RRA:MAX:0.5:1:600
164 RRA:MAX:0.5:6:700
165 RRA:MAX:0.5:24:775
166 RRA:MAX:0.5:288:795
167 """.strip().split("\n")
168         try:
169             rc = rrdtool.create(*create)
170             ru = rrdtool.update(*update)
171         except rrdtool.error as e:
172             print('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e))
173     except NameError:
174         pass
175
176 def get_upload_data(changesfn):
177     achanges = deb822.Changes(file(changesfn))
178     changesname = os.path.basename(changesfn)
179     delay = os.path.basename(os.path.dirname(changesfn))
180     m = re.match(r'([0-9]+)-day', delay)
181     if m:
182         delaydays = int(m.group(1))
183         remainingtime = (delaydays>0)*max(0,24*60*60+os.stat(changesfn).st_mtime-time.time())
184         delay = "%d days %02d:%02d" %(max(delaydays-1,0), int(remainingtime/3600),int(remainingtime/60)%60)
185     else:
186         delaydays = 0
187         remainingtime = 0
188
189     uploader = achanges.get('changed-by')
190     uploader = re.sub(r'^\s*(\S.*)\s+<.*>',r'\1',uploader)
191     if Cnf.has_key("Show-Deferred::LinkPath"):
192         isnew = 0
193         suites = get_suites_source_in(achanges['source'])
194         if 'unstable' not in suites and 'experimental' not in suites:
195             isnew = 1
196
197         for b in achanges['binary'].split():
198             suites = get_suites_binary_in(b)
199             if 'unstable' not in suites and 'experimental' not in suites:
200                 isnew = 1
201
202         if not isnew:
203             # we don't link .changes because we don't want other people to
204             # upload it with the existing signature.
205             for afn in map(lambda x: x['name'],achanges['files']):
206                 lfn = os.path.join(Cnf["Show-Deferred::LinkPath"],afn)
207                 qfn = os.path.join(os.path.dirname(changesfn),afn)
208                 if os.path.islink(lfn):
209                     os.unlink(lfn)
210                 if os.path.exists(qfn):
211                     os.symlink(qfn,lfn)
212                     os.chmod(qfn, 0o644)
213     return (max(delaydays-1,0)*24*60*60+remainingtime, changesname, delay, uploader, achanges.get('closes','').split(),achanges, delaydays)
214
215 def list_uploads(filelist, rrd_dir):
216     uploads = map(get_upload_data, filelist)
217     uploads.sort()
218     # print the summary page
219     print header()
220     if uploads:
221         print table_header()
222         print ''.join(map(lambda x: table_row(*x[1:5]), uploads)).encode('utf-8')
223         print table_footer()
224     else:
225         print '<h1>Currently no deferred uploads to Debian</h1>'
226     print footer()
227     # machine readable summary
228     if Cnf.has_key("Show-Deferred::LinkPath"):
229         fn = os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp')
230         f = open(fn,"w")
231         try:
232             counts = [0]*16
233             for u in uploads:
234                 counts[u[6]] += 1
235                 print >> f, "Changes-file: %s"%u[1]
236                 fields = """Location: DEFERRED
237 Delayed-Until: %s
238 Delay-Remaining: %s"""%(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time()+u[0])),u[2])
239                 print >> f, fields
240                 encoded = unicode(u[5]).encode('utf-8')
241                 print >> f, encoded.rstrip()
242                 open(os.path.join(Cnf["Show-Deferred::LinkPath"],u[1]),"w").write(encoded+fields+'\n')
243                 print >> f
244             f.close()
245             os.rename(os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp'),
246                       os.path.join(Cnf["Show-Deferred::LinkPath"],'status'))
247             update_graph_database(rrd_dir, *counts)
248         except:
249             os.unlink(fn)
250             raise
251
252 def usage (exit_code=0):
253     if exit_code:
254         f = sys.stderr
255     else:
256         f = sys.stdout
257     print >> f, """Usage: dak show-deferred
258   -h, --help                    show this help and exit.
259   -p, --link-path [path]        override output directory.
260   -d, --deferred-queue [path]   path to the deferred queue
261   -r, --rrd=key                 Directory where rrd files to be updated are stored
262   """
263     sys.exit(exit_code)
264
265 def init():
266     global Cnf, Options
267     Cnf = utils.get_conf()
268     Arguments = [('h',"help","Show-Deferred::Options::Help"),
269                  ("p","link-path","Show-Deferred::LinkPath","HasArg"),
270                  ("d","deferred-queue","Show-Deferred::DeferredQueue","HasArg"),
271                  ('r',"rrd","Show-Deferred::Options::Rrd", "HasArg")]
272     args = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
273     for i in ["help"]:
274         if not Cnf.has_key("Show-Deferred::Options::%s" % (i)):
275             Cnf["Show-Deferred::Options::%s" % (i)] = ""
276     for i,j in [("DeferredQueue","--deferred-queue")]:
277         if not Cnf.has_key("Show-Deferred::%s" % (i)):
278             print >> sys.stderr, """Show-Deferred::%s is mandatory.
279   set via config file or command-line option %s"""%(i,j)
280
281     Options = Cnf.subtree("Show-Deferred::Options")
282     if Options["help"]:
283         usage()
284
285     # Initialise database connection
286     DBConn()
287
288     return args
289
290 def main():
291     args = init()
292     if len(args)!=0:
293         usage(1)
294
295     if Cnf.has_key("Show-Deferred::Options::Rrd"):
296         rrd_dir = Cnf["Show-Deferred::Options::Rrd"]
297     elif Cnf.has_key("Dir::Rrd"):
298         rrd_dir = Cnf["Dir::Rrd"]
299     else:
300         rrd_dir = None
301
302     filelist = []
303     for r,d,f  in os.walk(Cnf["Show-Deferred::DeferredQueue"]):
304         filelist += map (lambda x: os.path.join(r,x),
305                          filter(lambda x: x.endswith('.changes'), f))
306     list_uploads(filelist, rrd_dir)
307
308     available_changes = set(map(os.path.basename,filelist))
309     if Cnf.has_key("Show-Deferred::LinkPath"):
310         # remove dead links
311         for r,d,f in os.walk(Cnf["Show-Deferred::LinkPath"]):
312             for af in f:
313                 afp = os.path.join(r,af)
314                 if (not os.path.exists(afp) or
315                     (af.endswith('.changes') and af not in available_changes)):
316                     os.unlink(afp)
317
318 if __name__ == '__main__':
319     main()