3 """ Overview of the DEFERRED queue, based on queue-report """
4 # Copyright (C) 2001, 2002, 2003, 2005, 2006 James Troup <james@nocrew.org>
5 # Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 import sys, os, re, time
27 from debian import deb822
29 from daklib.dbconn import *
30 from daklib.gpg import SignedFile
31 from daklib import utils
32 from daklib.regexes import re_html_escaping, html_escaping
34 ################################################################################
35 ### work around bug #487902 in debian-python 0.1.10
36 deb822.Changes._multivalued_fields = {
37 "files": [ "md5sum", "size", "section", "priority", "name" ],
38 "checksums-sha1": ["sha1", "size", "name"],
39 "checksums-sha256": ["sha256", "size", "name"],
42 ################################################################################
47 return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)
49 ################################################################################
52 return """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
53 <html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
54 <title>Deferred uploads to Debian</title>
55 <link type="text/css" rel="stylesheet" href="style.css">
56 <link rel="shortcut icon" href="https://www.debian.org/favicon.ico">
60 <a href="https://www.debian.org/">
61 <img src="https://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a>
62 <a href="https://www.debian.org/">
63 <img src="https://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a>
66 <table class="reddy" width="100%">
69 <img src="https://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0"
70 alt="" width="15" height="16"></td>
71 <td rowspan="2" class="reddy">Deferred uploads to Debian</td>
73 <img src="https://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0"
74 alt="" width="16" height="16"></td>
78 <img src="https://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0"
79 alt="" width="16" height="16"></td>
81 <img src="https://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0"
82 alt="" width="15" height="16"></td>
88 res = "<p class=\"validate\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
89 res += "<p class=\"timestamp\">There are <a href=\"/stat.html\">graphs about the queues</a> available.</p>"
90 res += "</body></html>"
91 return res.encode('utf-8')
94 return """<h1>Deferred uploads</h1>
95 <center><table border="0">
97 <th align="center">Change</th>
98 <th align="center">Time remaining</th>
99 <th align="center">Uploader</th>
100 <th align="center">Closes</th>
105 return '</table><br/><p>non-NEW uploads are <a href="/deferred/">available</a> (<a href="/deferred/status">machine readable version</a>), see the <a href="ftp://ftp-master.debian.org/pub/UploadQueue/README">UploadQueue-README</a> for more information.</p></center><br/>\n'
107 def table_row(changesname, delay, changed_by, closes, fingerprint):
110 res = '<tr class="%s">'%((row_number%2) and 'odd' or 'even')
111 res += (2*'<td valign="top">%s</td>')%tuple(map(html_escape,(changesname,delay)))
112 res += '<td valign="top">%s<br><span class=\"deferredfp\">Fingerprint: %s</span></td>' % (html_escape(changed_by), fingerprint)
113 res += ('<td valign="top">%s</td>' %
114 ''.join(map(lambda close: '<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>' % (close, close),closes)))
119 def update_graph_database(rrd_dir, *counts):
123 rrd_file = os.path.join(rrd_dir, 'deferred.rrd')
124 counts = [str(count) for count in counts]
125 update = [rrd_file, "N:"+":".join(counts)]
128 rrdtool.update(*update)
129 except rrdtool.error:
130 create = [rrd_file]+"""
135 DS:day0:GAUGE:7200:0:1000
136 DS:day1:GAUGE:7200:0:1000
137 DS:day2:GAUGE:7200:0:1000
138 DS:day3:GAUGE:7200:0:1000
139 DS:day4:GAUGE:7200:0:1000
140 DS:day5:GAUGE:7200:0:1000
141 DS:day6:GAUGE:7200:0:1000
142 DS:day7:GAUGE:7200:0:1000
143 DS:day8:GAUGE:7200:0:1000
144 DS:day9:GAUGE:7200:0:1000
145 DS:day10:GAUGE:7200:0:1000
146 DS:day11:GAUGE:7200:0:1000
147 DS:day12:GAUGE:7200:0:1000
148 DS:day13:GAUGE:7200:0:1000
149 DS:day14:GAUGE:7200:0:1000
150 DS:day15:GAUGE:7200:0:1000
151 RRA:AVERAGE:0.5:1:599
152 RRA:AVERAGE:0.5:6:700
153 RRA:AVERAGE:0.5:24:775
154 RRA:AVERAGE:0.5:288:795
163 """.strip().split("\n")
165 rc = rrdtool.create(*create)
166 ru = rrdtool.update(*update)
167 except rrdtool.error as e:
168 print('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e))
172 def get_upload_data(changesfn):
173 achanges = deb822.Changes(file(changesfn))
174 changesname = os.path.basename(changesfn)
175 delay = os.path.basename(os.path.dirname(changesfn))
176 m = re.match(r'([0-9]+)-day', delay)
178 delaydays = int(m.group(1))
179 remainingtime = (delaydays>0)*max(0,24*60*60+os.stat(changesfn).st_mtime-time.time())
180 delay = "%d days %02d:%02d" %(max(delaydays-1,0), int(remainingtime/3600),int(remainingtime/60)%60)
185 uploader = achanges.get('changed-by')
186 uploader = re.sub(r'^\s*(\S.*)\s+<.*>',r'\1',uploader)
187 with utils.open_file(changesfn) as f:
188 fingerprint = SignedFile(f.read(), keyrings=get_active_keyring_paths(), require_signature=False).fingerprint
189 if Cnf.has_key("Show-Deferred::LinkPath"):
191 suites = get_suites_source_in(achanges['source'])
192 if 'unstable' not in suites and 'experimental' not in suites:
195 for b in achanges['binary'].split():
196 suites = get_suites_binary_in(b)
197 if 'unstable' not in suites and 'experimental' not in suites:
201 # we don't link .changes because we don't want other people to
202 # upload it with the existing signature.
203 for afn in map(lambda x: x['name'],achanges['files']):
204 lfn = os.path.join(Cnf["Show-Deferred::LinkPath"],afn)
205 qfn = os.path.join(os.path.dirname(changesfn),afn)
206 if os.path.islink(lfn):
208 if os.path.exists(qfn):
211 return (max(delaydays-1,0)*24*60*60+remainingtime, changesname, delay, uploader, achanges.get('closes','').split(), fingerprint, achanges, delaydays)
213 def list_uploads(filelist, rrd_dir):
214 uploads = map(get_upload_data, filelist)
216 # print the summary page
220 print ''.join(map(lambda x: table_row(*x[1:6]), uploads)).encode('utf-8')
223 print '<h1>Currently no deferred uploads to Debian</h1>'
225 # machine readable summary
226 if Cnf.has_key("Show-Deferred::LinkPath"):
227 fn = os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp')
233 print >> f, "Changes-file: %s"%u[1]
234 fields = """Location: DEFERRED
237 Fingerprint: %s"""%(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time()+u[0])),u[2], u[5])
239 encoded = unicode(u[6]).encode('utf-8')
240 print >> f, encoded.rstrip()
241 open(os.path.join(Cnf["Show-Deferred::LinkPath"],u[1]),"w").write(encoded+fields+'\n')
244 os.rename(os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp'),
245 os.path.join(Cnf["Show-Deferred::LinkPath"],'status'))
246 update_graph_database(rrd_dir, *counts)
251 def usage (exit_code=0):
256 print >> f, """Usage: dak show-deferred
257 -h, --help show this help and exit.
258 -p, --link-path [path] override output directory.
259 -d, --deferred-queue [path] path to the deferred queue
260 -r, --rrd=key Directory where rrd files to be updated are stored
266 Cnf = utils.get_conf()
267 Arguments = [('h',"help","Show-Deferred::Options::Help"),
268 ("p","link-path","Show-Deferred::LinkPath","HasArg"),
269 ("d","deferred-queue","Show-Deferred::DeferredQueue","HasArg"),
270 ('r',"rrd","Show-Deferred::Options::Rrd", "HasArg")]
271 args = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
273 if not Cnf.has_key("Show-Deferred::Options::%s" % (i)):
274 Cnf["Show-Deferred::Options::%s" % (i)] = ""
275 for i,j in [("DeferredQueue","--deferred-queue")]:
276 if not Cnf.has_key("Show-Deferred::%s" % (i)):
277 print >> sys.stderr, """Show-Deferred::%s is mandatory.
278 set via config file or command-line option %s"""%(i,j)
280 Options = Cnf.subtree("Show-Deferred::Options")
284 # Initialise database connection
294 if Cnf.has_key("Show-Deferred::Options::Rrd"):
295 rrd_dir = Cnf["Show-Deferred::Options::Rrd"]
296 elif Cnf.has_key("Dir::Rrd"):
297 rrd_dir = Cnf["Dir::Rrd"]
302 for r,d,f in os.walk(Cnf["Show-Deferred::DeferredQueue"]):
303 filelist += map (lambda x: os.path.join(r,x),
304 filter(lambda x: x.endswith('.changes'), f))
305 list_uploads(filelist, rrd_dir)
307 available_changes = set(map(os.path.basename,filelist))
308 if Cnf.has_key("Show-Deferred::LinkPath"):
310 for r,d,f in os.walk(Cnf["Show-Deferred::LinkPath"]):
312 afp = os.path.join(r,af)
313 if (not os.path.exists(afp) or
314 (af.endswith('.changes') and af not in available_changes)):
317 if __name__ == '__main__':