X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=blobdiff_plain;f=tools%2Fqueue_rss.py;h=fd49c0dfe1f0173a7c0fd343e7ed37a1af5cd9bc;hp=9fcc1d8db5e8e0aa8f3c7807f051e4978ff5cd78;hb=HEAD;hpb=5bfb9fcc5aa4304b058f941c370e56f6dc0fa3e4 diff --git a/tools/queue_rss.py b/tools/queue_rss.py index 9fcc1d8d..fd49c0df 100755 --- a/tools/queue_rss.py +++ b/tools/queue_rss.py @@ -9,14 +9,16 @@ import cgi import os import os.path import cPickle +import re import sys -import encodings.ascii +import time from optparse import OptionParser from datetime import datetime +from email.utils import parseaddr import PyRSS2Gen -from debian_bundle.deb822 import Changes +from debian.deb822 import Changes inrss_filename = "NEW_in.rss" outrss_filename = "NEW_out.rss" @@ -24,7 +26,7 @@ db_filename = "status.db" parser = OptionParser() parser.set_defaults(queuedir="queue", outdir="out", datadir="status", - max_entries="30") + logdir="log", max_entries="30") parser.add_option("-q", "--queuedir", dest="queuedir", help="The queue dir (%default)") @@ -32,6 +34,8 @@ parser.add_option("-o", "--outdir", dest="outdir", help="The output directory (%default)") parser.add_option("-d", "--datadir", dest="datadir", help="The data dir (%default)") +parser.add_option("-l", "--logdir", dest="logdir", + help="The ACCEPT/REJECT dak log dir (%default)") parser.add_option("-m", "--max-entries", dest="max_entries", type="int", help="Max number of entries to keep (%default)") @@ -39,24 +43,16 @@ class Status: def __init__(self): self.feed_in = PyRSS2Gen.RSS2( title = "Packages entering NEW", - link = "http://ftp-master.debian.org/new.html", + link = "https://ftp-master.debian.org/new.html", description = "Debian packages entering the NEW queue" ) self.feed_out = PyRSS2Gen.RSS2( title = "Packages leaving NEW", - link = "http://ftp-master.debian.org/new.html", + link = "https://ftp-master.debian.org/new.html", description = "Debian packages leaving the NEW queue" ) self.queue = {} -def utf2ascii(src): - """ Return an ASCII encoded copy of the input UTF-8 string """ - try: - res = unicode(src, 'utf-8').encode('ascii', 'replace') - except UnicodeDecodeError: - res = None - return res - def purge_old_items(feed, max): """ Purge RSSItem from feed, no more than max. """ if feed.items is None or len(feed.items) == 0: @@ -99,6 +95,28 @@ def parse_queuedir(dir): return res +def parse_leave_reason(fname): + """ Parse a dak log file fname for ACCEPT/REJECT reason from process-new. + + Return a dictionary {filename: reason}""" + + reason_re = re.compile(".+\|process-new\|.+\|NEW (ACCEPT|REJECT): (\S+)") + + try: + f = open(fname) + except IOError as e: + sys.stderr.write("Can't open %s: %s\n" % (fname, e)) + return {} + + res = {} + for l in f.readlines(): + m = reason_re.search(l) + if m: + res[m.group(2)] = m.group(1) + + f.close() + return res + def add_rss_item(status, msg, direction): if direction == "in": feed = status.feed_in @@ -106,33 +124,46 @@ def add_rss_item(status, msg, direction): pubdate = msg['Date'] elif direction == "out": feed = status.feed_out - title = "%s %s left NEW" % (msg['Source'], msg['Version']) + if msg.has_key('Leave-Reason'): + title = "%s %s left NEW (%s)" % (msg['Source'], msg['Version'], + msg['Leave-Reason']) + else: + title = "%s %s left NEW" % (msg['Source'], msg['Version']) + + pubdate = datetime.utcnow() else: return False description = "
Description: %s\nChanges: %s\n
" % \ - (utf2ascii(cgi.escape(msg['Description'])), - utf2ascii(cgi.escape(msg['Changes']))) + (cgi.escape(msg['Description']), + cgi.escape(msg['Changes'])) - link = "http://ftp-master.debian.org/new/%s_%s.html" % \ + link = "https://ftp-master.debian.org/new/%s_%s.html" % \ (msg['Source'], msg['Version']) + maintainer = parseaddr(msg['Maintainer']) + author = "%s (%s)" % (maintainer[1], maintainer[0]) + feed.items.insert(0, PyRSS2Gen.RSSItem( title, pubDate = pubdate, description = description, - author = utf2ascii(cgi.escape(msg['Maintainer'])), + author = cgi.escape(author), link = link, guid = link ) ) -def update_feeds(curqueue, status): +def update_feeds(curqueue, status, settings): # inrss -> append all items in curqueue not in status.queue # outrss -> append all items in status.queue not in curqueue + leave_reason = None + # logfile from dak's process-new + reason_log = os.path.join(settings.logdir, time.strftime("%Y-%m")) + for (name, parsed) in curqueue.items(): if not status.queue.has_key(name): # new package @@ -140,7 +171,11 @@ def update_feeds(curqueue, status): for (name, parsed) in status.queue.items(): if not curqueue.has_key(name): - # removed package + # removed package, try to find out why + if leave_reason is None: + leave_reason = parse_leave_reason(reason_log) + if leave_reason and leave_reason.has_key(name): + parsed['Leave-Reason'] = leave_reason[name] add_rss_item(status, parsed, "out") @@ -167,12 +202,8 @@ if __name__ == "__main__": status = Status() current_queue = parse_queuedir(settings.queuedir) - if not current_queue: - sys.stderr.write("Unable to scan queuedir '%s'\n" % settings.queuedir) - parser.print_help() - sys.exit(1) - update_feeds(current_queue, status) + update_feeds(current_queue, status, settings) purge_old_items(status.feed_in, settings.max_entries) purge_old_items(status.feed_out, settings.max_entries) @@ -183,7 +214,7 @@ if __name__ == "__main__": try: status.feed_in.write_xml(file(feed_in_file, "w+"), "utf-8") status.feed_out.write_xml(file(feed_out_file, "w+"), "utf-8") - except IOError, why: + except IOError as why: sys.stderr.write("Unable to write feeds: %s\n", why) sys.exit(1) @@ -191,7 +222,7 @@ if __name__ == "__main__": try: cPickle.dump(status, open(status_db, "w+")) - except IOError, why: + except IOError as why: sys.stderr.write("Unable to save status: %s\n", why) sys.exit(1)