2 # Generate two rss feeds for a directory with .changes file
4 # License: GPL v2 or later
5 # Author: Filippo Giunchedi <filippo@debian.org>
15 import encodings.ascii
16 from optparse import OptionParser
17 from datetime import datetime
21 from debian_bundle.deb822 import Changes
23 inrss_filename = "NEW_in.rss"
24 outrss_filename = "NEW_out.rss"
25 db_filename = "status.db"
27 parser = OptionParser()
28 parser.set_defaults(queuedir="queue", outdir="out", datadir="status",
29 logdir="log", max_entries="30")
31 parser.add_option("-q", "--queuedir", dest="queuedir",
32 help="The queue dir (%default)")
33 parser.add_option("-o", "--outdir", dest="outdir",
34 help="The output directory (%default)")
35 parser.add_option("-d", "--datadir", dest="datadir",
36 help="The data dir (%default)")
37 parser.add_option("-l", "--logdir", dest="logdir",
38 help="The ACCEPT/REJECT dak log dir (%default)")
39 parser.add_option("-m", "--max-entries", dest="max_entries", type="int",
40 help="Max number of entries to keep (%default)")
44 self.feed_in = PyRSS2Gen.RSS2(
45 title = "Packages entering NEW",
46 link = "http://ftp-master.debian.org/new.html",
47 description = "Debian packages entering the NEW queue" )
49 self.feed_out = PyRSS2Gen.RSS2(
50 title = "Packages leaving NEW",
51 link = "http://ftp-master.debian.org/new.html",
52 description = "Debian packages leaving the NEW queue" )
57 """ Return an ASCII encoded copy of the input UTF-8 string """
59 res = unicode(src, 'utf-8').encode('ascii', 'replace')
60 except UnicodeDecodeError:
64 def purge_old_items(feed, max):
65 """ Purge RSSItem from feed, no more than max. """
66 if feed.items is None or len(feed.items) == 0:
69 feed.items = feed.items[:max]
72 def parse_changes(fname):
73 """ Parse a .changes file named fname.
75 Return {fname: parsed} """
77 m = Changes(open(fname))
79 wanted_fields = set(['Source', 'Version', 'Architecture', 'Distribution',
80 'Date', 'Maintainer', 'Description', 'Changes'])
82 if not set(m.keys()).issuperset(wanted_fields):
85 return {os.path.basename(fname): m}
87 def parse_queuedir(dir):
88 """ Parse dir for .changes files.
90 Return a dictionary {filename: parsed_file}"""
92 if not os.path.exists(dir):
96 for fname in os.listdir(dir):
97 if not fname.endswith(".changes"):
100 parsed = parse_changes(os.path.join(dir, fname))
106 def parse_leave_reason(fname):
107 """ Parse a dak log file fname for ACCEPT/REJECT reason from process-new.
109 Return a dictionary {filename: reason}"""
111 reason_re = re.compile(".+\|process-new\|.+\|NEW (ACCEPT|REJECT): (\S+)")
116 sys.stderr.write("Can't open %s: %s\n" % (fname, e))
120 for l in f.readlines():
121 m = reason_re.search(l)
123 res[m.group(2)] = m.group(1)
128 def add_rss_item(status, msg, direction):
129 if direction == "in":
130 feed = status.feed_in
131 title = "%s %s entered NEW" % (msg['Source'], msg['Version'])
132 pubdate = msg['Date']
133 elif direction == "out":
134 feed = status.feed_out
135 if msg.has_key('Leave-Reason'):
136 title = "%s %s left NEW (%s)" % (msg['Source'], msg['Version'],
139 title = "%s %s left NEW" % (msg['Source'], msg['Version'])
142 pubdate = datetime.utcnow()
146 description = "<pre>Description: %s\nChanges: %s\n</pre>" % \
147 (utf2ascii(cgi.escape(msg['Description'])),
148 utf2ascii(cgi.escape(msg['Changes'])))
150 link = "http://ftp-master.debian.org/new/%s_%s.html" % \
151 (msg['Source'], msg['Version'])
157 description = description,
158 author = utf2ascii(cgi.escape(msg['Maintainer'])),
164 def update_feeds(curqueue, status, settings):
165 # inrss -> append all items in curqueue not in status.queue
166 # outrss -> append all items in status.queue not in curqueue
169 # logfile from dak's process-new
170 reason_log = os.path.join(settings.logdir, time.strftime("%Y-%m"))
172 for (name, parsed) in curqueue.items():
173 if not status.queue.has_key(name):
175 add_rss_item(status, parsed, "in")
177 for (name, parsed) in status.queue.items():
178 if not curqueue.has_key(name):
179 # removed package, try to find out why
180 if leave_reason is None:
181 leave_reason = parse_leave_reason(reason_log)
182 if leave_reason and leave_reason.has_key(name):
183 parsed['Leave-Reason'] = leave_reason[name]
184 add_rss_item(status, parsed, "out")
188 if __name__ == "__main__":
190 (settings, args) = parser.parse_args()
192 if not os.path.exists(settings.outdir):
193 sys.stderr.write("Outdir '%s' does not exists\n" % settings.outdir)
197 if not os.path.exists(settings.datadir):
198 sys.stderr.write("Datadir '%s' does not exists\n" % settings.datadir)
202 status_db = os.path.join(settings.datadir, db_filename)
205 status = cPickle.load(open(status_db))
209 current_queue = parse_queuedir(settings.queuedir)
210 if not current_queue:
211 sys.stderr.write("Unable to scan queuedir '%s'\n" % settings.queuedir)
215 update_feeds(current_queue, status, settings)
217 purge_old_items(status.feed_in, settings.max_entries)
218 purge_old_items(status.feed_out, settings.max_entries)
220 feed_in_file = os.path.join(settings.outdir, inrss_filename)
221 feed_out_file = os.path.join(settings.outdir, outrss_filename)
224 status.feed_in.write_xml(file(feed_in_file, "w+"), "utf-8")
225 status.feed_out.write_xml(file(feed_out_file, "w+"), "utf-8")
227 sys.stderr.write("Unable to write feeds: %s\n", why)
230 status.queue = current_queue
233 cPickle.dump(status, open(status_db, "w+"))
235 sys.stderr.write("Unable to save status: %s\n", why)