%s - Debian NEW package overview

X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fshow_new.py;h=af1b8a6480b0b58ea6c1af99a90ea94fffa732b6;hb=026bdb9580e6d18bf504d2f44d46890df5d89f1a;hp=674eca391917220e7c297db232c72520be8d2fc6;hpb=57750c00438df3775be51ce86c4fd1ac36df98c3;p=dak.git diff --git a/dak/show_new.py b/dak/show_new.py index 674eca39..af1b8a64 100755 --- a/dak/show_new.py +++ b/dak/show_new.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# Output html for packages in NEW -# Copyright (C) 2007 Joerg Jaspert +""" Output html for packages in NEW """ +# Copyright (C) 2007, 2009 Joerg Jaspert # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,167 +25,243 @@ ################################################################################ -import copy, os, sys, time +from copy import copy +import os, sys, time import apt_pkg import examine_package -import daklib.database -import daklib.queue -import daklib.utils + +from daklib import policy +from daklib.dbconn import * +from daklib import utils +from daklib.config import Config +from daklib import daklog +from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED +from multiprocessing import Manager, TimeoutError # Globals Cnf = None Options = None -Upload = None -projectB = None -sources = set() +manager = Manager() +sources = manager.list() +htmlfiles_to_process = manager.list() +timeout_str = "Timed out while processing" ################################################################################ ################################################################################ ################################################################################ -def html_header(name): +def html_header(name, missing): if name.endswith('.changes'): name = ' '.join(name.split('_')[:2]) - print """ - """ - print "%s - Debian NEW package overview" % (name) - print """ - - - -

- -

-
- - - """ - print """""" % (name) - print """ - - - - - -

-	Debian NEW package overview for %s	-
-	Debian NEW package overview for %s	-

- """ + result = """ + + + + %(name)s - Debian NEW package overview + + + + + +

+ +

+ + Debian NEW package overview for %(name)s + +

+ + """%{"name":name} + + # we assume only one source (.dsc) per changes here + result += """ + " + return result def html_footer(): - print "

Timestamp: %s (UTC)

" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) - print """ -

- -

- """ - print "" - + result = """

Timestamp: %s (UTC)

+"""% (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) + result += "" + return result ################################################################################ -def do_pkg(changes_file): - Upload.pkg.changes_file = changes_file - Upload.init_vars() - Upload.update_vars() - files = Upload.pkg.files - changes = Upload.pkg.changes +def do_pkg(upload_id): + cnf = Config() + + session = DBConn().session() + upload = session.query(PolicyQueueUpload).filter_by(id=upload_id).one() + + queue = upload.policy_queue + changes = upload.changes + + origchanges = os.path.join(queue.path, changes.changesname) + print origchanges + + htmlname = "{0}_{1}.html".format(changes.source, changes.version) + htmlfile = os.path.join(cnf['Show-New::HTMLPath'], htmlname) + + # Have we already processed this? + if os.path.exists(htmlfile) and \ + os.stat(htmlfile).st_mtime > time.mktime(changes.created.timetuple()): + with open(htmlfile, "r") as fd: + if fd.read() != timeout_str: + sources.append(htmlname) + return (PROC_STATUS_SUCCESS, + '%s already up-to-date' % htmlfile) - changes["suite"] = copy.copy(changes["distribution"]) + # Go, process it... Now! + htmlfiles_to_process.append(htmlfile) + sources.append(htmlname) - # Find out what's new - new = daklib.queue.determine_new(changes, files, projectB, 0) + group = cnf.get('Dinstall::UnprivGroup') or None - stdout_fd = sys.stdout + with open(htmlfile, 'w') as outfile: + with policy.UploadCopy(upload, group=group) as upload_copy: + handler = policy.PolicyQueueUploadHandler(upload, session) + missing = [ (o['type'], o['package']) for o in handler.missing_overrides() ] + distribution = changes.distribution - htmlname = changes["source"] + "_" + changes["version"] + ".html" - sources.add(htmlname) - # do not generate html output if that source/version already has one. - if not os.path.exists(os.path.join(Cnf["Show-New::HTMLPath"],htmlname)): - sys.stdout = open(os.path.join(Cnf["Show-New::HTMLPath"],htmlname),"w") - html_header(changes["source"]) + print >>outfile, html_header(changes.source, missing) + print >>outfile, examine_package.display_changes(distribution, origchanges) - daklib.queue.check_valid(new) - examine_package.display_changes(Upload.pkg.changes_file) + if upload.source is not None and ('dsc', upload.source.source) in missing: + fn = os.path.join(upload_copy.directory, upload.source.poolfile.basename) + print >>outfile, examine_package.check_dsc(distribution, fn, session) + for binary in upload.binaries: + if (binary.binarytype, binary.package) not in missing: + continue + fn = os.path.join(upload_copy.directory, binary.poolfile.basename) + print >>outfile, examine_package.check_deb(distribution, fn, session) - for pkg in new.keys(): - for file in new[pkg]["files"]: - if ( files[file].has_key("new") and not - files[file]["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2"] ): - if file.endswith(".deb") or file.endswith(".udeb"): - examine_package.check_deb(file) - elif file.endswith(".dsc"): - examine_package.check_dsc(file) + print >>outfile, html_footer() - html_footer() - if sys.stdout != stdout_fd: - sys.stdout.close() - sys.stdout = stdout_fd + session.close() + htmlfiles_to_process.remove(htmlfile) + return (PROC_STATUS_SUCCESS, '{0} already updated'.format(htmlfile)) ################################################################################ def usage (exit_code=0): print """Usage: dak show-new [OPTION]... [CHANGES]... -h, --help show this help and exit. + -p, --html-path [path] override output directory. """ sys.exit(exit_code) ################################################################################ -def init(): - global Cnf, Options, Upload, projectB +def init(session): + global cnf, Options - Cnf = daklib.utils.get_conf() + cnf = Config() - Arguments = [('h',"help","Show-New::Options::Help")] + Arguments = [('h',"help","Show-New::Options::Help"), + ("p","html-path","Show-New::HTMLPath","HasArg"), + ('q','queue','Show-New::Options::Queue','HasArg')] for i in ["help"]: - if not Cnf.has_key("Show-New::Options::%s" % (i)): - Cnf["Show-New::Options::%s" % (i)] = "" + if not cnf.has_key("Show-New::Options::%s" % (i)): + cnf["Show-New::Options::%s" % (i)] = "" - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) - Options = Cnf.SubTree("Show-New::Options") + changesnames = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv) + Options = cnf.subtree("Show-New::Options") if Options["help"]: usage() - Upload = daklib.queue.Upload(Cnf) + queue_names = Options.find('Queue', 'new').split(',') + uploads = session.query(PolicyQueueUpload) \ + .join(PolicyQueueUpload.policy_queue).filter(PolicyQueue.queue_name.in_(queue_names)) \ + .join(PolicyQueueUpload.changes).order_by(DBChange.source) - projectB = Upload.projectB + if len(changesnames) > 0: + uploads = uploads.filter(DBChange.changesname.in_(changesnames)) - return changes_files + return uploads ################################################################################ ################################################################################ def main(): - changes_files = init() - - examine_package.use_html=1 - - for changes_file in changes_files: - changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0) - if not changes_file: - continue - print "\n" + changes_file - do_pkg (changes_file) - files = set(os.listdir(Cnf["Show-New::HTMLPath"])) - to_delete = files.difference(sources) - for file in to_delete: - os.remove(os.path.join(Cnf["Show-New::HTMLPath"],file)) + examine_package.use_html = True + pool = DakProcessPool(processes=5) + + session = DBConn().session() + upload_ids = [ u.id for u in init(session) ] + session.close() + + for upload_id in upload_ids: + pool.apply_async(do_pkg, [upload_id]) + pool.close() + + #p.wait(timeout=600) + pool.join() + for htmlfile in htmlfiles_to_process: + with open(htmlfile, "w") as fd: + fd.write(timeout_str) + + files = set(os.listdir(cnf["Show-New::HTMLPath"])) + to_delete = filter(lambda x: x.endswith(".html"), files.difference(set(sources))) + for f in to_delete: + os.remove(os.path.join(cnf["Show-New::HTMLPath"],f)) ################################################################################