X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fshow_new.py;h=2ec2c3518bae47d43c3104eba8f54561be416d7e;hb=d31ae14a9de247d47d9ff7b3f36705a58a065916;hp=b67930a2616e44ab25e0f8a7cea9c925f44db7c1;hpb=cc1fd76ee7812e4155271fc516736c587538fb5a;p=dak.git diff --git a/dak/show_new.py b/dak/show_new.py index b67930a2..2ec2c351 100755 --- a/dak/show_new.py +++ b/dak/show_new.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ Output html for packages in NEW """ -# Copyright (C) 2007 Joerg Jaspert +# Copyright (C) 2007, 2009 Joerg Jaspert # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,24 +30,32 @@ import os, sys, time import apt_pkg import examine_package -from daklib.queue import determine_new, check_valid, Upload +from daklib import policy +from daklib.dbconn import * from daklib import utils from daklib.regexes import re_source_ext +from daklib.config import Config +from daklib import daklog +from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED +from multiprocessing import Manager, TimeoutError # Globals Cnf = None Options = None -sources = set() +manager = Manager() +sources = manager.list() +htmlfiles_to_process = manager.list() +timeout_str = "Timed out while processing" ################################################################################ ################################################################################ ################################################################################ -def html_header(name, filestoexamine): +def html_header(name, missing): if name.endswith('.changes'): name = ' '.join(name.split('_')[:2]) - print """ + result = """ + """%{"name":name} # we assume only one source (.dsc) per changes here - print """ + result += """ " + +"""%{"pkg":packagename} + result += " " + return result def html_footer(): - print """

Timestamp: %s (UTC)

"""% (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) - print """

+ result = """

Timestamp: %s (UTC)

+"""% (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) + result += """

Valid HTML 4.01! @@ -135,51 +147,64 @@ def html_footer(): """ + return result ################################################################################ -def do_pkg(changes_file): - upload = Upload() - upload.load_changes(changes_file) +def do_pkg(upload_id): + cnf = Config() + + session = DBConn().session() + upload = session.query(PolicyQueueUpload).filter_by(id=upload_id).one() + + queue = upload.policy_queue + changes = upload.changes + + origchanges = os.path.join(queue.path, changes.changesname) + print origchanges - files = upload.pkg.files - changes = upload.pkg.changes - upload.pkg.changes["suite"] = copy(upload.pkg.changes["distribution"]) - distribution = upload.pkg.changes["distribution"].keys()[0] - # Find out what's new - new = determine_new(upload.pkg.changes, upload.pkg.files, 0) + htmlname = "{0}_{1}.html".format(changes.source, changes.version) + htmlfile = os.path.join(cnf['Show-New::HTMLPath'], htmlname) - stdout_fd = sys.stdout + # Have we already processed this? + if os.path.exists(htmlfile) and \ + os.stat(htmlfile).st_mtime > time.mktime(changes.created.timetuple()): + with open(htmlfile, "r") as fd: + if fd.read() != timeout_str: + sources.append(htmlname) + return (PROC_STATUS_SUCCESS, + '%s already up-to-date' % htmlfile) - htmlname = changes["source"] + "_" + changes["version"] + ".html" - sources.add(htmlname) - # do not generate html output if that source/version already has one. - if not os.path.exists(os.path.join(Cnf["Show-New::HTMLPath"],htmlname)): - sys.stdout = open(os.path.join(Cnf["Show-New::HTMLPath"],htmlname),"w") + # Go, process it... Now! + htmlfiles_to_process.append(htmlfile) + sources.append(htmlname) - filestoexamine = [] - for pkg in new.keys(): - for fn in new[pkg]["files"]: - if (files[fn].has_key("new") and - (files[fn]["type"] == "dsc" or - not re_source_ext.match(files[fn]["type"]))): - filestoexamine.append(fn) + group = cnf.get('Dinstall::UnprivGroup') or None - html_header(changes["source"], filestoexamine) + with open(htmlfile, 'w') as outfile: + with policy.UploadCopy(upload, group=group) as upload_copy: + handler = policy.PolicyQueueUploadHandler(upload, session) + missing = [ (o['type'], o['package']) for o in handler.missing_overrides() ] + distribution = changes.distribution - check_valid(new) - examine_package.display_changes( distribution, changes_file) + print >>outfile, html_header(changes.source, missing) + print >>outfile, examine_package.display_changes(distribution, origchanges) - for fn in filter(lambda fn: fn.endswith(".dsc"), filestoexamine): - examine_package.check_dsc(distribution, fn) - for fn in filter(lambda fn: fn.endswith(".deb") or fn.endswith(".udeb"), filestoexamine): - examine_package.check_deb(distribution, fn) + if upload.source is not None and ('dsc', upload.source.source) in missing: + fn = os.path.join(upload_copy.directory, upload.source.poolfile.basename) + print >>outfile, examine_package.check_dsc(distribution, fn, session) + for binary in upload.binaries: + if (binary.binarytype, binary.package) not in missing: + continue + fn = os.path.join(upload_copy.directory, binary.poolfile.basename) + print >>outfile, examine_package.check_deb(distribution, fn, session) - html_footer() - if sys.stdout != stdout_fd: - sys.stdout.close() - sys.stdout = stdout_fd + print >>outfile, html_footer() + + session.close() + htmlfiles_to_process.remove(htmlfile) + return (PROC_STATUS_SUCCESS, '{0} already updated'.format(htmlfile)) ################################################################################ @@ -192,45 +217,59 @@ def usage (exit_code=0): ################################################################################ -def init(): - global Cnf, Options +def init(session): + global cnf, Options - Cnf = utils.get_conf() + cnf = Config() Arguments = [('h',"help","Show-New::Options::Help"), ("p","html-path","Show-New::HTMLPath","HasArg")] for i in ["help"]: - if not Cnf.has_key("Show-New::Options::%s" % (i)): - Cnf["Show-New::Options::%s" % (i)] = "" + if not cnf.has_key("Show-New::Options::%s" % (i)): + cnf["Show-New::Options::%s" % (i)] = "" - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) - Options = Cnf.SubTree("Show-New::Options") + changesnames = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv) + Options = cnf.subtree("Show-New::Options") if Options["help"]: usage() - return changes_files + uploads = session.query(PolicyQueueUpload) \ + .join(PolicyQueueUpload.policy_queue).filter(PolicyQueue.queue_name == 'new') \ + .join(PolicyQueueUpload.changes).order_by(DBChange.source) + + if len(changesnames) > 0: + uploads = uploads.filter(DBChange.changesname.in_(changesnames)) + + return uploads ################################################################################ ################################################################################ def main(): - changes_files = init() - - examine_package.use_html=1 - - for changes_file in changes_files: - changes_file = utils.validate_changes_file_arg(changes_file, 0) - if not changes_file: - continue - print "\n" + changes_file - do_pkg (changes_file) - files = set(os.listdir(Cnf["Show-New::HTMLPath"])) - to_delete = filter(lambda x: x.endswith(".html"), files.difference(sources)) + examine_package.use_html = True + pool = DakProcessPool(processes=5) + + session = DBConn().session() + upload_ids = [ u.id for u in init(session) ] + session.close() + + for upload_id in upload_ids: + pool.apply_async(do_pkg, [upload_id]) + pool.close() + + #p.wait(timeout=600) + pool.join() + for htmlfile in htmlfiles_to_process: + with open(htmlfile, "w") as fd: + fd.write(timeout_str) + + files = set(os.listdir(cnf["Show-New::HTMLPath"])) + to_delete = filter(lambda x: x.endswith(".html"), files.difference(set(sources))) for f in to_delete: - os.remove(os.path.join(Cnf["Show-New::HTMLPath"],f)) + os.remove(os.path.join(cnf["Show-New::HTMLPath"],f)) ################################################################################