X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;ds=sidebyside;f=dak%2Fshow_new.py;h=2ec2c3518bae47d43c3104eba8f54561be416d7e;hb=f9a5d2fc1826cc796b822f1d21c394545c942227;hp=6d6469184b20d4af1e27a4e22a0c0331564d0139;hpb=027a0a56706486df9a4ccf783ef56cd172def363;p=dak.git diff --git a/dak/show_new.py b/dak/show_new.py index 6d646918..2ec2c351 100755 --- a/dak/show_new.py +++ b/dak/show_new.py @@ -30,26 +30,29 @@ import os, sys, time import apt_pkg import examine_package +from daklib import policy from daklib.dbconn import * -from daklib.queue import determine_new, check_valid, Upload, get_policy_queue from daklib import utils from daklib.regexes import re_source_ext from daklib.config import Config from daklib import daklog -from daklib.changesutils import * -from daklib.threadpool import ThreadPool +from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED +from multiprocessing import Manager, TimeoutError # Globals Cnf = None Options = None -sources = set() +manager = Manager() +sources = manager.list() +htmlfiles_to_process = manager.list() +timeout_str = "Timed out while processing" ################################################################################ ################################################################################ ################################################################################ -def html_header(name, filestoexamine): +def html_header(name, missing): if name.endswith('.changes'): name = ' '.join(name.split('_')[:2]) result = """ @@ -118,8 +121,7 @@ def html_header(name, filestoexamine):
""" - for fn in filter(lambda x: x.endswith('.deb') or x.endswith('.udeb'),filestoexamine): - packagename = fn.split('_')[0] + for binarytype, packagename in filter(lambda m: m[0] in ('deb', 'udeb'), missing): result += """%(pkg)s
@@ -150,56 +152,59 @@ def html_footer(): ################################################################################ -def do_pkg(changes_file): +def do_pkg(upload_id): + cnf = Config() + session = DBConn().session() - u = Upload() - u.pkg.changes_file = changes_file - (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file) - u.load_changes(changes_file) - new_queue = get_policy_queue('new', session ); - u.pkg.directory = new_queue.path - u.update_subst() - origchanges = os.path.abspath(u.pkg.changes_file) - files = u.pkg.files - changes = u.pkg.changes - - for deb_filename, f in files.items(): - if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"): - u.binary_file_checks(deb_filename, session) - u.check_binary_against_db(deb_filename, session) - else: - u.source_file_checks(deb_filename, session) - u.check_source_against_db(deb_filename, session) - u.pkg.changes["suite"] = u.pkg.changes["distribution"] - - new = determine_new(u.pkg.changes, files, 0, session) - - htmlname = changes["source"] + "_" + changes["version"] + ".html" - sources.add(htmlname) - # do not generate html output if that source/version already has one. - if not os.path.exists(os.path.join(cnf["Show-New::HTMLPath"],htmlname)): - outfile = open(os.path.join(cnf["Show-New::HTMLPath"],htmlname),"w") - - filestoexamine = [] - for pkg in new.keys(): - for fn in new[pkg]["files"]: - filestoexamine.append(fn) - - print >> outfile, html_header(changes["source"], filestoexamine) - - check_valid(new, session) - distribution = changes["distribution"].keys()[0] - print >> outfile, examine_package.display_changes(distribution, changes_file) - - for fn in filter(lambda fn: fn.endswith(".dsc"), filestoexamine): - print >> outfile, examine_package.check_dsc(distribution, fn, session) - for fn in filter(lambda fn: fn.endswith(".deb") or fn.endswith(".udeb"), filestoexamine): - print >> outfile, examine_package.check_deb(distribution, fn, session) - - print >> outfile, html_footer() - - outfile.close() + upload = session.query(PolicyQueueUpload).filter_by(id=upload_id).one() + + queue = upload.policy_queue + changes = upload.changes + + origchanges = os.path.join(queue.path, changes.changesname) + print origchanges + + htmlname = "{0}_{1}.html".format(changes.source, changes.version) + htmlfile = os.path.join(cnf['Show-New::HTMLPath'], htmlname) + + # Have we already processed this? + if os.path.exists(htmlfile) and \ + os.stat(htmlfile).st_mtime > time.mktime(changes.created.timetuple()): + with open(htmlfile, "r") as fd: + if fd.read() != timeout_str: + sources.append(htmlname) + return (PROC_STATUS_SUCCESS, + '%s already up-to-date' % htmlfile) + + # Go, process it... Now! + htmlfiles_to_process.append(htmlfile) + sources.append(htmlname) + + group = cnf.get('Dinstall::UnprivGroup') or None + + with open(htmlfile, 'w') as outfile: + with policy.UploadCopy(upload, group=group) as upload_copy: + handler = policy.PolicyQueueUploadHandler(upload, session) + missing = [ (o['type'], o['package']) for o in handler.missing_overrides() ] + distribution = changes.distribution + + print >>outfile, html_header(changes.source, missing) + print >>outfile, examine_package.display_changes(distribution, origchanges) + + if upload.source is not None and ('dsc', upload.source.source) in missing: + fn = os.path.join(upload_copy.directory, upload.source.poolfile.basename) + print >>outfile, examine_package.check_dsc(distribution, fn, session) + for binary in upload.binaries: + if (binary.binarytype, binary.package) not in missing: + continue + fn = os.path.join(upload_copy.directory, binary.poolfile.basename) + print >>outfile, examine_package.check_deb(distribution, fn, session) + + print >>outfile, html_footer() + session.close() + htmlfiles_to_process.remove(htmlfile) + return (PROC_STATUS_SUCCESS, '{0} already updated'.format(htmlfile)) ################################################################################ @@ -224,39 +229,45 @@ def init(session): if not cnf.has_key("Show-New::Options::%s" % (i)): cnf["Show-New::Options::%s" % (i)] = "" - changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv) - if len(changes_files) == 0: - new_queue = get_policy_queue('new', session ); - changes_files = utils.get_changes_files(new_queue.path) - - Options = cnf.SubTree("Show-New::Options") + changesnames = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv) + Options = cnf.subtree("Show-New::Options") if Options["help"]: usage() - return changes_files + uploads = session.query(PolicyQueueUpload) \ + .join(PolicyQueueUpload.policy_queue).filter(PolicyQueue.queue_name == 'new') \ + .join(PolicyQueueUpload.changes).order_by(DBChange.source) + + if len(changesnames) > 0: + uploads = uploads.filter(DBChange.changesname.in_(changesnames)) + + return uploads ################################################################################ ################################################################################ def main(): + examine_package.use_html = True + pool = DakProcessPool(processes=5) + session = DBConn().session() - changes_files = init(session) + upload_ids = [ u.id for u in init(session) ] + session.close() - examine_package.use_html=1 + for upload_id in upload_ids: + pool.apply_async(do_pkg, [upload_id]) + pool.close() - threadpool = ThreadPool() - for changes_file in changes_files: - changes_file = utils.validate_changes_file_arg(changes_file, 0) - if not changes_file: - continue - print "\n" + changes_file - threadpool.queueTask(do_pkg, changes_file) - threadpool.joinAll() + #p.wait(timeout=600) + pool.join() + for htmlfile in htmlfiles_to_process: + with open(htmlfile, "w") as fd: + fd.write(timeout_str) files = set(os.listdir(cnf["Show-New::HTMLPath"])) - to_delete = filter(lambda x: x.endswith(".html"), files.difference(sources)) + to_delete = filter(lambda x: x.endswith(".html"), files.difference(set(sources))) for f in to_delete: os.remove(os.path.join(cnf["Show-New::HTMLPath"],f))