X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fshow_new.py;h=64ea22874d0d36096e86f9b1c77c43444ce2bacd;hb=17c5cab4eb8d5181ec7a81267a4e2e6b43c0fc65;hp=34baba954940946f1184253da5e40a9cd40cf051;hpb=d9822f04453a1b62ca0aa66e2efeea35f654778f;p=dak.git diff --git a/dak/show_new.py b/dak/show_new.py old mode 100644 new mode 100755 index 34baba95..64ea2287 --- a/dak/show_new.py +++ b/dak/show_new.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# Output html for packages in NEW -# Copyright (C) 2007 Joerg Jaspert +""" Output html for packages in NEW """ +# Copyright (C) 2007, 2009 Joerg Jaspert # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,36 +25,44 @@ ################################################################################ -import copy, os, sys, time +from copy import copy +import os, sys, time import apt_pkg import examine_package -import daklib.database -import daklib.queue -import daklib.utils + +from daklib import policy +from daklib.dbconn import * +from daklib import utils +from daklib.regexes import re_source_ext +from daklib.config import Config +from daklib import daklog +from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED +from multiprocessing import Manager, TimeoutError # Globals Cnf = None Options = None -Upload = None -projectB = None -sources = set() +manager = Manager() +sources = manager.list() +htmlfiles_to_process = manager.list() +timeout_str = "Timed out while processing" ################################################################################ ################################################################################ ################################################################################ -def html_header(name, filestoexamine): +def html_header(name, missing): if name.endswith('.changes'): name = ' '.join(name.split('_')[:2]) - print """ + result = """ %(name)s - Debian NEW package overview - - + + - +
- - corner image - corner image - corner image - corner image Debian NEW package overview for %(name)s
+ """%{"name":name} # we assume only one source (.dsc) per changes here - print """ + result += """ " + +"""%{"pkg":packagename} + result += " " + return result def html_footer(): - print """

Timestamp: %s (UTC)

"""% (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) - print """

- Valid HTML 4.01! - - Valid CSS! -

- - -""" + result = """

Timestamp: %s (UTC)

+"""% (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) + result += "" + return result ################################################################################ -def do_pkg(changes_file): - Upload.pkg.changes_file = changes_file - Upload.init_vars() - Upload.update_vars() - files = Upload.pkg.files - changes = Upload.pkg.changes +def do_pkg(upload_id): + cnf = Config() + + session = DBConn().session() + upload = session.query(PolicyQueueUpload).filter_by(id=upload_id).one() + + queue = upload.policy_queue + changes = upload.changes + + origchanges = os.path.join(queue.path, changes.changesname) + print origchanges - changes["suite"] = copy.copy(changes["distribution"]) + htmlname = "{0}_{1}.html".format(changes.source, changes.version) + htmlfile = os.path.join(cnf['Show-New::HTMLPath'], htmlname) - # Find out what's new - new = daklib.queue.determine_new(changes, files, projectB, 0) + # Have we already processed this? + if os.path.exists(htmlfile) and \ + os.stat(htmlfile).st_mtime > time.mktime(changes.created.timetuple()): + with open(htmlfile, "r") as fd: + if fd.read() != timeout_str: + sources.append(htmlname) + return (PROC_STATUS_SUCCESS, + '%s already up-to-date' % htmlfile) - stdout_fd = sys.stdout + # Go, process it... Now! + htmlfiles_to_process.append(htmlfile) + sources.append(htmlname) - htmlname = changes["source"] + "_" + changes["version"] + ".html" - sources.add(htmlname) - # do not generate html output if that source/version already has one. - if not os.path.exists(os.path.join(Cnf["Show-New::HTMLPath"],htmlname)): - sys.stdout = open(os.path.join(Cnf["Show-New::HTMLPath"],htmlname),"w") + group = cnf.get('Dinstall::UnprivGroup') or None - filestoexamine = [] - for pkg in new.keys(): - for fn in new[pkg]["files"]: - if ( files[fn].has_key("new") and not - files[fn]["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2"] ): - filestoexamine.append(fn) + with open(htmlfile, 'w') as outfile: + with policy.UploadCopy(upload, group=group) as upload_copy: + handler = policy.PolicyQueueUploadHandler(upload, session) + missing = [ (o['type'], o['package']) for o in handler.missing_overrides() ] + distribution = changes.distribution - html_header(changes["source"], filestoexamine) + print >>outfile, html_header(changes.source, missing) + print >>outfile, examine_package.display_changes(distribution, origchanges) - daklib.queue.check_valid(new) - examine_package.display_changes(Upload.pkg.changes_file) + if upload.source is not None and ('dsc', upload.source.source) in missing: + fn = os.path.join(upload_copy.directory, upload.source.poolfile.basename) + print >>outfile, examine_package.check_dsc(distribution, fn, session) + for binary in upload.binaries: + if (binary.binarytype, binary.package) not in missing: + continue + fn = os.path.join(upload_copy.directory, binary.poolfile.basename) + print >>outfile, examine_package.check_deb(distribution, fn, session) - for fn in filter(lambda fn: fn.endswith(".dsc"), filestoexamine): - examine_package.check_dsc(fn) - for fn in filter(lambda fn: fn.endswith(".deb") or fn.endswith(".udeb"), filestoexamine): - examine_package.check_deb(fn) + print >>outfile, html_footer() - html_footer() - if sys.stdout != stdout_fd: - sys.stdout.close() - sys.stdout = stdout_fd + session.close() + htmlfiles_to_process.remove(htmlfile) + return (PROC_STATUS_SUCCESS, '{0} already updated'.format(htmlfile)) ################################################################################ @@ -193,49 +208,61 @@ def usage (exit_code=0): ################################################################################ -def init(): - global Cnf, Options, Upload, projectB +def init(session): + global cnf, Options - Cnf = daklib.utils.get_conf() + cnf = Config() Arguments = [('h',"help","Show-New::Options::Help"), - ("p","html-path","Show-New::HTMLPath","HasArg")] + ("p","html-path","Show-New::HTMLPath","HasArg"), + ('q','queue','Show-New::Options::Queue','HasArg')] for i in ["help"]: - if not Cnf.has_key("Show-New::Options::%s" % (i)): - Cnf["Show-New::Options::%s" % (i)] = "" + if not cnf.has_key("Show-New::Options::%s" % (i)): + cnf["Show-New::Options::%s" % (i)] = "" - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) - Options = Cnf.SubTree("Show-New::Options") + changesnames = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv) + Options = cnf.subtree("Show-New::Options") if Options["help"]: usage() - Upload = daklib.queue.Upload(Cnf) + queue_names = Options.find('Queue', 'new').split(',') + uploads = session.query(PolicyQueueUpload) \ + .join(PolicyQueueUpload.policy_queue).filter(PolicyQueue.queue_name.in_(queue_names)) \ + .join(PolicyQueueUpload.changes).order_by(DBChange.source) - projectB = Upload.projectB + if len(changesnames) > 0: + uploads = uploads.filter(DBChange.changesname.in_(changesnames)) - return changes_files + return uploads ################################################################################ ################################################################################ def main(): - changes_files = init() - - examine_package.use_html=1 - - for changes_file in changes_files: - changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0) - if not changes_file: - continue - print "\n" + changes_file - do_pkg (changes_file) - files = set(os.listdir(Cnf["Show-New::HTMLPath"])) - to_delete = filter(lambda x: x.endswith(".html"), files.difference(sources)) - for file in to_delete: - os.remove(os.path.join(Cnf["Show-New::HTMLPath"],file)) + examine_package.use_html = True + pool = DakProcessPool(processes=5) + + session = DBConn().session() + upload_ids = [ u.id for u in init(session) ] + session.close() + + for upload_id in upload_ids: + pool.apply_async(do_pkg, [upload_id]) + pool.close() + + #p.wait(timeout=600) + pool.join() + for htmlfile in htmlfiles_to_process: + with open(htmlfile, "w") as fd: + fd.write(timeout_str) + + files = set(os.listdir(cnf["Show-New::HTMLPath"])) + to_delete = filter(lambda x: x.endswith(".html"), files.difference(set(sources))) + for f in to_delete: + os.remove(os.path.join(cnf["Show-New::HTMLPath"],f)) ################################################################################