From 2cf4d2fa796002258fe11b0ed67b85e12f07d2c1 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 21 Mar 2011 14:17:45 +0100 Subject: [PATCH] gps change the way we run the pool once more, hopefully being less bad now. also, this tool is EOL somehwat, so who cares how bad it looks Signed-off-by: Joerg Jaspert --- dak/generate_packages_sources.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py index 07a64b6e..3c02e0dd 100755 --- a/dak/generate_packages_sources.py +++ b/dak/generate_packages_sources.py @@ -41,6 +41,7 @@ from daklib.config import Config Options = None #: Commandline arguments parsed into this Logger = None #: Our logging object +results = [] #: Results of the subprocesses ################################################################################ @@ -360,11 +361,16 @@ tree "dists/oldstable-proposed-updates/main" def sname(arch): return arch.arch_string +def get_result(arg): + global results + if arg: + results.append(arg) + ######################################################################## ######################################################################## def main (): - global Options, Logger + global Options, Logger, results cnf = Config() @@ -405,40 +411,28 @@ def main (): broken=[] # For each given suite, each architecture, run one apt-ftparchive for s in suites: + results=[] # Setup a multiprocessing Pool. As many workers as we have CPU cores. pool = Pool() arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session) Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))]) for a in arch_list: - try: - result=pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"])) - # Get the result. Should it take too long (a-f hanging), break out. - r=result.get(timeout=3600) - except TimeoutError: - broken.append("Timeout: %s - %s" % (s.suite_name, a.arch_string)) - # Now try the next architecture - continue - - if r: - # As long as we get 0, we are fine. Otherwise we yell about it later. - broken.append("Breakage: %s - %s returned %s" % (s.suite_name, a.arch_string, r)) + pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result) # No more work will be added to our pool, close it and then wait for all to finish pool.close() pool.join() - if len(broken) > 0: - Logger.log(['Trouble: %s' % (broken)]) - print "Trouble: %s" % (broken) + if len(results) > 0: + Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)]) + print "Trouble, something with a-f broke, resultcodes: %s" % (results) + sys.exit(1) os.chdir(startdir) # this script doesn't change the database session.close() Logger.close() - if len(broken) > 0: - sys.exit(1) - ####################################################################################### if __name__ == '__main__': -- 2.39.5