X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fgenerate_filelist.py;h=faf08bd7a8680e59802d4bdf8c12d4ec0a68a21b;hb=b3d7c0ea3704fa38d91b47023e232b94ce05735d;hp=1f4d665495ed21497b421f2d8ea5d4c7b6080570;hpb=6b1a0d77983a7a2dd80630cc46ad659fa4cc3b7a;p=dak.git diff --git a/dak/generate_filelist.py b/dak/generate_filelist.py index 1f4d6654..faf08bd7 100755 --- a/dak/generate_filelist.py +++ b/dak/generate_filelist.py @@ -5,6 +5,7 @@ Generate file lists for apt-ftparchive. @contact: Debian FTP Master @copyright: 2009 Torsten Werner +@copyright: 2011 Ansgar Burchardt @license: GNU General Public License version 2 or later """ @@ -37,8 +38,8 @@ Generate file lists for apt-ftparchive. from daklib.dbconn import * from daklib.config import Config -from daklib.threadpool import ThreadPool -from daklib import utils +from daklib import utils, daklog +from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED import apt_pkg, os, stat, sys from daklib.lists import getSources, getBinaries, getArchAll @@ -64,37 +65,54 @@ def listPath(suite, component, architecture = None, type = None, file.truncate() return (file, timestamp) -def writeSourceList(args): - (suite, component, incremental_mode) = args +def writeSourceList(suite_id, component_id, incremental_mode): + session = DBConn().session() + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) (file, timestamp) = listPath(suite, component, incremental_mode = incremental_mode) - session = DBConn().session() + + message = "sources list for %s %s" % (suite.suite_name, component.component_name) + for _, filename in getSources(suite, component, session, timestamp): file.write(filename + '\n') - session.close() + session.rollback() file.close() + return (PROC_STATUS_SUCCESS, message) -def writeAllList(args): - (suite, component, architecture, type, incremental_mode) = args +def writeAllList(suite_id, component_id, architecture_id, type, incremental_mode): + session = DBConn().session() + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) + architecture = Architecture.get(architecture_id, session) (file, timestamp) = listPath(suite, component, architecture, type, incremental_mode) - session = DBConn().session() + + message = "all list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type) + for _, filename in getArchAll(suite, component, architecture, type, session, timestamp): file.write(filename + '\n') - session.close() + session.rollback() file.close() + return (PROC_STATUS_SUCCESS, message) -def writeBinaryList(args): - (suite, component, architecture, type, incremental_mode) = args +def writeBinaryList(suite_id, component_id, architecture_id, type, incremental_mode): + session = DBConn().session() + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) + architecture = Architecture.get(architecture_id, session) (file, timestamp) = listPath(suite, component, architecture, type, incremental_mode) - session = DBConn().session() + + message = "binary list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type) + for _, filename in getBinaries(suite, component, architecture, type, session, timestamp): file.write(filename + '\n') - session.close() + session.rollback() file.close() + return (PROC_STATUS_SUCCESS, message) def usage(): print """Usage: dak generate_filelist [OPTIONS] @@ -114,6 +132,7 @@ Incremental mode appends only newer files to existing lists.""" def main(): cnf = Config() + Logger = daklog.Logger('generate-filelist') Arguments = [('h', "help", "Filelist::Options::Help"), ('s', "suite", "Filelist::Options::Suite", "HasArg"), ('c', "component", "Filelist::Options::Component", "HasArg"), @@ -136,43 +155,63 @@ def main(): cnf['Filelist::Options::Architecture'] = ','.join(architectures).encode() cnf['Filelist::Options::Help'] = '' cnf['Filelist::Options::Incremental'] = '' - apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) - Options = cnf.SubTree("Filelist::Options") + apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) + Options = cnf.subtree("Filelist::Options") if Options['Help']: usage() - threadpool = ThreadPool() + pool = DakProcessPool() query_suites = query_suites. \ filter(Suite.suite_name.in_(utils.split_args(Options['Suite']))) query_components = query_components. \ filter(Component.component_name.in_(utils.split_args(Options['Component']))) query_architectures = query_architectures. \ filter(Architecture.arch_string.in_(utils.split_args(Options['Architecture']))) + + def parse_results(message): + # Split out into (code, msg) + code, msg = message + if code == PROC_STATUS_SUCCESS: + Logger.log([msg]) + elif code == PROC_STATUS_SIGNALRAISED: + Logger.log(['E: Subprocess recieved signal ', msg]) + else: + Logger.log(['E: ', msg]) + for suite in query_suites: + suite_id = suite.suite_id for component in query_components: + component_id = component.component_id for architecture in query_architectures: + architecture_id = architecture.arch_id if architecture not in suite.architectures: pass elif architecture.arch_string == 'source': - threadpool.queueTask(writeSourceList, - (suite, component, Options['Incremental'])) + pool.apply_async(writeSourceList, + (suite_id, component_id, Options['Incremental']), callback=parse_results) elif architecture.arch_string == 'all': - threadpool.queueTask(writeAllList, - (suite, component, architecture, 'deb', - Options['Incremental'])) - threadpool.queueTask(writeAllList, - (suite, component, architecture, 'udeb', - Options['Incremental'])) + pool.apply_async(writeAllList, + (suite_id, component_id, architecture_id, 'deb', + Options['Incremental']), callback=parse_results) + pool.apply_async(writeAllList, + (suite_id, component_id, architecture_id, 'udeb', + Options['Incremental']), callback=parse_results) else: # arch any - threadpool.queueTask(writeBinaryList, - (suite, component, architecture, 'deb', - Options['Incremental'])) - threadpool.queueTask(writeBinaryList, - (suite, component, architecture, 'udeb', - Options['Incremental'])) - threadpool.joinAll() + pool.apply_async(writeBinaryList, + (suite_id, component_id, architecture_id, 'deb', + Options['Incremental']), callback=parse_results) + pool.apply_async(writeBinaryList, + (suite_id, component_id, architecture_id, 'udeb', + Options['Incremental']), callback=parse_results) + pool.close() + pool.join() + # this script doesn't change the database session.close() + Logger.close() + + sys.exit(pool.overall_status()) + if __name__ == '__main__': main()