X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fgenerate_filelist.py;h=a2246f5cd3707dbf656886ffcee2bb356140064f;hb=39cdc8cc1607dbacfcdbf8182e3a024c05737fe4;hp=50a0949a9040712f64bb1f5007cdc887aaf36149;hpb=b5feab8ab22835ba9a01e96ab9a8088986bd493c;p=dak.git diff --git a/dak/generate_filelist.py b/dak/generate_filelist.py index 50a0949a..a2246f5c 100755 --- a/dak/generate_filelist.py +++ b/dak/generate_filelist.py @@ -22,28 +22,65 @@ Generate file lists for apt-ftparchive. # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +################################################################################ + +# Ganneff> Please go and try to lock mhy now. After than try to lock NEW. +# twerner> !lock mhy +# dak> twerner: You suck, this is already locked by Ganneff +# Ganneff> now try with NEW +# twerner> !lock NEW +# dak> twerner: also locked NEW +# mhy> Ganneff: oy, stop using me for locks and highlighting me you tall muppet +# Ganneff> hehe :) + +################################################################################ + from daklib.dbconn import * from daklib.config import Config +from daklib.threadpool import ThreadPool from daklib import utils -import apt_pkg, os, sys +import apt_pkg, os, stat, sys def fetch(query, args, session): return [path + filename for (path, filename) in \ session.execute(query, args).fetchall()] -def getSources(suite, component, session): +def getSources(suite, component, session, timestamp): + extra_cond = "" + if timestamp: + extra_cond = "AND extract(epoch from sa.created) > %d" % timestamp query = """ - SELECT path, filename - FROM srcfiles_suite_component - WHERE suite = :suite AND component = :component + SELECT l.path, f.filename + FROM source s + JOIN src_associations sa + ON s.id = sa.source AND sa.suite = :suite %s + JOIN files f + ON s.file = f.id + JOIN location l + ON f.location = l.id AND l.component = :component ORDER BY filename - """ + """ % extra_cond args = { 'suite': suite.suite_id, 'component': component.component_id } return fetch(query, args, session) -def getBinaries(suite, component, architecture, type, session): +def getBinaries(suite, component, architecture, type, session, timestamp): + extra_cond = "" + if timestamp: + extra_cond = "AND extract(epoch from ba.created) > %d" % timestamp query = """ +CREATE TEMP TABLE b_candidates ( + source integer, + file integer, + architecture integer); + +INSERT INTO b_candidates (source, file, architecture) + SELECT b.source, b.file, b.architecture + FROM binaries b + JOIN bin_associations ba ON b.id = ba.bin + WHERE b.type = :type AND ba.suite = :suite AND + b.architecture IN (2, :architecture) %s; + CREATE TEMP TABLE gf_candidates ( filename text, path text, @@ -52,29 +89,27 @@ CREATE TEMP TABLE gf_candidates ( source text); INSERT INTO gf_candidates (filename, path, architecture, src, source) - SELECT f.filename, l.path, b.architecture, b.source as src, s.source - FROM binaries b - JOIN bin_associations ba ON b.id = ba.bin - JOIN source s ON b.source = s.id - JOIN files f ON b.file = f.id + SELECT f.filename, l.path, bc.architecture, bc.source as src, s.source + FROM b_candidates bc + JOIN source s ON bc.source = s.id + JOIN files f ON bc.file = f.id JOIN location l ON f.location = l.id - WHERE ba.suite = :suite AND b.type = :type AND - l.component = :component AND b.architecture IN (2, :architecture); + WHERE l.component = :component; WITH arch_any AS (SELECT path, filename FROM gf_candidates - WHERE architecture > 2), + WHERE architecture > 2), arch_all_with_any AS (SELECT path, filename FROM gf_candidates - WHERE architecture = 2 AND - src IN (SELECT src FROM gf_candidates WHERE architecture > 2)), + WHERE architecture = 2 AND + src IN (SELECT src FROM gf_candidates WHERE architecture > 2)), arch_all_without_any AS (SELECT path, filename FROM gf_candidates - WHERE architecture = 2 AND - source NOT IN (SELECT DISTINCT source FROM gf_candidates WHERE architecture > 2)), + WHERE architecture = 2 AND + source NOT IN (SELECT DISTINCT source FROM gf_candidates WHERE architecture > 2)), filelist AS (SELECT * FROM arch_any @@ -84,14 +119,15 @@ WITH arch_any AS SELECT * FROM arch_all_without_any) SELECT * FROM filelist ORDER BY filename - """ + """ % extra_cond args = { 'suite': suite.suite_id, 'component': component.component_id, 'architecture': architecture.arch_id, 'type': type } return fetch(query, args, session) -def listPath(suite, component, architecture = None, type = None): +def listPath(suite, component, architecture = None, type = None, + incremental_mode = False): """returns full path to the list file""" suffixMap = { 'deb': "binary-", 'udeb': "debian-installer_binary-" } @@ -102,18 +138,32 @@ def listPath(suite, component, architecture = None, type = None): filename = "%s_%s_%s.list" % \ (suite.suite_name, component.component_name, suffix) pathname = os.path.join(Config()["Dir::Lists"], filename) - return utils.open_file(pathname, "w") - -def writeSourceList(suite, component, session): - file = listPath(suite, component) - for filename in getSources(suite, component, session): + file = utils.open_file(pathname, "a") + timestamp = None + if incremental_mode: + timestamp = os.fstat(file.fileno())[stat.ST_MTIME] + else: + file.seek(0) + file.truncate() + return (file, timestamp) + +def writeSourceList(args): + (suite, component, incremental_mode) = args + (file, timestamp) = listPath(suite, component, + incremental_mode = incremental_mode) + session = DBConn().session() + for filename in getSources(suite, component, session, timestamp): file.write(filename + '\n') + session.close() file.close() -def writeBinaryList(suite, component, architecture, type): - file = listPath(suite, component, architecture, type) +def writeBinaryList(args): + (suite, component, architecture, type, incremental_mode) = args + (file, timestamp) = listPath(suite, component, architecture, type, + incremental_mode) session = DBConn().session() - for filename in getBinaries(suite, component, architecture, type, session): + for filename in getBinaries(suite, component, architecture, type, + session, timestamp): file.write(filename + '\n') session.close() file.close() @@ -126,9 +176,12 @@ Create filename lists for apt-ftparchive. -c, --component=COMPONENT act on this component -a, --architecture=ARCH act on this architecture -h, --help show this help and exit + -i, --incremental activate incremental mode ARCH, COMPONENT and SUITE can be comma (or space) separated list, e.g. - --suite=testing,unstable""" + --suite=testing,unstable + +Incremental mode appends only newer files to existing lists.""" sys.exit() def main(): @@ -136,26 +189,29 @@ def main(): Arguments = [('h', "help", "Filelist::Options::Help"), ('s', "suite", "Filelist::Options::Suite", "HasArg"), ('c', "component", "Filelist::Options::Component", "HasArg"), - ('a', "architecture", "Filelist::Options::Architecture", "HasArg")] - query_suites = DBConn().session().query(Suite) + ('a', "architecture", "Filelist::Options::Architecture", "HasArg"), + ('i', "incremental", "Filelist::Options::Incremental")] + session = DBConn().session() + query_suites = session.query(Suite) suites = [suite.suite_name for suite in query_suites.all()] if not cnf.has_key('Filelist::Options::Suite'): cnf['Filelist::Options::Suite'] = ','.join(suites) # we can ask the database for components if 'mixed' is gone if not cnf.has_key('Filelist::Options::Component'): cnf['Filelist::Options::Component'] = 'main,contrib,non-free' - query_architectures = DBConn().session().query(Architecture) + query_architectures = session.query(Architecture) architectures = \ [architecture.arch_string for architecture in query_architectures.all()] if not cnf.has_key('Filelist::Options::Architecture'): cnf['Filelist::Options::Architecture'] = ','.join(architectures) cnf['Filelist::Options::Help'] = '' + cnf['Filelist::Options::Incremental'] = '' apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) Options = cnf.SubTree("Filelist::Options") if Options['Help']: usage() - session = DBConn().session() suite_arch = session.query(SuiteArchitecture) + threadpool = ThreadPool() for suite_name in utils.split_args(Options['Suite']): suite = query_suites.filter_by(suite_name = suite_name).one() join = suite_arch.filter_by(suite_id = suite.suite_id) @@ -168,12 +224,18 @@ def main(): try: join.filter_by(arch_id = architecture.arch_id).one() if architecture_name == 'source': - writeSourceList(suite, component, session) + threadpool.queueTask(writeSourceList, + (suite, component, Options['Incremental'])) elif architecture_name != 'all': - writeBinaryList(suite, component, architecture, 'deb') - writeBinaryList(suite, component, architecture, 'udeb') + threadpool.queueTask(writeBinaryList, + (suite, component, architecture, 'deb', + Options['Incremental'])) + threadpool.queueTask(writeBinaryList, + (suite, component, architecture, 'udeb', + Options['Incremental'])) except: pass + threadpool.joinAll() # this script doesn't change the database session.close()