X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fcontents.py;h=99e9575d874d39ac0d4745b217ee529b2373037e;hb=9ca85bf3b8790a8d9e1daa4a9dd70019f55f6e08;hp=53d742270dda4a87f47f6ef1dc3bb1b31a375501;hpb=1cc079eee7fce279ec6e9701d82f69a1567955cd;p=dak.git diff --git a/dak/contents.py b/dak/contents.py index 53d74227..99e9575d 100755 --- a/dak/contents.py +++ b/dak/contents.py @@ -48,6 +48,10 @@ from daklib import utils from daklib.binary import Binary from daklib.config import Config from daklib.dbconn import * +from daklib.contents import ContentsScanner +from daklib import daklog + +Logger = None ################################################################################ @@ -79,6 +83,9 @@ OPTIONS -s, --suite={stable,testing,unstable,...} only operate on a single suite + + -l, --limit=NUMBER + optional package limit for bootstrap_bin """ sys.exit(exit_code) @@ -215,7 +222,7 @@ class OutputThread(ContentsWorkThread): contents_file.filehandle.write("%s\t%s\n" % (fname,contents_file.filenames[fname])) contents_file.sorted_keys = None contents_file.filenames.clear() - + class GzipThread(ContentsWorkThread): def __init__(self, upstream, downstream): ContentsWorkThread.__init__(self, upstream, downstream) @@ -263,28 +270,14 @@ class ContentFile(object): self.session.close() -# def ingest(self): -# while True: -# r = self.results.fetchone() -# if not r: -# break -# filename, package = r -# if self.filenames.has_key(filename): -# self.filenames[filename] += ",%s" % (package) -# else: -# self.filenames[filename] = "%s" % (package) -# self.session.close() - def open_file(self): """ opens a gzip stream to the contents file """ -# filepath = Config()["Contents::Root"] + self.filename - self.filename = "/home/stew/contents/" + self.filename + filepath = Config()["Contents::Root"] + self.filename filedir = os.path.dirname(self.filename) if not os.path.isdir(filedir): os.makedirs(filedir) -# self.filehandle = gzip.open(self.filename, "w") self.filehandle = open(self.filename, "w") self._write_header() @@ -482,7 +475,7 @@ class Contents(object): Generate contents files for both deb and udeb """ self.deb_generate() -# self.udeb_generate() + self.udeb_generate() def deb_generate(self): """ @@ -500,23 +493,16 @@ class Contents(object): qt = QueryThread(inputtoquery,querytoingest) it = IngestThread(querytoingest,ingesttosort) -# these actually make things worse -# it2 = IngestThread(querytoingest,ingesttosort) -# it3 = IngestThread(querytoingest,ingesttosort) -# it4 = IngestThread(querytoingest,ingesttosort) st = SortThread(ingesttosort,sorttooutput) ot = OutputThread(sorttooutput,outputtogzip) gt = GzipThread(outputtogzip, None) qt.start() it.start() -# it2.start() -# it3.start() -# it2.start() st.start() ot.start() gt.start() - + # Get our suites, and the architectures for suite in [i.lower() for i in suites]: suite_id = get_suite(suite, session).suite_id @@ -526,7 +512,6 @@ class Contents(object): for (arch_id,arch_str) in arch_list: print( "suite: %s, arch: %s time: %s" %(suite_id, arch_id, datetime.datetime.now().isoformat()) ) -# filename = "dists/%s/Contents-%s.gz" % (suite, arch_str) filename = "dists/%s/Contents-%s" % (suite, arch_str) cf = DebContentFile(filename, suite, suite_id, arch_str, arch_id) inputtoquery.enqueue( cf ) @@ -550,54 +535,15 @@ class Contents(object): qt = QueryThread(inputtoquery,querytoingest) it = IngestThread(querytoingest,ingesttosort) -# these actually make things worse -# it2 = IngestThread(querytoingest,ingesttosort) -# it3 = IngestThread(querytoingest,ingesttosort) -# it4 = IngestThread(querytoingest,ingesttosort) st = SortThread(ingesttosort,sorttooutput) ot = OutputThread(sorttooutput,outputtogzip) gt = GzipThread(outputtogzip, None) qt.start() it.start() -# it2.start() -# it3.start() -# it2.start() st.start() ot.start() gt.start() - -# for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s"), -# ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s")]: - -# section_id = DBConn().get_section_id(section) # all udebs should be here) -# if section_id != -1: - - - -# # Get our suites, and the architectures -# for suite in [i.lower() for i in suites]: -# suite_id = DBConn().get_suite_id(suite) -# arch_list = self._arches(suite_id, session) - -# for arch_id in arch_list: - -# writer = GzippedContentWriter(fn_pattern % (suite, arch_id[1])) -# try: - -# results = session.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (suite_id, udebtype_id, section_id, arch_id)) - -# while True: -# r = cursor.fetchone() -# if not r: -# break - -# filename, section, package, arch = r -# writer.write(filename, section, package) -# finally: -# writer.close() - - def generate(self): @@ -655,8 +601,7 @@ class Contents(object): if Config().has_key( "%s::%s" %(options_prefix,"Suite")): suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")]) else: - suites = [ 'unstable', 'testing' ] -# suites = Config().SubTree("Suite").List() + suites = Config().SubTree("Suite").List() return suites @@ -683,6 +628,12 @@ class Contents(object): return arch_list +def scan_all(limit): + result = ContentsScanner.scan_all(limit) + processed = '%(processed)d packages processed' % result + remaining = '%(remaining)d packages remaining' % result + Logger.log([processed, remaining]) + ################################################################################ def main(): @@ -690,11 +641,12 @@ def main(): arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")), ('s',"suite", "%s::%s" % (options_prefix,"Suite"),"HasArg"), + ('l',"limit", "%s::%s" % (options_prefix,"Limit"),"HasArg"), ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")), ('v',"verbose", "%s::%s" % (options_prefix,"Verbose")), ] - commands = {'generate' : Contents.deb_generate, + commands = {'generate' : Contents.generate, 'bootstrap_bin' : Contents.bootstrap_bin, 'bootstrap' : Contents.bootstrap, 'cruft' : Contents.cruft, @@ -720,7 +672,19 @@ def main(): format='%(asctime)s %(levelname)s %(message)s', stream = sys.stderr ) - commands[args[0]](Contents()) + global Logger + Logger = daklog.Logger(cnf.Cnf, 'contents') + + limit = None + if cnf.has_key("%s::%s" % (options_prefix,"Limit")): + limit = cnf["%s::%s" % (options_prefix,"Limit")] + + if args[0] == 'bootstrap_bin': + scan_all(limit) + else: + commands[args[0]](Contents()) + + Logger.close() def which_suites(session): """