import logging
import gzip
import threading
+import traceback
import Queue
import apt_pkg
import datetime
contents_file.filehandle.write("%s\t%s\n" % (fname,contents_file.filenames[fname]))
contents_file.sorted_keys = None
contents_file.filenames.clear()
-
+
class GzipThread(ContentsWorkThread):
def __init__(self, upstream, downstream):
ContentsWorkThread.__init__(self, upstream, downstream)
self.session.close()
-# def ingest(self):
-# while True:
-# r = self.results.fetchone()
-# if not r:
-# break
-# filename, package = r
-# if self.filenames.has_key(filename):
-# self.filenames[filename] += ",%s" % (package)
-# else:
-# self.filenames[filename] = "%s" % (package)
-# self.session.close()
-
def open_file(self):
"""
opens a gzip stream to the contents file
"""
-# filepath = Config()["Contents::Root"] + self.filename
- self.filename = "/home/stew/contents/" + self.filename
+ filepath = Config()["Contents::Root"] + self.filename
filedir = os.path.dirname(self.filename)
if not os.path.isdir(filedir):
os.makedirs(filedir)
-# self.filehandle = gzip.open(self.filename, "w")
self.filehandle = open(self.filename, "w")
self._write_header()
Generate contents files for both deb and udeb
"""
self.deb_generate()
-# self.udeb_generate()
+ self.udeb_generate()
def deb_generate(self):
"""
qt = QueryThread(inputtoquery,querytoingest)
it = IngestThread(querytoingest,ingesttosort)
-# these actually make things worse
-# it2 = IngestThread(querytoingest,ingesttosort)
-# it3 = IngestThread(querytoingest,ingesttosort)
-# it4 = IngestThread(querytoingest,ingesttosort)
st = SortThread(ingesttosort,sorttooutput)
ot = OutputThread(sorttooutput,outputtogzip)
gt = GzipThread(outputtogzip, None)
qt.start()
it.start()
-# it2.start()
-# it3.start()
-# it2.start()
st.start()
ot.start()
gt.start()
-
+
# Get our suites, and the architectures
for suite in [i.lower() for i in suites]:
suite_id = get_suite(suite, session).suite_id
for (arch_id,arch_str) in arch_list:
print( "suite: %s, arch: %s time: %s" %(suite_id, arch_id, datetime.datetime.now().isoformat()) )
-# filename = "dists/%s/Contents-%s.gz" % (suite, arch_str)
filename = "dists/%s/Contents-%s" % (suite, arch_str)
cf = DebContentFile(filename, suite, suite_id, arch_str, arch_id)
inputtoquery.enqueue( cf )
qt = QueryThread(inputtoquery,querytoingest)
it = IngestThread(querytoingest,ingesttosort)
-# these actually make things worse
-# it2 = IngestThread(querytoingest,ingesttosort)
-# it3 = IngestThread(querytoingest,ingesttosort)
-# it4 = IngestThread(querytoingest,ingesttosort)
st = SortThread(ingesttosort,sorttooutput)
ot = OutputThread(sorttooutput,outputtogzip)
gt = GzipThread(outputtogzip, None)
qt.start()
it.start()
-# it2.start()
-# it3.start()
-# it2.start()
st.start()
ot.start()
gt.start()
-
-# for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s"),
-# ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s")]:
-
-# section_id = DBConn().get_section_id(section) # all udebs should be here)
-# if section_id != -1:
-
-
-
-# # Get our suites, and the architectures
-# for suite in [i.lower() for i in suites]:
-# suite_id = DBConn().get_suite_id(suite)
-# arch_list = self._arches(suite_id, session)
-
-# for arch_id in arch_list:
-
-# writer = GzippedContentWriter(fn_pattern % (suite, arch_id[1]))
-# try:
-
-# results = session.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (suite_id, udebtype_id, section_id, arch_id))
-
-# while True:
-# r = cursor.fetchone()
-# if not r:
-# break
-
-# filename, section, package, arch = r
-# writer.write(filename, section, package)
-# finally:
-# writer.close()
-
-
def generate(self):
if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
else:
- suites = [ 'unstable', 'testing' ]
-# suites = Config().SubTree("Suite").List()
+ suites = Config().SubTree("Suite").List()
return suites
('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
]
- commands = {'generate' : Contents.deb_generate,
+ commands = {'generate' : Contents.generate,
'bootstrap_bin' : Contents.bootstrap_bin,
'bootstrap' : Contents.bootstrap,
'cruft' : Contents.cruft,