-# we unfortunately still have broken stuff in headers
-latin1_q = """SET CLIENT_ENCODING TO 'LATIN1'"""
-
-# get all the arches delivered for a given suite
-# this should probably exist somehere common
-arches_q = """PREPARE arches_q as
- SELECT s.architecture, a.arch_string
- FROM suite_architectures s
- JOIN architecture a ON (s.architecture=a.id)
- WHERE suite = $1"""
-
-# find me the .deb for a given binary id
-debs_q = """PREPARE debs_q as
- SELECT b.id, f.filename FROM bin_assoc_by_arch baa
- JOIN binaries b ON baa.bin=b.id
- JOIN files f ON b.file=f.id
- WHERE suite = $1
- AND arch = $2"""
-
-# ask if we already have contents associated with this binary
-olddeb_q = """PREPARE olddeb_q as
- SELECT 1 FROM content_associations
- WHERE binary_pkg = $1
- LIMIT 1"""
-
-# find me all of the contents for a given .deb
-contents_q = """PREPARE contents_q as
- SELECT (p.path||'/'||n.file) AS fn,
- comma_separated_list(s.section||'/'||b.package)
- FROM content_associations c
- JOIN content_file_paths p ON (c.filepath=p.id)
- JOIN content_file_names n ON (c.filename=n.id)
- JOIN binaries b ON (b.id=c.binary_pkg)
- JOIN bin_associations ba ON (b.id=ba.bin)
- JOIN override o ON (o.package=b.package)
- JOIN section s ON (s.id=o.section)
- WHERE (b.architecture = $1 OR b.architecture = $2)
- AND ba.suite = $3
- AND o.suite = $4
- AND b.type = 'deb'
- AND o.type = '7'
- GROUP BY fn
- ORDER BY fn"""
-
-# find me all of the contents for a given .udeb
-udeb_contents_q = """PREPARE udeb_contents_q as
- SELECT (p.path||'/'||n.file) as fn,
- comma_separated_list(s.section||'/'||b.package)
- FROM content_associations c
- JOIN content_file_paths p ON (c.filepath=p.id)
- JOIN content_file_names n ON (c.filename=n.id)
- JOIN binaries b ON (b.id=c.binary_pkg)
- JOIN bin_associations ba ON (b.id=ba.bin)
- JOIN override o ON (o.package=b.package)
- JOIN section s ON (s.id=o.section)
- WHERE s.id = $1
- AND ba.suite = $2
- AND o.suite = $3
- AND b.type = 'udeb'
- AND o.type = '8'
- GROUP BY fn
- ORDER BY fn"""
-
-# clear out all of the temporarily stored content associations
-# this should be run only after p-a has run. after a p-a
-# run we should have either accepted or rejected every package
-# so there should no longer be anything in the queue
-remove_pending_contents_cruft_q = """DELETE FROM pending_content_associations"""
-
-# delete any filenames we are storing which have no binary associated with them
-remove_filename_cruft_q = """DELETE FROM content_file_names
- WHERE id IN (SELECT cfn.id FROM content_file_names cfn
- LEFT JOIN content_associations ca
- ON ca.filename=cfn.id
- WHERE ca.id IS NULL)""" );
-
-# delete any paths we are storing which have no binary associated with them
-remove_filepath_cruft_q = """DELETE FROM content_file_paths
- WHERE id IN (SELECT cfn.id FROM content_file_paths cfn
- LEFT JOIN content_associations ca
- ON ca.filepath=cfn.id
- WHERE ca.id IS NULL)"""
-class Contents(object):
- """
- Class capable of generating Contents-$arch.gz files
-
- Usage GenerateContents().generateContents( ["main","contrib","non-free"] )
- """
-
- def __init__(self):
- self.header = None
-
- def _getHeader(self):
- """
- Internal method to return the header for Contents.gz files
-
- This is boilerplate which explains the contents of the file and how
- it can be used.
- """
- if self.header == None:
- if Config().has_key("Contents::Header"):
- try:
- h = open(os.path.join( Config()["Dir::Templates"],
- Config()["Contents::Header"] ), "r")
- self.header = h.read()
- print( "header: %s" % self.header )
- h.close()
- except:
- log.error( "error opening header file: %d\n%s" % (Config()["Contents::Header"],
- traceback.format_exc() ))
- self.header = False
- else:
- print( "no header" )
- self.header = False
-
- return self.header
-
- # goal column for section column
- _goal_column = 54
-
- def _write_content_file(self, cursor, filename):
- """
- Internal method for writing all the results to a given file.
- The cursor should have a result set generated from a query already.
- """
- f = gzip.open(Config()["Dir::Root"] + filename, "w")
- try:
- header = self._getHeader()
-
- if header:
- f.write(header)
-
- while True:
- contents = cursor.fetchone()
- if not contents:
- return
-
- num_tabs = max(1,
- int( math.ceil( (self._goal_column - len(contents[0])) / 8) ) )
- f.write(contents[0] + ( '\t' * num_tabs ) + contents[-1] + "\n")
-
- finally:
- f.close()
-
- def cruft(self):
- """
- remove files/paths from the DB which are no longer referenced
- by binaries and clean the temporary table
- """
- cursor = DBConn().cursor();
- cursor.execute( "BEGIN WORK" )
- cursor.execute( remove_pending_contents_cruft_q )
- cursor.execute( remove_filename_cruft_q )
- cursor.execute( remove_filepath_cruft_q )
- cursor.execute( "COMMIT" )
-
-
- def bootstrap(self):
- """
- scan the existing debs in the pool to populate the contents database tables
- """
- pooldir = Config()[ 'Dir::Pool' ]
-
- cursor = DBConn().cursor();
- cursor.execute( latin1_q )
- cursor.execute( debs_q )
- cursor.execute( olddeb_q )
- cursor.execute( arches_q )
-
- suites = self._suites()
- for suite in [i.lower() for i in suites]:
- suite_id = DBConn().get_suite_id(suite)
-
- arch_list = self._arches(cursor, suite_id)
- arch_all_id = DBConn().get_architecture_id("all")
- for arch_id in arch_list:
- cursor.execute( "EXECUTE debs_q(%d, %d)" % ( suite_id, arch_id[0] ) )
-
- debs = cursor.fetchall()
- count = 0
- for deb in debs:
- count += 1
- cursor.execute( "EXECUTE olddeb_q(%d)" % (deb[0] ) )
- old = cursor.fetchone()
- if old:
- log.debug( "already imported: %s" % deb[1] )
- else:
- debfile = os.path.join( pooldir, deb[1] )
- if os.path.exists( debfile ):
- contents = utils.generate_contents_information( debfile )
- DBConn().insert_content_paths(deb[0], contents)
- log.info( "imported (%d/%d): %s" % (count,len(debs),deb[1] ) )
- else:
- log.error( "missing .deb: %s" % deb[1] )
-
- def generate(self):
- """
- Generate Contents-$arch.gz files for every aviailable arch in each given suite.
- """
- cursor = DBConn().cursor();
-
- cursor.execute( arches_q )
- cursor.execute( contents_q )
- cursor.execute( udeb_contents_q )
-
- suites = self._suites()
-
- # Get our suites, and the architectures
- for suite in [i.lower() for i in suites]:
- suite_id = DBConn().get_suite_id(suite)
- arch_list = self._arches(cursor, suite_id)
-
- arch_all_id = DBConn().get_architecture_id("all")
-
- for arch_id in arch_list:
- cursor.execute( "EXECUTE contents_q(%d,%d,%d,%d)" % (arch_id[0], arch_all_id, suite_id, suite_id))
- self._write_content_file(cursor, "dists/%s/Contents-%s.gz" % (suite, arch_id[1]))
-
- # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free)
- # This is HORRIBLY debian specific :-/
- # First off, udeb
- section_id = DBConn().get_section_id('debian-installer') # all udebs should be here)
- if section_id != -1:
- cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (section_id, suite_id, suite_id))
- self._write_content_file(cursor, "dists/%s/Contents-udeb.gz" % suite)
-
- # Once more, with non-free
- section_id = DBConn().get_section_id('non-free/debian-installer') # all udebs should be here)
-
- if section_id != -1:
- cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (section_id, suite_id, suite_id))
- self._write_content_file(cursor, "dists/%s/Contents-udeb-nf.gz" % suite)
-