From dab0414eff3741bbdf9f9e294292f20fb2a4d129 Mon Sep 17 00:00:00 2001 From: Mike O'Connor Date: Wed, 25 Feb 2009 15:09:14 -0500 Subject: [PATCH] get bootstrap working again Signed-off-by: Mike O'Connor --- dak/contents.py | 17 ++++++++-------- daklib/binary.py | 17 ++++++++++------ daklib/dbconn.py | 38 +++++++++++++++++++++++++++++++++++- daklib/utils.py | 50 ------------------------------------------------ 4 files changed, 56 insertions(+), 66 deletions(-) diff --git a/dak/contents.py b/dak/contents.py index 963bf494..5ff121bc 100644 --- a/dak/contents.py +++ b/dak/contents.py @@ -42,6 +42,7 @@ import math import gzip import apt_pkg from daklib import utils +from daklib.binary import Binary from daklib.config import Config from daklib.dbconn import DBConn ################################################################################ @@ -159,7 +160,7 @@ remove_filename_cruft_q = """DELETE FROM content_file_names WHERE id IN (SELECT cfn.id FROM content_file_names cfn LEFT JOIN content_associations ca ON ca.filename=cfn.id - WHERE ca.id IS NULL)""" ); + WHERE ca.id IS NULL)""" # delete any paths we are storing which have no binary associated with them remove_filepath_cruft_q = """DELETE FROM content_file_paths @@ -273,9 +274,7 @@ class Contents(object): else: debfile = os.path.join( pooldir, deb[1] ) if os.path.exists( debfile ): - contents = utils.generate_contents_information( debfile ) - DBConn().insert_content_paths(deb[0], contents) - log.info( "imported (%d/%d): %s" % (count,len(debs),deb[1] ) ) + Binary(f).scan_package( deb[0] ) else: log.error( "missing .deb: %s" % deb[1] ) @@ -305,12 +304,12 @@ class Contents(object): # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free) # This is HORRIBLY debian specific :-/ for section_id, fn_pattern in [("debian-installer","dists/%s/Contents-udeb.gz"), - ("non-free/debian-installer", "dists/%s/Contents-udeb-nf.gz")] + ("non-free/debian-installer", "dists/%s/Contents-udeb-nf.gz")]: - section_id = DBConn().get_section_id(section_id) # all udebs should be here) - if section_id != -1: - cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (section_id, suite_id, suite_id)) - self._write_content_file(cursor, fn_pattern % suite) + section_id = DBConn().get_section_id(section_id) # all udebs should be here) + if section_id != -1: + cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (section_id, suite_id, suite_id)) + self._write_content_file(cursor, fn_pattern % suite) ################################################################################ diff --git a/daklib/binary.py b/daklib/binary.py index f9cf2437..042cd266 100755 --- a/daklib/binary.py +++ b/daklib/binary.py @@ -115,7 +115,7 @@ class Binary(object): return not rejected - def scan_package(self): + def scan_package(self, bootstrap_id=0): """ Unpack the .deb, do sanity checking, and gather info from it. @@ -123,6 +123,11 @@ class Binary(object): the hopefully near future, it should also include gathering info from the control file. + @ptype bootstrap_id: int + @param bootstrap_id: the id of the binary these packages + should be associated or zero meaning we are not bootstrapping + so insert into a temporary table + @return True if the deb is valid and contents were imported """ rejected = not self.valid_deb() @@ -135,14 +140,17 @@ class Binary(object): if self.chunks[1] == "control.tar.gz": control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz") - pkg = deb822.Packages.iter_paragraphs( control.extractfile('./control') ).next() if self.chunks[2] == "data.tar.gz": data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz") elif self.chunks[2] == "data.tar.bz2": data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2") - return DBConn().insert_content_paths(pkg, [ tarinfo.name for tarinfo in data if not tarinfo.isdir()]) + if bootstrap_id: + return DBConn().insert_content_paths(bootstrap_id, [ tarinfo.name for tarinfo in data if not tarinfo.isdir()]) + else: + pkg = deb822.Packages.iter_paragraphs( control.extractfile('./control') ).next() + return DBConn().insert_pending_content_paths(pkg, [ tarinfo.name for tarinfo in data if not tarinfo.isdir()]) except: traceback.print_exc() @@ -152,9 +160,6 @@ class Binary(object): finally: os.chdir( cwd ) - - - if __name__ == "__main__": Binary( "/srv/ftp.debian.org/queue/accepted/halevt_0.1.3-2_amd64.deb" ).scan_package() diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 21c9cc62..3fad3f50 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -478,7 +478,7 @@ class DBConn(Singleton): return map(lambda x: x[0], c.fetchall()) - def insert_content_paths(self, package, fullpaths): + def insert_content_paths(self, bin_id, fullpaths): """ Make sure given path is associated with given binary id @@ -492,6 +492,42 @@ class DBConn(Singleton): c = self.db_con.cursor() + c.execute("BEGIN WORK") + try: + + for fullpath in fullpaths: + (path, file) = os.path.split(fullpath) + + # Get the necessary IDs ... + file_id = self.get_or_set_contents_file_id(file) + path_id = self.get_or_set_contents_path_id(path) + + c.execute("""INSERT INTO content_associations + (binary_pkg, filepath, filename) + VALUES ( '%d', '%d', '%d')""" % (bin_id, path_id, file_id) ) + + c.execute("COMMIT") + return True + except: + traceback.print_exc() + c.execute("ROLLBACK") + return False + + def insert_pending_content_paths(self, package, fullpaths): + """ + Make sure given paths are temporarily associated with given + package + + @type package: dict + @param package: the package to associate with should have been read in from the binary control file + @type fullpaths: list + @param fullpaths: the list of paths of the file being associated with the binary + + @return True upon success + """ + + c = self.db_con.cursor() + c.execute("BEGIN WORK") try: diff --git a/daklib/utils.py b/daklib/utils.py index 27c3af38..c6300539 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -1526,54 +1526,4 @@ apt_pkg.ReadConfigFileISC(Cnf,default_config) if which_conf_file() != default_config: apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()) -################################################################################ - -def generate_contents_information(filename): - """ - Generate a list of flies contained in a .deb - - @ptype filename: string - @param filename: the path to a .deb - - @rtype: list - @return: a list of files in the data.tar.* portion of the .deb - """ - cmd = "ar t %s" % (filename) - (result, output) = commands.getstatusoutput(cmd) - if result != 0: - reject("%s: 'ar t' invocation failed." % (filename)) - reject(utils.prefix_multi_line_string(output, " [ar output:] "), "") - - # Ugh ... this is ugly ... Code ripped from process_unchecked.py - chunks = output.split('\n') - - contents = [] - try: - cmd = "ar x %s %s" % (filename, chunks[2]) - (result, output) = commands.getstatusoutput(cmd) - if result != 0: - reject("%s: '%s' invocation failed." % (filename, cmd)) - reject(utils.prefix_multi_line_string(output, " [ar output:] "), "") - - # Got deb tarballs, now lets go through and determine what bits - # and pieces the deb had ... - if chunks[2] == "data.tar.gz": - data = tarfile.open("data.tar.gz", "r:gz") - elif chunks[2] == "data.tar.bz2": - data = tarfile.open("data.tar.bz2", "r:bz2") - else: - os.remove(chunks[2]) - reject("couldn't find data.tar.*") - - for tarinfo in data: - if not tarinfo.isdir(): - contents.append(tarinfo.name[2:]) - - finally: - if os.path.exists( chunks[2] ): - shutil.rmtree( chunks[2] ) - os.remove( chunks[2] ) - - return contents - ############################################################################### -- 2.39.2