X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=18f427d424222d02fd9e74b4ade827950679003f;hb=535e3199605d07efb6bc936213aa2db1266646b2;hp=39285f0872673cd28c07fd59303bdabb97fa417d;hpb=25a56c7669cfdd660472b1bbc6a674113f3346ef;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 39285f08..18f427d4 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -39,7 +39,7 @@ import traceback from inspect import getargspec -from sqlalchemy import create_engine, Table, MetaData, select +from sqlalchemy import create_engine, Table, MetaData from sqlalchemy.orm import sessionmaker, mapper, relation # Don't remove this, we re-export the exceptions to scripts which import us @@ -83,6 +83,10 @@ def session_wrapper(fn): else: # Session is last argument in args session = args[-1] + if session is None: + args = list(args) + session = args[-1] = DBConn().session() + private_transaction = True if private_transaction: session.commit_or_flush = session.commit @@ -226,6 +230,17 @@ __all__.append('BinAssociation') ################################################################################ +class BinContents(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.binary, self.filename) + +__all__.append('BinContents') + +################################################################################ + class DBBinary(object): def __init__(self, *args, **kwargs): pass @@ -431,15 +446,6 @@ __all__.append('DBConfig') ################################################################################ -class ContentFilename(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.filename - -__all__.append('ContentFilename') - @session_wrapper def get_or_set_contents_file_id(filename, session=None): """ @@ -605,29 +611,20 @@ def insert_content_paths(binary_id, fullpaths, session=None): try: # Insert paths pathcache = {} - for fullpath in fullpaths: - # Get the necessary IDs ... - (path, file) = os.path.split(fullpath) - filepath_id = get_or_set_contents_path_id(path, session) - filename_id = get_or_set_contents_file_id(file, session) + def generate_path_dicts(): + for fullpath in fullpaths: + if fullpath.startswith( './' ): + fullpath = fullpath[2:] - pathcache[fullpath] = (filepath_id, filename_id) + yield {'fulename':fullpath, 'id': binary_id } - for fullpath, dat in pathcache.items(): - ca = ContentAssociation() - ca.binary_id = binary_id - ca.filepath_id = dat[0] - ca.filename_id = dat[1] - session.add(ca) + session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", + generate_path_dicts() ) - # Only commit if we set up the session ourself + session.commit() if privatetrans: - session.commit() session.close() - else: - session.flush() - return True except: @@ -1175,16 +1172,38 @@ __all__.append('get_override_type') ################################################################################ -class PendingContentAssociation(object): +class DebContents(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.package.package,self.file) + +__all__.append('DebContents') + + +class UdebContents(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.package.package,self.file) + +__all__.append('UdebContents') + +class PendingBinContents(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % self.pca_id + return '' % self.contents_id -__all__.append('PendingContentAssociation') +__all__.append('PendingBinContents') -def insert_pending_content_paths(package, fullpaths, session=None): +def insert_pending_content_paths(package, + is_udeb, + fullpaths, + session=None): """ Make sure given paths are temporarily associated with given package @@ -1213,32 +1232,27 @@ def insert_pending_content_paths(package, fullpaths, session=None): arch_id = arch.arch_id # Remove any already existing recorded files for this package - q = session.query(PendingContentAssociation) + q = session.query(PendingBinContents) q = q.filter_by(package=package['Package']) q = q.filter_by(version=package['Version']) q = q.filter_by(architecture=arch_id) q.delete() - # Insert paths - pathcache = {} for fullpath in fullpaths: - (path, file) = os.path.split(fullpath) - - if path.startswith( "./" ): - path = path[2:] - filepath_id = get_or_set_contents_path_id(path, session) - filename_id = get_or_set_contents_file_id(file, session) + if fullpath.startswith( "./" ): + fullpath = fullpath[2:] - pathcache[fullpath] = (filepath_id, filename_id) - - for fullpath, dat in pathcache.items(): - pca = PendingContentAssociation() + pca = PendingBinContents() pca.package = package['Package'] pca.version = package['Version'] - pca.filepath_id = dat[0] - pca.filename_id = dat[1] + pca.file = fullpath pca.architecture = arch_id + + if isudeb: + pca.type = 8 # gross + else: + pca.type = 7 # also gross session.add(pca) # Only commit if we set up the session ourself @@ -1410,46 +1424,26 @@ class Queue(object): session.add(qb) - # If the .orig tarballs are in the pool, create a symlink to - # them (if one doesn't already exist) - for dsc_file in changes.dsc_files.keys(): - # Skip all files except orig tarballs - if not re_is_orig_source.match(dsc_file): - continue - # Skip orig files not identified in the pool - if not (changes.orig_files.has_key(dsc_file) and - changes.orig_files[dsc_file].has_key("id")): - continue - orig_file_id = changes.orig_files[dsc_file]["id"] - dest = os.path.join(dest_dir, dsc_file) - - # If it doesn't exist, create a symlink - if not os.path.exists(dest): - q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id", - {'id': orig_file_id}) - res = q.fetchone() - if not res: - return "[INTERNAL ERROR] Couldn't find id %s in files table." % (orig_file_id) - - src = os.path.join(res[0], res[1]) - os.symlink(src, dest) + exists, symlinked = utils.ensure_orig_files(changes, dest, session) + + # Add symlinked files to the list of packages for later processing + # by apt-ftparchive + for filename in symlinked: + qb = QueueBuild() + qb.suite_id = s.suite_id + qb.queue_id = self.queue_id + qb.filename = filename + qb.in_queue = True + session.add(qb) - # Add it to the list of packages for later processing by apt-ftparchive - qb = QueueBuild() - qb.suite_id = s.suite_id - qb.queue_id = self.queue_id - qb.filename = dest + # Update files to ensure they are not removed prematurely + for filename in exists: + qb = get_queue_build(filename, s.suite_id, session) + if qb is None: qb.in_queue = True + qb.last_used = None session.add(qb) - # If it does, update things to ensure it's not removed prematurely - else: - qb = get_queue_build(dest, s.suite_id, session) - if qb is None: - qb.in_queue = True - qb.last_used = None - session.add(qb) - if privatetrans: session.commit() session.close() @@ -1459,9 +1453,10 @@ class Queue(object): __all__.append('Queue') @session_wrapper -def get_queue(queuename, session=None): +def get_or_set_queue(queuename, session=None): """ - Returns Queue object for given C{queue name}. + Returns Queue object for given C{queue name}, creating it if it does not + exist. @type queuename: string @param queuename: The name of the queue @@ -1477,11 +1472,17 @@ def get_queue(queuename, session=None): q = session.query(Queue).filter_by(queue_name=queuename) try: - return q.one() + ret = q.one() except NoResultFound: - return None + queue = Queue() + queue.queue_name = queuename + session.add(queue) + session.commit_or_flush() + ret = queue -__all__.append('get_queue') + return ret + +__all__.append('get_or_set_queue') ################################################################################ @@ -2093,6 +2094,7 @@ class DBConn(Singleton): def __setuptables(self): self.tbl_architecture = Table('architecture', self.db_meta, autoload=True) self.tbl_archive = Table('archive', self.db_meta, autoload=True) + self.tbl_bin_contents = Table('bin_contents', self.db_meta, autoload=True) self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True) self.tbl_binaries = Table('binaries', self.db_meta, autoload=True) self.tbl_component = Table('component', self.db_meta, autoload=True) @@ -2101,6 +2103,7 @@ class DBConn(Singleton): self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True) self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True) self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True) + self.tbl_deb_contents = Table('deb_contents', self.db_meta, autoload=True) self.tbl_files = Table('files', self.db_meta, autoload=True) self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True) self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True) @@ -2109,7 +2112,7 @@ class DBConn(Singleton): self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True) self.tbl_override = Table('override', self.db_meta, autoload=True) self.tbl_override_type = Table('override_type', self.db_meta, autoload=True) - self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True) + self.tbl_pending_bin_contents = Table('pending_bin_contents', self.db_meta, autoload=True) self.tbl_priority = Table('priority', self.db_meta, autoload=True) self.tbl_queue = Table('queue', self.db_meta, autoload=True) self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True) @@ -2121,6 +2124,7 @@ class DBConn(Singleton): self.tbl_suite = Table('suite', self.db_meta, autoload=True) self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True) self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True) + self.tbl_udeb_contents = Table('udeb_contents', self.db_meta, autoload=True) self.tbl_uid = Table('uid', self.db_meta, autoload=True) def __setupmappers(self): @@ -2138,6 +2142,30 @@ class DBConn(Singleton): binary_id = self.tbl_bin_associations.c.bin, binary = relation(DBBinary))) + mapper(PendingBinContents, self.tbl_pending_bin_contents, + properties = dict(contents_id =self.tbl_pending_bin_contents.c.id, + filename = self.tbl_pending_bin_contents.c.filename, + package = self.tbl_pending_bin_contents.c.package, + version = self.tbl_pending_bin_contents.c.version, + arch = self.tbl_pending_bin_contents.c.arch, + otype = self.tbl_pending_bin_contents.c.type)) + + mapper(DebContents, self.tbl_deb_contents, + properties = dict(binary_id=self.tbl_deb_contents.c.binary_id, + package=self.tbl_deb_contents.c.package, + component=self.tbl_deb_contents.c.component, + arch=self.tbl_deb_contents.c.arch, + section=self.tbl_deb_contents.c.section, + filename=self.tbl_deb_contents.c.filename)) + + mapper(UdebContents, self.tbl_udeb_contents, + properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id, + package=self.tbl_udeb_contents.c.package, + component=self.tbl_udeb_contents.c.component, + arch=self.tbl_udeb_contents.c.arch, + section=self.tbl_udeb_contents.c.section, + filename=self.tbl_udeb_contents.c.filename)) + mapper(DBBinary, self.tbl_binaries, properties = dict(binary_id = self.tbl_binaries.c.id, package = self.tbl_binaries.c.package, @@ -2164,24 +2192,6 @@ class DBConn(Singleton): mapper(DBConfig, self.tbl_config, properties = dict(config_id = self.tbl_config.c.id)) - mapper(ContentAssociation, self.tbl_content_associations, - properties = dict(ca_id = self.tbl_content_associations.c.id, - filename_id = self.tbl_content_associations.c.filename, - filename = relation(ContentFilename), - filepath_id = self.tbl_content_associations.c.filepath, - filepath = relation(ContentFilepath), - binary_id = self.tbl_content_associations.c.binary_pkg, - binary = relation(DBBinary))) - - - mapper(ContentFilename, self.tbl_content_file_names, - properties = dict(cafilename_id = self.tbl_content_file_names.c.id, - filename = self.tbl_content_file_names.c.file)) - - mapper(ContentFilepath, self.tbl_content_file_paths, - properties = dict(cafilepath_id = self.tbl_content_file_paths.c.id, - filepath = self.tbl_content_file_paths.c.path)) - mapper(DSCFile, self.tbl_dsc_files, properties = dict(dscfile_id = self.tbl_dsc_files.c.id, source_id = self.tbl_dsc_files.c.source, @@ -2223,6 +2233,7 @@ class DBConn(Singleton): mapper(Override, self.tbl_override, properties = dict(suite_id = self.tbl_override.c.suite, suite = relation(Suite), + package = self.tbl_override.c.package, component_id = self.tbl_override.c.component, component = relation(Component), priority_id = self.tbl_override.c.priority, @@ -2236,13 +2247,6 @@ class DBConn(Singleton): properties = dict(overridetype = self.tbl_override_type.c.type, overridetype_id = self.tbl_override_type.c.id)) - mapper(PendingContentAssociation, self.tbl_pending_content_associations, - properties = dict(pca_id = self.tbl_pending_content_associations.c.id, - filepath_id = self.tbl_pending_content_associations.c.filepath, - filepath = relation(ContentFilepath), - filename_id = self.tbl_pending_content_associations.c.filename, - filename = relation(ContentFilename))) - mapper(Priority, self.tbl_priority, properties = dict(priority_id = self.tbl_priority.c.id)) @@ -2255,7 +2259,8 @@ class DBConn(Singleton): queue = relation(Queue, backref='queuebuild'))) mapper(Section, self.tbl_section, - properties = dict(section_id = self.tbl_section.c.id)) + properties = dict(section_id = self.tbl_section.c.id, + section=self.tbl_section.c.section)) mapper(DBSource, self.tbl_source, properties = dict(source_id = self.tbl_source.c.id,