5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
37 from os.path import normpath
48 import simplejson as json
50 from datetime import datetime, timedelta
51 from errno import ENOENT
52 from tempfile import mkstemp, mkdtemp
53 from subprocess import Popen, PIPE
54 from tarfile import TarFile
56 from inspect import getargspec
59 from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \
61 from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
62 backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers
63 from sqlalchemy import types as sqltypes
65 # Don't remove this, we re-export the exceptions to scripts which import us
66 from sqlalchemy.exc import *
67 from sqlalchemy.orm.exc import NoResultFound
69 # Only import Config until Queue stuff is changed to store its config
71 from config import Config
72 from textutils import fix_maintainer
73 from dak_exceptions import DBUpdateError, NoSourceFieldError
75 # suppress some deprecation warnings in squeeze related to sqlalchemy
77 warnings.filterwarnings('ignore', \
78 "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
82 ################################################################################
84 # Patch in support for the debversion field type so that it works during
88 # that is for sqlalchemy 0.6
89 UserDefinedType = sqltypes.UserDefinedType
91 # this one for sqlalchemy 0.5
92 UserDefinedType = sqltypes.TypeEngine
94 class DebVersion(UserDefinedType):
95 def get_col_spec(self):
98 def bind_processor(self, dialect):
101 # ' = None' is needed for sqlalchemy 0.5:
102 def result_processor(self, dialect, coltype = None):
105 sa_major_version = sqlalchemy.__version__[0:3]
106 if sa_major_version in ["0.5", "0.6"]:
107 from sqlalchemy.databases import postgres
108 postgres.ischema_names['debversion'] = DebVersion
110 raise Exception("dak only ported to SQLA versions 0.5 and 0.6. See daklib/dbconn.py")
112 ################################################################################
114 __all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion']
116 ################################################################################
118 def session_wrapper(fn):
120 Wrapper around common ".., session=None):" handling. If the wrapped
121 function is called without passing 'session', we create a local one
122 and destroy it when the function ends.
124 Also attaches a commit_or_flush method to the session; if we created a
125 local session, this is a synonym for session.commit(), otherwise it is a
126 synonym for session.flush().
129 def wrapped(*args, **kwargs):
130 private_transaction = False
132 # Find the session object
133 session = kwargs.get('session')
136 if len(args) <= len(getargspec(fn)[0]) - 1:
137 # No session specified as last argument or in kwargs
138 private_transaction = True
139 session = kwargs['session'] = DBConn().session()
141 # Session is last argument in args
145 session = args[-1] = DBConn().session()
146 private_transaction = True
148 if private_transaction:
149 session.commit_or_flush = session.commit
151 session.commit_or_flush = session.flush
154 return fn(*args, **kwargs)
156 if private_transaction:
157 # We created a session; close it.
160 wrapped.__doc__ = fn.__doc__
161 wrapped.func_name = fn.func_name
165 __all__.append('session_wrapper')
167 ################################################################################
169 class ORMObject(object):
171 ORMObject is a base class for all ORM classes mapped by SQLalchemy. All
172 derived classes must implement the properties() method.
175 def properties(self):
177 This method should be implemented by all derived classes and returns a
178 list of the important properties. The properties 'created' and
179 'modified' will be added automatically. A suffix '_count' should be
180 added to properties that are lists or query objects. The most important
181 property name should be returned as the first element in the list
182 because it is used by repr().
188 Returns a JSON representation of the object based on the properties
189 returned from the properties() method.
192 # add created and modified
193 all_properties = self.properties() + ['created', 'modified']
194 for property in all_properties:
195 # check for list or query
196 if property[-6:] == '_count':
197 real_property = property[:-6]
198 if not hasattr(self, real_property):
200 value = getattr(self, real_property)
201 if hasattr(value, '__len__'):
204 elif hasattr(value, 'count'):
206 value = value.count()
208 raise KeyError('Do not understand property %s.' % property)
210 if not hasattr(self, property):
213 value = getattr(self, property)
217 elif isinstance(value, ORMObject):
218 # use repr() for ORMObject types
221 # we want a string for all other types because json cannot
224 data[property] = value
225 return json.dumps(data)
229 Returns the name of the class.
231 return type(self).__name__
235 Returns a short string representation of the object using the first
236 element from the properties() method.
238 primary_property = self.properties()[0]
239 value = getattr(self, primary_property)
240 return '<%s %s>' % (self.classname(), str(value))
244 Returns a human readable form of the object using the properties()
247 return '<%s %s>' % (self.classname(), self.json())
249 def not_null_constraints(self):
251 Returns a list of properties that must be not NULL. Derived classes
252 should override this method if needed.
256 validation_message = \
257 "Validation failed because property '%s' must not be empty in object\n%s"
261 This function validates the not NULL constraints as returned by
262 not_null_constraints(). It raises the DBUpdateError exception if
265 for property in self.not_null_constraints():
266 # TODO: It is a bit awkward that the mapper configuration allow
267 # directly setting the numeric _id columns. We should get rid of it
269 if hasattr(self, property + '_id') and \
270 getattr(self, property + '_id') is not None:
272 if not hasattr(self, property) or getattr(self, property) is None:
273 raise DBUpdateError(self.validation_message % \
274 (property, str(self)))
278 def get(cls, primary_key, session = None):
280 This is a support function that allows getting an object by its primary
283 Architecture.get(3[, session])
285 instead of the more verbose
287 session.query(Architecture).get(3)
289 return session.query(cls).get(primary_key)
291 def session(self, replace = False):
293 Returns the current session that is associated with the object. May
294 return None is object is in detached state.
297 return object_session(self)
299 def clone(self, session = None):
301 Clones the current object in a new session and returns the new clone. A
302 fresh session is created if the optional session parameter is not
303 provided. The function will fail if a session is provided and has
306 RATIONALE: SQLAlchemy's session is not thread safe. This method clones
307 an existing object to allow several threads to work with their own
308 instances of an ORMObject.
310 WARNING: Only persistent (committed) objects can be cloned. Changes
311 made to the original object that are not committed yet will get lost.
312 The session of the new object will always be rolled back to avoid
316 if self.session() is None:
317 raise RuntimeError( \
318 'Method clone() failed for detached object:\n%s' % self)
319 self.session().flush()
320 mapper = object_mapper(self)
321 primary_key = mapper.primary_key_from_instance(self)
322 object_class = self.__class__
324 session = DBConn().session()
325 elif len(session.new) + len(session.dirty) + len(session.deleted) > 0:
326 raise RuntimeError( \
327 'Method clone() failed due to unflushed changes in session.')
328 new_object = session.query(object_class).get(primary_key)
330 if new_object is None:
331 raise RuntimeError( \
332 'Method clone() failed for non-persistent object:\n%s' % self)
335 __all__.append('ORMObject')
337 ################################################################################
339 class Validator(MapperExtension):
341 This class calls the validate() method for each instance for the
342 'before_update' and 'before_insert' events. A global object validator is
343 used for configuring the individual mappers.
346 def before_update(self, mapper, connection, instance):
350 def before_insert(self, mapper, connection, instance):
354 validator = Validator()
356 ################################################################################
358 class Architecture(ORMObject):
359 def __init__(self, arch_string = None, description = None):
360 self.arch_string = arch_string
361 self.description = description
363 def __eq__(self, val):
364 if isinstance(val, str):
365 return (self.arch_string== val)
366 # This signals to use the normal comparison operator
367 return NotImplemented
369 def __ne__(self, val):
370 if isinstance(val, str):
371 return (self.arch_string != val)
372 # This signals to use the normal comparison operator
373 return NotImplemented
375 def properties(self):
376 return ['arch_string', 'arch_id', 'suites_count']
378 def not_null_constraints(self):
379 return ['arch_string']
381 __all__.append('Architecture')
384 def get_architecture(architecture, session=None):
386 Returns database id for given C{architecture}.
388 @type architecture: string
389 @param architecture: The name of the architecture
391 @type session: Session
392 @param session: Optional SQLA session object (a temporary one will be
393 generated if not supplied)
396 @return: Architecture object for the given arch (None if not present)
399 q = session.query(Architecture).filter_by(arch_string=architecture)
403 except NoResultFound:
406 __all__.append('get_architecture')
408 # TODO: should be removed because the implementation is too trivial
410 def get_architecture_suites(architecture, session=None):
412 Returns list of Suite objects for given C{architecture} name
414 @type architecture: str
415 @param architecture: Architecture name to search for
417 @type session: Session
418 @param session: Optional SQL session object (a temporary one will be
419 generated if not supplied)
422 @return: list of Suite objects for the given name (may be empty)
425 return get_architecture(architecture, session).suites
427 __all__.append('get_architecture_suites')
429 ################################################################################
431 class Archive(object):
432 def __init__(self, *args, **kwargs):
436 return '<Archive %s>' % self.archive_name
438 __all__.append('Archive')
441 def get_archive(archive, session=None):
443 returns database id for given C{archive}.
445 @type archive: string
446 @param archive: the name of the arhive
448 @type session: Session
449 @param session: Optional SQLA session object (a temporary one will be
450 generated if not supplied)
453 @return: Archive object for the given name (None if not present)
456 archive = archive.lower()
458 q = session.query(Archive).filter_by(archive_name=archive)
462 except NoResultFound:
465 __all__.append('get_archive')
467 ################################################################################
469 class BinContents(ORMObject):
470 def __init__(self, file = None, binary = None):
474 def properties(self):
475 return ['file', 'binary']
477 __all__.append('BinContents')
479 ################################################################################
481 class DBBinary(ORMObject):
482 def __init__(self, package = None, source = None, version = None, \
483 maintainer = None, architecture = None, poolfile = None, \
485 self.package = package
487 self.version = version
488 self.maintainer = maintainer
489 self.architecture = architecture
490 self.poolfile = poolfile
491 self.binarytype = binarytype
493 def properties(self):
494 return ['package', 'version', 'maintainer', 'source', 'architecture', \
495 'poolfile', 'binarytype', 'fingerprint', 'install_date', \
496 'suites_count', 'binary_id', 'contents_count', 'extra_sources']
498 def not_null_constraints(self):
499 return ['package', 'version', 'maintainer', 'source', 'poolfile', \
502 def get_component_name(self):
503 return self.poolfile.location.component.component_name
505 def scan_contents(self):
507 Yields the contents of the package. Only regular files are yielded and
508 the path names are normalized after converting them from either utf-8
509 or iso8859-1 encoding. It yields the string ' <EMPTY PACKAGE>' if the
510 package does not contain any regular file.
512 fullpath = self.poolfile.fullpath
513 dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE)
514 tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
515 for member in tar.getmembers():
516 if not member.isdir():
517 name = normpath(member.name)
518 # enforce proper utf-8 encoding
521 except UnicodeDecodeError:
522 name = name.decode('iso8859-1').encode('utf-8')
528 __all__.append('DBBinary')
531 def get_suites_binary_in(package, session=None):
533 Returns list of Suite objects which given C{package} name is in
536 @param package: DBBinary package name to search for
539 @return: list of Suite objects for the given package
542 return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all()
544 __all__.append('get_suites_binary_in')
547 def get_component_by_package_suite(package, suite_list, arch_list=[], session=None):
549 Returns the component name of the newest binary package in suite_list or
550 None if no package is found. The result can be optionally filtered by a list
551 of architecture names.
554 @param package: DBBinary package name to search for
556 @type suite_list: list of str
557 @param suite_list: list of suite_name items
559 @type arch_list: list of str
560 @param arch_list: optional list of arch_string items that defaults to []
562 @rtype: str or NoneType
563 @return: name of component or None
566 q = session.query(DBBinary).filter_by(package = package). \
567 join(DBBinary.suites).filter(Suite.suite_name.in_(suite_list))
568 if len(arch_list) > 0:
569 q = q.join(DBBinary.architecture). \
570 filter(Architecture.arch_string.in_(arch_list))
571 binary = q.order_by(desc(DBBinary.version)).first()
575 return binary.get_component_name()
577 __all__.append('get_component_by_package_suite')
579 ################################################################################
581 class BinaryACL(object):
582 def __init__(self, *args, **kwargs):
586 return '<BinaryACL %s>' % self.binary_acl_id
588 __all__.append('BinaryACL')
590 ################################################################################
592 class BinaryACLMap(object):
593 def __init__(self, *args, **kwargs):
597 return '<BinaryACLMap %s>' % self.binary_acl_map_id
599 __all__.append('BinaryACLMap')
601 ################################################################################
606 ArchiveDir "%(archivepath)s";
607 OverrideDir "%(overridedir)s";
608 CacheDir "%(cachedir)s";
613 Packages::Compress ". bzip2 gzip";
614 Sources::Compress ". bzip2 gzip";
619 bindirectory "incoming"
624 BinOverride "override.sid.all3";
625 BinCacheDB "packages-accepted.db";
627 FileList "%(filelist)s";
630 Packages::Extensions ".deb .udeb";
633 bindirectory "incoming/"
636 BinOverride "override.sid.all3";
637 SrcOverride "override.sid.all3.src";
638 FileList "%(filelist)s";
642 class BuildQueue(object):
643 def __init__(self, *args, **kwargs):
647 return '<BuildQueue %s>' % self.queue_name
649 def write_metadata(self, starttime, force=False):
650 # Do we write out metafiles?
651 if not (force or self.generate_metadata):
654 session = DBConn().session().object_session(self)
656 fl_fd = fl_name = ac_fd = ac_name = None
658 arches = " ".join([ a.arch_string for a in session.query(Architecture).all() if a.arch_string != 'source' ])
659 startdir = os.getcwd()
662 # Grab files we want to include
663 newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
664 # Write file list with newer files
665 (fl_fd, fl_name) = mkstemp()
667 os.write(fl_fd, '%s\n' % n.fullpath)
672 # Write minimal apt.conf
673 # TODO: Remove hardcoding from template
674 (ac_fd, ac_name) = mkstemp()
675 os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path,
677 'cachedir': cnf["Dir::Cache"],
678 'overridedir': cnf["Dir::Override"],
682 # Run apt-ftparchive generate
683 os.chdir(os.path.dirname(ac_name))
684 os.system('apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
686 # Run apt-ftparchive release
687 # TODO: Eww - fix this
688 bname = os.path.basename(self.path)
692 # We have to remove the Release file otherwise it'll be included in the
695 os.unlink(os.path.join(bname, 'Release'))
699 os.system("""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="%s" release %s > Release""" % (self.origin, self.label, self.releasedescription, arches, bname))
701 # Crude hack with open and append, but this whole section is and should be redone.
702 if self.notautomatic:
703 release=open("Release", "a")
704 release.write("NotAutomatic: yes")
709 keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
710 if cnf.has_key("Dinstall::SigningPubKeyring"):
711 keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
713 os.system("gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey))
715 # Move the files if we got this far
716 os.rename('Release', os.path.join(bname, 'Release'))
718 os.rename('Release.gpg', os.path.join(bname, 'Release.gpg'))
720 # Clean up any left behind files
747 def clean_and_update(self, starttime, Logger, dryrun=False):
748 """WARNING: This routine commits for you"""
749 session = DBConn().session().object_session(self)
751 if self.generate_metadata and not dryrun:
752 self.write_metadata(starttime)
754 # Grab files older than our execution time
755 older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
761 Logger.log(["I: Would have removed %s from the queue" % o.fullpath])
763 Logger.log(["I: Removing %s from the queue" % o.fullpath])
764 os.unlink(o.fullpath)
767 # If it wasn't there, don't worry
768 if e.errno == ENOENT:
771 # TODO: Replace with proper logging call
772 Logger.log(["E: Could not remove %s" % o.fullpath])
779 for f in os.listdir(self.path):
780 if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release') or f.startswith('advisory'):
784 r = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter_by(filename = f).one()
785 except NoResultFound:
786 fp = os.path.join(self.path, f)
788 Logger.log(["I: Would remove unused link %s" % fp])
790 Logger.log(["I: Removing unused link %s" % fp])
794 Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath])
796 def add_file_from_pool(self, poolfile):
797 """Copies a file into the pool. Assumes that the PoolFile object is
798 attached to the same SQLAlchemy session as the Queue object is.
800 The caller is responsible for committing after calling this function."""
801 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
803 # Check if we have a file of this name or this ID already
804 for f in self.queuefiles:
805 if f.fileid is not None and f.fileid == poolfile.file_id or \
806 f.poolfile.filename == poolfile_basename:
807 # In this case, update the BuildQueueFile entry so we
808 # don't remove it too early
809 f.lastused = datetime.now()
810 DBConn().session().object_session(poolfile).add(f)
813 # Prepare BuildQueueFile object
814 qf = BuildQueueFile()
815 qf.build_queue_id = self.queue_id
816 qf.lastused = datetime.now()
817 qf.filename = poolfile_basename
819 targetpath = poolfile.fullpath
820 queuepath = os.path.join(self.path, poolfile_basename)
824 # We need to copy instead of symlink
826 utils.copy(targetpath, queuepath)
827 # NULL in the fileid field implies a copy
830 os.symlink(targetpath, queuepath)
831 qf.fileid = poolfile.file_id
835 # Get the same session as the PoolFile is using and add the qf to it
836 DBConn().session().object_session(poolfile).add(qf)
841 __all__.append('BuildQueue')
844 def get_build_queue(queuename, session=None):
846 Returns BuildQueue object for given C{queue name}, creating it if it does not
849 @type queuename: string
850 @param queuename: The name of the queue
852 @type session: Session
853 @param session: Optional SQLA session object (a temporary one will be
854 generated if not supplied)
857 @return: BuildQueue object for the given queue
860 q = session.query(BuildQueue).filter_by(queue_name=queuename)
864 except NoResultFound:
867 __all__.append('get_build_queue')
869 ################################################################################
871 class BuildQueueFile(object):
872 def __init__(self, *args, **kwargs):
876 return '<BuildQueueFile %s (%s)>' % (self.filename, self.build_queue_id)
880 return os.path.join(self.buildqueue.path, self.filename)
883 __all__.append('BuildQueueFile')
885 ################################################################################
887 class ChangePendingBinary(object):
888 def __init__(self, *args, **kwargs):
892 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
894 __all__.append('ChangePendingBinary')
896 ################################################################################
898 class ChangePendingFile(object):
899 def __init__(self, *args, **kwargs):
903 return '<ChangePendingFile %s>' % self.change_pending_file_id
905 __all__.append('ChangePendingFile')
907 ################################################################################
909 class ChangePendingSource(object):
910 def __init__(self, *args, **kwargs):
914 return '<ChangePendingSource %s>' % self.change_pending_source_id
916 __all__.append('ChangePendingSource')
918 ################################################################################
920 class Component(ORMObject):
921 def __init__(self, component_name = None):
922 self.component_name = component_name
924 def __eq__(self, val):
925 if isinstance(val, str):
926 return (self.component_name == val)
927 # This signals to use the normal comparison operator
928 return NotImplemented
930 def __ne__(self, val):
931 if isinstance(val, str):
932 return (self.component_name != val)
933 # This signals to use the normal comparison operator
934 return NotImplemented
936 def properties(self):
937 return ['component_name', 'component_id', 'description', \
938 'location_count', 'meets_dfsg', 'overrides_count']
940 def not_null_constraints(self):
941 return ['component_name']
944 __all__.append('Component')
947 def get_component(component, session=None):
949 Returns database id for given C{component}.
951 @type component: string
952 @param component: The name of the override type
955 @return: the database id for the given component
958 component = component.lower()
960 q = session.query(Component).filter_by(component_name=component)
964 except NoResultFound:
967 __all__.append('get_component')
969 ################################################################################
971 class DBConfig(object):
972 def __init__(self, *args, **kwargs):
976 return '<DBConfig %s>' % self.name
978 __all__.append('DBConfig')
980 ################################################################################
983 def get_or_set_contents_file_id(filename, session=None):
985 Returns database id for given filename.
987 If no matching file is found, a row is inserted.
989 @type filename: string
990 @param filename: The filename
991 @type session: SQLAlchemy
992 @param session: Optional SQL session object (a temporary one will be
993 generated if not supplied). If not passed, a commit will be performed at
994 the end of the function, otherwise the caller is responsible for commiting.
997 @return: the database id for the given component
1000 q = session.query(ContentFilename).filter_by(filename=filename)
1003 ret = q.one().cafilename_id
1004 except NoResultFound:
1005 cf = ContentFilename()
1006 cf.filename = filename
1008 session.commit_or_flush()
1009 ret = cf.cafilename_id
1013 __all__.append('get_or_set_contents_file_id')
1016 def get_contents(suite, overridetype, section=None, session=None):
1018 Returns contents for a suite / overridetype combination, limiting
1019 to a section if not None.
1022 @param suite: Suite object
1024 @type overridetype: OverrideType
1025 @param overridetype: OverrideType object
1027 @type section: Section
1028 @param section: Optional section object to limit results to
1030 @type session: SQLAlchemy
1031 @param session: Optional SQL session object (a temporary one will be
1032 generated if not supplied)
1034 @rtype: ResultsProxy
1035 @return: ResultsProxy object set up to return tuples of (filename, section,
1039 # find me all of the contents for a given suite
1040 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
1044 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
1045 JOIN content_file_names n ON (c.filename=n.id)
1046 JOIN binaries b ON (b.id=c.binary_pkg)
1047 JOIN override o ON (o.package=b.package)
1048 JOIN section s ON (s.id=o.section)
1049 WHERE o.suite = :suiteid AND o.type = :overridetypeid
1050 AND b.type=:overridetypename"""
1052 vals = {'suiteid': suite.suite_id,
1053 'overridetypeid': overridetype.overridetype_id,
1054 'overridetypename': overridetype.overridetype}
1056 if section is not None:
1057 contents_q += " AND s.id = :sectionid"
1058 vals['sectionid'] = section.section_id
1060 contents_q += " ORDER BY fn"
1062 return session.execute(contents_q, vals)
1064 __all__.append('get_contents')
1066 ################################################################################
1068 class ContentFilepath(object):
1069 def __init__(self, *args, **kwargs):
1073 return '<ContentFilepath %s>' % self.filepath
1075 __all__.append('ContentFilepath')
1078 def get_or_set_contents_path_id(filepath, session=None):
1080 Returns database id for given path.
1082 If no matching file is found, a row is inserted.
1084 @type filepath: string
1085 @param filepath: The filepath
1087 @type session: SQLAlchemy
1088 @param session: Optional SQL session object (a temporary one will be
1089 generated if not supplied). If not passed, a commit will be performed at
1090 the end of the function, otherwise the caller is responsible for commiting.
1093 @return: the database id for the given path
1096 q = session.query(ContentFilepath).filter_by(filepath=filepath)
1099 ret = q.one().cafilepath_id
1100 except NoResultFound:
1101 cf = ContentFilepath()
1102 cf.filepath = filepath
1104 session.commit_or_flush()
1105 ret = cf.cafilepath_id
1109 __all__.append('get_or_set_contents_path_id')
1111 ################################################################################
1113 class ContentAssociation(object):
1114 def __init__(self, *args, **kwargs):
1118 return '<ContentAssociation %s>' % self.ca_id
1120 __all__.append('ContentAssociation')
1122 def insert_content_paths(binary_id, fullpaths, session=None):
1124 Make sure given path is associated with given binary id
1126 @type binary_id: int
1127 @param binary_id: the id of the binary
1128 @type fullpaths: list
1129 @param fullpaths: the list of paths of the file being associated with the binary
1130 @type session: SQLAlchemy session
1131 @param session: Optional SQLAlchemy session. If this is passed, the caller
1132 is responsible for ensuring a transaction has begun and committing the
1133 results or rolling back based on the result code. If not passed, a commit
1134 will be performed at the end of the function, otherwise the caller is
1135 responsible for commiting.
1137 @return: True upon success
1140 privatetrans = False
1142 session = DBConn().session()
1147 def generate_path_dicts():
1148 for fullpath in fullpaths:
1149 if fullpath.startswith( './' ):
1150 fullpath = fullpath[2:]
1152 yield {'filename':fullpath, 'id': binary_id }
1154 for d in generate_path_dicts():
1155 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
1164 traceback.print_exc()
1166 # Only rollback if we set up the session ourself
1173 __all__.append('insert_content_paths')
1175 ################################################################################
1177 class DSCFile(object):
1178 def __init__(self, *args, **kwargs):
1182 return '<DSCFile %s>' % self.dscfile_id
1184 __all__.append('DSCFile')
1187 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
1189 Returns a list of DSCFiles which may be empty
1191 @type dscfile_id: int (optional)
1192 @param dscfile_id: the dscfile_id of the DSCFiles to find
1194 @type source_id: int (optional)
1195 @param source_id: the source id related to the DSCFiles to find
1197 @type poolfile_id: int (optional)
1198 @param poolfile_id: the poolfile id related to the DSCFiles to find
1201 @return: Possibly empty list of DSCFiles
1204 q = session.query(DSCFile)
1206 if dscfile_id is not None:
1207 q = q.filter_by(dscfile_id=dscfile_id)
1209 if source_id is not None:
1210 q = q.filter_by(source_id=source_id)
1212 if poolfile_id is not None:
1213 q = q.filter_by(poolfile_id=poolfile_id)
1217 __all__.append('get_dscfiles')
1219 ################################################################################
1221 class PoolFile(ORMObject):
1222 def __init__(self, filename = None, location = None, filesize = -1, \
1224 self.filename = filename
1225 self.location = location
1226 self.filesize = filesize
1227 self.md5sum = md5sum
1231 return os.path.join(self.location.path, self.filename)
1233 def is_valid(self, filesize = -1, md5sum = None):
1234 return self.filesize == long(filesize) and self.md5sum == md5sum
1236 def properties(self):
1237 return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \
1238 'sha256sum', 'location', 'source', 'binary', 'last_used']
1240 def not_null_constraints(self):
1241 return ['filename', 'md5sum', 'location']
1243 __all__.append('PoolFile')
1246 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
1249 (ValidFileFound [boolean], PoolFile object or None)
1251 @type filename: string
1252 @param filename: the filename of the file to check against the DB
1255 @param filesize: the size of the file to check against the DB
1257 @type md5sum: string
1258 @param md5sum: the md5sum of the file to check against the DB
1260 @type location_id: int
1261 @param location_id: the id of the location to look in
1264 @return: Tuple of length 2.
1265 - If valid pool file found: (C{True}, C{PoolFile object})
1266 - If valid pool file not found:
1267 - (C{False}, C{None}) if no file found
1268 - (C{False}, C{PoolFile object}) if file found with size/md5sum mismatch
1271 poolfile = session.query(Location).get(location_id). \
1272 files.filter_by(filename=filename).first()
1274 if poolfile and poolfile.is_valid(filesize = filesize, md5sum = md5sum):
1277 return (valid, poolfile)
1279 __all__.append('check_poolfile')
1281 # TODO: the implementation can trivially be inlined at the place where the
1282 # function is called
1284 def get_poolfile_by_id(file_id, session=None):
1286 Returns a PoolFile objects or None for the given id
1289 @param file_id: the id of the file to look for
1291 @rtype: PoolFile or None
1292 @return: either the PoolFile object or None
1295 return session.query(PoolFile).get(file_id)
1297 __all__.append('get_poolfile_by_id')
1300 def get_poolfile_like_name(filename, session=None):
1302 Returns an array of PoolFile objects which are like the given name
1304 @type filename: string
1305 @param filename: the filename of the file to check against the DB
1308 @return: array of PoolFile objects
1311 # TODO: There must be a way of properly using bind parameters with %FOO%
1312 q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
1316 __all__.append('get_poolfile_like_name')
1319 def add_poolfile(filename, datadict, location_id, session=None):
1321 Add a new file to the pool
1323 @type filename: string
1324 @param filename: filename
1326 @type datadict: dict
1327 @param datadict: dict with needed data
1329 @type location_id: int
1330 @param location_id: database id of the location
1333 @return: the PoolFile object created
1335 poolfile = PoolFile()
1336 poolfile.filename = filename
1337 poolfile.filesize = datadict["size"]
1338 poolfile.md5sum = datadict["md5sum"]
1339 poolfile.sha1sum = datadict["sha1sum"]
1340 poolfile.sha256sum = datadict["sha256sum"]
1341 poolfile.location_id = location_id
1343 session.add(poolfile)
1344 # Flush to get a file id (NB: This is not a commit)
1349 __all__.append('add_poolfile')
1351 ################################################################################
1353 class Fingerprint(ORMObject):
1354 def __init__(self, fingerprint = None):
1355 self.fingerprint = fingerprint
1357 def properties(self):
1358 return ['fingerprint', 'fingerprint_id', 'keyring', 'uid', \
1361 def not_null_constraints(self):
1362 return ['fingerprint']
1364 __all__.append('Fingerprint')
1367 def get_fingerprint(fpr, session=None):
1369 Returns Fingerprint object for given fpr.
1372 @param fpr: The fpr to find / add
1374 @type session: SQLAlchemy
1375 @param session: Optional SQL session object (a temporary one will be
1376 generated if not supplied).
1379 @return: the Fingerprint object for the given fpr or None
1382 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1386 except NoResultFound:
1391 __all__.append('get_fingerprint')
1394 def get_or_set_fingerprint(fpr, session=None):
1396 Returns Fingerprint object for given fpr.
1398 If no matching fpr is found, a row is inserted.
1401 @param fpr: The fpr to find / add
1403 @type session: SQLAlchemy
1404 @param session: Optional SQL session object (a temporary one will be
1405 generated if not supplied). If not passed, a commit will be performed at
1406 the end of the function, otherwise the caller is responsible for commiting.
1407 A flush will be performed either way.
1410 @return: the Fingerprint object for the given fpr
1413 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1417 except NoResultFound:
1418 fingerprint = Fingerprint()
1419 fingerprint.fingerprint = fpr
1420 session.add(fingerprint)
1421 session.commit_or_flush()
1426 __all__.append('get_or_set_fingerprint')
1428 ################################################################################
1430 # Helper routine for Keyring class
1431 def get_ldap_name(entry):
1433 for k in ["cn", "mn", "sn"]:
1435 if ret and ret[0] != "" and ret[0] != "-":
1437 return " ".join(name)
1439 ################################################################################
1441 class Keyring(object):
1442 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1443 " --with-colons --fingerprint --fingerprint"
1448 def __init__(self, *args, **kwargs):
1452 return '<Keyring %s>' % self.keyring_name
1454 def de_escape_gpg_str(self, txt):
1455 esclist = re.split(r'(\\x..)', txt)
1456 for x in range(1,len(esclist),2):
1457 esclist[x] = "%c" % (int(esclist[x][2:],16))
1458 return "".join(esclist)
1460 def parse_address(self, uid):
1461 """parses uid and returns a tuple of real name and email address"""
1463 (name, address) = email.Utils.parseaddr(uid)
1464 name = re.sub(r"\s*[(].*[)]", "", name)
1465 name = self.de_escape_gpg_str(name)
1468 return (name, address)
1470 def load_keys(self, keyring):
1471 if not self.keyring_id:
1472 raise Exception('Must be initialized with database information')
1474 k = os.popen(self.gpg_invocation % keyring, "r")
1478 for line in k.xreadlines():
1479 field = line.split(":")
1480 if field[0] == "pub":
1483 (name, addr) = self.parse_address(field[9])
1485 self.keys[key]["email"] = addr
1486 self.keys[key]["name"] = name
1487 self.keys[key]["fingerprints"] = []
1489 elif key and field[0] == "sub" and len(field) >= 12:
1490 signingkey = ("s" in field[11])
1491 elif key and field[0] == "uid":
1492 (name, addr) = self.parse_address(field[9])
1493 if "email" not in self.keys[key] and "@" in addr:
1494 self.keys[key]["email"] = addr
1495 self.keys[key]["name"] = name
1496 elif signingkey and field[0] == "fpr":
1497 self.keys[key]["fingerprints"].append(field[9])
1498 self.fpr_lookup[field[9]] = key
1500 def import_users_from_ldap(self, session):
1504 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1505 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1507 l = ldap.open(LDAPServer)
1508 l.simple_bind_s("","")
1509 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1510 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1511 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1513 ldap_fin_uid_id = {}
1520 uid = entry["uid"][0]
1521 name = get_ldap_name(entry)
1522 fingerprints = entry["keyFingerPrint"]
1524 for f in fingerprints:
1525 key = self.fpr_lookup.get(f, None)
1526 if key not in self.keys:
1528 self.keys[key]["uid"] = uid
1532 keyid = get_or_set_uid(uid, session).uid_id
1533 byuid[keyid] = (uid, name)
1534 byname[uid] = (keyid, name)
1536 return (byname, byuid)
1538 def generate_users_from_keyring(self, format, session):
1542 for x in self.keys.keys():
1543 if "email" not in self.keys[x]:
1545 self.keys[x]["uid"] = format % "invalid-uid"
1547 uid = format % self.keys[x]["email"]
1548 keyid = get_or_set_uid(uid, session).uid_id
1549 byuid[keyid] = (uid, self.keys[x]["name"])
1550 byname[uid] = (keyid, self.keys[x]["name"])
1551 self.keys[x]["uid"] = uid
1554 uid = format % "invalid-uid"
1555 keyid = get_or_set_uid(uid, session).uid_id
1556 byuid[keyid] = (uid, "ungeneratable user id")
1557 byname[uid] = (keyid, "ungeneratable user id")
1559 return (byname, byuid)
1561 __all__.append('Keyring')
1564 def get_keyring(keyring, session=None):
1566 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1567 If C{keyring} already has an entry, simply return the existing Keyring
1569 @type keyring: string
1570 @param keyring: the keyring name
1573 @return: the Keyring object for this keyring
1576 q = session.query(Keyring).filter_by(keyring_name=keyring)
1580 except NoResultFound:
1583 __all__.append('get_keyring')
1585 ################################################################################
1587 class KeyringACLMap(object):
1588 def __init__(self, *args, **kwargs):
1592 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1594 __all__.append('KeyringACLMap')
1596 ################################################################################
1598 class DBChange(object):
1599 def __init__(self, *args, **kwargs):
1603 return '<DBChange %s>' % self.changesname
1605 def clean_from_queue(self):
1606 session = DBConn().session().object_session(self)
1608 # Remove changes_pool_files entries
1611 # Remove changes_pending_files references
1614 # Clear out of queue
1615 self.in_queue = None
1616 self.approved_for_id = None
1618 __all__.append('DBChange')
1621 def get_dbchange(filename, session=None):
1623 returns DBChange object for given C{filename}.
1625 @type filename: string
1626 @param filename: the name of the file
1628 @type session: Session
1629 @param session: Optional SQLA session object (a temporary one will be
1630 generated if not supplied)
1633 @return: DBChange object for the given filename (C{None} if not present)
1636 q = session.query(DBChange).filter_by(changesname=filename)
1640 except NoResultFound:
1643 __all__.append('get_dbchange')
1645 ################################################################################
1647 class Location(ORMObject):
1648 def __init__(self, path = None, component = None):
1650 self.component = component
1651 # the column 'type' should go away, see comment at mapper
1652 self.archive_type = 'pool'
1654 def properties(self):
1655 return ['path', 'location_id', 'archive_type', 'component', \
1658 def not_null_constraints(self):
1659 return ['path', 'archive_type']
1661 __all__.append('Location')
1664 def get_location(location, component=None, archive=None, session=None):
1666 Returns Location object for the given combination of location, component
1669 @type location: string
1670 @param location: the path of the location, e.g. I{/srv/ftp-master.debian.org/ftp/pool/}
1672 @type component: string
1673 @param component: the component name (if None, no restriction applied)
1675 @type archive: string
1676 @param archive: the archive name (if None, no restriction applied)
1678 @rtype: Location / None
1679 @return: Either a Location object or None if one can't be found
1682 q = session.query(Location).filter_by(path=location)
1684 if archive is not None:
1685 q = q.join(Archive).filter_by(archive_name=archive)
1687 if component is not None:
1688 q = q.join(Component).filter_by(component_name=component)
1692 except NoResultFound:
1695 __all__.append('get_location')
1697 ################################################################################
1699 class Maintainer(ORMObject):
1700 def __init__(self, name = None):
1703 def properties(self):
1704 return ['name', 'maintainer_id']
1706 def not_null_constraints(self):
1709 def get_split_maintainer(self):
1710 if not hasattr(self, 'name') or self.name is None:
1711 return ('', '', '', '')
1713 return fix_maintainer(self.name.strip())
1715 __all__.append('Maintainer')
1718 def get_or_set_maintainer(name, session=None):
1720 Returns Maintainer object for given maintainer name.
1722 If no matching maintainer name is found, a row is inserted.
1725 @param name: The maintainer name to add
1727 @type session: SQLAlchemy
1728 @param session: Optional SQL session object (a temporary one will be
1729 generated if not supplied). If not passed, a commit will be performed at
1730 the end of the function, otherwise the caller is responsible for commiting.
1731 A flush will be performed either way.
1734 @return: the Maintainer object for the given maintainer
1737 q = session.query(Maintainer).filter_by(name=name)
1740 except NoResultFound:
1741 maintainer = Maintainer()
1742 maintainer.name = name
1743 session.add(maintainer)
1744 session.commit_or_flush()
1749 __all__.append('get_or_set_maintainer')
1752 def get_maintainer(maintainer_id, session=None):
1754 Return the name of the maintainer behind C{maintainer_id} or None if that
1755 maintainer_id is invalid.
1757 @type maintainer_id: int
1758 @param maintainer_id: the id of the maintainer
1761 @return: the Maintainer with this C{maintainer_id}
1764 return session.query(Maintainer).get(maintainer_id)
1766 __all__.append('get_maintainer')
1768 ################################################################################
1770 class NewComment(object):
1771 def __init__(self, *args, **kwargs):
1775 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1777 __all__.append('NewComment')
1780 def has_new_comment(package, version, session=None):
1782 Returns true if the given combination of C{package}, C{version} has a comment.
1784 @type package: string
1785 @param package: name of the package
1787 @type version: string
1788 @param version: package version
1790 @type session: Session
1791 @param session: Optional SQLA session object (a temporary one will be
1792 generated if not supplied)
1798 q = session.query(NewComment)
1799 q = q.filter_by(package=package)
1800 q = q.filter_by(version=version)
1802 return bool(q.count() > 0)
1804 __all__.append('has_new_comment')
1807 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1809 Returns (possibly empty) list of NewComment objects for the given
1812 @type package: string (optional)
1813 @param package: name of the package
1815 @type version: string (optional)
1816 @param version: package version
1818 @type comment_id: int (optional)
1819 @param comment_id: An id of a comment
1821 @type session: Session
1822 @param session: Optional SQLA session object (a temporary one will be
1823 generated if not supplied)
1826 @return: A (possibly empty) list of NewComment objects will be returned
1829 q = session.query(NewComment)
1830 if package is not None: q = q.filter_by(package=package)
1831 if version is not None: q = q.filter_by(version=version)
1832 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1836 __all__.append('get_new_comments')
1838 ################################################################################
1840 class Override(ORMObject):
1841 def __init__(self, package = None, suite = None, component = None, overridetype = None, \
1842 section = None, priority = None):
1843 self.package = package
1845 self.component = component
1846 self.overridetype = overridetype
1847 self.section = section
1848 self.priority = priority
1850 def properties(self):
1851 return ['package', 'suite', 'component', 'overridetype', 'section', \
1854 def not_null_constraints(self):
1855 return ['package', 'suite', 'component', 'overridetype', 'section']
1857 __all__.append('Override')
1860 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1862 Returns Override object for the given parameters
1864 @type package: string
1865 @param package: The name of the package
1867 @type suite: string, list or None
1868 @param suite: The name of the suite (or suites if a list) to limit to. If
1869 None, don't limit. Defaults to None.
1871 @type component: string, list or None
1872 @param component: The name of the component (or components if a list) to
1873 limit to. If None, don't limit. Defaults to None.
1875 @type overridetype: string, list or None
1876 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1877 limit to. If None, don't limit. Defaults to None.
1879 @type session: Session
1880 @param session: Optional SQLA session object (a temporary one will be
1881 generated if not supplied)
1884 @return: A (possibly empty) list of Override objects will be returned
1887 q = session.query(Override)
1888 q = q.filter_by(package=package)
1890 if suite is not None:
1891 if not isinstance(suite, list): suite = [suite]
1892 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1894 if component is not None:
1895 if not isinstance(component, list): component = [component]
1896 q = q.join(Component).filter(Component.component_name.in_(component))
1898 if overridetype is not None:
1899 if not isinstance(overridetype, list): overridetype = [overridetype]
1900 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1904 __all__.append('get_override')
1907 ################################################################################
1909 class OverrideType(ORMObject):
1910 def __init__(self, overridetype = None):
1911 self.overridetype = overridetype
1913 def properties(self):
1914 return ['overridetype', 'overridetype_id', 'overrides_count']
1916 def not_null_constraints(self):
1917 return ['overridetype']
1919 __all__.append('OverrideType')
1922 def get_override_type(override_type, session=None):
1924 Returns OverrideType object for given C{override type}.
1926 @type override_type: string
1927 @param override_type: The name of the override type
1929 @type session: Session
1930 @param session: Optional SQLA session object (a temporary one will be
1931 generated if not supplied)
1934 @return: the database id for the given override type
1937 q = session.query(OverrideType).filter_by(overridetype=override_type)
1941 except NoResultFound:
1944 __all__.append('get_override_type')
1946 ################################################################################
1948 class PolicyQueue(object):
1949 def __init__(self, *args, **kwargs):
1953 return '<PolicyQueue %s>' % self.queue_name
1955 __all__.append('PolicyQueue')
1958 def get_policy_queue(queuename, session=None):
1960 Returns PolicyQueue object for given C{queue name}
1962 @type queuename: string
1963 @param queuename: The name of the queue
1965 @type session: Session
1966 @param session: Optional SQLA session object (a temporary one will be
1967 generated if not supplied)
1970 @return: PolicyQueue object for the given queue
1973 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
1977 except NoResultFound:
1980 __all__.append('get_policy_queue')
1983 def get_policy_queue_from_path(pathname, session=None):
1985 Returns PolicyQueue object for given C{path name}
1987 @type queuename: string
1988 @param queuename: The path
1990 @type session: Session
1991 @param session: Optional SQLA session object (a temporary one will be
1992 generated if not supplied)
1995 @return: PolicyQueue object for the given queue
1998 q = session.query(PolicyQueue).filter_by(path=pathname)
2002 except NoResultFound:
2005 __all__.append('get_policy_queue_from_path')
2007 ################################################################################
2009 class Priority(ORMObject):
2010 def __init__(self, priority = None, level = None):
2011 self.priority = priority
2014 def properties(self):
2015 return ['priority', 'priority_id', 'level', 'overrides_count']
2017 def not_null_constraints(self):
2018 return ['priority', 'level']
2020 def __eq__(self, val):
2021 if isinstance(val, str):
2022 return (self.priority == val)
2023 # This signals to use the normal comparison operator
2024 return NotImplemented
2026 def __ne__(self, val):
2027 if isinstance(val, str):
2028 return (self.priority != val)
2029 # This signals to use the normal comparison operator
2030 return NotImplemented
2032 __all__.append('Priority')
2035 def get_priority(priority, session=None):
2037 Returns Priority object for given C{priority name}.
2039 @type priority: string
2040 @param priority: The name of the priority
2042 @type session: Session
2043 @param session: Optional SQLA session object (a temporary one will be
2044 generated if not supplied)
2047 @return: Priority object for the given priority
2050 q = session.query(Priority).filter_by(priority=priority)
2054 except NoResultFound:
2057 __all__.append('get_priority')
2060 def get_priorities(session=None):
2062 Returns dictionary of priority names -> id mappings
2064 @type session: Session
2065 @param session: Optional SQL session object (a temporary one will be
2066 generated if not supplied)
2069 @return: dictionary of priority names -> id mappings
2073 q = session.query(Priority)
2075 ret[x.priority] = x.priority_id
2079 __all__.append('get_priorities')
2081 ################################################################################
2083 class Section(ORMObject):
2084 def __init__(self, section = None):
2085 self.section = section
2087 def properties(self):
2088 return ['section', 'section_id', 'overrides_count']
2090 def not_null_constraints(self):
2093 def __eq__(self, val):
2094 if isinstance(val, str):
2095 return (self.section == val)
2096 # This signals to use the normal comparison operator
2097 return NotImplemented
2099 def __ne__(self, val):
2100 if isinstance(val, str):
2101 return (self.section != val)
2102 # This signals to use the normal comparison operator
2103 return NotImplemented
2105 __all__.append('Section')
2108 def get_section(section, session=None):
2110 Returns Section object for given C{section name}.
2112 @type section: string
2113 @param section: The name of the section
2115 @type session: Session
2116 @param session: Optional SQLA session object (a temporary one will be
2117 generated if not supplied)
2120 @return: Section object for the given section name
2123 q = session.query(Section).filter_by(section=section)
2127 except NoResultFound:
2130 __all__.append('get_section')
2133 def get_sections(session=None):
2135 Returns dictionary of section names -> id mappings
2137 @type session: Session
2138 @param session: Optional SQL session object (a temporary one will be
2139 generated if not supplied)
2142 @return: dictionary of section names -> id mappings
2146 q = session.query(Section)
2148 ret[x.section] = x.section_id
2152 __all__.append('get_sections')
2154 ################################################################################
2156 class DBSource(ORMObject):
2157 def __init__(self, source = None, version = None, maintainer = None, \
2158 changedby = None, poolfile = None, install_date = None):
2159 self.source = source
2160 self.version = version
2161 self.maintainer = maintainer
2162 self.changedby = changedby
2163 self.poolfile = poolfile
2164 self.install_date = install_date
2166 def properties(self):
2167 return ['source', 'source_id', 'maintainer', 'changedby', \
2168 'fingerprint', 'poolfile', 'version', 'suites_count', \
2169 'install_date', 'binaries_count']
2171 def not_null_constraints(self):
2172 return ['source', 'version', 'install_date', 'maintainer', \
2173 'changedby', 'poolfile', 'install_date']
2175 __all__.append('DBSource')
2178 def source_exists(source, source_version, suites = ["any"], session=None):
2180 Ensure that source exists somewhere in the archive for the binary
2181 upload being processed.
2182 1. exact match => 1.0-3
2183 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
2185 @type source: string
2186 @param source: source name
2188 @type source_version: string
2189 @param source_version: expected source version
2192 @param suites: list of suites to check in, default I{any}
2194 @type session: Session
2195 @param session: Optional SQLA session object (a temporary one will be
2196 generated if not supplied)
2199 @return: returns 1 if a source with expected version is found, otherwise 0
2206 from daklib.regexes import re_bin_only_nmu
2207 orig_source_version = re_bin_only_nmu.sub('', source_version)
2209 for suite in suites:
2210 q = session.query(DBSource).filter_by(source=source). \
2211 filter(DBSource.version.in_([source_version, orig_source_version]))
2213 # source must exist in suite X, or in some other suite that's
2214 # mapped to X, recursively... silent-maps are counted too,
2215 # unreleased-maps aren't.
2216 maps = cnf.ValueList("SuiteMappings")[:]
2218 maps = [ m.split() for m in maps ]
2219 maps = [ (x[1], x[2]) for x in maps
2220 if x[0] == "map" or x[0] == "silent-map" ]
2222 for (from_, to) in maps:
2223 if from_ in s and to not in s:
2226 q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
2231 # No source found so return not ok
2236 __all__.append('source_exists')
2239 def get_suites_source_in(source, session=None):
2241 Returns list of Suite objects which given C{source} name is in
2244 @param source: DBSource package name to search for
2247 @return: list of Suite objects for the given source
2250 return session.query(Suite).filter(Suite.sources.any(source=source)).all()
2252 __all__.append('get_suites_source_in')
2255 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
2257 Returns list of DBSource objects for given C{source} name and other parameters
2260 @param source: DBSource package name to search for
2262 @type version: str or None
2263 @param version: DBSource version name to search for or None if not applicable
2265 @type dm_upload_allowed: bool
2266 @param dm_upload_allowed: If None, no effect. If True or False, only
2267 return packages with that dm_upload_allowed setting
2269 @type session: Session
2270 @param session: Optional SQL session object (a temporary one will be
2271 generated if not supplied)
2274 @return: list of DBSource objects for the given name (may be empty)
2277 q = session.query(DBSource).filter_by(source=source)
2279 if version is not None:
2280 q = q.filter_by(version=version)
2282 if dm_upload_allowed is not None:
2283 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
2287 __all__.append('get_sources_from_name')
2289 # FIXME: This function fails badly if it finds more than 1 source package and
2290 # its implementation is trivial enough to be inlined.
2292 def get_source_in_suite(source, suite, session=None):
2294 Returns a DBSource object for a combination of C{source} and C{suite}.
2296 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2297 - B{suite} - a suite name, eg. I{unstable}
2299 @type source: string
2300 @param source: source package name
2303 @param suite: the suite name
2306 @return: the version for I{source} in I{suite}
2310 q = get_suite(suite, session).get_sources(source)
2313 except NoResultFound:
2316 __all__.append('get_source_in_suite')
2318 ################################################################################
2321 def add_dsc_to_db(u, filename, session=None):
2322 entry = u.pkg.files[filename]
2326 source.source = u.pkg.dsc["source"]
2327 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2328 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2329 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2330 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2331 source.install_date = datetime.now().date()
2333 dsc_component = entry["component"]
2334 dsc_location_id = entry["location id"]
2336 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2338 # Set up a new poolfile if necessary
2339 if not entry.has_key("files id") or not entry["files id"]:
2340 filename = entry["pool name"] + filename
2341 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2343 pfs.append(poolfile)
2344 entry["files id"] = poolfile.file_id
2346 source.poolfile_id = entry["files id"]
2349 suite_names = u.pkg.changes["distribution"].keys()
2350 source.suites = session.query(Suite). \
2351 filter(Suite.suite_name.in_(suite_names)).all()
2353 # Add the source files to the DB (files and dsc_files)
2355 dscfile.source_id = source.source_id
2356 dscfile.poolfile_id = entry["files id"]
2357 session.add(dscfile)
2359 for dsc_file, dentry in u.pkg.dsc_files.items():
2361 df.source_id = source.source_id
2363 # If the .orig tarball is already in the pool, it's
2364 # files id is stored in dsc_files by check_dsc().
2365 files_id = dentry.get("files id", None)
2367 # Find the entry in the files hash
2368 # TODO: Bail out here properly
2370 for f, e in u.pkg.files.items():
2375 if files_id is None:
2376 filename = dfentry["pool name"] + dsc_file
2378 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2379 # FIXME: needs to check for -1/-2 and or handle exception
2380 if found and obj is not None:
2381 files_id = obj.file_id
2384 # If still not found, add it
2385 if files_id is None:
2386 # HACK: Force sha1sum etc into dentry
2387 dentry["sha1sum"] = dfentry["sha1sum"]
2388 dentry["sha256sum"] = dfentry["sha256sum"]
2389 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2390 pfs.append(poolfile)
2391 files_id = poolfile.file_id
2393 poolfile = get_poolfile_by_id(files_id, session)
2394 if poolfile is None:
2395 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2396 pfs.append(poolfile)
2398 df.poolfile_id = files_id
2401 # Add the src_uploaders to the DB
2402 uploader_ids = [source.maintainer_id]
2403 if u.pkg.dsc.has_key("uploaders"):
2404 for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
2406 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2409 for up_id in uploader_ids:
2410 if added_ids.has_key(up_id):
2412 utils.warn("Already saw uploader %s for source %s" % (up_id, source.source))
2418 su.maintainer_id = up_id
2419 su.source_id = source.source_id
2424 return source, dsc_component, dsc_location_id, pfs
2426 __all__.append('add_dsc_to_db')
2429 def add_deb_to_db(u, filename, session=None):
2431 Contrary to what you might expect, this routine deals with both
2432 debs and udebs. That info is in 'dbtype', whilst 'type' is
2433 'deb' for both of them
2436 entry = u.pkg.files[filename]
2439 bin.package = entry["package"]
2440 bin.version = entry["version"]
2441 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2442 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2443 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2444 bin.binarytype = entry["dbtype"]
2447 filename = entry["pool name"] + filename
2448 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2449 if not entry.get("location id", None):
2450 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2452 if entry.get("files id", None):
2453 poolfile = get_poolfile_by_id(bin.poolfile_id)
2454 bin.poolfile_id = entry["files id"]
2456 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2457 bin.poolfile_id = entry["files id"] = poolfile.file_id
2460 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2461 if len(bin_sources) != 1:
2462 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2463 (bin.package, bin.version, entry["architecture"],
2464 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2466 bin.source_id = bin_sources[0].source_id
2468 if entry.has_key("built-using"):
2469 for srcname, version in entry["built-using"]:
2470 exsources = get_sources_from_name(srcname, version, session=session)
2471 if len(exsources) != 1:
2472 raise NoSourceFieldError, "Unable to find source package (%s = %s) in Built-Using for %s (%s), %s, file %s, type %s, signed by %s" % \
2473 (srcname, version, bin.package, bin.version, entry["architecture"],
2474 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2476 bin.extra_sources.append(exsources[0])
2478 # Add and flush object so it has an ID
2481 suite_names = u.pkg.changes["distribution"].keys()
2482 bin.suites = session.query(Suite). \
2483 filter(Suite.suite_name.in_(suite_names)).all()
2487 # Deal with contents - disabled for now
2488 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2490 # print "REJECT\nCould not determine contents of package %s" % bin.package
2491 # session.rollback()
2492 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2496 __all__.append('add_deb_to_db')
2498 ################################################################################
2500 class SourceACL(object):
2501 def __init__(self, *args, **kwargs):
2505 return '<SourceACL %s>' % self.source_acl_id
2507 __all__.append('SourceACL')
2509 ################################################################################
2511 class SrcFormat(object):
2512 def __init__(self, *args, **kwargs):
2516 return '<SrcFormat %s>' % (self.format_name)
2518 __all__.append('SrcFormat')
2520 ################################################################################
2522 class SrcUploader(object):
2523 def __init__(self, *args, **kwargs):
2527 return '<SrcUploader %s>' % self.uploader_id
2529 __all__.append('SrcUploader')
2531 ################################################################################
2533 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2534 ('SuiteID', 'suite_id'),
2535 ('Version', 'version'),
2536 ('Origin', 'origin'),
2538 ('Description', 'description'),
2539 ('Untouchable', 'untouchable'),
2540 ('Announce', 'announce'),
2541 ('Codename', 'codename'),
2542 ('OverrideCodename', 'overridecodename'),
2543 ('ValidTime', 'validtime'),
2544 ('Priority', 'priority'),
2545 ('NotAutomatic', 'notautomatic'),
2546 ('CopyChanges', 'copychanges'),
2547 ('OverrideSuite', 'overridesuite')]
2549 # Why the heck don't we have any UNIQUE constraints in table suite?
2550 # TODO: Add UNIQUE constraints for appropriate columns.
2551 class Suite(ORMObject):
2552 def __init__(self, suite_name = None, version = None):
2553 self.suite_name = suite_name
2554 self.version = version
2556 def properties(self):
2557 return ['suite_name', 'version', 'sources_count', 'binaries_count', \
2560 def not_null_constraints(self):
2561 return ['suite_name', 'version']
2563 def __eq__(self, val):
2564 if isinstance(val, str):
2565 return (self.suite_name == val)
2566 # This signals to use the normal comparison operator
2567 return NotImplemented
2569 def __ne__(self, val):
2570 if isinstance(val, str):
2571 return (self.suite_name != val)
2572 # This signals to use the normal comparison operator
2573 return NotImplemented
2577 for disp, field in SUITE_FIELDS:
2578 val = getattr(self, field, None)
2580 ret.append("%s: %s" % (disp, val))
2582 return "\n".join(ret)
2584 def get_architectures(self, skipsrc=False, skipall=False):
2586 Returns list of Architecture objects
2588 @type skipsrc: boolean
2589 @param skipsrc: Whether to skip returning the 'source' architecture entry
2592 @type skipall: boolean
2593 @param skipall: Whether to skip returning the 'all' architecture entry
2597 @return: list of Architecture objects for the given name (may be empty)
2600 q = object_session(self).query(Architecture).with_parent(self)
2602 q = q.filter(Architecture.arch_string != 'source')
2604 q = q.filter(Architecture.arch_string != 'all')
2605 return q.order_by(Architecture.arch_string).all()
2607 def get_sources(self, source):
2609 Returns a query object representing DBSource that is part of C{suite}.
2611 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2613 @type source: string
2614 @param source: source package name
2616 @rtype: sqlalchemy.orm.query.Query
2617 @return: a query of DBSource
2621 session = object_session(self)
2622 return session.query(DBSource).filter_by(source = source). \
2625 __all__.append('Suite')
2628 def get_suite(suite, session=None):
2630 Returns Suite object for given C{suite name}.
2633 @param suite: The name of the suite
2635 @type session: Session
2636 @param session: Optional SQLA session object (a temporary one will be
2637 generated if not supplied)
2640 @return: Suite object for the requested suite name (None if not present)
2643 q = session.query(Suite).filter_by(suite_name=suite)
2647 except NoResultFound:
2650 __all__.append('get_suite')
2652 ################################################################################
2654 # TODO: should be removed because the implementation is too trivial
2656 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2658 Returns list of Architecture objects for given C{suite} name
2661 @param suite: Suite name to search for
2663 @type skipsrc: boolean
2664 @param skipsrc: Whether to skip returning the 'source' architecture entry
2667 @type skipall: boolean
2668 @param skipall: Whether to skip returning the 'all' architecture entry
2671 @type session: Session
2672 @param session: Optional SQL session object (a temporary one will be
2673 generated if not supplied)
2676 @return: list of Architecture objects for the given name (may be empty)
2679 return get_suite(suite, session).get_architectures(skipsrc, skipall)
2681 __all__.append('get_suite_architectures')
2683 ################################################################################
2685 class SuiteSrcFormat(object):
2686 def __init__(self, *args, **kwargs):
2690 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2692 __all__.append('SuiteSrcFormat')
2695 def get_suite_src_formats(suite, session=None):
2697 Returns list of allowed SrcFormat for C{suite}.
2700 @param suite: Suite name to search for
2702 @type session: Session
2703 @param session: Optional SQL session object (a temporary one will be
2704 generated if not supplied)
2707 @return: the list of allowed source formats for I{suite}
2710 q = session.query(SrcFormat)
2711 q = q.join(SuiteSrcFormat)
2712 q = q.join(Suite).filter_by(suite_name=suite)
2713 q = q.order_by('format_name')
2717 __all__.append('get_suite_src_formats')
2719 ################################################################################
2721 class Uid(ORMObject):
2722 def __init__(self, uid = None, name = None):
2726 def __eq__(self, val):
2727 if isinstance(val, str):
2728 return (self.uid == val)
2729 # This signals to use the normal comparison operator
2730 return NotImplemented
2732 def __ne__(self, val):
2733 if isinstance(val, str):
2734 return (self.uid != val)
2735 # This signals to use the normal comparison operator
2736 return NotImplemented
2738 def properties(self):
2739 return ['uid', 'name', 'fingerprint']
2741 def not_null_constraints(self):
2744 __all__.append('Uid')
2747 def get_or_set_uid(uidname, session=None):
2749 Returns uid object for given uidname.
2751 If no matching uidname is found, a row is inserted.
2753 @type uidname: string
2754 @param uidname: The uid to add
2756 @type session: SQLAlchemy
2757 @param session: Optional SQL session object (a temporary one will be
2758 generated if not supplied). If not passed, a commit will be performed at
2759 the end of the function, otherwise the caller is responsible for commiting.
2762 @return: the uid object for the given uidname
2765 q = session.query(Uid).filter_by(uid=uidname)
2769 except NoResultFound:
2773 session.commit_or_flush()
2778 __all__.append('get_or_set_uid')
2781 def get_uid_from_fingerprint(fpr, session=None):
2782 q = session.query(Uid)
2783 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2787 except NoResultFound:
2790 __all__.append('get_uid_from_fingerprint')
2792 ################################################################################
2794 class UploadBlock(object):
2795 def __init__(self, *args, **kwargs):
2799 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2801 __all__.append('UploadBlock')
2803 ################################################################################
2805 class DBConn(object):
2807 database module init.
2811 def __init__(self, *args, **kwargs):
2812 self.__dict__ = self.__shared_state
2814 if not getattr(self, 'initialised', False):
2815 self.initialised = True
2816 self.debug = kwargs.has_key('debug')
2819 def __setuptables(self):
2820 tables_with_primary = (
2828 'build_queue_files',
2833 'changes_pending_binaries',
2834 'changes_pending_files',
2835 'changes_pending_source',
2858 tables_no_primary = (
2859 'changes_pending_files_map',
2860 'changes_pending_source_files',
2861 'changes_pool_files',
2862 'extra_src_references',
2863 # TODO: the maintainer column in table override should be removed.
2865 'suite_architectures',
2866 'suite_src_formats',
2867 'suite_build_queue_copy',
2871 'almost_obsolete_all_associations',
2872 'almost_obsolete_src_associations',
2873 'any_associations_source',
2874 'bin_assoc_by_arch',
2875 'bin_associations_binaries',
2876 'binaries_suite_arch',
2877 'binfiles_suite_component_arch',
2880 'newest_all_associations',
2881 'newest_any_associations',
2883 'newest_src_association',
2884 'obsolete_all_associations',
2885 'obsolete_any_associations',
2886 'obsolete_any_by_all_associations',
2887 'obsolete_src_associations',
2889 'src_associations_bin',
2890 'src_associations_src',
2891 'suite_arch_by_name',
2894 # Sqlalchemy version 0.5 fails to reflect the SERIAL type
2895 # correctly and that is why we have to use a workaround. It can
2896 # be removed as soon as we switch to version 0.6.
2897 for table_name in tables_with_primary:
2898 table = Table(table_name, self.db_meta, \
2899 Column('id', Integer, primary_key = True), \
2900 autoload=True, useexisting=True)
2901 setattr(self, 'tbl_%s' % table_name, table)
2903 for table_name in tables_no_primary:
2904 table = Table(table_name, self.db_meta, autoload=True)
2905 setattr(self, 'tbl_%s' % table_name, table)
2907 # bin_contents needs special attention until the SERIAL type is
2908 # correctly detected and the workaround has been removed; see comment
2910 self.tbl_bin_contents = Table('bin_contents', self.db_meta, \
2911 Column('file', Text, primary_key = True),
2912 Column('binary_id', Integer, ForeignKey('binaries.id'), \
2913 primary_key = True),
2914 autoload=True, useexisting=True)
2916 for view_name in views:
2917 view = Table(view_name, self.db_meta, autoload=True)
2918 setattr(self, 'view_%s' % view_name, view)
2920 def __setupmappers(self):
2921 mapper(Architecture, self.tbl_architecture,
2922 properties = dict(arch_id = self.tbl_architecture.c.id,
2923 suites = relation(Suite, secondary=self.tbl_suite_architectures,
2924 order_by='suite_name',
2925 backref=backref('architectures', order_by='arch_string'))),
2926 extension = validator)
2928 mapper(Archive, self.tbl_archive,
2929 properties = dict(archive_id = self.tbl_archive.c.id,
2930 archive_name = self.tbl_archive.c.name))
2932 mapper(BuildQueue, self.tbl_build_queue,
2933 properties = dict(queue_id = self.tbl_build_queue.c.id))
2935 mapper(BuildQueueFile, self.tbl_build_queue_files,
2936 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
2937 poolfile = relation(PoolFile, backref='buildqueueinstances')))
2939 mapper(DBBinary, self.tbl_binaries,
2940 properties = dict(binary_id = self.tbl_binaries.c.id,
2941 package = self.tbl_binaries.c.package,
2942 version = self.tbl_binaries.c.version,
2943 maintainer_id = self.tbl_binaries.c.maintainer,
2944 maintainer = relation(Maintainer),
2945 source_id = self.tbl_binaries.c.source,
2946 source = relation(DBSource, backref='binaries'),
2947 arch_id = self.tbl_binaries.c.architecture,
2948 architecture = relation(Architecture),
2949 poolfile_id = self.tbl_binaries.c.file,
2950 poolfile = relation(PoolFile, backref=backref('binary', uselist = False)),
2951 binarytype = self.tbl_binaries.c.type,
2952 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2953 fingerprint = relation(Fingerprint),
2954 install_date = self.tbl_binaries.c.install_date,
2955 suites = relation(Suite, secondary=self.tbl_bin_associations,
2956 backref=backref('binaries', lazy='dynamic')),
2957 extra_sources = relation(DBSource, secondary=self.tbl_extra_src_references,
2958 backref=backref('extra_binary_references', lazy='dynamic'))),
2959 extension = validator)
2961 mapper(BinaryACL, self.tbl_binary_acl,
2962 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2964 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2965 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2966 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2967 architecture = relation(Architecture)))
2969 mapper(Component, self.tbl_component,
2970 properties = dict(component_id = self.tbl_component.c.id,
2971 component_name = self.tbl_component.c.name),
2972 extension = validator)
2974 mapper(DBConfig, self.tbl_config,
2975 properties = dict(config_id = self.tbl_config.c.id))
2977 mapper(DSCFile, self.tbl_dsc_files,
2978 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2979 source_id = self.tbl_dsc_files.c.source,
2980 source = relation(DBSource),
2981 poolfile_id = self.tbl_dsc_files.c.file,
2982 poolfile = relation(PoolFile)))
2984 mapper(PoolFile, self.tbl_files,
2985 properties = dict(file_id = self.tbl_files.c.id,
2986 filesize = self.tbl_files.c.size,
2987 location_id = self.tbl_files.c.location,
2988 location = relation(Location,
2989 # using lazy='dynamic' in the back
2990 # reference because we have A LOT of
2991 # files in one location
2992 backref=backref('files', lazy='dynamic'))),
2993 extension = validator)
2995 mapper(Fingerprint, self.tbl_fingerprint,
2996 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2997 uid_id = self.tbl_fingerprint.c.uid,
2998 uid = relation(Uid),
2999 keyring_id = self.tbl_fingerprint.c.keyring,
3000 keyring = relation(Keyring),
3001 source_acl = relation(SourceACL),
3002 binary_acl = relation(BinaryACL)),
3003 extension = validator)
3005 mapper(Keyring, self.tbl_keyrings,
3006 properties = dict(keyring_name = self.tbl_keyrings.c.name,
3007 keyring_id = self.tbl_keyrings.c.id))
3009 mapper(DBChange, self.tbl_changes,
3010 properties = dict(change_id = self.tbl_changes.c.id,
3011 poolfiles = relation(PoolFile,
3012 secondary=self.tbl_changes_pool_files,
3013 backref="changeslinks"),
3014 seen = self.tbl_changes.c.seen,
3015 source = self.tbl_changes.c.source,
3016 binaries = self.tbl_changes.c.binaries,
3017 architecture = self.tbl_changes.c.architecture,
3018 distribution = self.tbl_changes.c.distribution,
3019 urgency = self.tbl_changes.c.urgency,
3020 maintainer = self.tbl_changes.c.maintainer,
3021 changedby = self.tbl_changes.c.changedby,
3022 date = self.tbl_changes.c.date,
3023 version = self.tbl_changes.c.version,
3024 files = relation(ChangePendingFile,
3025 secondary=self.tbl_changes_pending_files_map,
3026 backref="changesfile"),
3027 in_queue_id = self.tbl_changes.c.in_queue,
3028 in_queue = relation(PolicyQueue,
3029 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
3030 approved_for_id = self.tbl_changes.c.approved_for))
3032 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
3033 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
3035 mapper(ChangePendingFile, self.tbl_changes_pending_files,
3036 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id,
3037 filename = self.tbl_changes_pending_files.c.filename,
3038 size = self.tbl_changes_pending_files.c.size,
3039 md5sum = self.tbl_changes_pending_files.c.md5sum,
3040 sha1sum = self.tbl_changes_pending_files.c.sha1sum,
3041 sha256sum = self.tbl_changes_pending_files.c.sha256sum))
3043 mapper(ChangePendingSource, self.tbl_changes_pending_source,
3044 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
3045 change = relation(DBChange),
3046 maintainer = relation(Maintainer,
3047 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
3048 changedby = relation(Maintainer,
3049 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
3050 fingerprint = relation(Fingerprint),
3051 source_files = relation(ChangePendingFile,
3052 secondary=self.tbl_changes_pending_source_files,
3053 backref="pending_sources")))
3056 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
3057 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
3058 keyring = relation(Keyring, backref="keyring_acl_map"),
3059 architecture = relation(Architecture)))
3061 mapper(Location, self.tbl_location,
3062 properties = dict(location_id = self.tbl_location.c.id,
3063 component_id = self.tbl_location.c.component,
3064 component = relation(Component, backref='location'),
3065 archive_id = self.tbl_location.c.archive,
3066 archive = relation(Archive),
3067 # FIXME: the 'type' column is old cruft and
3068 # should be removed in the future.
3069 archive_type = self.tbl_location.c.type),
3070 extension = validator)
3072 mapper(Maintainer, self.tbl_maintainer,
3073 properties = dict(maintainer_id = self.tbl_maintainer.c.id,
3074 maintains_sources = relation(DBSource, backref='maintainer',
3075 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.maintainer)),
3076 changed_sources = relation(DBSource, backref='changedby',
3077 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.changedby))),
3078 extension = validator)
3080 mapper(NewComment, self.tbl_new_comments,
3081 properties = dict(comment_id = self.tbl_new_comments.c.id))
3083 mapper(Override, self.tbl_override,
3084 properties = dict(suite_id = self.tbl_override.c.suite,
3085 suite = relation(Suite, \
3086 backref=backref('overrides', lazy='dynamic')),
3087 package = self.tbl_override.c.package,
3088 component_id = self.tbl_override.c.component,
3089 component = relation(Component, \
3090 backref=backref('overrides', lazy='dynamic')),
3091 priority_id = self.tbl_override.c.priority,
3092 priority = relation(Priority, \
3093 backref=backref('overrides', lazy='dynamic')),
3094 section_id = self.tbl_override.c.section,
3095 section = relation(Section, \
3096 backref=backref('overrides', lazy='dynamic')),
3097 overridetype_id = self.tbl_override.c.type,
3098 overridetype = relation(OverrideType, \
3099 backref=backref('overrides', lazy='dynamic'))))
3101 mapper(OverrideType, self.tbl_override_type,
3102 properties = dict(overridetype = self.tbl_override_type.c.type,
3103 overridetype_id = self.tbl_override_type.c.id))
3105 mapper(PolicyQueue, self.tbl_policy_queue,
3106 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
3108 mapper(Priority, self.tbl_priority,
3109 properties = dict(priority_id = self.tbl_priority.c.id))
3111 mapper(Section, self.tbl_section,
3112 properties = dict(section_id = self.tbl_section.c.id,
3113 section=self.tbl_section.c.section))
3115 mapper(DBSource, self.tbl_source,
3116 properties = dict(source_id = self.tbl_source.c.id,
3117 version = self.tbl_source.c.version,
3118 maintainer_id = self.tbl_source.c.maintainer,
3119 poolfile_id = self.tbl_source.c.file,
3120 poolfile = relation(PoolFile, backref=backref('source', uselist = False)),
3121 fingerprint_id = self.tbl_source.c.sig_fpr,
3122 fingerprint = relation(Fingerprint),
3123 changedby_id = self.tbl_source.c.changedby,
3124 srcfiles = relation(DSCFile,
3125 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
3126 suites = relation(Suite, secondary=self.tbl_src_associations,
3127 backref=backref('sources', lazy='dynamic')),
3128 srcuploaders = relation(SrcUploader)),
3129 extension = validator)
3131 mapper(SourceACL, self.tbl_source_acl,
3132 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
3134 mapper(SrcFormat, self.tbl_src_format,
3135 properties = dict(src_format_id = self.tbl_src_format.c.id,
3136 format_name = self.tbl_src_format.c.format_name))
3138 mapper(SrcUploader, self.tbl_src_uploaders,
3139 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
3140 source_id = self.tbl_src_uploaders.c.source,
3141 source = relation(DBSource,
3142 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
3143 maintainer_id = self.tbl_src_uploaders.c.maintainer,
3144 maintainer = relation(Maintainer,
3145 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
3147 mapper(Suite, self.tbl_suite,
3148 properties = dict(suite_id = self.tbl_suite.c.id,
3149 policy_queue = relation(PolicyQueue),
3150 copy_queues = relation(BuildQueue,
3151 secondary=self.tbl_suite_build_queue_copy)),
3152 extension = validator)
3154 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
3155 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
3156 suite = relation(Suite, backref='suitesrcformats'),
3157 src_format_id = self.tbl_suite_src_formats.c.src_format,
3158 src_format = relation(SrcFormat)))
3160 mapper(Uid, self.tbl_uid,
3161 properties = dict(uid_id = self.tbl_uid.c.id,
3162 fingerprint = relation(Fingerprint)),
3163 extension = validator)
3165 mapper(UploadBlock, self.tbl_upload_blocks,
3166 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
3167 fingerprint = relation(Fingerprint, backref="uploadblocks"),
3168 uid = relation(Uid, backref="uploadblocks")))
3170 mapper(BinContents, self.tbl_bin_contents,
3172 binary = relation(DBBinary,
3173 backref=backref('contents', lazy='dynamic', cascade='all')),
3174 file = self.tbl_bin_contents.c.file))
3176 ## Connection functions
3177 def __createconn(self):
3178 from config import Config
3180 if cnf.has_key("DB::Service"):
3181 connstr = "postgresql://service=%s" % cnf["DB::Service"]
3182 elif cnf.has_key("DB::Host"):
3184 connstr = "postgresql://%s" % cnf["DB::Host"]
3185 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
3186 connstr += ":%s" % cnf["DB::Port"]
3187 connstr += "/%s" % cnf["DB::Name"]
3190 connstr = "postgresql:///%s" % cnf["DB::Name"]
3191 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
3192 connstr += "?port=%s" % cnf["DB::Port"]
3194 engine_args = { 'echo': self.debug }
3195 if cnf.has_key('DB::PoolSize'):
3196 engine_args['pool_size'] = int(cnf['DB::PoolSize'])
3197 if cnf.has_key('DB::MaxOverflow'):
3198 engine_args['max_overflow'] = int(cnf['DB::MaxOverflow'])
3199 if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \
3200 cnf['DB::Unicode'] == 'false':
3201 engine_args['use_native_unicode'] = False
3203 # Monkey patch a new dialect in in order to support service= syntax
3204 import sqlalchemy.dialects.postgresql
3205 from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
3206 class PGDialect_psycopg2_dak(PGDialect_psycopg2):
3207 def create_connect_args(self, url):
3208 if str(url).startswith('postgresql://service='):
3210 servicename = str(url)[21:]
3211 return (['service=%s' % servicename], {})
3213 return PGDialect_psycopg2.create_connect_args(self, url)
3215 sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak
3217 self.db_pg = create_engine(connstr, **engine_args)
3218 self.db_meta = MetaData()
3219 self.db_meta.bind = self.db_pg
3220 self.db_smaker = sessionmaker(bind=self.db_pg,
3224 self.__setuptables()
3225 self.__setupmappers()
3228 return self.db_smaker()
3232 Resets the DBConn object. This function must be called by subprocesses
3233 created by the multiprocessing module. See tests/dbtest_multiproc.py
3239 __all__.append('DBConn')