5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
37 from os.path import normpath
48 import simplejson as json
50 from datetime import datetime, timedelta
51 from errno import ENOENT
52 from tempfile import mkstemp, mkdtemp
53 from subprocess import Popen, PIPE
54 from tarfile import TarFile
56 from inspect import getargspec
59 from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \
61 from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
62 backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers
63 from sqlalchemy import types as sqltypes
65 # Don't remove this, we re-export the exceptions to scripts which import us
66 from sqlalchemy.exc import *
67 from sqlalchemy.orm.exc import NoResultFound
69 # Only import Config until Queue stuff is changed to store its config
71 from config import Config
72 from textutils import fix_maintainer
73 from dak_exceptions import DBUpdateError, NoSourceFieldError
75 # suppress some deprecation warnings in squeeze related to sqlalchemy
77 warnings.filterwarnings('ignore', \
78 "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
82 ################################################################################
84 # Patch in support for the debversion field type so that it works during
88 # that is for sqlalchemy 0.6
89 UserDefinedType = sqltypes.UserDefinedType
91 # this one for sqlalchemy 0.5
92 UserDefinedType = sqltypes.TypeEngine
94 class DebVersion(UserDefinedType):
95 def get_col_spec(self):
98 def bind_processor(self, dialect):
101 # ' = None' is needed for sqlalchemy 0.5:
102 def result_processor(self, dialect, coltype = None):
105 sa_major_version = sqlalchemy.__version__[0:3]
106 if sa_major_version in ["0.5", "0.6"]:
107 from sqlalchemy.databases import postgres
108 postgres.ischema_names['debversion'] = DebVersion
110 raise Exception("dak only ported to SQLA versions 0.5 and 0.6. See daklib/dbconn.py")
112 ################################################################################
114 __all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion']
116 ################################################################################
118 def session_wrapper(fn):
120 Wrapper around common ".., session=None):" handling. If the wrapped
121 function is called without passing 'session', we create a local one
122 and destroy it when the function ends.
124 Also attaches a commit_or_flush method to the session; if we created a
125 local session, this is a synonym for session.commit(), otherwise it is a
126 synonym for session.flush().
129 def wrapped(*args, **kwargs):
130 private_transaction = False
132 # Find the session object
133 session = kwargs.get('session')
136 if len(args) <= len(getargspec(fn)[0]) - 1:
137 # No session specified as last argument or in kwargs
138 private_transaction = True
139 session = kwargs['session'] = DBConn().session()
141 # Session is last argument in args
145 session = args[-1] = DBConn().session()
146 private_transaction = True
148 if private_transaction:
149 session.commit_or_flush = session.commit
151 session.commit_or_flush = session.flush
154 return fn(*args, **kwargs)
156 if private_transaction:
157 # We created a session; close it.
160 wrapped.__doc__ = fn.__doc__
161 wrapped.func_name = fn.func_name
165 __all__.append('session_wrapper')
167 ################################################################################
169 class ORMObject(object):
171 ORMObject is a base class for all ORM classes mapped by SQLalchemy. All
172 derived classes must implement the properties() method.
175 def properties(self):
177 This method should be implemented by all derived classes and returns a
178 list of the important properties. The properties 'created' and
179 'modified' will be added automatically. A suffix '_count' should be
180 added to properties that are lists or query objects. The most important
181 property name should be returned as the first element in the list
182 because it is used by repr().
188 Returns a JSON representation of the object based on the properties
189 returned from the properties() method.
192 # add created and modified
193 all_properties = self.properties() + ['created', 'modified']
194 for property in all_properties:
195 # check for list or query
196 if property[-6:] == '_count':
197 real_property = property[:-6]
198 if not hasattr(self, real_property):
200 value = getattr(self, real_property)
201 if hasattr(value, '__len__'):
204 elif hasattr(value, 'count'):
206 value = value.count()
208 raise KeyError('Do not understand property %s.' % property)
210 if not hasattr(self, property):
213 value = getattr(self, property)
217 elif isinstance(value, ORMObject):
218 # use repr() for ORMObject types
221 # we want a string for all other types because json cannot
224 data[property] = value
225 return json.dumps(data)
229 Returns the name of the class.
231 return type(self).__name__
235 Returns a short string representation of the object using the first
236 element from the properties() method.
238 primary_property = self.properties()[0]
239 value = getattr(self, primary_property)
240 return '<%s %s>' % (self.classname(), str(value))
244 Returns a human readable form of the object using the properties()
247 return '<%s %s>' % (self.classname(), self.json())
249 def not_null_constraints(self):
251 Returns a list of properties that must be not NULL. Derived classes
252 should override this method if needed.
256 validation_message = \
257 "Validation failed because property '%s' must not be empty in object\n%s"
261 This function validates the not NULL constraints as returned by
262 not_null_constraints(). It raises the DBUpdateError exception if
265 for property in self.not_null_constraints():
266 # TODO: It is a bit awkward that the mapper configuration allow
267 # directly setting the numeric _id columns. We should get rid of it
269 if hasattr(self, property + '_id') and \
270 getattr(self, property + '_id') is not None:
272 if not hasattr(self, property) or getattr(self, property) is None:
273 raise DBUpdateError(self.validation_message % \
274 (property, str(self)))
278 def get(cls, primary_key, session = None):
280 This is a support function that allows getting an object by its primary
283 Architecture.get(3[, session])
285 instead of the more verbose
287 session.query(Architecture).get(3)
289 return session.query(cls).get(primary_key)
291 def session(self, replace = False):
293 Returns the current session that is associated with the object. May
294 return None is object is in detached state.
297 return object_session(self)
299 def clone(self, session = None):
301 Clones the current object in a new session and returns the new clone. A
302 fresh session is created if the optional session parameter is not
303 provided. The function will fail if a session is provided and has
306 RATIONALE: SQLAlchemy's session is not thread safe. This method clones
307 an existing object to allow several threads to work with their own
308 instances of an ORMObject.
310 WARNING: Only persistent (committed) objects can be cloned. Changes
311 made to the original object that are not committed yet will get lost.
312 The session of the new object will always be rolled back to avoid
316 if self.session() is None:
317 raise RuntimeError( \
318 'Method clone() failed for detached object:\n%s' % self)
319 self.session().flush()
320 mapper = object_mapper(self)
321 primary_key = mapper.primary_key_from_instance(self)
322 object_class = self.__class__
324 session = DBConn().session()
325 elif len(session.new) + len(session.dirty) + len(session.deleted) > 0:
326 raise RuntimeError( \
327 'Method clone() failed due to unflushed changes in session.')
328 new_object = session.query(object_class).get(primary_key)
330 if new_object is None:
331 raise RuntimeError( \
332 'Method clone() failed for non-persistent object:\n%s' % self)
335 __all__.append('ORMObject')
337 ################################################################################
339 class Validator(MapperExtension):
341 This class calls the validate() method for each instance for the
342 'before_update' and 'before_insert' events. A global object validator is
343 used for configuring the individual mappers.
346 def before_update(self, mapper, connection, instance):
350 def before_insert(self, mapper, connection, instance):
354 validator = Validator()
356 ################################################################################
358 class Architecture(ORMObject):
359 def __init__(self, arch_string = None, description = None):
360 self.arch_string = arch_string
361 self.description = description
363 def __eq__(self, val):
364 if isinstance(val, str):
365 return (self.arch_string== val)
366 # This signals to use the normal comparison operator
367 return NotImplemented
369 def __ne__(self, val):
370 if isinstance(val, str):
371 return (self.arch_string != val)
372 # This signals to use the normal comparison operator
373 return NotImplemented
375 def properties(self):
376 return ['arch_string', 'arch_id', 'suites_count']
378 def not_null_constraints(self):
379 return ['arch_string']
381 __all__.append('Architecture')
384 def get_architecture(architecture, session=None):
386 Returns database id for given C{architecture}.
388 @type architecture: string
389 @param architecture: The name of the architecture
391 @type session: Session
392 @param session: Optional SQLA session object (a temporary one will be
393 generated if not supplied)
396 @return: Architecture object for the given arch (None if not present)
399 q = session.query(Architecture).filter_by(arch_string=architecture)
403 except NoResultFound:
406 __all__.append('get_architecture')
408 # TODO: should be removed because the implementation is too trivial
410 def get_architecture_suites(architecture, session=None):
412 Returns list of Suite objects for given C{architecture} name
414 @type architecture: str
415 @param architecture: Architecture name to search for
417 @type session: Session
418 @param session: Optional SQL session object (a temporary one will be
419 generated if not supplied)
422 @return: list of Suite objects for the given name (may be empty)
425 return get_architecture(architecture, session).suites
427 __all__.append('get_architecture_suites')
429 ################################################################################
431 class Archive(object):
432 def __init__(self, *args, **kwargs):
436 return '<Archive %s>' % self.archive_name
438 __all__.append('Archive')
441 def get_archive(archive, session=None):
443 returns database id for given C{archive}.
445 @type archive: string
446 @param archive: the name of the arhive
448 @type session: Session
449 @param session: Optional SQLA session object (a temporary one will be
450 generated if not supplied)
453 @return: Archive object for the given name (None if not present)
456 archive = archive.lower()
458 q = session.query(Archive).filter_by(archive_name=archive)
462 except NoResultFound:
465 __all__.append('get_archive')
467 ################################################################################
469 class BinContents(ORMObject):
470 def __init__(self, file = None, binary = None):
474 def properties(self):
475 return ['file', 'binary']
477 __all__.append('BinContents')
479 ################################################################################
481 class DBBinary(ORMObject):
482 def __init__(self, package = None, source = None, version = None, \
483 maintainer = None, architecture = None, poolfile = None, \
485 self.package = package
487 self.version = version
488 self.maintainer = maintainer
489 self.architecture = architecture
490 self.poolfile = poolfile
491 self.binarytype = binarytype
493 def properties(self):
494 return ['package', 'version', 'maintainer', 'source', 'architecture', \
495 'poolfile', 'binarytype', 'fingerprint', 'install_date', \
496 'suites_count', 'binary_id', 'contents_count']
498 def not_null_constraints(self):
499 return ['package', 'version', 'maintainer', 'source', 'poolfile', \
502 def get_component_name(self):
503 return self.poolfile.location.component.component_name
505 def scan_contents(self):
507 Yields the contents of the package. Only regular files are yielded and
508 the path names are normalized after converting them from either utf-8
509 or iso8859-1 encoding. It yields the string ' <EMPTY PACKAGE>' if the
510 package does not contain any regular file.
512 fullpath = self.poolfile.fullpath
513 dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE)
514 tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
515 for member in tar.getmembers():
516 if not member.isdir():
517 name = normpath(member.name)
518 # enforce proper utf-8 encoding
521 except UnicodeDecodeError:
522 name = name.decode('iso8859-1').encode('utf-8')
528 __all__.append('DBBinary')
531 def get_suites_binary_in(package, session=None):
533 Returns list of Suite objects which given C{package} name is in
536 @param package: DBBinary package name to search for
539 @return: list of Suite objects for the given package
542 return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all()
544 __all__.append('get_suites_binary_in')
547 def get_component_by_package_suite(package, suite_list, arch_list=[], session=None):
549 Returns the component name of the newest binary package in suite_list or
550 None if no package is found. The result can be optionally filtered by a list
551 of architecture names.
554 @param package: DBBinary package name to search for
556 @type suite_list: list of str
557 @param suite_list: list of suite_name items
559 @type arch_list: list of str
560 @param arch_list: optional list of arch_string items that defaults to []
562 @rtype: str or NoneType
563 @return: name of component or None
566 q = session.query(DBBinary).filter_by(package = package). \
567 join(DBBinary.suites).filter(Suite.suite_name.in_(suite_list))
568 if len(arch_list) > 0:
569 q = q.join(DBBinary.architecture). \
570 filter(Architecture.arch_string.in_(arch_list))
571 binary = q.order_by(desc(DBBinary.version)).first()
575 return binary.get_component_name()
577 __all__.append('get_component_by_package_suite')
579 ################################################################################
581 class BinaryACL(object):
582 def __init__(self, *args, **kwargs):
586 return '<BinaryACL %s>' % self.binary_acl_id
588 __all__.append('BinaryACL')
590 ################################################################################
592 class BinaryACLMap(object):
593 def __init__(self, *args, **kwargs):
597 return '<BinaryACLMap %s>' % self.binary_acl_map_id
599 __all__.append('BinaryACLMap')
601 ################################################################################
606 ArchiveDir "%(archivepath)s";
607 OverrideDir "%(overridedir)s";
608 CacheDir "%(cachedir)s";
613 Packages::Compress ". bzip2 gzip";
614 Sources::Compress ". bzip2 gzip";
619 bindirectory "incoming"
624 BinOverride "override.sid.all3";
625 BinCacheDB "packages-accepted.db";
627 FileList "%(filelist)s";
630 Packages::Extensions ".deb .udeb";
633 bindirectory "incoming/"
636 BinOverride "override.sid.all3";
637 SrcOverride "override.sid.all3.src";
638 FileList "%(filelist)s";
642 class BuildQueue(object):
643 def __init__(self, *args, **kwargs):
647 return '<BuildQueue %s>' % self.queue_name
649 def write_metadata(self, starttime, force=False):
650 # Do we write out metafiles?
651 if not (force or self.generate_metadata):
654 session = DBConn().session().object_session(self)
656 fl_fd = fl_name = ac_fd = ac_name = None
658 arches = " ".join([ a.arch_string for a in session.query(Architecture).all() if a.arch_string != 'source' ])
659 startdir = os.getcwd()
662 # Grab files we want to include
663 newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
664 # Write file list with newer files
665 (fl_fd, fl_name) = mkstemp()
667 os.write(fl_fd, '%s\n' % n.fullpath)
672 # Write minimal apt.conf
673 # TODO: Remove hardcoding from template
674 (ac_fd, ac_name) = mkstemp()
675 os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path,
677 'cachedir': cnf["Dir::Cache"],
678 'overridedir': cnf["Dir::Override"],
682 # Run apt-ftparchive generate
683 os.chdir(os.path.dirname(ac_name))
684 os.system('apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
686 # Run apt-ftparchive release
687 # TODO: Eww - fix this
688 bname = os.path.basename(self.path)
692 # We have to remove the Release file otherwise it'll be included in the
695 os.unlink(os.path.join(bname, 'Release'))
699 os.system("""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="%s" release %s > Release""" % (self.origin, self.label, self.releasedescription, arches, bname))
701 # Crude hack with open and append, but this whole section is and should be redone.
702 if self.notautomatic:
703 release=open("Release", "a")
704 release.write("NotAutomatic: yes")
709 keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
710 if cnf.has_key("Dinstall::SigningPubKeyring"):
711 keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
713 os.system("gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey))
715 # Move the files if we got this far
716 os.rename('Release', os.path.join(bname, 'Release'))
718 os.rename('Release.gpg', os.path.join(bname, 'Release.gpg'))
720 # Clean up any left behind files
747 def clean_and_update(self, starttime, Logger, dryrun=False):
748 """WARNING: This routine commits for you"""
749 session = DBConn().session().object_session(self)
751 if self.generate_metadata and not dryrun:
752 self.write_metadata(starttime)
754 # Grab files older than our execution time
755 older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
761 Logger.log(["I: Would have removed %s from the queue" % o.fullpath])
763 Logger.log(["I: Removing %s from the queue" % o.fullpath])
764 os.unlink(o.fullpath)
767 # If it wasn't there, don't worry
768 if e.errno == ENOENT:
771 # TODO: Replace with proper logging call
772 Logger.log(["E: Could not remove %s" % o.fullpath])
779 for f in os.listdir(self.path):
780 if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release') or f.startswith('advisory'):
784 r = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter_by(filename = f).one()
785 except NoResultFound:
786 fp = os.path.join(self.path, f)
788 Logger.log(["I: Would remove unused link %s" % fp])
790 Logger.log(["I: Removing unused link %s" % fp])
794 Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath])
796 def add_file_from_pool(self, poolfile):
797 """Copies a file into the pool. Assumes that the PoolFile object is
798 attached to the same SQLAlchemy session as the Queue object is.
800 The caller is responsible for committing after calling this function."""
801 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
803 # Check if we have a file of this name or this ID already
804 for f in self.queuefiles:
805 if f.fileid is not None and f.fileid == poolfile.file_id or \
806 f.poolfile.filename == poolfile_basename:
807 # In this case, update the BuildQueueFile entry so we
808 # don't remove it too early
809 f.lastused = datetime.now()
810 DBConn().session().object_session(poolfile).add(f)
813 # Prepare BuildQueueFile object
814 qf = BuildQueueFile()
815 qf.build_queue_id = self.queue_id
816 qf.lastused = datetime.now()
817 qf.filename = poolfile_basename
819 targetpath = poolfile.fullpath
820 queuepath = os.path.join(self.path, poolfile_basename)
824 # We need to copy instead of symlink
826 utils.copy(targetpath, queuepath)
827 # NULL in the fileid field implies a copy
830 os.symlink(targetpath, queuepath)
831 qf.fileid = poolfile.file_id
835 # Get the same session as the PoolFile is using and add the qf to it
836 DBConn().session().object_session(poolfile).add(qf)
841 __all__.append('BuildQueue')
844 def get_build_queue(queuename, session=None):
846 Returns BuildQueue object for given C{queue name}, creating it if it does not
849 @type queuename: string
850 @param queuename: The name of the queue
852 @type session: Session
853 @param session: Optional SQLA session object (a temporary one will be
854 generated if not supplied)
857 @return: BuildQueue object for the given queue
860 q = session.query(BuildQueue).filter_by(queue_name=queuename)
864 except NoResultFound:
867 __all__.append('get_build_queue')
869 ################################################################################
871 class BuildQueueFile(object):
872 def __init__(self, *args, **kwargs):
876 return '<BuildQueueFile %s (%s)>' % (self.filename, self.build_queue_id)
880 return os.path.join(self.buildqueue.path, self.filename)
883 __all__.append('BuildQueueFile')
885 ################################################################################
887 class ChangePendingBinary(object):
888 def __init__(self, *args, **kwargs):
892 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
894 __all__.append('ChangePendingBinary')
896 ################################################################################
898 class ChangePendingFile(object):
899 def __init__(self, *args, **kwargs):
903 return '<ChangePendingFile %s>' % self.change_pending_file_id
905 __all__.append('ChangePendingFile')
907 ################################################################################
909 class ChangePendingSource(object):
910 def __init__(self, *args, **kwargs):
914 return '<ChangePendingSource %s>' % self.change_pending_source_id
916 __all__.append('ChangePendingSource')
918 ################################################################################
920 class Component(ORMObject):
921 def __init__(self, component_name = None):
922 self.component_name = component_name
924 def __eq__(self, val):
925 if isinstance(val, str):
926 return (self.component_name == val)
927 # This signals to use the normal comparison operator
928 return NotImplemented
930 def __ne__(self, val):
931 if isinstance(val, str):
932 return (self.component_name != val)
933 # This signals to use the normal comparison operator
934 return NotImplemented
936 def properties(self):
937 return ['component_name', 'component_id', 'description', \
938 'location_count', 'meets_dfsg', 'overrides_count']
940 def not_null_constraints(self):
941 return ['component_name']
944 __all__.append('Component')
947 def get_component(component, session=None):
949 Returns database id for given C{component}.
951 @type component: string
952 @param component: The name of the override type
955 @return: the database id for the given component
958 component = component.lower()
960 q = session.query(Component).filter_by(component_name=component)
964 except NoResultFound:
967 __all__.append('get_component')
969 ################################################################################
971 class DBConfig(object):
972 def __init__(self, *args, **kwargs):
976 return '<DBConfig %s>' % self.name
978 __all__.append('DBConfig')
980 ################################################################################
983 def get_or_set_contents_file_id(filename, session=None):
985 Returns database id for given filename.
987 If no matching file is found, a row is inserted.
989 @type filename: string
990 @param filename: The filename
991 @type session: SQLAlchemy
992 @param session: Optional SQL session object (a temporary one will be
993 generated if not supplied). If not passed, a commit will be performed at
994 the end of the function, otherwise the caller is responsible for commiting.
997 @return: the database id for the given component
1000 q = session.query(ContentFilename).filter_by(filename=filename)
1003 ret = q.one().cafilename_id
1004 except NoResultFound:
1005 cf = ContentFilename()
1006 cf.filename = filename
1008 session.commit_or_flush()
1009 ret = cf.cafilename_id
1013 __all__.append('get_or_set_contents_file_id')
1016 def get_contents(suite, overridetype, section=None, session=None):
1018 Returns contents for a suite / overridetype combination, limiting
1019 to a section if not None.
1022 @param suite: Suite object
1024 @type overridetype: OverrideType
1025 @param overridetype: OverrideType object
1027 @type section: Section
1028 @param section: Optional section object to limit results to
1030 @type session: SQLAlchemy
1031 @param session: Optional SQL session object (a temporary one will be
1032 generated if not supplied)
1034 @rtype: ResultsProxy
1035 @return: ResultsProxy object set up to return tuples of (filename, section,
1039 # find me all of the contents for a given suite
1040 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
1044 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
1045 JOIN content_file_names n ON (c.filename=n.id)
1046 JOIN binaries b ON (b.id=c.binary_pkg)
1047 JOIN override o ON (o.package=b.package)
1048 JOIN section s ON (s.id=o.section)
1049 WHERE o.suite = :suiteid AND o.type = :overridetypeid
1050 AND b.type=:overridetypename"""
1052 vals = {'suiteid': suite.suite_id,
1053 'overridetypeid': overridetype.overridetype_id,
1054 'overridetypename': overridetype.overridetype}
1056 if section is not None:
1057 contents_q += " AND s.id = :sectionid"
1058 vals['sectionid'] = section.section_id
1060 contents_q += " ORDER BY fn"
1062 return session.execute(contents_q, vals)
1064 __all__.append('get_contents')
1066 ################################################################################
1068 class ContentFilepath(object):
1069 def __init__(self, *args, **kwargs):
1073 return '<ContentFilepath %s>' % self.filepath
1075 __all__.append('ContentFilepath')
1078 def get_or_set_contents_path_id(filepath, session=None):
1080 Returns database id for given path.
1082 If no matching file is found, a row is inserted.
1084 @type filepath: string
1085 @param filepath: The filepath
1087 @type session: SQLAlchemy
1088 @param session: Optional SQL session object (a temporary one will be
1089 generated if not supplied). If not passed, a commit will be performed at
1090 the end of the function, otherwise the caller is responsible for commiting.
1093 @return: the database id for the given path
1096 q = session.query(ContentFilepath).filter_by(filepath=filepath)
1099 ret = q.one().cafilepath_id
1100 except NoResultFound:
1101 cf = ContentFilepath()
1102 cf.filepath = filepath
1104 session.commit_or_flush()
1105 ret = cf.cafilepath_id
1109 __all__.append('get_or_set_contents_path_id')
1111 ################################################################################
1113 class ContentAssociation(object):
1114 def __init__(self, *args, **kwargs):
1118 return '<ContentAssociation %s>' % self.ca_id
1120 __all__.append('ContentAssociation')
1122 def insert_content_paths(binary_id, fullpaths, session=None):
1124 Make sure given path is associated with given binary id
1126 @type binary_id: int
1127 @param binary_id: the id of the binary
1128 @type fullpaths: list
1129 @param fullpaths: the list of paths of the file being associated with the binary
1130 @type session: SQLAlchemy session
1131 @param session: Optional SQLAlchemy session. If this is passed, the caller
1132 is responsible for ensuring a transaction has begun and committing the
1133 results or rolling back based on the result code. If not passed, a commit
1134 will be performed at the end of the function, otherwise the caller is
1135 responsible for commiting.
1137 @return: True upon success
1140 privatetrans = False
1142 session = DBConn().session()
1147 def generate_path_dicts():
1148 for fullpath in fullpaths:
1149 if fullpath.startswith( './' ):
1150 fullpath = fullpath[2:]
1152 yield {'filename':fullpath, 'id': binary_id }
1154 for d in generate_path_dicts():
1155 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
1164 traceback.print_exc()
1166 # Only rollback if we set up the session ourself
1173 __all__.append('insert_content_paths')
1175 ################################################################################
1177 class DSCFile(object):
1178 def __init__(self, *args, **kwargs):
1182 return '<DSCFile %s>' % self.dscfile_id
1184 __all__.append('DSCFile')
1187 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
1189 Returns a list of DSCFiles which may be empty
1191 @type dscfile_id: int (optional)
1192 @param dscfile_id: the dscfile_id of the DSCFiles to find
1194 @type source_id: int (optional)
1195 @param source_id: the source id related to the DSCFiles to find
1197 @type poolfile_id: int (optional)
1198 @param poolfile_id: the poolfile id related to the DSCFiles to find
1201 @return: Possibly empty list of DSCFiles
1204 q = session.query(DSCFile)
1206 if dscfile_id is not None:
1207 q = q.filter_by(dscfile_id=dscfile_id)
1209 if source_id is not None:
1210 q = q.filter_by(source_id=source_id)
1212 if poolfile_id is not None:
1213 q = q.filter_by(poolfile_id=poolfile_id)
1217 __all__.append('get_dscfiles')
1219 ################################################################################
1221 class PoolFile(ORMObject):
1222 def __init__(self, filename = None, location = None, filesize = -1, \
1224 self.filename = filename
1225 self.location = location
1226 self.filesize = filesize
1227 self.md5sum = md5sum
1231 return os.path.join(self.location.path, self.filename)
1233 def is_valid(self, filesize = -1, md5sum = None):
1234 return self.filesize == long(filesize) and self.md5sum == md5sum
1236 def properties(self):
1237 return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \
1238 'sha256sum', 'location', 'source', 'binary', 'last_used']
1240 def not_null_constraints(self):
1241 return ['filename', 'md5sum', 'location']
1243 __all__.append('PoolFile')
1246 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
1249 (ValidFileFound [boolean], PoolFile object or None)
1251 @type filename: string
1252 @param filename: the filename of the file to check against the DB
1255 @param filesize: the size of the file to check against the DB
1257 @type md5sum: string
1258 @param md5sum: the md5sum of the file to check against the DB
1260 @type location_id: int
1261 @param location_id: the id of the location to look in
1264 @return: Tuple of length 2.
1265 - If valid pool file found: (C{True}, C{PoolFile object})
1266 - If valid pool file not found:
1267 - (C{False}, C{None}) if no file found
1268 - (C{False}, C{PoolFile object}) if file found with size/md5sum mismatch
1271 poolfile = session.query(Location).get(location_id). \
1272 files.filter_by(filename=filename).first()
1274 if poolfile and poolfile.is_valid(filesize = filesize, md5sum = md5sum):
1277 return (valid, poolfile)
1279 __all__.append('check_poolfile')
1281 # TODO: the implementation can trivially be inlined at the place where the
1282 # function is called
1284 def get_poolfile_by_id(file_id, session=None):
1286 Returns a PoolFile objects or None for the given id
1289 @param file_id: the id of the file to look for
1291 @rtype: PoolFile or None
1292 @return: either the PoolFile object or None
1295 return session.query(PoolFile).get(file_id)
1297 __all__.append('get_poolfile_by_id')
1300 def get_poolfile_like_name(filename, session=None):
1302 Returns an array of PoolFile objects which are like the given name
1304 @type filename: string
1305 @param filename: the filename of the file to check against the DB
1308 @return: array of PoolFile objects
1311 # TODO: There must be a way of properly using bind parameters with %FOO%
1312 q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
1316 __all__.append('get_poolfile_like_name')
1319 def add_poolfile(filename, datadict, location_id, session=None):
1321 Add a new file to the pool
1323 @type filename: string
1324 @param filename: filename
1326 @type datadict: dict
1327 @param datadict: dict with needed data
1329 @type location_id: int
1330 @param location_id: database id of the location
1333 @return: the PoolFile object created
1335 poolfile = PoolFile()
1336 poolfile.filename = filename
1337 poolfile.filesize = datadict["size"]
1338 poolfile.md5sum = datadict["md5sum"]
1339 poolfile.sha1sum = datadict["sha1sum"]
1340 poolfile.sha256sum = datadict["sha256sum"]
1341 poolfile.location_id = location_id
1343 session.add(poolfile)
1344 # Flush to get a file id (NB: This is not a commit)
1349 __all__.append('add_poolfile')
1351 ################################################################################
1353 class Fingerprint(ORMObject):
1354 def __init__(self, fingerprint = None):
1355 self.fingerprint = fingerprint
1357 def properties(self):
1358 return ['fingerprint', 'fingerprint_id', 'keyring', 'uid', \
1361 def not_null_constraints(self):
1362 return ['fingerprint']
1364 __all__.append('Fingerprint')
1367 def get_fingerprint(fpr, session=None):
1369 Returns Fingerprint object for given fpr.
1372 @param fpr: The fpr to find / add
1374 @type session: SQLAlchemy
1375 @param session: Optional SQL session object (a temporary one will be
1376 generated if not supplied).
1379 @return: the Fingerprint object for the given fpr or None
1382 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1386 except NoResultFound:
1391 __all__.append('get_fingerprint')
1394 def get_or_set_fingerprint(fpr, session=None):
1396 Returns Fingerprint object for given fpr.
1398 If no matching fpr is found, a row is inserted.
1401 @param fpr: The fpr to find / add
1403 @type session: SQLAlchemy
1404 @param session: Optional SQL session object (a temporary one will be
1405 generated if not supplied). If not passed, a commit will be performed at
1406 the end of the function, otherwise the caller is responsible for commiting.
1407 A flush will be performed either way.
1410 @return: the Fingerprint object for the given fpr
1413 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1417 except NoResultFound:
1418 fingerprint = Fingerprint()
1419 fingerprint.fingerprint = fpr
1420 session.add(fingerprint)
1421 session.commit_or_flush()
1426 __all__.append('get_or_set_fingerprint')
1428 ################################################################################
1430 # Helper routine for Keyring class
1431 def get_ldap_name(entry):
1433 for k in ["cn", "mn", "sn"]:
1435 if ret and ret[0] != "" and ret[0] != "-":
1437 return " ".join(name)
1439 ################################################################################
1441 class Keyring(object):
1442 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1443 " --with-colons --fingerprint --fingerprint"
1448 def __init__(self, *args, **kwargs):
1452 return '<Keyring %s>' % self.keyring_name
1454 def de_escape_gpg_str(self, txt):
1455 esclist = re.split(r'(\\x..)', txt)
1456 for x in range(1,len(esclist),2):
1457 esclist[x] = "%c" % (int(esclist[x][2:],16))
1458 return "".join(esclist)
1460 def parse_address(self, uid):
1461 """parses uid and returns a tuple of real name and email address"""
1463 (name, address) = email.Utils.parseaddr(uid)
1464 name = re.sub(r"\s*[(].*[)]", "", name)
1465 name = self.de_escape_gpg_str(name)
1468 return (name, address)
1470 def load_keys(self, keyring):
1471 if not self.keyring_id:
1472 raise Exception('Must be initialized with database information')
1474 k = os.popen(self.gpg_invocation % keyring, "r")
1478 for line in k.xreadlines():
1479 field = line.split(":")
1480 if field[0] == "pub":
1483 (name, addr) = self.parse_address(field[9])
1485 self.keys[key]["email"] = addr
1486 self.keys[key]["name"] = name
1487 self.keys[key]["fingerprints"] = []
1489 elif key and field[0] == "sub" and len(field) >= 12:
1490 signingkey = ("s" in field[11])
1491 elif key and field[0] == "uid":
1492 (name, addr) = self.parse_address(field[9])
1493 if "email" not in self.keys[key] and "@" in addr:
1494 self.keys[key]["email"] = addr
1495 self.keys[key]["name"] = name
1496 elif signingkey and field[0] == "fpr":
1497 self.keys[key]["fingerprints"].append(field[9])
1498 self.fpr_lookup[field[9]] = key
1500 def import_users_from_ldap(self, session):
1504 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1505 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1507 l = ldap.open(LDAPServer)
1508 l.simple_bind_s("","")
1509 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1510 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1511 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1513 ldap_fin_uid_id = {}
1520 uid = entry["uid"][0]
1521 name = get_ldap_name(entry)
1522 fingerprints = entry["keyFingerPrint"]
1524 for f in fingerprints:
1525 key = self.fpr_lookup.get(f, None)
1526 if key not in self.keys:
1528 self.keys[key]["uid"] = uid
1532 keyid = get_or_set_uid(uid, session).uid_id
1533 byuid[keyid] = (uid, name)
1534 byname[uid] = (keyid, name)
1536 return (byname, byuid)
1538 def generate_users_from_keyring(self, format, session):
1542 for x in self.keys.keys():
1543 if "email" not in self.keys[x]:
1545 self.keys[x]["uid"] = format % "invalid-uid"
1547 uid = format % self.keys[x]["email"]
1548 keyid = get_or_set_uid(uid, session).uid_id
1549 byuid[keyid] = (uid, self.keys[x]["name"])
1550 byname[uid] = (keyid, self.keys[x]["name"])
1551 self.keys[x]["uid"] = uid
1554 uid = format % "invalid-uid"
1555 keyid = get_or_set_uid(uid, session).uid_id
1556 byuid[keyid] = (uid, "ungeneratable user id")
1557 byname[uid] = (keyid, "ungeneratable user id")
1559 return (byname, byuid)
1561 __all__.append('Keyring')
1564 def get_keyring(keyring, session=None):
1566 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1567 If C{keyring} already has an entry, simply return the existing Keyring
1569 @type keyring: string
1570 @param keyring: the keyring name
1573 @return: the Keyring object for this keyring
1576 q = session.query(Keyring).filter_by(keyring_name=keyring)
1580 except NoResultFound:
1583 __all__.append('get_keyring')
1585 ################################################################################
1587 class KeyringACLMap(object):
1588 def __init__(self, *args, **kwargs):
1592 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1594 __all__.append('KeyringACLMap')
1596 ################################################################################
1598 class DBChange(object):
1599 def __init__(self, *args, **kwargs):
1603 return '<DBChange %s>' % self.changesname
1605 def clean_from_queue(self):
1606 session = DBConn().session().object_session(self)
1608 # Remove changes_pool_files entries
1611 # Remove changes_pending_files references
1614 # Clear out of queue
1615 self.in_queue = None
1616 self.approved_for_id = None
1618 __all__.append('DBChange')
1621 def get_dbchange(filename, session=None):
1623 returns DBChange object for given C{filename}.
1625 @type filename: string
1626 @param filename: the name of the file
1628 @type session: Session
1629 @param session: Optional SQLA session object (a temporary one will be
1630 generated if not supplied)
1633 @return: DBChange object for the given filename (C{None} if not present)
1636 q = session.query(DBChange).filter_by(changesname=filename)
1640 except NoResultFound:
1643 __all__.append('get_dbchange')
1645 ################################################################################
1647 class Location(ORMObject):
1648 def __init__(self, path = None, component = None):
1650 self.component = component
1651 # the column 'type' should go away, see comment at mapper
1652 self.archive_type = 'pool'
1654 def properties(self):
1655 return ['path', 'location_id', 'archive_type', 'component', \
1658 def not_null_constraints(self):
1659 return ['path', 'archive_type']
1661 __all__.append('Location')
1664 def get_location(location, component=None, archive=None, session=None):
1666 Returns Location object for the given combination of location, component
1669 @type location: string
1670 @param location: the path of the location, e.g. I{/srv/ftp-master.debian.org/ftp/pool/}
1672 @type component: string
1673 @param component: the component name (if None, no restriction applied)
1675 @type archive: string
1676 @param archive: the archive name (if None, no restriction applied)
1678 @rtype: Location / None
1679 @return: Either a Location object or None if one can't be found
1682 q = session.query(Location).filter_by(path=location)
1684 if archive is not None:
1685 q = q.join(Archive).filter_by(archive_name=archive)
1687 if component is not None:
1688 q = q.join(Component).filter_by(component_name=component)
1692 except NoResultFound:
1695 __all__.append('get_location')
1697 ################################################################################
1699 class Maintainer(ORMObject):
1700 def __init__(self, name = None):
1703 def properties(self):
1704 return ['name', 'maintainer_id']
1706 def not_null_constraints(self):
1709 def get_split_maintainer(self):
1710 if not hasattr(self, 'name') or self.name is None:
1711 return ('', '', '', '')
1713 return fix_maintainer(self.name.strip())
1715 __all__.append('Maintainer')
1718 def get_or_set_maintainer(name, session=None):
1720 Returns Maintainer object for given maintainer name.
1722 If no matching maintainer name is found, a row is inserted.
1725 @param name: The maintainer name to add
1727 @type session: SQLAlchemy
1728 @param session: Optional SQL session object (a temporary one will be
1729 generated if not supplied). If not passed, a commit will be performed at
1730 the end of the function, otherwise the caller is responsible for commiting.
1731 A flush will be performed either way.
1734 @return: the Maintainer object for the given maintainer
1737 q = session.query(Maintainer).filter_by(name=name)
1740 except NoResultFound:
1741 maintainer = Maintainer()
1742 maintainer.name = name
1743 session.add(maintainer)
1744 session.commit_or_flush()
1749 __all__.append('get_or_set_maintainer')
1752 def get_maintainer(maintainer_id, session=None):
1754 Return the name of the maintainer behind C{maintainer_id} or None if that
1755 maintainer_id is invalid.
1757 @type maintainer_id: int
1758 @param maintainer_id: the id of the maintainer
1761 @return: the Maintainer with this C{maintainer_id}
1764 return session.query(Maintainer).get(maintainer_id)
1766 __all__.append('get_maintainer')
1768 ################################################################################
1770 class NewComment(object):
1771 def __init__(self, *args, **kwargs):
1775 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1777 __all__.append('NewComment')
1780 def has_new_comment(package, version, session=None):
1782 Returns true if the given combination of C{package}, C{version} has a comment.
1784 @type package: string
1785 @param package: name of the package
1787 @type version: string
1788 @param version: package version
1790 @type session: Session
1791 @param session: Optional SQLA session object (a temporary one will be
1792 generated if not supplied)
1798 q = session.query(NewComment)
1799 q = q.filter_by(package=package)
1800 q = q.filter_by(version=version)
1802 return bool(q.count() > 0)
1804 __all__.append('has_new_comment')
1807 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1809 Returns (possibly empty) list of NewComment objects for the given
1812 @type package: string (optional)
1813 @param package: name of the package
1815 @type version: string (optional)
1816 @param version: package version
1818 @type comment_id: int (optional)
1819 @param comment_id: An id of a comment
1821 @type session: Session
1822 @param session: Optional SQLA session object (a temporary one will be
1823 generated if not supplied)
1826 @return: A (possibly empty) list of NewComment objects will be returned
1829 q = session.query(NewComment)
1830 if package is not None: q = q.filter_by(package=package)
1831 if version is not None: q = q.filter_by(version=version)
1832 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1836 __all__.append('get_new_comments')
1838 ################################################################################
1840 class Override(ORMObject):
1841 def __init__(self, package = None, suite = None, component = None, overridetype = None, \
1842 section = None, priority = None):
1843 self.package = package
1845 self.component = component
1846 self.overridetype = overridetype
1847 self.section = section
1848 self.priority = priority
1850 def properties(self):
1851 return ['package', 'suite', 'component', 'overridetype', 'section', \
1854 def not_null_constraints(self):
1855 return ['package', 'suite', 'component', 'overridetype', 'section']
1857 __all__.append('Override')
1860 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1862 Returns Override object for the given parameters
1864 @type package: string
1865 @param package: The name of the package
1867 @type suite: string, list or None
1868 @param suite: The name of the suite (or suites if a list) to limit to. If
1869 None, don't limit. Defaults to None.
1871 @type component: string, list or None
1872 @param component: The name of the component (or components if a list) to
1873 limit to. If None, don't limit. Defaults to None.
1875 @type overridetype: string, list or None
1876 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1877 limit to. If None, don't limit. Defaults to None.
1879 @type session: Session
1880 @param session: Optional SQLA session object (a temporary one will be
1881 generated if not supplied)
1884 @return: A (possibly empty) list of Override objects will be returned
1887 q = session.query(Override)
1888 q = q.filter_by(package=package)
1890 if suite is not None:
1891 if not isinstance(suite, list): suite = [suite]
1892 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1894 if component is not None:
1895 if not isinstance(component, list): component = [component]
1896 q = q.join(Component).filter(Component.component_name.in_(component))
1898 if overridetype is not None:
1899 if not isinstance(overridetype, list): overridetype = [overridetype]
1900 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1904 __all__.append('get_override')
1907 ################################################################################
1909 class OverrideType(ORMObject):
1910 def __init__(self, overridetype = None):
1911 self.overridetype = overridetype
1913 def properties(self):
1914 return ['overridetype', 'overridetype_id', 'overrides_count']
1916 def not_null_constraints(self):
1917 return ['overridetype']
1919 __all__.append('OverrideType')
1922 def get_override_type(override_type, session=None):
1924 Returns OverrideType object for given C{override type}.
1926 @type override_type: string
1927 @param override_type: The name of the override type
1929 @type session: Session
1930 @param session: Optional SQLA session object (a temporary one will be
1931 generated if not supplied)
1934 @return: the database id for the given override type
1937 q = session.query(OverrideType).filter_by(overridetype=override_type)
1941 except NoResultFound:
1944 __all__.append('get_override_type')
1946 ################################################################################
1948 class PolicyQueue(object):
1949 def __init__(self, *args, **kwargs):
1953 return '<PolicyQueue %s>' % self.queue_name
1955 __all__.append('PolicyQueue')
1958 def get_policy_queue(queuename, session=None):
1960 Returns PolicyQueue object for given C{queue name}
1962 @type queuename: string
1963 @param queuename: The name of the queue
1965 @type session: Session
1966 @param session: Optional SQLA session object (a temporary one will be
1967 generated if not supplied)
1970 @return: PolicyQueue object for the given queue
1973 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
1977 except NoResultFound:
1980 __all__.append('get_policy_queue')
1983 def get_policy_queue_from_path(pathname, session=None):
1985 Returns PolicyQueue object for given C{path name}
1987 @type queuename: string
1988 @param queuename: The path
1990 @type session: Session
1991 @param session: Optional SQLA session object (a temporary one will be
1992 generated if not supplied)
1995 @return: PolicyQueue object for the given queue
1998 q = session.query(PolicyQueue).filter_by(path=pathname)
2002 except NoResultFound:
2005 __all__.append('get_policy_queue_from_path')
2007 ################################################################################
2009 class Priority(ORMObject):
2010 def __init__(self, priority = None, level = None):
2011 self.priority = priority
2014 def properties(self):
2015 return ['priority', 'priority_id', 'level', 'overrides_count']
2017 def not_null_constraints(self):
2018 return ['priority', 'level']
2020 def __eq__(self, val):
2021 if isinstance(val, str):
2022 return (self.priority == val)
2023 # This signals to use the normal comparison operator
2024 return NotImplemented
2026 def __ne__(self, val):
2027 if isinstance(val, str):
2028 return (self.priority != val)
2029 # This signals to use the normal comparison operator
2030 return NotImplemented
2032 __all__.append('Priority')
2035 def get_priority(priority, session=None):
2037 Returns Priority object for given C{priority name}.
2039 @type priority: string
2040 @param priority: The name of the priority
2042 @type session: Session
2043 @param session: Optional SQLA session object (a temporary one will be
2044 generated if not supplied)
2047 @return: Priority object for the given priority
2050 q = session.query(Priority).filter_by(priority=priority)
2054 except NoResultFound:
2057 __all__.append('get_priority')
2060 def get_priorities(session=None):
2062 Returns dictionary of priority names -> id mappings
2064 @type session: Session
2065 @param session: Optional SQL session object (a temporary one will be
2066 generated if not supplied)
2069 @return: dictionary of priority names -> id mappings
2073 q = session.query(Priority)
2075 ret[x.priority] = x.priority_id
2079 __all__.append('get_priorities')
2081 ################################################################################
2083 class Section(ORMObject):
2084 def __init__(self, section = None):
2085 self.section = section
2087 def properties(self):
2088 return ['section', 'section_id', 'overrides_count']
2090 def not_null_constraints(self):
2093 def __eq__(self, val):
2094 if isinstance(val, str):
2095 return (self.section == val)
2096 # This signals to use the normal comparison operator
2097 return NotImplemented
2099 def __ne__(self, val):
2100 if isinstance(val, str):
2101 return (self.section != val)
2102 # This signals to use the normal comparison operator
2103 return NotImplemented
2105 __all__.append('Section')
2108 def get_section(section, session=None):
2110 Returns Section object for given C{section name}.
2112 @type section: string
2113 @param section: The name of the section
2115 @type session: Session
2116 @param session: Optional SQLA session object (a temporary one will be
2117 generated if not supplied)
2120 @return: Section object for the given section name
2123 q = session.query(Section).filter_by(section=section)
2127 except NoResultFound:
2130 __all__.append('get_section')
2133 def get_sections(session=None):
2135 Returns dictionary of section names -> id mappings
2137 @type session: Session
2138 @param session: Optional SQL session object (a temporary one will be
2139 generated if not supplied)
2142 @return: dictionary of section names -> id mappings
2146 q = session.query(Section)
2148 ret[x.section] = x.section_id
2152 __all__.append('get_sections')
2154 ################################################################################
2156 class DBSource(ORMObject):
2157 def __init__(self, source = None, version = None, maintainer = None, \
2158 changedby = None, poolfile = None, install_date = None):
2159 self.source = source
2160 self.version = version
2161 self.maintainer = maintainer
2162 self.changedby = changedby
2163 self.poolfile = poolfile
2164 self.install_date = install_date
2166 def properties(self):
2167 return ['source', 'source_id', 'maintainer', 'changedby', \
2168 'fingerprint', 'poolfile', 'version', 'suites_count', \
2169 'install_date', 'binaries_count']
2171 def not_null_constraints(self):
2172 return ['source', 'version', 'install_date', 'maintainer', \
2173 'changedby', 'poolfile', 'install_date']
2175 __all__.append('DBSource')
2178 def source_exists(source, source_version, suites = ["any"], session=None):
2180 Ensure that source exists somewhere in the archive for the binary
2181 upload being processed.
2182 1. exact match => 1.0-3
2183 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
2185 @type source: string
2186 @param source: source name
2188 @type source_version: string
2189 @param source_version: expected source version
2192 @param suites: list of suites to check in, default I{any}
2194 @type session: Session
2195 @param session: Optional SQLA session object (a temporary one will be
2196 generated if not supplied)
2199 @return: returns 1 if a source with expected version is found, otherwise 0
2206 from daklib.regexes import re_bin_only_nmu
2207 orig_source_version = re_bin_only_nmu.sub('', source_version)
2209 for suite in suites:
2210 q = session.query(DBSource).filter_by(source=source). \
2211 filter(DBSource.version.in_([source_version, orig_source_version]))
2213 # source must exist in suite X, or in some other suite that's
2214 # mapped to X, recursively... silent-maps are counted too,
2215 # unreleased-maps aren't.
2216 maps = cnf.ValueList("SuiteMappings")[:]
2218 maps = [ m.split() for m in maps ]
2219 maps = [ (x[1], x[2]) for x in maps
2220 if x[0] == "map" or x[0] == "silent-map" ]
2222 for (from_, to) in maps:
2223 if from_ in s and to not in s:
2226 q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
2231 # No source found so return not ok
2236 __all__.append('source_exists')
2239 def get_suites_source_in(source, session=None):
2241 Returns list of Suite objects which given C{source} name is in
2244 @param source: DBSource package name to search for
2247 @return: list of Suite objects for the given source
2250 return session.query(Suite).filter(Suite.sources.any(source=source)).all()
2252 __all__.append('get_suites_source_in')
2255 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
2257 Returns list of DBSource objects for given C{source} name and other parameters
2260 @param source: DBSource package name to search for
2262 @type version: str or None
2263 @param version: DBSource version name to search for or None if not applicable
2265 @type dm_upload_allowed: bool
2266 @param dm_upload_allowed: If None, no effect. If True or False, only
2267 return packages with that dm_upload_allowed setting
2269 @type session: Session
2270 @param session: Optional SQL session object (a temporary one will be
2271 generated if not supplied)
2274 @return: list of DBSource objects for the given name (may be empty)
2277 q = session.query(DBSource).filter_by(source=source)
2279 if version is not None:
2280 q = q.filter_by(version=version)
2282 if dm_upload_allowed is not None:
2283 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
2287 __all__.append('get_sources_from_name')
2289 # FIXME: This function fails badly if it finds more than 1 source package and
2290 # its implementation is trivial enough to be inlined.
2292 def get_source_in_suite(source, suite, session=None):
2294 Returns a DBSource object for a combination of C{source} and C{suite}.
2296 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2297 - B{suite} - a suite name, eg. I{unstable}
2299 @type source: string
2300 @param source: source package name
2303 @param suite: the suite name
2306 @return: the version for I{source} in I{suite}
2310 q = get_suite(suite, session).get_sources(source)
2313 except NoResultFound:
2316 __all__.append('get_source_in_suite')
2318 ################################################################################
2321 def add_dsc_to_db(u, filename, session=None):
2322 entry = u.pkg.files[filename]
2326 source.source = u.pkg.dsc["source"]
2327 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2328 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2329 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2330 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2331 source.install_date = datetime.now().date()
2333 dsc_component = entry["component"]
2334 dsc_location_id = entry["location id"]
2336 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2338 # Set up a new poolfile if necessary
2339 if not entry.has_key("files id") or not entry["files id"]:
2340 filename = entry["pool name"] + filename
2341 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2343 pfs.append(poolfile)
2344 entry["files id"] = poolfile.file_id
2346 source.poolfile_id = entry["files id"]
2349 suite_names = u.pkg.changes["distribution"].keys()
2350 source.suites = session.query(Suite). \
2351 filter(Suite.suite_name.in_(suite_names)).all()
2353 # Add the source files to the DB (files and dsc_files)
2355 dscfile.source_id = source.source_id
2356 dscfile.poolfile_id = entry["files id"]
2357 session.add(dscfile)
2359 for dsc_file, dentry in u.pkg.dsc_files.items():
2361 df.source_id = source.source_id
2363 # If the .orig tarball is already in the pool, it's
2364 # files id is stored in dsc_files by check_dsc().
2365 files_id = dentry.get("files id", None)
2367 # Find the entry in the files hash
2368 # TODO: Bail out here properly
2370 for f, e in u.pkg.files.items():
2375 if files_id is None:
2376 filename = dfentry["pool name"] + dsc_file
2378 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2379 # FIXME: needs to check for -1/-2 and or handle exception
2380 if found and obj is not None:
2381 files_id = obj.file_id
2384 # If still not found, add it
2385 if files_id is None:
2386 # HACK: Force sha1sum etc into dentry
2387 dentry["sha1sum"] = dfentry["sha1sum"]
2388 dentry["sha256sum"] = dfentry["sha256sum"]
2389 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2390 pfs.append(poolfile)
2391 files_id = poolfile.file_id
2393 poolfile = get_poolfile_by_id(files_id, session)
2394 if poolfile is None:
2395 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2396 pfs.append(poolfile)
2398 df.poolfile_id = files_id
2401 # Add the src_uploaders to the DB
2402 uploader_ids = [source.maintainer_id]
2403 if u.pkg.dsc.has_key("uploaders"):
2404 for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
2406 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2409 for up_id in uploader_ids:
2410 if added_ids.has_key(up_id):
2412 utils.warn("Already saw uploader %s for source %s" % (up_id, source.source))
2418 su.maintainer_id = up_id
2419 su.source_id = source.source_id
2424 return source, dsc_component, dsc_location_id, pfs
2426 __all__.append('add_dsc_to_db')
2429 def add_deb_to_db(u, filename, session=None):
2431 Contrary to what you might expect, this routine deals with both
2432 debs and udebs. That info is in 'dbtype', whilst 'type' is
2433 'deb' for both of them
2436 entry = u.pkg.files[filename]
2439 bin.package = entry["package"]
2440 bin.version = entry["version"]
2441 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2442 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2443 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2444 bin.binarytype = entry["dbtype"]
2447 filename = entry["pool name"] + filename
2448 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2449 if not entry.get("location id", None):
2450 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2452 if entry.get("files id", None):
2453 poolfile = get_poolfile_by_id(bin.poolfile_id)
2454 bin.poolfile_id = entry["files id"]
2456 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2457 bin.poolfile_id = entry["files id"] = poolfile.file_id
2460 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2461 if len(bin_sources) != 1:
2462 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2463 (bin.package, bin.version, entry["architecture"],
2464 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2466 bin.source_id = bin_sources[0].source_id
2468 # Add and flush object so it has an ID
2471 suite_names = u.pkg.changes["distribution"].keys()
2472 bin.suites = session.query(Suite). \
2473 filter(Suite.suite_name.in_(suite_names)).all()
2477 # Deal with contents - disabled for now
2478 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2480 # print "REJECT\nCould not determine contents of package %s" % bin.package
2481 # session.rollback()
2482 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2486 __all__.append('add_deb_to_db')
2488 ################################################################################
2490 class SourceACL(object):
2491 def __init__(self, *args, **kwargs):
2495 return '<SourceACL %s>' % self.source_acl_id
2497 __all__.append('SourceACL')
2499 ################################################################################
2501 class SrcFormat(object):
2502 def __init__(self, *args, **kwargs):
2506 return '<SrcFormat %s>' % (self.format_name)
2508 __all__.append('SrcFormat')
2510 ################################################################################
2512 class SrcUploader(object):
2513 def __init__(self, *args, **kwargs):
2517 return '<SrcUploader %s>' % self.uploader_id
2519 __all__.append('SrcUploader')
2521 ################################################################################
2523 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2524 ('SuiteID', 'suite_id'),
2525 ('Version', 'version'),
2526 ('Origin', 'origin'),
2528 ('Description', 'description'),
2529 ('Untouchable', 'untouchable'),
2530 ('Announce', 'announce'),
2531 ('Codename', 'codename'),
2532 ('OverrideCodename', 'overridecodename'),
2533 ('ValidTime', 'validtime'),
2534 ('Priority', 'priority'),
2535 ('NotAutomatic', 'notautomatic'),
2536 ('CopyChanges', 'copychanges'),
2537 ('OverrideSuite', 'overridesuite')]
2539 # Why the heck don't we have any UNIQUE constraints in table suite?
2540 # TODO: Add UNIQUE constraints for appropriate columns.
2541 class Suite(ORMObject):
2542 def __init__(self, suite_name = None, version = None):
2543 self.suite_name = suite_name
2544 self.version = version
2546 def properties(self):
2547 return ['suite_name', 'version', 'sources_count', 'binaries_count', \
2550 def not_null_constraints(self):
2551 return ['suite_name', 'version']
2553 def __eq__(self, val):
2554 if isinstance(val, str):
2555 return (self.suite_name == val)
2556 # This signals to use the normal comparison operator
2557 return NotImplemented
2559 def __ne__(self, val):
2560 if isinstance(val, str):
2561 return (self.suite_name != val)
2562 # This signals to use the normal comparison operator
2563 return NotImplemented
2567 for disp, field in SUITE_FIELDS:
2568 val = getattr(self, field, None)
2570 ret.append("%s: %s" % (disp, val))
2572 return "\n".join(ret)
2574 def get_architectures(self, skipsrc=False, skipall=False):
2576 Returns list of Architecture objects
2578 @type skipsrc: boolean
2579 @param skipsrc: Whether to skip returning the 'source' architecture entry
2582 @type skipall: boolean
2583 @param skipall: Whether to skip returning the 'all' architecture entry
2587 @return: list of Architecture objects for the given name (may be empty)
2590 q = object_session(self).query(Architecture).with_parent(self)
2592 q = q.filter(Architecture.arch_string != 'source')
2594 q = q.filter(Architecture.arch_string != 'all')
2595 return q.order_by(Architecture.arch_string).all()
2597 def get_sources(self, source):
2599 Returns a query object representing DBSource that is part of C{suite}.
2601 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2603 @type source: string
2604 @param source: source package name
2606 @rtype: sqlalchemy.orm.query.Query
2607 @return: a query of DBSource
2611 session = object_session(self)
2612 return session.query(DBSource).filter_by(source = source). \
2615 __all__.append('Suite')
2618 def get_suite(suite, session=None):
2620 Returns Suite object for given C{suite name}.
2623 @param suite: The name of the suite
2625 @type session: Session
2626 @param session: Optional SQLA session object (a temporary one will be
2627 generated if not supplied)
2630 @return: Suite object for the requested suite name (None if not present)
2633 q = session.query(Suite).filter_by(suite_name=suite)
2637 except NoResultFound:
2640 __all__.append('get_suite')
2642 ################################################################################
2644 # TODO: should be removed because the implementation is too trivial
2646 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2648 Returns list of Architecture objects for given C{suite} name
2651 @param suite: Suite name to search for
2653 @type skipsrc: boolean
2654 @param skipsrc: Whether to skip returning the 'source' architecture entry
2657 @type skipall: boolean
2658 @param skipall: Whether to skip returning the 'all' architecture entry
2661 @type session: Session
2662 @param session: Optional SQL session object (a temporary one will be
2663 generated if not supplied)
2666 @return: list of Architecture objects for the given name (may be empty)
2669 return get_suite(suite, session).get_architectures(skipsrc, skipall)
2671 __all__.append('get_suite_architectures')
2673 ################################################################################
2675 class SuiteSrcFormat(object):
2676 def __init__(self, *args, **kwargs):
2680 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2682 __all__.append('SuiteSrcFormat')
2685 def get_suite_src_formats(suite, session=None):
2687 Returns list of allowed SrcFormat for C{suite}.
2690 @param suite: Suite name to search for
2692 @type session: Session
2693 @param session: Optional SQL session object (a temporary one will be
2694 generated if not supplied)
2697 @return: the list of allowed source formats for I{suite}
2700 q = session.query(SrcFormat)
2701 q = q.join(SuiteSrcFormat)
2702 q = q.join(Suite).filter_by(suite_name=suite)
2703 q = q.order_by('format_name')
2707 __all__.append('get_suite_src_formats')
2709 ################################################################################
2711 class Uid(ORMObject):
2712 def __init__(self, uid = None, name = None):
2716 def __eq__(self, val):
2717 if isinstance(val, str):
2718 return (self.uid == val)
2719 # This signals to use the normal comparison operator
2720 return NotImplemented
2722 def __ne__(self, val):
2723 if isinstance(val, str):
2724 return (self.uid != val)
2725 # This signals to use the normal comparison operator
2726 return NotImplemented
2728 def properties(self):
2729 return ['uid', 'name', 'fingerprint']
2731 def not_null_constraints(self):
2734 __all__.append('Uid')
2737 def get_or_set_uid(uidname, session=None):
2739 Returns uid object for given uidname.
2741 If no matching uidname is found, a row is inserted.
2743 @type uidname: string
2744 @param uidname: The uid to add
2746 @type session: SQLAlchemy
2747 @param session: Optional SQL session object (a temporary one will be
2748 generated if not supplied). If not passed, a commit will be performed at
2749 the end of the function, otherwise the caller is responsible for commiting.
2752 @return: the uid object for the given uidname
2755 q = session.query(Uid).filter_by(uid=uidname)
2759 except NoResultFound:
2763 session.commit_or_flush()
2768 __all__.append('get_or_set_uid')
2771 def get_uid_from_fingerprint(fpr, session=None):
2772 q = session.query(Uid)
2773 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2777 except NoResultFound:
2780 __all__.append('get_uid_from_fingerprint')
2782 ################################################################################
2784 class UploadBlock(object):
2785 def __init__(self, *args, **kwargs):
2789 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2791 __all__.append('UploadBlock')
2793 ################################################################################
2795 class DBConn(object):
2797 database module init.
2801 def __init__(self, *args, **kwargs):
2802 self.__dict__ = self.__shared_state
2804 if not getattr(self, 'initialised', False):
2805 self.initialised = True
2806 self.debug = kwargs.has_key('debug')
2809 def __setuptables(self):
2810 tables_with_primary = (
2818 'build_queue_files',
2823 'changes_pending_binaries',
2824 'changes_pending_files',
2825 'changes_pending_source',
2848 tables_no_primary = (
2849 'changes_pending_files_map',
2850 'changes_pending_source_files',
2851 'changes_pool_files',
2852 # TODO: the maintainer column in table override should be removed.
2854 'suite_architectures',
2855 'suite_src_formats',
2856 'suite_build_queue_copy',
2860 'almost_obsolete_all_associations',
2861 'almost_obsolete_src_associations',
2862 'any_associations_source',
2863 'bin_assoc_by_arch',
2864 'bin_associations_binaries',
2865 'binaries_suite_arch',
2866 'binfiles_suite_component_arch',
2869 'newest_all_associations',
2870 'newest_any_associations',
2872 'newest_src_association',
2873 'obsolete_all_associations',
2874 'obsolete_any_associations',
2875 'obsolete_any_by_all_associations',
2876 'obsolete_src_associations',
2878 'src_associations_bin',
2879 'src_associations_src',
2880 'suite_arch_by_name',
2883 # Sqlalchemy version 0.5 fails to reflect the SERIAL type
2884 # correctly and that is why we have to use a workaround. It can
2885 # be removed as soon as we switch to version 0.6.
2886 for table_name in tables_with_primary:
2887 table = Table(table_name, self.db_meta, \
2888 Column('id', Integer, primary_key = True), \
2889 autoload=True, useexisting=True)
2890 setattr(self, 'tbl_%s' % table_name, table)
2892 for table_name in tables_no_primary:
2893 table = Table(table_name, self.db_meta, autoload=True)
2894 setattr(self, 'tbl_%s' % table_name, table)
2896 # bin_contents needs special attention until the SERIAL type is
2897 # correctly detected and the workaround has been removed; see comment
2899 self.tbl_bin_contents = Table('bin_contents', self.db_meta, \
2900 Column('file', Text, primary_key = True),
2901 Column('binary_id', Integer, ForeignKey('binaries.id'), \
2902 primary_key = True),
2903 autoload=True, useexisting=True)
2905 for view_name in views:
2906 view = Table(view_name, self.db_meta, autoload=True)
2907 setattr(self, 'view_%s' % view_name, view)
2909 def __setupmappers(self):
2910 mapper(Architecture, self.tbl_architecture,
2911 properties = dict(arch_id = self.tbl_architecture.c.id,
2912 suites = relation(Suite, secondary=self.tbl_suite_architectures,
2913 order_by='suite_name',
2914 backref=backref('architectures', order_by='arch_string'))),
2915 extension = validator)
2917 mapper(Archive, self.tbl_archive,
2918 properties = dict(archive_id = self.tbl_archive.c.id,
2919 archive_name = self.tbl_archive.c.name))
2921 mapper(BuildQueue, self.tbl_build_queue,
2922 properties = dict(queue_id = self.tbl_build_queue.c.id))
2924 mapper(BuildQueueFile, self.tbl_build_queue_files,
2925 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
2926 poolfile = relation(PoolFile, backref='buildqueueinstances')))
2928 mapper(DBBinary, self.tbl_binaries,
2929 properties = dict(binary_id = self.tbl_binaries.c.id,
2930 package = self.tbl_binaries.c.package,
2931 version = self.tbl_binaries.c.version,
2932 maintainer_id = self.tbl_binaries.c.maintainer,
2933 maintainer = relation(Maintainer),
2934 source_id = self.tbl_binaries.c.source,
2935 source = relation(DBSource, backref='binaries'),
2936 arch_id = self.tbl_binaries.c.architecture,
2937 architecture = relation(Architecture),
2938 poolfile_id = self.tbl_binaries.c.file,
2939 poolfile = relation(PoolFile, backref=backref('binary', uselist = False)),
2940 binarytype = self.tbl_binaries.c.type,
2941 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2942 fingerprint = relation(Fingerprint),
2943 install_date = self.tbl_binaries.c.install_date,
2944 suites = relation(Suite, secondary=self.tbl_bin_associations,
2945 backref=backref('binaries', lazy='dynamic'))),
2946 extension = validator)
2948 mapper(BinaryACL, self.tbl_binary_acl,
2949 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2951 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2952 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2953 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2954 architecture = relation(Architecture)))
2956 mapper(Component, self.tbl_component,
2957 properties = dict(component_id = self.tbl_component.c.id,
2958 component_name = self.tbl_component.c.name),
2959 extension = validator)
2961 mapper(DBConfig, self.tbl_config,
2962 properties = dict(config_id = self.tbl_config.c.id))
2964 mapper(DSCFile, self.tbl_dsc_files,
2965 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2966 source_id = self.tbl_dsc_files.c.source,
2967 source = relation(DBSource),
2968 poolfile_id = self.tbl_dsc_files.c.file,
2969 poolfile = relation(PoolFile)))
2971 mapper(PoolFile, self.tbl_files,
2972 properties = dict(file_id = self.tbl_files.c.id,
2973 filesize = self.tbl_files.c.size,
2974 location_id = self.tbl_files.c.location,
2975 location = relation(Location,
2976 # using lazy='dynamic' in the back
2977 # reference because we have A LOT of
2978 # files in one location
2979 backref=backref('files', lazy='dynamic'))),
2980 extension = validator)
2982 mapper(Fingerprint, self.tbl_fingerprint,
2983 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2984 uid_id = self.tbl_fingerprint.c.uid,
2985 uid = relation(Uid),
2986 keyring_id = self.tbl_fingerprint.c.keyring,
2987 keyring = relation(Keyring),
2988 source_acl = relation(SourceACL),
2989 binary_acl = relation(BinaryACL)),
2990 extension = validator)
2992 mapper(Keyring, self.tbl_keyrings,
2993 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2994 keyring_id = self.tbl_keyrings.c.id))
2996 mapper(DBChange, self.tbl_changes,
2997 properties = dict(change_id = self.tbl_changes.c.id,
2998 poolfiles = relation(PoolFile,
2999 secondary=self.tbl_changes_pool_files,
3000 backref="changeslinks"),
3001 seen = self.tbl_changes.c.seen,
3002 source = self.tbl_changes.c.source,
3003 binaries = self.tbl_changes.c.binaries,
3004 architecture = self.tbl_changes.c.architecture,
3005 distribution = self.tbl_changes.c.distribution,
3006 urgency = self.tbl_changes.c.urgency,
3007 maintainer = self.tbl_changes.c.maintainer,
3008 changedby = self.tbl_changes.c.changedby,
3009 date = self.tbl_changes.c.date,
3010 version = self.tbl_changes.c.version,
3011 files = relation(ChangePendingFile,
3012 secondary=self.tbl_changes_pending_files_map,
3013 backref="changesfile"),
3014 in_queue_id = self.tbl_changes.c.in_queue,
3015 in_queue = relation(PolicyQueue,
3016 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
3017 approved_for_id = self.tbl_changes.c.approved_for))
3019 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
3020 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
3022 mapper(ChangePendingFile, self.tbl_changes_pending_files,
3023 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id,
3024 filename = self.tbl_changes_pending_files.c.filename,
3025 size = self.tbl_changes_pending_files.c.size,
3026 md5sum = self.tbl_changes_pending_files.c.md5sum,
3027 sha1sum = self.tbl_changes_pending_files.c.sha1sum,
3028 sha256sum = self.tbl_changes_pending_files.c.sha256sum))
3030 mapper(ChangePendingSource, self.tbl_changes_pending_source,
3031 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
3032 change = relation(DBChange),
3033 maintainer = relation(Maintainer,
3034 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
3035 changedby = relation(Maintainer,
3036 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
3037 fingerprint = relation(Fingerprint),
3038 source_files = relation(ChangePendingFile,
3039 secondary=self.tbl_changes_pending_source_files,
3040 backref="pending_sources")))
3043 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
3044 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
3045 keyring = relation(Keyring, backref="keyring_acl_map"),
3046 architecture = relation(Architecture)))
3048 mapper(Location, self.tbl_location,
3049 properties = dict(location_id = self.tbl_location.c.id,
3050 component_id = self.tbl_location.c.component,
3051 component = relation(Component, backref='location'),
3052 archive_id = self.tbl_location.c.archive,
3053 archive = relation(Archive),
3054 # FIXME: the 'type' column is old cruft and
3055 # should be removed in the future.
3056 archive_type = self.tbl_location.c.type),
3057 extension = validator)
3059 mapper(Maintainer, self.tbl_maintainer,
3060 properties = dict(maintainer_id = self.tbl_maintainer.c.id,
3061 maintains_sources = relation(DBSource, backref='maintainer',
3062 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.maintainer)),
3063 changed_sources = relation(DBSource, backref='changedby',
3064 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.changedby))),
3065 extension = validator)
3067 mapper(NewComment, self.tbl_new_comments,
3068 properties = dict(comment_id = self.tbl_new_comments.c.id))
3070 mapper(Override, self.tbl_override,
3071 properties = dict(suite_id = self.tbl_override.c.suite,
3072 suite = relation(Suite, \
3073 backref=backref('overrides', lazy='dynamic')),
3074 package = self.tbl_override.c.package,
3075 component_id = self.tbl_override.c.component,
3076 component = relation(Component, \
3077 backref=backref('overrides', lazy='dynamic')),
3078 priority_id = self.tbl_override.c.priority,
3079 priority = relation(Priority, \
3080 backref=backref('overrides', lazy='dynamic')),
3081 section_id = self.tbl_override.c.section,
3082 section = relation(Section, \
3083 backref=backref('overrides', lazy='dynamic')),
3084 overridetype_id = self.tbl_override.c.type,
3085 overridetype = relation(OverrideType, \
3086 backref=backref('overrides', lazy='dynamic'))))
3088 mapper(OverrideType, self.tbl_override_type,
3089 properties = dict(overridetype = self.tbl_override_type.c.type,
3090 overridetype_id = self.tbl_override_type.c.id))
3092 mapper(PolicyQueue, self.tbl_policy_queue,
3093 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
3095 mapper(Priority, self.tbl_priority,
3096 properties = dict(priority_id = self.tbl_priority.c.id))
3098 mapper(Section, self.tbl_section,
3099 properties = dict(section_id = self.tbl_section.c.id,
3100 section=self.tbl_section.c.section))
3102 mapper(DBSource, self.tbl_source,
3103 properties = dict(source_id = self.tbl_source.c.id,
3104 version = self.tbl_source.c.version,
3105 maintainer_id = self.tbl_source.c.maintainer,
3106 poolfile_id = self.tbl_source.c.file,
3107 poolfile = relation(PoolFile, backref=backref('source', uselist = False)),
3108 fingerprint_id = self.tbl_source.c.sig_fpr,
3109 fingerprint = relation(Fingerprint),
3110 changedby_id = self.tbl_source.c.changedby,
3111 srcfiles = relation(DSCFile,
3112 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
3113 suites = relation(Suite, secondary=self.tbl_src_associations,
3114 backref=backref('sources', lazy='dynamic')),
3115 srcuploaders = relation(SrcUploader)),
3116 extension = validator)
3118 mapper(SourceACL, self.tbl_source_acl,
3119 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
3121 mapper(SrcFormat, self.tbl_src_format,
3122 properties = dict(src_format_id = self.tbl_src_format.c.id,
3123 format_name = self.tbl_src_format.c.format_name))
3125 mapper(SrcUploader, self.tbl_src_uploaders,
3126 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
3127 source_id = self.tbl_src_uploaders.c.source,
3128 source = relation(DBSource,
3129 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
3130 maintainer_id = self.tbl_src_uploaders.c.maintainer,
3131 maintainer = relation(Maintainer,
3132 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
3134 mapper(Suite, self.tbl_suite,
3135 properties = dict(suite_id = self.tbl_suite.c.id,
3136 policy_queue = relation(PolicyQueue),
3137 copy_queues = relation(BuildQueue,
3138 secondary=self.tbl_suite_build_queue_copy)),
3139 extension = validator)
3141 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
3142 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
3143 suite = relation(Suite, backref='suitesrcformats'),
3144 src_format_id = self.tbl_suite_src_formats.c.src_format,
3145 src_format = relation(SrcFormat)))
3147 mapper(Uid, self.tbl_uid,
3148 properties = dict(uid_id = self.tbl_uid.c.id,
3149 fingerprint = relation(Fingerprint)),
3150 extension = validator)
3152 mapper(UploadBlock, self.tbl_upload_blocks,
3153 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
3154 fingerprint = relation(Fingerprint, backref="uploadblocks"),
3155 uid = relation(Uid, backref="uploadblocks")))
3157 mapper(BinContents, self.tbl_bin_contents,
3159 binary = relation(DBBinary,
3160 backref=backref('contents', lazy='dynamic', cascade='all')),
3161 file = self.tbl_bin_contents.c.file))
3163 ## Connection functions
3164 def __createconn(self):
3165 from config import Config
3167 if cnf.has_key("DB::Service"):
3168 connstr = "postgresql://service=%s" % cnf["DB::Service"]
3169 elif cnf.has_key("DB::Host"):
3171 connstr = "postgresql://%s" % cnf["DB::Host"]
3172 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
3173 connstr += ":%s" % cnf["DB::Port"]
3174 connstr += "/%s" % cnf["DB::Name"]
3177 connstr = "postgresql:///%s" % cnf["DB::Name"]
3178 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
3179 connstr += "?port=%s" % cnf["DB::Port"]
3181 engine_args = { 'echo': self.debug }
3182 if cnf.has_key('DB::PoolSize'):
3183 engine_args['pool_size'] = int(cnf['DB::PoolSize'])
3184 if cnf.has_key('DB::MaxOverflow'):
3185 engine_args['max_overflow'] = int(cnf['DB::MaxOverflow'])
3186 if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \
3187 cnf['DB::Unicode'] == 'false':
3188 engine_args['use_native_unicode'] = False
3190 # Monkey patch a new dialect in in order to support service= syntax
3191 import sqlalchemy.dialects.postgresql
3192 from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
3193 class PGDialect_psycopg2_dak(PGDialect_psycopg2):
3194 def create_connect_args(self, url):
3195 if str(url).startswith('postgresql://service='):
3197 servicename = str(url)[21:]
3198 return (['service=%s' % servicename], {})
3200 return PGDialect_psycopg2.create_connect_args(self, url)
3202 sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak
3204 self.db_pg = create_engine(connstr, **engine_args)
3205 self.db_meta = MetaData()
3206 self.db_meta.bind = self.db_pg
3207 self.db_smaker = sessionmaker(bind=self.db_pg,
3211 self.__setuptables()
3212 self.__setupmappers()
3215 return self.db_smaker()
3219 Resets the DBConn object. This function must be called by subprocesses
3220 created by the multiprocessing module. See tests/dbtest_multiproc.py
3226 __all__.append('DBConn')