5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
37 from os.path import normpath
48 import simplejson as json
50 from datetime import datetime, timedelta
51 from errno import ENOENT
52 from tempfile import mkstemp, mkdtemp
53 from subprocess import Popen, PIPE
54 from tarfile import TarFile
56 from inspect import getargspec
59 from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \
61 from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
62 backref, MapperExtension, EXT_CONTINUE, object_mapper
63 from sqlalchemy import types as sqltypes
65 # Don't remove this, we re-export the exceptions to scripts which import us
66 from sqlalchemy.exc import *
67 from sqlalchemy.orm.exc import NoResultFound
69 # Only import Config until Queue stuff is changed to store its config
71 from config import Config
72 from textutils import fix_maintainer
73 from dak_exceptions import DBUpdateError, NoSourceFieldError
75 # suppress some deprecation warnings in squeeze related to sqlalchemy
77 warnings.filterwarnings('ignore', \
78 "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
80 # TODO: sqlalchemy needs some extra configuration to correctly reflect
81 # the ind_deb_contents_* indexes - we ignore the warnings at the moment
82 warnings.filterwarnings("ignore", 'Predicate of partial index', SAWarning)
85 ################################################################################
87 # Patch in support for the debversion field type so that it works during
91 # that is for sqlalchemy 0.6
92 UserDefinedType = sqltypes.UserDefinedType
94 # this one for sqlalchemy 0.5
95 UserDefinedType = sqltypes.TypeEngine
97 class DebVersion(UserDefinedType):
98 def get_col_spec(self):
101 def bind_processor(self, dialect):
104 # ' = None' is needed for sqlalchemy 0.5:
105 def result_processor(self, dialect, coltype = None):
108 sa_major_version = sqlalchemy.__version__[0:3]
109 if sa_major_version in ["0.5", "0.6"]:
110 from sqlalchemy.databases import postgres
111 postgres.ischema_names['debversion'] = DebVersion
113 raise Exception("dak only ported to SQLA versions 0.5 and 0.6. See daklib/dbconn.py")
115 ################################################################################
117 __all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion']
119 ################################################################################
121 def session_wrapper(fn):
123 Wrapper around common ".., session=None):" handling. If the wrapped
124 function is called without passing 'session', we create a local one
125 and destroy it when the function ends.
127 Also attaches a commit_or_flush method to the session; if we created a
128 local session, this is a synonym for session.commit(), otherwise it is a
129 synonym for session.flush().
132 def wrapped(*args, **kwargs):
133 private_transaction = False
135 # Find the session object
136 session = kwargs.get('session')
139 if len(args) <= len(getargspec(fn)[0]) - 1:
140 # No session specified as last argument or in kwargs
141 private_transaction = True
142 session = kwargs['session'] = DBConn().session()
144 # Session is last argument in args
148 session = args[-1] = DBConn().session()
149 private_transaction = True
151 if private_transaction:
152 session.commit_or_flush = session.commit
154 session.commit_or_flush = session.flush
157 return fn(*args, **kwargs)
159 if private_transaction:
160 # We created a session; close it.
163 wrapped.__doc__ = fn.__doc__
164 wrapped.func_name = fn.func_name
168 __all__.append('session_wrapper')
170 ################################################################################
172 class ORMObject(object):
174 ORMObject is a base class for all ORM classes mapped by SQLalchemy. All
175 derived classes must implement the properties() method.
178 def properties(self):
180 This method should be implemented by all derived classes and returns a
181 list of the important properties. The properties 'created' and
182 'modified' will be added automatically. A suffix '_count' should be
183 added to properties that are lists or query objects. The most important
184 property name should be returned as the first element in the list
185 because it is used by repr().
191 Returns a JSON representation of the object based on the properties
192 returned from the properties() method.
195 # add created and modified
196 all_properties = self.properties() + ['created', 'modified']
197 for property in all_properties:
198 # check for list or query
199 if property[-6:] == '_count':
200 real_property = property[:-6]
201 if not hasattr(self, real_property):
203 value = getattr(self, real_property)
204 if hasattr(value, '__len__'):
207 elif hasattr(value, 'count'):
209 value = value.count()
211 raise KeyError('Do not understand property %s.' % property)
213 if not hasattr(self, property):
216 value = getattr(self, property)
220 elif isinstance(value, ORMObject):
221 # use repr() for ORMObject types
224 # we want a string for all other types because json cannot
227 data[property] = value
228 return json.dumps(data)
232 Returns the name of the class.
234 return type(self).__name__
238 Returns a short string representation of the object using the first
239 element from the properties() method.
241 primary_property = self.properties()[0]
242 value = getattr(self, primary_property)
243 return '<%s %s>' % (self.classname(), str(value))
247 Returns a human readable form of the object using the properties()
250 return '<%s %s>' % (self.classname(), self.json())
252 def not_null_constraints(self):
254 Returns a list of properties that must be not NULL. Derived classes
255 should override this method if needed.
259 validation_message = \
260 "Validation failed because property '%s' must not be empty in object\n%s"
264 This function validates the not NULL constraints as returned by
265 not_null_constraints(). It raises the DBUpdateError exception if
268 for property in self.not_null_constraints():
269 # TODO: It is a bit awkward that the mapper configuration allow
270 # directly setting the numeric _id columns. We should get rid of it
272 if hasattr(self, property + '_id') and \
273 getattr(self, property + '_id') is not None:
275 if not hasattr(self, property) or getattr(self, property) is None:
276 raise DBUpdateError(self.validation_message % \
277 (property, str(self)))
281 def get(cls, primary_key, session = None):
283 This is a support function that allows getting an object by its primary
286 Architecture.get(3[, session])
288 instead of the more verbose
290 session.query(Architecture).get(3)
292 return session.query(cls).get(primary_key)
294 def session(self, replace = False):
296 Returns the current session that is associated with the object. May
297 return None is object is in detached state.
300 return object_session(self)
302 def clone(self, session = None):
304 Clones the current object in a new session and returns the new clone. A
305 fresh session is created if the optional session parameter is not
306 provided. The function will fail if a session is provided and has
309 RATIONALE: SQLAlchemy's session is not thread safe. This method clones
310 an existing object to allow several threads to work with their own
311 instances of an ORMObject.
313 WARNING: Only persistent (committed) objects can be cloned. Changes
314 made to the original object that are not committed yet will get lost.
315 The session of the new object will always be rolled back to avoid
319 if self.session() is None:
320 raise RuntimeError( \
321 'Method clone() failed for detached object:\n%s' % self)
322 self.session().flush()
323 mapper = object_mapper(self)
324 primary_key = mapper.primary_key_from_instance(self)
325 object_class = self.__class__
327 session = DBConn().session()
328 elif len(session.new) + len(session.dirty) + len(session.deleted) > 0:
329 raise RuntimeError( \
330 'Method clone() failed due to unflushed changes in session.')
331 new_object = session.query(object_class).get(primary_key)
333 if new_object is None:
334 raise RuntimeError( \
335 'Method clone() failed for non-persistent object:\n%s' % self)
338 __all__.append('ORMObject')
340 ################################################################################
342 class Validator(MapperExtension):
344 This class calls the validate() method for each instance for the
345 'before_update' and 'before_insert' events. A global object validator is
346 used for configuring the individual mappers.
349 def before_update(self, mapper, connection, instance):
353 def before_insert(self, mapper, connection, instance):
357 validator = Validator()
359 ################################################################################
361 class Architecture(ORMObject):
362 def __init__(self, arch_string = None, description = None):
363 self.arch_string = arch_string
364 self.description = description
366 def __eq__(self, val):
367 if isinstance(val, str):
368 return (self.arch_string== val)
369 # This signals to use the normal comparison operator
370 return NotImplemented
372 def __ne__(self, val):
373 if isinstance(val, str):
374 return (self.arch_string != val)
375 # This signals to use the normal comparison operator
376 return NotImplemented
378 def properties(self):
379 return ['arch_string', 'arch_id', 'suites_count']
381 def not_null_constraints(self):
382 return ['arch_string']
384 __all__.append('Architecture')
387 def get_architecture(architecture, session=None):
389 Returns database id for given C{architecture}.
391 @type architecture: string
392 @param architecture: The name of the architecture
394 @type session: Session
395 @param session: Optional SQLA session object (a temporary one will be
396 generated if not supplied)
399 @return: Architecture object for the given arch (None if not present)
402 q = session.query(Architecture).filter_by(arch_string=architecture)
406 except NoResultFound:
409 __all__.append('get_architecture')
411 # TODO: should be removed because the implementation is too trivial
413 def get_architecture_suites(architecture, session=None):
415 Returns list of Suite objects for given C{architecture} name
417 @type architecture: str
418 @param architecture: Architecture name to search for
420 @type session: Session
421 @param session: Optional SQL session object (a temporary one will be
422 generated if not supplied)
425 @return: list of Suite objects for the given name (may be empty)
428 return get_architecture(architecture, session).suites
430 __all__.append('get_architecture_suites')
432 ################################################################################
434 class Archive(object):
435 def __init__(self, *args, **kwargs):
439 return '<Archive %s>' % self.archive_name
441 __all__.append('Archive')
444 def get_archive(archive, session=None):
446 returns database id for given C{archive}.
448 @type archive: string
449 @param archive: the name of the arhive
451 @type session: Session
452 @param session: Optional SQLA session object (a temporary one will be
453 generated if not supplied)
456 @return: Archive object for the given name (None if not present)
459 archive = archive.lower()
461 q = session.query(Archive).filter_by(archive_name=archive)
465 except NoResultFound:
468 __all__.append('get_archive')
470 ################################################################################
472 class BinContents(ORMObject):
473 def __init__(self, file = None, binary = None):
477 def properties(self):
478 return ['file', 'binary']
480 __all__.append('BinContents')
482 ################################################################################
484 class DBBinary(ORMObject):
485 def __init__(self, package = None, source = None, version = None, \
486 maintainer = None, architecture = None, poolfile = None, \
488 self.package = package
490 self.version = version
491 self.maintainer = maintainer
492 self.architecture = architecture
493 self.poolfile = poolfile
494 self.binarytype = binarytype
496 def properties(self):
497 return ['package', 'version', 'maintainer', 'source', 'architecture', \
498 'poolfile', 'binarytype', 'fingerprint', 'install_date', \
499 'suites_count', 'binary_id', 'contents_count']
501 def not_null_constraints(self):
502 return ['package', 'version', 'maintainer', 'source', 'poolfile', \
505 def get_component_name(self):
506 return self.poolfile.location.component.component_name
508 def scan_contents(self):
510 Yields the contents of the package. Only regular files are yielded and
511 the path names are normalized after converting them from either utf-8
512 or iso8859-1 encoding. It yields the string ' <EMPTY PACKAGE>' if the
513 package does not contain any regular file.
515 fullpath = self.poolfile.fullpath
516 dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE)
517 tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
518 anything_yielded = False
519 for member in tar.getmembers():
521 name = normpath(member.name)
522 # enforce proper utf-8 encoding
525 except UnicodeDecodeError:
526 name = name.decode('iso8859-1').encode('utf-8')
528 anything_yielded = True
529 if not anything_yielded:
530 yield ' <EMPTY PACKAGE>'
535 __all__.append('DBBinary')
538 def get_suites_binary_in(package, session=None):
540 Returns list of Suite objects which given C{package} name is in
543 @param package: DBBinary package name to search for
546 @return: list of Suite objects for the given package
549 return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all()
551 __all__.append('get_suites_binary_in')
554 def get_component_by_package_suite(package, suite_list, arch_list=[], session=None):
556 Returns the component name of the newest binary package in suite_list or
557 None if no package is found. The result can be optionally filtered by a list
558 of architecture names.
561 @param package: DBBinary package name to search for
563 @type suite_list: list of str
564 @param suite_list: list of suite_name items
566 @type arch_list: list of str
567 @param arch_list: optional list of arch_string items that defaults to []
569 @rtype: str or NoneType
570 @return: name of component or None
573 q = session.query(DBBinary).filter_by(package = package). \
574 join(DBBinary.suites).filter(Suite.suite_name.in_(suite_list))
575 if len(arch_list) > 0:
576 q = q.join(DBBinary.architecture). \
577 filter(Architecture.arch_string.in_(arch_list))
578 binary = q.order_by(desc(DBBinary.version)).first()
582 return binary.get_component_name()
584 __all__.append('get_component_by_package_suite')
586 ################################################################################
588 class BinaryACL(object):
589 def __init__(self, *args, **kwargs):
593 return '<BinaryACL %s>' % self.binary_acl_id
595 __all__.append('BinaryACL')
597 ################################################################################
599 class BinaryACLMap(object):
600 def __init__(self, *args, **kwargs):
604 return '<BinaryACLMap %s>' % self.binary_acl_map_id
606 __all__.append('BinaryACLMap')
608 ################################################################################
613 ArchiveDir "%(archivepath)s";
614 OverrideDir "%(overridedir)s";
615 CacheDir "%(cachedir)s";
620 Packages::Compress ". bzip2 gzip";
621 Sources::Compress ". bzip2 gzip";
626 bindirectory "incoming"
631 BinOverride "override.sid.all3";
632 BinCacheDB "packages-accepted.db";
634 FileList "%(filelist)s";
637 Packages::Extensions ".deb .udeb";
640 bindirectory "incoming/"
643 BinOverride "override.sid.all3";
644 SrcOverride "override.sid.all3.src";
645 FileList "%(filelist)s";
649 class BuildQueue(object):
650 def __init__(self, *args, **kwargs):
654 return '<BuildQueue %s>' % self.queue_name
656 def write_metadata(self, starttime, force=False):
657 # Do we write out metafiles?
658 if not (force or self.generate_metadata):
661 session = DBConn().session().object_session(self)
663 fl_fd = fl_name = ac_fd = ac_name = None
665 arches = " ".join([ a.arch_string for a in session.query(Architecture).all() if a.arch_string != 'source' ])
666 startdir = os.getcwd()
669 # Grab files we want to include
670 newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
671 # Write file list with newer files
672 (fl_fd, fl_name) = mkstemp()
674 os.write(fl_fd, '%s\n' % n.fullpath)
679 # Write minimal apt.conf
680 # TODO: Remove hardcoding from template
681 (ac_fd, ac_name) = mkstemp()
682 os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path,
684 'cachedir': cnf["Dir::Cache"],
685 'overridedir': cnf["Dir::Override"],
689 # Run apt-ftparchive generate
690 os.chdir(os.path.dirname(ac_name))
691 os.system('apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
693 # Run apt-ftparchive release
694 # TODO: Eww - fix this
695 bname = os.path.basename(self.path)
699 # We have to remove the Release file otherwise it'll be included in the
702 os.unlink(os.path.join(bname, 'Release'))
706 os.system("""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="%s" release %s > Release""" % (self.origin, self.label, self.releasedescription, arches, bname))
708 # Crude hack with open and append, but this whole section is and should be redone.
709 if self.notautomatic:
710 release=open("Release", "a")
711 release.write("NotAutomatic: yes")
716 keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
717 if cnf.has_key("Dinstall::SigningPubKeyring"):
718 keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
720 os.system("gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey))
722 # Move the files if we got this far
723 os.rename('Release', os.path.join(bname, 'Release'))
725 os.rename('Release.gpg', os.path.join(bname, 'Release.gpg'))
727 # Clean up any left behind files
754 def clean_and_update(self, starttime, Logger, dryrun=False):
755 """WARNING: This routine commits for you"""
756 session = DBConn().session().object_session(self)
758 if self.generate_metadata and not dryrun:
759 self.write_metadata(starttime)
761 # Grab files older than our execution time
762 older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
768 Logger.log(["I: Would have removed %s from the queue" % o.fullpath])
770 Logger.log(["I: Removing %s from the queue" % o.fullpath])
771 os.unlink(o.fullpath)
774 # If it wasn't there, don't worry
775 if e.errno == ENOENT:
778 # TODO: Replace with proper logging call
779 Logger.log(["E: Could not remove %s" % o.fullpath])
786 for f in os.listdir(self.path):
787 if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release') or f.startswith('advisory'):
791 r = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter_by(filename = f).one()
792 except NoResultFound:
793 fp = os.path.join(self.path, f)
795 Logger.log(["I: Would remove unused link %s" % fp])
797 Logger.log(["I: Removing unused link %s" % fp])
801 Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath])
803 def add_file_from_pool(self, poolfile):
804 """Copies a file into the pool. Assumes that the PoolFile object is
805 attached to the same SQLAlchemy session as the Queue object is.
807 The caller is responsible for committing after calling this function."""
808 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
810 # Check if we have a file of this name or this ID already
811 for f in self.queuefiles:
812 if f.fileid is not None and f.fileid == poolfile.file_id or \
813 f.poolfile.filename == poolfile_basename:
814 # In this case, update the BuildQueueFile entry so we
815 # don't remove it too early
816 f.lastused = datetime.now()
817 DBConn().session().object_session(poolfile).add(f)
820 # Prepare BuildQueueFile object
821 qf = BuildQueueFile()
822 qf.build_queue_id = self.queue_id
823 qf.lastused = datetime.now()
824 qf.filename = poolfile_basename
826 targetpath = poolfile.fullpath
827 queuepath = os.path.join(self.path, poolfile_basename)
831 # We need to copy instead of symlink
833 utils.copy(targetpath, queuepath)
834 # NULL in the fileid field implies a copy
837 os.symlink(targetpath, queuepath)
838 qf.fileid = poolfile.file_id
842 # Get the same session as the PoolFile is using and add the qf to it
843 DBConn().session().object_session(poolfile).add(qf)
848 __all__.append('BuildQueue')
851 def get_build_queue(queuename, session=None):
853 Returns BuildQueue object for given C{queue name}, creating it if it does not
856 @type queuename: string
857 @param queuename: The name of the queue
859 @type session: Session
860 @param session: Optional SQLA session object (a temporary one will be
861 generated if not supplied)
864 @return: BuildQueue object for the given queue
867 q = session.query(BuildQueue).filter_by(queue_name=queuename)
871 except NoResultFound:
874 __all__.append('get_build_queue')
876 ################################################################################
878 class BuildQueueFile(object):
879 def __init__(self, *args, **kwargs):
883 return '<BuildQueueFile %s (%s)>' % (self.filename, self.build_queue_id)
887 return os.path.join(self.buildqueue.path, self.filename)
890 __all__.append('BuildQueueFile')
892 ################################################################################
894 class ChangePendingBinary(object):
895 def __init__(self, *args, **kwargs):
899 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
901 __all__.append('ChangePendingBinary')
903 ################################################################################
905 class ChangePendingFile(object):
906 def __init__(self, *args, **kwargs):
910 return '<ChangePendingFile %s>' % self.change_pending_file_id
912 __all__.append('ChangePendingFile')
914 ################################################################################
916 class ChangePendingSource(object):
917 def __init__(self, *args, **kwargs):
921 return '<ChangePendingSource %s>' % self.change_pending_source_id
923 __all__.append('ChangePendingSource')
925 ################################################################################
927 class Component(ORMObject):
928 def __init__(self, component_name = None):
929 self.component_name = component_name
931 def __eq__(self, val):
932 if isinstance(val, str):
933 return (self.component_name == val)
934 # This signals to use the normal comparison operator
935 return NotImplemented
937 def __ne__(self, val):
938 if isinstance(val, str):
939 return (self.component_name != val)
940 # This signals to use the normal comparison operator
941 return NotImplemented
943 def properties(self):
944 return ['component_name', 'component_id', 'description', \
945 'location_count', 'meets_dfsg', 'overrides_count']
947 def not_null_constraints(self):
948 return ['component_name']
951 __all__.append('Component')
954 def get_component(component, session=None):
956 Returns database id for given C{component}.
958 @type component: string
959 @param component: The name of the override type
962 @return: the database id for the given component
965 component = component.lower()
967 q = session.query(Component).filter_by(component_name=component)
971 except NoResultFound:
974 __all__.append('get_component')
976 ################################################################################
978 class DBConfig(object):
979 def __init__(self, *args, **kwargs):
983 return '<DBConfig %s>' % self.name
985 __all__.append('DBConfig')
987 ################################################################################
990 def get_or_set_contents_file_id(filename, session=None):
992 Returns database id for given filename.
994 If no matching file is found, a row is inserted.
996 @type filename: string
997 @param filename: The filename
998 @type session: SQLAlchemy
999 @param session: Optional SQL session object (a temporary one will be
1000 generated if not supplied). If not passed, a commit will be performed at
1001 the end of the function, otherwise the caller is responsible for commiting.
1004 @return: the database id for the given component
1007 q = session.query(ContentFilename).filter_by(filename=filename)
1010 ret = q.one().cafilename_id
1011 except NoResultFound:
1012 cf = ContentFilename()
1013 cf.filename = filename
1015 session.commit_or_flush()
1016 ret = cf.cafilename_id
1020 __all__.append('get_or_set_contents_file_id')
1023 def get_contents(suite, overridetype, section=None, session=None):
1025 Returns contents for a suite / overridetype combination, limiting
1026 to a section if not None.
1029 @param suite: Suite object
1031 @type overridetype: OverrideType
1032 @param overridetype: OverrideType object
1034 @type section: Section
1035 @param section: Optional section object to limit results to
1037 @type session: SQLAlchemy
1038 @param session: Optional SQL session object (a temporary one will be
1039 generated if not supplied)
1041 @rtype: ResultsProxy
1042 @return: ResultsProxy object set up to return tuples of (filename, section,
1046 # find me all of the contents for a given suite
1047 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
1051 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
1052 JOIN content_file_names n ON (c.filename=n.id)
1053 JOIN binaries b ON (b.id=c.binary_pkg)
1054 JOIN override o ON (o.package=b.package)
1055 JOIN section s ON (s.id=o.section)
1056 WHERE o.suite = :suiteid AND o.type = :overridetypeid
1057 AND b.type=:overridetypename"""
1059 vals = {'suiteid': suite.suite_id,
1060 'overridetypeid': overridetype.overridetype_id,
1061 'overridetypename': overridetype.overridetype}
1063 if section is not None:
1064 contents_q += " AND s.id = :sectionid"
1065 vals['sectionid'] = section.section_id
1067 contents_q += " ORDER BY fn"
1069 return session.execute(contents_q, vals)
1071 __all__.append('get_contents')
1073 ################################################################################
1075 class ContentFilepath(object):
1076 def __init__(self, *args, **kwargs):
1080 return '<ContentFilepath %s>' % self.filepath
1082 __all__.append('ContentFilepath')
1085 def get_or_set_contents_path_id(filepath, session=None):
1087 Returns database id for given path.
1089 If no matching file is found, a row is inserted.
1091 @type filepath: string
1092 @param filepath: The filepath
1094 @type session: SQLAlchemy
1095 @param session: Optional SQL session object (a temporary one will be
1096 generated if not supplied). If not passed, a commit will be performed at
1097 the end of the function, otherwise the caller is responsible for commiting.
1100 @return: the database id for the given path
1103 q = session.query(ContentFilepath).filter_by(filepath=filepath)
1106 ret = q.one().cafilepath_id
1107 except NoResultFound:
1108 cf = ContentFilepath()
1109 cf.filepath = filepath
1111 session.commit_or_flush()
1112 ret = cf.cafilepath_id
1116 __all__.append('get_or_set_contents_path_id')
1118 ################################################################################
1120 class ContentAssociation(object):
1121 def __init__(self, *args, **kwargs):
1125 return '<ContentAssociation %s>' % self.ca_id
1127 __all__.append('ContentAssociation')
1129 def insert_content_paths(binary_id, fullpaths, session=None):
1131 Make sure given path is associated with given binary id
1133 @type binary_id: int
1134 @param binary_id: the id of the binary
1135 @type fullpaths: list
1136 @param fullpaths: the list of paths of the file being associated with the binary
1137 @type session: SQLAlchemy session
1138 @param session: Optional SQLAlchemy session. If this is passed, the caller
1139 is responsible for ensuring a transaction has begun and committing the
1140 results or rolling back based on the result code. If not passed, a commit
1141 will be performed at the end of the function, otherwise the caller is
1142 responsible for commiting.
1144 @return: True upon success
1147 privatetrans = False
1149 session = DBConn().session()
1154 def generate_path_dicts():
1155 for fullpath in fullpaths:
1156 if fullpath.startswith( './' ):
1157 fullpath = fullpath[2:]
1159 yield {'filename':fullpath, 'id': binary_id }
1161 for d in generate_path_dicts():
1162 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
1171 traceback.print_exc()
1173 # Only rollback if we set up the session ourself
1180 __all__.append('insert_content_paths')
1182 ################################################################################
1184 class DSCFile(object):
1185 def __init__(self, *args, **kwargs):
1189 return '<DSCFile %s>' % self.dscfile_id
1191 __all__.append('DSCFile')
1194 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
1196 Returns a list of DSCFiles which may be empty
1198 @type dscfile_id: int (optional)
1199 @param dscfile_id: the dscfile_id of the DSCFiles to find
1201 @type source_id: int (optional)
1202 @param source_id: the source id related to the DSCFiles to find
1204 @type poolfile_id: int (optional)
1205 @param poolfile_id: the poolfile id related to the DSCFiles to find
1208 @return: Possibly empty list of DSCFiles
1211 q = session.query(DSCFile)
1213 if dscfile_id is not None:
1214 q = q.filter_by(dscfile_id=dscfile_id)
1216 if source_id is not None:
1217 q = q.filter_by(source_id=source_id)
1219 if poolfile_id is not None:
1220 q = q.filter_by(poolfile_id=poolfile_id)
1224 __all__.append('get_dscfiles')
1226 ################################################################################
1228 class PoolFile(ORMObject):
1229 def __init__(self, filename = None, location = None, filesize = -1, \
1231 self.filename = filename
1232 self.location = location
1233 self.filesize = filesize
1234 self.md5sum = md5sum
1238 return os.path.join(self.location.path, self.filename)
1240 def is_valid(self, filesize = -1, md5sum = None):
1241 return self.filesize == long(filesize) and self.md5sum == md5sum
1243 def properties(self):
1244 return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \
1245 'sha256sum', 'location', 'source', 'binary', 'last_used']
1247 def not_null_constraints(self):
1248 return ['filename', 'md5sum', 'location']
1250 __all__.append('PoolFile')
1253 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
1256 (ValidFileFound [boolean], PoolFile object or None)
1258 @type filename: string
1259 @param filename: the filename of the file to check against the DB
1262 @param filesize: the size of the file to check against the DB
1264 @type md5sum: string
1265 @param md5sum: the md5sum of the file to check against the DB
1267 @type location_id: int
1268 @param location_id: the id of the location to look in
1271 @return: Tuple of length 2.
1272 - If valid pool file found: (C{True}, C{PoolFile object})
1273 - If valid pool file not found:
1274 - (C{False}, C{None}) if no file found
1275 - (C{False}, C{PoolFile object}) if file found with size/md5sum mismatch
1278 poolfile = session.query(Location).get(location_id). \
1279 files.filter_by(filename=filename).first()
1281 if poolfile and poolfile.is_valid(filesize = filesize, md5sum = md5sum):
1284 return (valid, poolfile)
1286 __all__.append('check_poolfile')
1288 # TODO: the implementation can trivially be inlined at the place where the
1289 # function is called
1291 def get_poolfile_by_id(file_id, session=None):
1293 Returns a PoolFile objects or None for the given id
1296 @param file_id: the id of the file to look for
1298 @rtype: PoolFile or None
1299 @return: either the PoolFile object or None
1302 return session.query(PoolFile).get(file_id)
1304 __all__.append('get_poolfile_by_id')
1307 def get_poolfile_like_name(filename, session=None):
1309 Returns an array of PoolFile objects which are like the given name
1311 @type filename: string
1312 @param filename: the filename of the file to check against the DB
1315 @return: array of PoolFile objects
1318 # TODO: There must be a way of properly using bind parameters with %FOO%
1319 q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
1323 __all__.append('get_poolfile_like_name')
1326 def add_poolfile(filename, datadict, location_id, session=None):
1328 Add a new file to the pool
1330 @type filename: string
1331 @param filename: filename
1333 @type datadict: dict
1334 @param datadict: dict with needed data
1336 @type location_id: int
1337 @param location_id: database id of the location
1340 @return: the PoolFile object created
1342 poolfile = PoolFile()
1343 poolfile.filename = filename
1344 poolfile.filesize = datadict["size"]
1345 poolfile.md5sum = datadict["md5sum"]
1346 poolfile.sha1sum = datadict["sha1sum"]
1347 poolfile.sha256sum = datadict["sha256sum"]
1348 poolfile.location_id = location_id
1350 session.add(poolfile)
1351 # Flush to get a file id (NB: This is not a commit)
1356 __all__.append('add_poolfile')
1358 ################################################################################
1360 class Fingerprint(ORMObject):
1361 def __init__(self, fingerprint = None):
1362 self.fingerprint = fingerprint
1364 def properties(self):
1365 return ['fingerprint', 'fingerprint_id', 'keyring', 'uid', \
1368 def not_null_constraints(self):
1369 return ['fingerprint']
1371 __all__.append('Fingerprint')
1374 def get_fingerprint(fpr, session=None):
1376 Returns Fingerprint object for given fpr.
1379 @param fpr: The fpr to find / add
1381 @type session: SQLAlchemy
1382 @param session: Optional SQL session object (a temporary one will be
1383 generated if not supplied).
1386 @return: the Fingerprint object for the given fpr or None
1389 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1393 except NoResultFound:
1398 __all__.append('get_fingerprint')
1401 def get_or_set_fingerprint(fpr, session=None):
1403 Returns Fingerprint object for given fpr.
1405 If no matching fpr is found, a row is inserted.
1408 @param fpr: The fpr to find / add
1410 @type session: SQLAlchemy
1411 @param session: Optional SQL session object (a temporary one will be
1412 generated if not supplied). If not passed, a commit will be performed at
1413 the end of the function, otherwise the caller is responsible for commiting.
1414 A flush will be performed either way.
1417 @return: the Fingerprint object for the given fpr
1420 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1424 except NoResultFound:
1425 fingerprint = Fingerprint()
1426 fingerprint.fingerprint = fpr
1427 session.add(fingerprint)
1428 session.commit_or_flush()
1433 __all__.append('get_or_set_fingerprint')
1435 ################################################################################
1437 # Helper routine for Keyring class
1438 def get_ldap_name(entry):
1440 for k in ["cn", "mn", "sn"]:
1442 if ret and ret[0] != "" and ret[0] != "-":
1444 return " ".join(name)
1446 ################################################################################
1448 class Keyring(object):
1449 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1450 " --with-colons --fingerprint --fingerprint"
1455 def __init__(self, *args, **kwargs):
1459 return '<Keyring %s>' % self.keyring_name
1461 def de_escape_gpg_str(self, txt):
1462 esclist = re.split(r'(\\x..)', txt)
1463 for x in range(1,len(esclist),2):
1464 esclist[x] = "%c" % (int(esclist[x][2:],16))
1465 return "".join(esclist)
1467 def parse_address(self, uid):
1468 """parses uid and returns a tuple of real name and email address"""
1470 (name, address) = email.Utils.parseaddr(uid)
1471 name = re.sub(r"\s*[(].*[)]", "", name)
1472 name = self.de_escape_gpg_str(name)
1475 return (name, address)
1477 def load_keys(self, keyring):
1478 if not self.keyring_id:
1479 raise Exception('Must be initialized with database information')
1481 k = os.popen(self.gpg_invocation % keyring, "r")
1485 for line in k.xreadlines():
1486 field = line.split(":")
1487 if field[0] == "pub":
1490 (name, addr) = self.parse_address(field[9])
1492 self.keys[key]["email"] = addr
1493 self.keys[key]["name"] = name
1494 self.keys[key]["fingerprints"] = []
1496 elif key and field[0] == "sub" and len(field) >= 12:
1497 signingkey = ("s" in field[11])
1498 elif key and field[0] == "uid":
1499 (name, addr) = self.parse_address(field[9])
1500 if "email" not in self.keys[key] and "@" in addr:
1501 self.keys[key]["email"] = addr
1502 self.keys[key]["name"] = name
1503 elif signingkey and field[0] == "fpr":
1504 self.keys[key]["fingerprints"].append(field[9])
1505 self.fpr_lookup[field[9]] = key
1507 def import_users_from_ldap(self, session):
1511 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1512 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1514 l = ldap.open(LDAPServer)
1515 l.simple_bind_s("","")
1516 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1517 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1518 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1520 ldap_fin_uid_id = {}
1527 uid = entry["uid"][0]
1528 name = get_ldap_name(entry)
1529 fingerprints = entry["keyFingerPrint"]
1531 for f in fingerprints:
1532 key = self.fpr_lookup.get(f, None)
1533 if key not in self.keys:
1535 self.keys[key]["uid"] = uid
1539 keyid = get_or_set_uid(uid, session).uid_id
1540 byuid[keyid] = (uid, name)
1541 byname[uid] = (keyid, name)
1543 return (byname, byuid)
1545 def generate_users_from_keyring(self, format, session):
1549 for x in self.keys.keys():
1550 if "email" not in self.keys[x]:
1552 self.keys[x]["uid"] = format % "invalid-uid"
1554 uid = format % self.keys[x]["email"]
1555 keyid = get_or_set_uid(uid, session).uid_id
1556 byuid[keyid] = (uid, self.keys[x]["name"])
1557 byname[uid] = (keyid, self.keys[x]["name"])
1558 self.keys[x]["uid"] = uid
1561 uid = format % "invalid-uid"
1562 keyid = get_or_set_uid(uid, session).uid_id
1563 byuid[keyid] = (uid, "ungeneratable user id")
1564 byname[uid] = (keyid, "ungeneratable user id")
1566 return (byname, byuid)
1568 __all__.append('Keyring')
1571 def get_keyring(keyring, session=None):
1573 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1574 If C{keyring} already has an entry, simply return the existing Keyring
1576 @type keyring: string
1577 @param keyring: the keyring name
1580 @return: the Keyring object for this keyring
1583 q = session.query(Keyring).filter_by(keyring_name=keyring)
1587 except NoResultFound:
1590 __all__.append('get_keyring')
1592 ################################################################################
1594 class KeyringACLMap(object):
1595 def __init__(self, *args, **kwargs):
1599 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1601 __all__.append('KeyringACLMap')
1603 ################################################################################
1605 class DBChange(object):
1606 def __init__(self, *args, **kwargs):
1610 return '<DBChange %s>' % self.changesname
1612 def clean_from_queue(self):
1613 session = DBConn().session().object_session(self)
1615 # Remove changes_pool_files entries
1618 # Remove changes_pending_files references
1621 # Clear out of queue
1622 self.in_queue = None
1623 self.approved_for_id = None
1625 __all__.append('DBChange')
1628 def get_dbchange(filename, session=None):
1630 returns DBChange object for given C{filename}.
1632 @type filename: string
1633 @param filename: the name of the file
1635 @type session: Session
1636 @param session: Optional SQLA session object (a temporary one will be
1637 generated if not supplied)
1640 @return: DBChange object for the given filename (C{None} if not present)
1643 q = session.query(DBChange).filter_by(changesname=filename)
1647 except NoResultFound:
1650 __all__.append('get_dbchange')
1652 ################################################################################
1654 class Location(ORMObject):
1655 def __init__(self, path = None, component = None):
1657 self.component = component
1658 # the column 'type' should go away, see comment at mapper
1659 self.archive_type = 'pool'
1661 def properties(self):
1662 return ['path', 'location_id', 'archive_type', 'component', \
1665 def not_null_constraints(self):
1666 return ['path', 'archive_type']
1668 __all__.append('Location')
1671 def get_location(location, component=None, archive=None, session=None):
1673 Returns Location object for the given combination of location, component
1676 @type location: string
1677 @param location: the path of the location, e.g. I{/srv/ftp-master.debian.org/ftp/pool/}
1679 @type component: string
1680 @param component: the component name (if None, no restriction applied)
1682 @type archive: string
1683 @param archive: the archive name (if None, no restriction applied)
1685 @rtype: Location / None
1686 @return: Either a Location object or None if one can't be found
1689 q = session.query(Location).filter_by(path=location)
1691 if archive is not None:
1692 q = q.join(Archive).filter_by(archive_name=archive)
1694 if component is not None:
1695 q = q.join(Component).filter_by(component_name=component)
1699 except NoResultFound:
1702 __all__.append('get_location')
1704 ################################################################################
1706 class Maintainer(ORMObject):
1707 def __init__(self, name = None):
1710 def properties(self):
1711 return ['name', 'maintainer_id']
1713 def not_null_constraints(self):
1716 def get_split_maintainer(self):
1717 if not hasattr(self, 'name') or self.name is None:
1718 return ('', '', '', '')
1720 return fix_maintainer(self.name.strip())
1722 __all__.append('Maintainer')
1725 def get_or_set_maintainer(name, session=None):
1727 Returns Maintainer object for given maintainer name.
1729 If no matching maintainer name is found, a row is inserted.
1732 @param name: The maintainer name to add
1734 @type session: SQLAlchemy
1735 @param session: Optional SQL session object (a temporary one will be
1736 generated if not supplied). If not passed, a commit will be performed at
1737 the end of the function, otherwise the caller is responsible for commiting.
1738 A flush will be performed either way.
1741 @return: the Maintainer object for the given maintainer
1744 q = session.query(Maintainer).filter_by(name=name)
1747 except NoResultFound:
1748 maintainer = Maintainer()
1749 maintainer.name = name
1750 session.add(maintainer)
1751 session.commit_or_flush()
1756 __all__.append('get_or_set_maintainer')
1759 def get_maintainer(maintainer_id, session=None):
1761 Return the name of the maintainer behind C{maintainer_id} or None if that
1762 maintainer_id is invalid.
1764 @type maintainer_id: int
1765 @param maintainer_id: the id of the maintainer
1768 @return: the Maintainer with this C{maintainer_id}
1771 return session.query(Maintainer).get(maintainer_id)
1773 __all__.append('get_maintainer')
1775 ################################################################################
1777 class NewComment(object):
1778 def __init__(self, *args, **kwargs):
1782 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1784 __all__.append('NewComment')
1787 def has_new_comment(package, version, session=None):
1789 Returns true if the given combination of C{package}, C{version} has a comment.
1791 @type package: string
1792 @param package: name of the package
1794 @type version: string
1795 @param version: package version
1797 @type session: Session
1798 @param session: Optional SQLA session object (a temporary one will be
1799 generated if not supplied)
1805 q = session.query(NewComment)
1806 q = q.filter_by(package=package)
1807 q = q.filter_by(version=version)
1809 return bool(q.count() > 0)
1811 __all__.append('has_new_comment')
1814 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1816 Returns (possibly empty) list of NewComment objects for the given
1819 @type package: string (optional)
1820 @param package: name of the package
1822 @type version: string (optional)
1823 @param version: package version
1825 @type comment_id: int (optional)
1826 @param comment_id: An id of a comment
1828 @type session: Session
1829 @param session: Optional SQLA session object (a temporary one will be
1830 generated if not supplied)
1833 @return: A (possibly empty) list of NewComment objects will be returned
1836 q = session.query(NewComment)
1837 if package is not None: q = q.filter_by(package=package)
1838 if version is not None: q = q.filter_by(version=version)
1839 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1843 __all__.append('get_new_comments')
1845 ################################################################################
1847 class Override(ORMObject):
1848 def __init__(self, package = None, suite = None, component = None, overridetype = None, \
1849 section = None, priority = None):
1850 self.package = package
1852 self.component = component
1853 self.overridetype = overridetype
1854 self.section = section
1855 self.priority = priority
1857 def properties(self):
1858 return ['package', 'suite', 'component', 'overridetype', 'section', \
1861 def not_null_constraints(self):
1862 return ['package', 'suite', 'component', 'overridetype', 'section']
1864 __all__.append('Override')
1867 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1869 Returns Override object for the given parameters
1871 @type package: string
1872 @param package: The name of the package
1874 @type suite: string, list or None
1875 @param suite: The name of the suite (or suites if a list) to limit to. If
1876 None, don't limit. Defaults to None.
1878 @type component: string, list or None
1879 @param component: The name of the component (or components if a list) to
1880 limit to. If None, don't limit. Defaults to None.
1882 @type overridetype: string, list or None
1883 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1884 limit to. If None, don't limit. Defaults to None.
1886 @type session: Session
1887 @param session: Optional SQLA session object (a temporary one will be
1888 generated if not supplied)
1891 @return: A (possibly empty) list of Override objects will be returned
1894 q = session.query(Override)
1895 q = q.filter_by(package=package)
1897 if suite is not None:
1898 if not isinstance(suite, list): suite = [suite]
1899 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1901 if component is not None:
1902 if not isinstance(component, list): component = [component]
1903 q = q.join(Component).filter(Component.component_name.in_(component))
1905 if overridetype is not None:
1906 if not isinstance(overridetype, list): overridetype = [overridetype]
1907 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1911 __all__.append('get_override')
1914 ################################################################################
1916 class OverrideType(ORMObject):
1917 def __init__(self, overridetype = None):
1918 self.overridetype = overridetype
1920 def properties(self):
1921 return ['overridetype', 'overridetype_id', 'overrides_count']
1923 def not_null_constraints(self):
1924 return ['overridetype']
1926 __all__.append('OverrideType')
1929 def get_override_type(override_type, session=None):
1931 Returns OverrideType object for given C{override type}.
1933 @type override_type: string
1934 @param override_type: The name of the override type
1936 @type session: Session
1937 @param session: Optional SQLA session object (a temporary one will be
1938 generated if not supplied)
1941 @return: the database id for the given override type
1944 q = session.query(OverrideType).filter_by(overridetype=override_type)
1948 except NoResultFound:
1951 __all__.append('get_override_type')
1953 ################################################################################
1955 class DebContents(object):
1956 def __init__(self, *args, **kwargs):
1960 return '<DebConetnts %s: %s>' % (self.package.package,self.file)
1962 __all__.append('DebContents')
1965 class UdebContents(object):
1966 def __init__(self, *args, **kwargs):
1970 return '<UdebConetnts %s: %s>' % (self.package.package,self.file)
1972 __all__.append('UdebContents')
1974 class PendingBinContents(object):
1975 def __init__(self, *args, **kwargs):
1979 return '<PendingBinContents %s>' % self.contents_id
1981 __all__.append('PendingBinContents')
1983 def insert_pending_content_paths(package,
1988 Make sure given paths are temporarily associated with given
1992 @param package: the package to associate with should have been read in from the binary control file
1993 @type fullpaths: list
1994 @param fullpaths: the list of paths of the file being associated with the binary
1995 @type session: SQLAlchemy session
1996 @param session: Optional SQLAlchemy session. If this is passed, the caller
1997 is responsible for ensuring a transaction has begun and committing the
1998 results or rolling back based on the result code. If not passed, a commit
1999 will be performed at the end of the function
2001 @return: True upon success, False if there is a problem
2004 privatetrans = False
2007 session = DBConn().session()
2011 arch = get_architecture(package['Architecture'], session)
2012 arch_id = arch.arch_id
2014 # Remove any already existing recorded files for this package
2015 q = session.query(PendingBinContents)
2016 q = q.filter_by(package=package['Package'])
2017 q = q.filter_by(version=package['Version'])
2018 q = q.filter_by(architecture=arch_id)
2021 for fullpath in fullpaths:
2023 if fullpath.startswith( "./" ):
2024 fullpath = fullpath[2:]
2026 pca = PendingBinContents()
2027 pca.package = package['Package']
2028 pca.version = package['Version']
2030 pca.architecture = arch_id
2033 pca.type = 8 # gross
2035 pca.type = 7 # also gross
2038 # Only commit if we set up the session ourself
2046 except Exception, e:
2047 traceback.print_exc()
2049 # Only rollback if we set up the session ourself
2056 __all__.append('insert_pending_content_paths')
2058 ################################################################################
2060 class PolicyQueue(object):
2061 def __init__(self, *args, **kwargs):
2065 return '<PolicyQueue %s>' % self.queue_name
2067 __all__.append('PolicyQueue')
2070 def get_policy_queue(queuename, session=None):
2072 Returns PolicyQueue object for given C{queue name}
2074 @type queuename: string
2075 @param queuename: The name of the queue
2077 @type session: Session
2078 @param session: Optional SQLA session object (a temporary one will be
2079 generated if not supplied)
2082 @return: PolicyQueue object for the given queue
2085 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
2089 except NoResultFound:
2092 __all__.append('get_policy_queue')
2095 def get_policy_queue_from_path(pathname, session=None):
2097 Returns PolicyQueue object for given C{path name}
2099 @type queuename: string
2100 @param queuename: The path
2102 @type session: Session
2103 @param session: Optional SQLA session object (a temporary one will be
2104 generated if not supplied)
2107 @return: PolicyQueue object for the given queue
2110 q = session.query(PolicyQueue).filter_by(path=pathname)
2114 except NoResultFound:
2117 __all__.append('get_policy_queue_from_path')
2119 ################################################################################
2121 class Priority(ORMObject):
2122 def __init__(self, priority = None, level = None):
2123 self.priority = priority
2126 def properties(self):
2127 return ['priority', 'priority_id', 'level', 'overrides_count']
2129 def not_null_constraints(self):
2130 return ['priority', 'level']
2132 def __eq__(self, val):
2133 if isinstance(val, str):
2134 return (self.priority == val)
2135 # This signals to use the normal comparison operator
2136 return NotImplemented
2138 def __ne__(self, val):
2139 if isinstance(val, str):
2140 return (self.priority != val)
2141 # This signals to use the normal comparison operator
2142 return NotImplemented
2144 __all__.append('Priority')
2147 def get_priority(priority, session=None):
2149 Returns Priority object for given C{priority name}.
2151 @type priority: string
2152 @param priority: The name of the priority
2154 @type session: Session
2155 @param session: Optional SQLA session object (a temporary one will be
2156 generated if not supplied)
2159 @return: Priority object for the given priority
2162 q = session.query(Priority).filter_by(priority=priority)
2166 except NoResultFound:
2169 __all__.append('get_priority')
2172 def get_priorities(session=None):
2174 Returns dictionary of priority names -> id mappings
2176 @type session: Session
2177 @param session: Optional SQL session object (a temporary one will be
2178 generated if not supplied)
2181 @return: dictionary of priority names -> id mappings
2185 q = session.query(Priority)
2187 ret[x.priority] = x.priority_id
2191 __all__.append('get_priorities')
2193 ################################################################################
2195 class Section(ORMObject):
2196 def __init__(self, section = None):
2197 self.section = section
2199 def properties(self):
2200 return ['section', 'section_id', 'overrides_count']
2202 def not_null_constraints(self):
2205 def __eq__(self, val):
2206 if isinstance(val, str):
2207 return (self.section == val)
2208 # This signals to use the normal comparison operator
2209 return NotImplemented
2211 def __ne__(self, val):
2212 if isinstance(val, str):
2213 return (self.section != val)
2214 # This signals to use the normal comparison operator
2215 return NotImplemented
2217 __all__.append('Section')
2220 def get_section(section, session=None):
2222 Returns Section object for given C{section name}.
2224 @type section: string
2225 @param section: The name of the section
2227 @type session: Session
2228 @param session: Optional SQLA session object (a temporary one will be
2229 generated if not supplied)
2232 @return: Section object for the given section name
2235 q = session.query(Section).filter_by(section=section)
2239 except NoResultFound:
2242 __all__.append('get_section')
2245 def get_sections(session=None):
2247 Returns dictionary of section names -> id mappings
2249 @type session: Session
2250 @param session: Optional SQL session object (a temporary one will be
2251 generated if not supplied)
2254 @return: dictionary of section names -> id mappings
2258 q = session.query(Section)
2260 ret[x.section] = x.section_id
2264 __all__.append('get_sections')
2266 ################################################################################
2268 class DBSource(ORMObject):
2269 def __init__(self, source = None, version = None, maintainer = None, \
2270 changedby = None, poolfile = None, install_date = None):
2271 self.source = source
2272 self.version = version
2273 self.maintainer = maintainer
2274 self.changedby = changedby
2275 self.poolfile = poolfile
2276 self.install_date = install_date
2278 def properties(self):
2279 return ['source', 'source_id', 'maintainer', 'changedby', \
2280 'fingerprint', 'poolfile', 'version', 'suites_count', \
2281 'install_date', 'binaries_count']
2283 def not_null_constraints(self):
2284 return ['source', 'version', 'install_date', 'maintainer', \
2285 'changedby', 'poolfile', 'install_date']
2287 __all__.append('DBSource')
2290 def source_exists(source, source_version, suites = ["any"], session=None):
2292 Ensure that source exists somewhere in the archive for the binary
2293 upload being processed.
2294 1. exact match => 1.0-3
2295 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
2297 @type source: string
2298 @param source: source name
2300 @type source_version: string
2301 @param source_version: expected source version
2304 @param suites: list of suites to check in, default I{any}
2306 @type session: Session
2307 @param session: Optional SQLA session object (a temporary one will be
2308 generated if not supplied)
2311 @return: returns 1 if a source with expected version is found, otherwise 0
2318 from daklib.regexes import re_bin_only_nmu
2319 orig_source_version = re_bin_only_nmu.sub('', source_version)
2321 for suite in suites:
2322 q = session.query(DBSource).filter_by(source=source). \
2323 filter(DBSource.version.in_([source_version, orig_source_version]))
2325 # source must exist in suite X, or in some other suite that's
2326 # mapped to X, recursively... silent-maps are counted too,
2327 # unreleased-maps aren't.
2328 maps = cnf.ValueList("SuiteMappings")[:]
2330 maps = [ m.split() for m in maps ]
2331 maps = [ (x[1], x[2]) for x in maps
2332 if x[0] == "map" or x[0] == "silent-map" ]
2334 for (from_, to) in maps:
2335 if from_ in s and to not in s:
2338 q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
2343 # No source found so return not ok
2348 __all__.append('source_exists')
2351 def get_suites_source_in(source, session=None):
2353 Returns list of Suite objects which given C{source} name is in
2356 @param source: DBSource package name to search for
2359 @return: list of Suite objects for the given source
2362 return session.query(Suite).filter(Suite.sources.any(source=source)).all()
2364 __all__.append('get_suites_source_in')
2367 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
2369 Returns list of DBSource objects for given C{source} name and other parameters
2372 @param source: DBSource package name to search for
2374 @type version: str or None
2375 @param version: DBSource version name to search for or None if not applicable
2377 @type dm_upload_allowed: bool
2378 @param dm_upload_allowed: If None, no effect. If True or False, only
2379 return packages with that dm_upload_allowed setting
2381 @type session: Session
2382 @param session: Optional SQL session object (a temporary one will be
2383 generated if not supplied)
2386 @return: list of DBSource objects for the given name (may be empty)
2389 q = session.query(DBSource).filter_by(source=source)
2391 if version is not None:
2392 q = q.filter_by(version=version)
2394 if dm_upload_allowed is not None:
2395 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
2399 __all__.append('get_sources_from_name')
2401 # FIXME: This function fails badly if it finds more than 1 source package and
2402 # its implementation is trivial enough to be inlined.
2404 def get_source_in_suite(source, suite, session=None):
2406 Returns a DBSource object for a combination of C{source} and C{suite}.
2408 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2409 - B{suite} - a suite name, eg. I{unstable}
2411 @type source: string
2412 @param source: source package name
2415 @param suite: the suite name
2418 @return: the version for I{source} in I{suite}
2422 q = get_suite(suite, session).get_sources(source)
2425 except NoResultFound:
2428 __all__.append('get_source_in_suite')
2430 ################################################################################
2433 def add_dsc_to_db(u, filename, session=None):
2434 entry = u.pkg.files[filename]
2438 source.source = u.pkg.dsc["source"]
2439 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2440 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2441 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2442 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2443 source.install_date = datetime.now().date()
2445 dsc_component = entry["component"]
2446 dsc_location_id = entry["location id"]
2448 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2450 # Set up a new poolfile if necessary
2451 if not entry.has_key("files id") or not entry["files id"]:
2452 filename = entry["pool name"] + filename
2453 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2455 pfs.append(poolfile)
2456 entry["files id"] = poolfile.file_id
2458 source.poolfile_id = entry["files id"]
2461 suite_names = u.pkg.changes["distribution"].keys()
2462 source.suites = session.query(Suite). \
2463 filter(Suite.suite_name.in_(suite_names)).all()
2465 # Add the source files to the DB (files and dsc_files)
2467 dscfile.source_id = source.source_id
2468 dscfile.poolfile_id = entry["files id"]
2469 session.add(dscfile)
2471 for dsc_file, dentry in u.pkg.dsc_files.items():
2473 df.source_id = source.source_id
2475 # If the .orig tarball is already in the pool, it's
2476 # files id is stored in dsc_files by check_dsc().
2477 files_id = dentry.get("files id", None)
2479 # Find the entry in the files hash
2480 # TODO: Bail out here properly
2482 for f, e in u.pkg.files.items():
2487 if files_id is None:
2488 filename = dfentry["pool name"] + dsc_file
2490 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2491 # FIXME: needs to check for -1/-2 and or handle exception
2492 if found and obj is not None:
2493 files_id = obj.file_id
2496 # If still not found, add it
2497 if files_id is None:
2498 # HACK: Force sha1sum etc into dentry
2499 dentry["sha1sum"] = dfentry["sha1sum"]
2500 dentry["sha256sum"] = dfentry["sha256sum"]
2501 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2502 pfs.append(poolfile)
2503 files_id = poolfile.file_id
2505 poolfile = get_poolfile_by_id(files_id, session)
2506 if poolfile is None:
2507 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2508 pfs.append(poolfile)
2510 df.poolfile_id = files_id
2513 # Add the src_uploaders to the DB
2514 uploader_ids = [source.maintainer_id]
2515 if u.pkg.dsc.has_key("uploaders"):
2516 for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
2518 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2521 for up_id in uploader_ids:
2522 if added_ids.has_key(up_id):
2524 utils.warn("Already saw uploader %s for source %s" % (up_id, source.source))
2530 su.maintainer_id = up_id
2531 su.source_id = source.source_id
2536 return source, dsc_component, dsc_location_id, pfs
2538 __all__.append('add_dsc_to_db')
2541 def add_deb_to_db(u, filename, session=None):
2543 Contrary to what you might expect, this routine deals with both
2544 debs and udebs. That info is in 'dbtype', whilst 'type' is
2545 'deb' for both of them
2548 entry = u.pkg.files[filename]
2551 bin.package = entry["package"]
2552 bin.version = entry["version"]
2553 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2554 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2555 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2556 bin.binarytype = entry["dbtype"]
2559 filename = entry["pool name"] + filename
2560 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2561 if not entry.get("location id", None):
2562 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2564 if entry.get("files id", None):
2565 poolfile = get_poolfile_by_id(bin.poolfile_id)
2566 bin.poolfile_id = entry["files id"]
2568 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2569 bin.poolfile_id = entry["files id"] = poolfile.file_id
2572 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2573 if len(bin_sources) != 1:
2574 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2575 (bin.package, bin.version, entry["architecture"],
2576 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2578 bin.source_id = bin_sources[0].source_id
2580 # Add and flush object so it has an ID
2583 suite_names = u.pkg.changes["distribution"].keys()
2584 bin.suites = session.query(Suite). \
2585 filter(Suite.suite_name.in_(suite_names)).all()
2589 # Deal with contents - disabled for now
2590 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2592 # print "REJECT\nCould not determine contents of package %s" % bin.package
2593 # session.rollback()
2594 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2598 __all__.append('add_deb_to_db')
2600 ################################################################################
2602 class SourceACL(object):
2603 def __init__(self, *args, **kwargs):
2607 return '<SourceACL %s>' % self.source_acl_id
2609 __all__.append('SourceACL')
2611 ################################################################################
2613 class SrcFormat(object):
2614 def __init__(self, *args, **kwargs):
2618 return '<SrcFormat %s>' % (self.format_name)
2620 __all__.append('SrcFormat')
2622 ################################################################################
2624 class SrcUploader(object):
2625 def __init__(self, *args, **kwargs):
2629 return '<SrcUploader %s>' % self.uploader_id
2631 __all__.append('SrcUploader')
2633 ################################################################################
2635 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2636 ('SuiteID', 'suite_id'),
2637 ('Version', 'version'),
2638 ('Origin', 'origin'),
2640 ('Description', 'description'),
2641 ('Untouchable', 'untouchable'),
2642 ('Announce', 'announce'),
2643 ('Codename', 'codename'),
2644 ('OverrideCodename', 'overridecodename'),
2645 ('ValidTime', 'validtime'),
2646 ('Priority', 'priority'),
2647 ('NotAutomatic', 'notautomatic'),
2648 ('CopyChanges', 'copychanges'),
2649 ('OverrideSuite', 'overridesuite')]
2651 # Why the heck don't we have any UNIQUE constraints in table suite?
2652 # TODO: Add UNIQUE constraints for appropriate columns.
2653 class Suite(ORMObject):
2654 def __init__(self, suite_name = None, version = None):
2655 self.suite_name = suite_name
2656 self.version = version
2658 def properties(self):
2659 return ['suite_name', 'version', 'sources_count', 'binaries_count', \
2662 def not_null_constraints(self):
2663 return ['suite_name', 'version']
2665 def __eq__(self, val):
2666 if isinstance(val, str):
2667 return (self.suite_name == val)
2668 # This signals to use the normal comparison operator
2669 return NotImplemented
2671 def __ne__(self, val):
2672 if isinstance(val, str):
2673 return (self.suite_name != val)
2674 # This signals to use the normal comparison operator
2675 return NotImplemented
2679 for disp, field in SUITE_FIELDS:
2680 val = getattr(self, field, None)
2682 ret.append("%s: %s" % (disp, val))
2684 return "\n".join(ret)
2686 def get_architectures(self, skipsrc=False, skipall=False):
2688 Returns list of Architecture objects
2690 @type skipsrc: boolean
2691 @param skipsrc: Whether to skip returning the 'source' architecture entry
2694 @type skipall: boolean
2695 @param skipall: Whether to skip returning the 'all' architecture entry
2699 @return: list of Architecture objects for the given name (may be empty)
2702 q = object_session(self).query(Architecture).with_parent(self)
2704 q = q.filter(Architecture.arch_string != 'source')
2706 q = q.filter(Architecture.arch_string != 'all')
2707 return q.order_by(Architecture.arch_string).all()
2709 def get_sources(self, source):
2711 Returns a query object representing DBSource that is part of C{suite}.
2713 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2715 @type source: string
2716 @param source: source package name
2718 @rtype: sqlalchemy.orm.query.Query
2719 @return: a query of DBSource
2723 session = object_session(self)
2724 return session.query(DBSource).filter_by(source = source). \
2727 __all__.append('Suite')
2730 def get_suite(suite, session=None):
2732 Returns Suite object for given C{suite name}.
2735 @param suite: The name of the suite
2737 @type session: Session
2738 @param session: Optional SQLA session object (a temporary one will be
2739 generated if not supplied)
2742 @return: Suite object for the requested suite name (None if not present)
2745 q = session.query(Suite).filter_by(suite_name=suite)
2749 except NoResultFound:
2752 __all__.append('get_suite')
2754 ################################################################################
2756 # TODO: should be removed because the implementation is too trivial
2758 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2760 Returns list of Architecture objects for given C{suite} name
2763 @param suite: Suite name to search for
2765 @type skipsrc: boolean
2766 @param skipsrc: Whether to skip returning the 'source' architecture entry
2769 @type skipall: boolean
2770 @param skipall: Whether to skip returning the 'all' architecture entry
2773 @type session: Session
2774 @param session: Optional SQL session object (a temporary one will be
2775 generated if not supplied)
2778 @return: list of Architecture objects for the given name (may be empty)
2781 return get_suite(suite, session).get_architectures(skipsrc, skipall)
2783 __all__.append('get_suite_architectures')
2785 ################################################################################
2787 class SuiteSrcFormat(object):
2788 def __init__(self, *args, **kwargs):
2792 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2794 __all__.append('SuiteSrcFormat')
2797 def get_suite_src_formats(suite, session=None):
2799 Returns list of allowed SrcFormat for C{suite}.
2802 @param suite: Suite name to search for
2804 @type session: Session
2805 @param session: Optional SQL session object (a temporary one will be
2806 generated if not supplied)
2809 @return: the list of allowed source formats for I{suite}
2812 q = session.query(SrcFormat)
2813 q = q.join(SuiteSrcFormat)
2814 q = q.join(Suite).filter_by(suite_name=suite)
2815 q = q.order_by('format_name')
2819 __all__.append('get_suite_src_formats')
2821 ################################################################################
2823 class Uid(ORMObject):
2824 def __init__(self, uid = None, name = None):
2828 def __eq__(self, val):
2829 if isinstance(val, str):
2830 return (self.uid == val)
2831 # This signals to use the normal comparison operator
2832 return NotImplemented
2834 def __ne__(self, val):
2835 if isinstance(val, str):
2836 return (self.uid != val)
2837 # This signals to use the normal comparison operator
2838 return NotImplemented
2840 def properties(self):
2841 return ['uid', 'name', 'fingerprint']
2843 def not_null_constraints(self):
2846 __all__.append('Uid')
2849 def get_or_set_uid(uidname, session=None):
2851 Returns uid object for given uidname.
2853 If no matching uidname is found, a row is inserted.
2855 @type uidname: string
2856 @param uidname: The uid to add
2858 @type session: SQLAlchemy
2859 @param session: Optional SQL session object (a temporary one will be
2860 generated if not supplied). If not passed, a commit will be performed at
2861 the end of the function, otherwise the caller is responsible for commiting.
2864 @return: the uid object for the given uidname
2867 q = session.query(Uid).filter_by(uid=uidname)
2871 except NoResultFound:
2875 session.commit_or_flush()
2880 __all__.append('get_or_set_uid')
2883 def get_uid_from_fingerprint(fpr, session=None):
2884 q = session.query(Uid)
2885 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2889 except NoResultFound:
2892 __all__.append('get_uid_from_fingerprint')
2894 ################################################################################
2896 class UploadBlock(object):
2897 def __init__(self, *args, **kwargs):
2901 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2903 __all__.append('UploadBlock')
2905 ################################################################################
2907 class DBConn(object):
2909 database module init.
2913 def __init__(self, *args, **kwargs):
2914 self.__dict__ = self.__shared_state
2916 if not getattr(self, 'initialised', False):
2917 self.initialised = True
2918 self.debug = kwargs.has_key('debug')
2921 def __setuptables(self):
2922 tables_with_primary = (
2930 'build_queue_files',
2935 'changes_pending_binaries',
2936 'changes_pending_files',
2937 'changes_pending_source',
2947 'pending_bin_contents',
2961 tables_no_primary = (
2962 'changes_pending_files_map',
2963 'changes_pending_source_files',
2964 'changes_pool_files',
2966 # TODO: the maintainer column in table override should be removed.
2968 'suite_architectures',
2969 'suite_src_formats',
2970 'suite_build_queue_copy',
2975 'almost_obsolete_all_associations',
2976 'almost_obsolete_src_associations',
2977 'any_associations_source',
2978 'bin_assoc_by_arch',
2979 'bin_associations_binaries',
2980 'binaries_suite_arch',
2981 'binfiles_suite_component_arch',
2984 'newest_all_associations',
2985 'newest_any_associations',
2987 'newest_src_association',
2988 'obsolete_all_associations',
2989 'obsolete_any_associations',
2990 'obsolete_any_by_all_associations',
2991 'obsolete_src_associations',
2993 'src_associations_bin',
2994 'src_associations_src',
2995 'suite_arch_by_name',
2998 # Sqlalchemy version 0.5 fails to reflect the SERIAL type
2999 # correctly and that is why we have to use a workaround. It can
3000 # be removed as soon as we switch to version 0.6.
3001 for table_name in tables_with_primary:
3002 table = Table(table_name, self.db_meta, \
3003 Column('id', Integer, primary_key = True), \
3004 autoload=True, useexisting=True)
3005 setattr(self, 'tbl_%s' % table_name, table)
3007 for table_name in tables_no_primary:
3008 table = Table(table_name, self.db_meta, autoload=True)
3009 setattr(self, 'tbl_%s' % table_name, table)
3011 # bin_contents needs special attention until update #41 has been
3013 self.tbl_bin_contents = Table('bin_contents', self.db_meta, \
3014 Column('file', Text, primary_key = True),
3015 Column('binary_id', Integer, ForeignKey('binaries.id'), \
3016 primary_key = True),
3017 autoload=True, useexisting=True)
3019 for view_name in views:
3020 view = Table(view_name, self.db_meta, autoload=True)
3021 setattr(self, 'view_%s' % view_name, view)
3023 def __setupmappers(self):
3024 mapper(Architecture, self.tbl_architecture,
3025 properties = dict(arch_id = self.tbl_architecture.c.id,
3026 suites = relation(Suite, secondary=self.tbl_suite_architectures,
3027 order_by='suite_name',
3028 backref=backref('architectures', order_by='arch_string'))),
3029 extension = validator)
3031 mapper(Archive, self.tbl_archive,
3032 properties = dict(archive_id = self.tbl_archive.c.id,
3033 archive_name = self.tbl_archive.c.name))
3035 mapper(PendingBinContents, self.tbl_pending_bin_contents,
3036 properties = dict(contents_id =self.tbl_pending_bin_contents.c.id,
3037 filename = self.tbl_pending_bin_contents.c.filename,
3038 package = self.tbl_pending_bin_contents.c.package,
3039 version = self.tbl_pending_bin_contents.c.version,
3040 arch = self.tbl_pending_bin_contents.c.arch,
3041 otype = self.tbl_pending_bin_contents.c.type))
3043 mapper(DebContents, self.tbl_deb_contents,
3044 properties = dict(binary_id=self.tbl_deb_contents.c.binary_id,
3045 package=self.tbl_deb_contents.c.package,
3046 suite=self.tbl_deb_contents.c.suite,
3047 arch=self.tbl_deb_contents.c.arch,
3048 section=self.tbl_deb_contents.c.section,
3049 filename=self.tbl_deb_contents.c.filename))
3051 mapper(UdebContents, self.tbl_udeb_contents,
3052 properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id,
3053 package=self.tbl_udeb_contents.c.package,
3054 suite=self.tbl_udeb_contents.c.suite,
3055 arch=self.tbl_udeb_contents.c.arch,
3056 section=self.tbl_udeb_contents.c.section,
3057 filename=self.tbl_udeb_contents.c.filename))
3059 mapper(BuildQueue, self.tbl_build_queue,
3060 properties = dict(queue_id = self.tbl_build_queue.c.id))
3062 mapper(BuildQueueFile, self.tbl_build_queue_files,
3063 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
3064 poolfile = relation(PoolFile, backref='buildqueueinstances')))
3066 mapper(DBBinary, self.tbl_binaries,
3067 properties = dict(binary_id = self.tbl_binaries.c.id,
3068 package = self.tbl_binaries.c.package,
3069 version = self.tbl_binaries.c.version,
3070 maintainer_id = self.tbl_binaries.c.maintainer,
3071 maintainer = relation(Maintainer),
3072 source_id = self.tbl_binaries.c.source,
3073 source = relation(DBSource, backref='binaries'),
3074 arch_id = self.tbl_binaries.c.architecture,
3075 architecture = relation(Architecture),
3076 poolfile_id = self.tbl_binaries.c.file,
3077 poolfile = relation(PoolFile, backref=backref('binary', uselist = False)),
3078 binarytype = self.tbl_binaries.c.type,
3079 fingerprint_id = self.tbl_binaries.c.sig_fpr,
3080 fingerprint = relation(Fingerprint),
3081 install_date = self.tbl_binaries.c.install_date,
3082 suites = relation(Suite, secondary=self.tbl_bin_associations,
3083 backref=backref('binaries', lazy='dynamic'))),
3084 extension = validator)
3086 mapper(BinaryACL, self.tbl_binary_acl,
3087 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
3089 mapper(BinaryACLMap, self.tbl_binary_acl_map,
3090 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
3091 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
3092 architecture = relation(Architecture)))
3094 mapper(Component, self.tbl_component,
3095 properties = dict(component_id = self.tbl_component.c.id,
3096 component_name = self.tbl_component.c.name),
3097 extension = validator)
3099 mapper(DBConfig, self.tbl_config,
3100 properties = dict(config_id = self.tbl_config.c.id))
3102 mapper(DSCFile, self.tbl_dsc_files,
3103 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
3104 source_id = self.tbl_dsc_files.c.source,
3105 source = relation(DBSource),
3106 poolfile_id = self.tbl_dsc_files.c.file,
3107 poolfile = relation(PoolFile)))
3109 mapper(PoolFile, self.tbl_files,
3110 properties = dict(file_id = self.tbl_files.c.id,
3111 filesize = self.tbl_files.c.size,
3112 location_id = self.tbl_files.c.location,
3113 location = relation(Location,
3114 # using lazy='dynamic' in the back
3115 # reference because we have A LOT of
3116 # files in one location
3117 backref=backref('files', lazy='dynamic'))),
3118 extension = validator)
3120 mapper(Fingerprint, self.tbl_fingerprint,
3121 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
3122 uid_id = self.tbl_fingerprint.c.uid,
3123 uid = relation(Uid),
3124 keyring_id = self.tbl_fingerprint.c.keyring,
3125 keyring = relation(Keyring),
3126 source_acl = relation(SourceACL),
3127 binary_acl = relation(BinaryACL)),
3128 extension = validator)
3130 mapper(Keyring, self.tbl_keyrings,
3131 properties = dict(keyring_name = self.tbl_keyrings.c.name,
3132 keyring_id = self.tbl_keyrings.c.id))
3134 mapper(DBChange, self.tbl_changes,
3135 properties = dict(change_id = self.tbl_changes.c.id,
3136 poolfiles = relation(PoolFile,
3137 secondary=self.tbl_changes_pool_files,
3138 backref="changeslinks"),
3139 seen = self.tbl_changes.c.seen,
3140 source = self.tbl_changes.c.source,
3141 binaries = self.tbl_changes.c.binaries,
3142 architecture = self.tbl_changes.c.architecture,
3143 distribution = self.tbl_changes.c.distribution,
3144 urgency = self.tbl_changes.c.urgency,
3145 maintainer = self.tbl_changes.c.maintainer,
3146 changedby = self.tbl_changes.c.changedby,
3147 date = self.tbl_changes.c.date,
3148 version = self.tbl_changes.c.version,
3149 files = relation(ChangePendingFile,
3150 secondary=self.tbl_changes_pending_files_map,
3151 backref="changesfile"),
3152 in_queue_id = self.tbl_changes.c.in_queue,
3153 in_queue = relation(PolicyQueue,
3154 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
3155 approved_for_id = self.tbl_changes.c.approved_for))
3157 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
3158 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
3160 mapper(ChangePendingFile, self.tbl_changes_pending_files,
3161 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id,
3162 filename = self.tbl_changes_pending_files.c.filename,
3163 size = self.tbl_changes_pending_files.c.size,
3164 md5sum = self.tbl_changes_pending_files.c.md5sum,
3165 sha1sum = self.tbl_changes_pending_files.c.sha1sum,
3166 sha256sum = self.tbl_changes_pending_files.c.sha256sum))
3168 mapper(ChangePendingSource, self.tbl_changes_pending_source,
3169 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
3170 change = relation(DBChange),
3171 maintainer = relation(Maintainer,
3172 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
3173 changedby = relation(Maintainer,
3174 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
3175 fingerprint = relation(Fingerprint),
3176 source_files = relation(ChangePendingFile,
3177 secondary=self.tbl_changes_pending_source_files,
3178 backref="pending_sources")))
3181 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
3182 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
3183 keyring = relation(Keyring, backref="keyring_acl_map"),
3184 architecture = relation(Architecture)))
3186 mapper(Location, self.tbl_location,
3187 properties = dict(location_id = self.tbl_location.c.id,
3188 component_id = self.tbl_location.c.component,
3189 component = relation(Component, backref='location'),
3190 archive_id = self.tbl_location.c.archive,
3191 archive = relation(Archive),
3192 # FIXME: the 'type' column is old cruft and
3193 # should be removed in the future.
3194 archive_type = self.tbl_location.c.type),
3195 extension = validator)
3197 mapper(Maintainer, self.tbl_maintainer,
3198 properties = dict(maintainer_id = self.tbl_maintainer.c.id,
3199 maintains_sources = relation(DBSource, backref='maintainer',
3200 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.maintainer)),
3201 changed_sources = relation(DBSource, backref='changedby',
3202 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.changedby))),
3203 extension = validator)
3205 mapper(NewComment, self.tbl_new_comments,
3206 properties = dict(comment_id = self.tbl_new_comments.c.id))
3208 mapper(Override, self.tbl_override,
3209 properties = dict(suite_id = self.tbl_override.c.suite,
3210 suite = relation(Suite, \
3211 backref=backref('overrides', lazy='dynamic')),
3212 package = self.tbl_override.c.package,
3213 component_id = self.tbl_override.c.component,
3214 component = relation(Component, \
3215 backref=backref('overrides', lazy='dynamic')),
3216 priority_id = self.tbl_override.c.priority,
3217 priority = relation(Priority, \
3218 backref=backref('overrides', lazy='dynamic')),
3219 section_id = self.tbl_override.c.section,
3220 section = relation(Section, \
3221 backref=backref('overrides', lazy='dynamic')),
3222 overridetype_id = self.tbl_override.c.type,
3223 overridetype = relation(OverrideType, \
3224 backref=backref('overrides', lazy='dynamic'))))
3226 mapper(OverrideType, self.tbl_override_type,
3227 properties = dict(overridetype = self.tbl_override_type.c.type,
3228 overridetype_id = self.tbl_override_type.c.id))
3230 mapper(PolicyQueue, self.tbl_policy_queue,
3231 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
3233 mapper(Priority, self.tbl_priority,
3234 properties = dict(priority_id = self.tbl_priority.c.id))
3236 mapper(Section, self.tbl_section,
3237 properties = dict(section_id = self.tbl_section.c.id,
3238 section=self.tbl_section.c.section))
3240 mapper(DBSource, self.tbl_source,
3241 properties = dict(source_id = self.tbl_source.c.id,
3242 version = self.tbl_source.c.version,
3243 maintainer_id = self.tbl_source.c.maintainer,
3244 poolfile_id = self.tbl_source.c.file,
3245 poolfile = relation(PoolFile, backref=backref('source', uselist = False)),
3246 fingerprint_id = self.tbl_source.c.sig_fpr,
3247 fingerprint = relation(Fingerprint),
3248 changedby_id = self.tbl_source.c.changedby,
3249 srcfiles = relation(DSCFile,
3250 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
3251 suites = relation(Suite, secondary=self.tbl_src_associations,
3252 backref=backref('sources', lazy='dynamic')),
3253 srcuploaders = relation(SrcUploader)),
3254 extension = validator)
3256 mapper(SourceACL, self.tbl_source_acl,
3257 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
3259 mapper(SrcFormat, self.tbl_src_format,
3260 properties = dict(src_format_id = self.tbl_src_format.c.id,
3261 format_name = self.tbl_src_format.c.format_name))
3263 mapper(SrcUploader, self.tbl_src_uploaders,
3264 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
3265 source_id = self.tbl_src_uploaders.c.source,
3266 source = relation(DBSource,
3267 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
3268 maintainer_id = self.tbl_src_uploaders.c.maintainer,
3269 maintainer = relation(Maintainer,
3270 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
3272 mapper(Suite, self.tbl_suite,
3273 properties = dict(suite_id = self.tbl_suite.c.id,
3274 policy_queue = relation(PolicyQueue),
3275 copy_queues = relation(BuildQueue,
3276 secondary=self.tbl_suite_build_queue_copy)),
3277 extension = validator)
3279 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
3280 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
3281 suite = relation(Suite, backref='suitesrcformats'),
3282 src_format_id = self.tbl_suite_src_formats.c.src_format,
3283 src_format = relation(SrcFormat)))
3285 mapper(Uid, self.tbl_uid,
3286 properties = dict(uid_id = self.tbl_uid.c.id,
3287 fingerprint = relation(Fingerprint)),
3288 extension = validator)
3290 mapper(UploadBlock, self.tbl_upload_blocks,
3291 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
3292 fingerprint = relation(Fingerprint, backref="uploadblocks"),
3293 uid = relation(Uid, backref="uploadblocks")))
3295 mapper(BinContents, self.tbl_bin_contents,
3297 binary = relation(DBBinary,
3298 backref=backref('contents', lazy='dynamic', cascade='all')),
3299 file = self.tbl_bin_contents.c.file))
3301 ## Connection functions
3302 def __createconn(self):
3303 from config import Config
3307 connstr = "postgres://%s" % cnf["DB::Host"]
3308 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
3309 connstr += ":%s" % cnf["DB::Port"]
3310 connstr += "/%s" % cnf["DB::Name"]
3313 connstr = "postgres:///%s" % cnf["DB::Name"]
3314 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
3315 connstr += "?port=%s" % cnf["DB::Port"]
3317 engine_args = { 'echo': self.debug }
3318 if cnf.has_key('DB::PoolSize'):
3319 engine_args['pool_size'] = int(cnf['DB::PoolSize'])
3320 if cnf.has_key('DB::MaxOverflow'):
3321 engine_args['max_overflow'] = int(cnf['DB::MaxOverflow'])
3322 if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \
3323 cnf['DB::Unicode'] == 'false':
3324 engine_args['use_native_unicode'] = False
3326 self.db_pg = create_engine(connstr, **engine_args)
3327 self.db_meta = MetaData()
3328 self.db_meta.bind = self.db_pg
3329 self.db_smaker = sessionmaker(bind=self.db_pg,
3333 self.__setuptables()
3334 self.__setupmappers()
3337 return self.db_smaker()
3339 __all__.append('DBConn')