5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
38 from os.path import normpath
45 from daklib.gpg import SignedFile
52 import simplejson as json
54 from datetime import datetime, timedelta
55 from errno import ENOENT
56 from tempfile import mkstemp, mkdtemp
57 from subprocess import Popen, PIPE
58 from tarfile import TarFile
60 from inspect import getargspec
63 from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \
65 from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
66 backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers
67 from sqlalchemy import types as sqltypes
68 from sqlalchemy.orm.collections import attribute_mapped_collection
69 from sqlalchemy.ext.associationproxy import association_proxy
71 # Don't remove this, we re-export the exceptions to scripts which import us
72 from sqlalchemy.exc import *
73 from sqlalchemy.orm.exc import NoResultFound
75 # Only import Config until Queue stuff is changed to store its config
77 from config import Config
78 from textutils import fix_maintainer
79 from dak_exceptions import DBUpdateError, NoSourceFieldError, FileExistsError
81 # suppress some deprecation warnings in squeeze related to sqlalchemy
83 warnings.filterwarnings('ignore', \
84 "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
86 warnings.filterwarnings('ignore', \
87 "Predicate of partial index .* ignored during reflection", \
91 ################################################################################
93 # Patch in support for the debversion field type so that it works during
97 # that is for sqlalchemy 0.6
98 UserDefinedType = sqltypes.UserDefinedType
100 # this one for sqlalchemy 0.5
101 UserDefinedType = sqltypes.TypeEngine
103 class DebVersion(UserDefinedType):
104 def get_col_spec(self):
107 def bind_processor(self, dialect):
110 # ' = None' is needed for sqlalchemy 0.5:
111 def result_processor(self, dialect, coltype = None):
114 sa_major_version = sqlalchemy.__version__[0:3]
115 if sa_major_version in ["0.5", "0.6", "0.7"]:
116 from sqlalchemy.databases import postgres
117 postgres.ischema_names['debversion'] = DebVersion
119 raise Exception("dak only ported to SQLA versions 0.5 to 0.7. See daklib/dbconn.py")
121 ################################################################################
123 __all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion']
125 ################################################################################
127 def session_wrapper(fn):
129 Wrapper around common ".., session=None):" handling. If the wrapped
130 function is called without passing 'session', we create a local one
131 and destroy it when the function ends.
133 Also attaches a commit_or_flush method to the session; if we created a
134 local session, this is a synonym for session.commit(), otherwise it is a
135 synonym for session.flush().
138 def wrapped(*args, **kwargs):
139 private_transaction = False
141 # Find the session object
142 session = kwargs.get('session')
145 if len(args) <= len(getargspec(fn)[0]) - 1:
146 # No session specified as last argument or in kwargs
147 private_transaction = True
148 session = kwargs['session'] = DBConn().session()
150 # Session is last argument in args
154 session = args[-1] = DBConn().session()
155 private_transaction = True
157 if private_transaction:
158 session.commit_or_flush = session.commit
160 session.commit_or_flush = session.flush
163 return fn(*args, **kwargs)
165 if private_transaction:
166 # We created a session; close it.
169 wrapped.__doc__ = fn.__doc__
170 wrapped.func_name = fn.func_name
174 __all__.append('session_wrapper')
176 ################################################################################
178 class ORMObject(object):
180 ORMObject is a base class for all ORM classes mapped by SQLalchemy. All
181 derived classes must implement the properties() method.
184 def properties(self):
186 This method should be implemented by all derived classes and returns a
187 list of the important properties. The properties 'created' and
188 'modified' will be added automatically. A suffix '_count' should be
189 added to properties that are lists or query objects. The most important
190 property name should be returned as the first element in the list
191 because it is used by repr().
197 Returns a JSON representation of the object based on the properties
198 returned from the properties() method.
201 # add created and modified
202 all_properties = self.properties() + ['created', 'modified']
203 for property in all_properties:
204 # check for list or query
205 if property[-6:] == '_count':
206 real_property = property[:-6]
207 if not hasattr(self, real_property):
209 value = getattr(self, real_property)
210 if hasattr(value, '__len__'):
213 elif hasattr(value, 'count'):
214 # query (but not during validation)
215 if self.in_validation:
217 value = value.count()
219 raise KeyError('Do not understand property %s.' % property)
221 if not hasattr(self, property):
224 value = getattr(self, property)
228 elif isinstance(value, ORMObject):
229 # use repr() for ORMObject types
232 # we want a string for all other types because json cannot
235 data[property] = value
236 return json.dumps(data)
240 Returns the name of the class.
242 return type(self).__name__
246 Returns a short string representation of the object using the first
247 element from the properties() method.
249 primary_property = self.properties()[0]
250 value = getattr(self, primary_property)
251 return '<%s %s>' % (self.classname(), str(value))
255 Returns a human readable form of the object using the properties()
258 return '<%s %s>' % (self.classname(), self.json())
260 def not_null_constraints(self):
262 Returns a list of properties that must be not NULL. Derived classes
263 should override this method if needed.
267 validation_message = \
268 "Validation failed because property '%s' must not be empty in object\n%s"
270 in_validation = False
274 This function validates the not NULL constraints as returned by
275 not_null_constraints(). It raises the DBUpdateError exception if
278 for property in self.not_null_constraints():
279 # TODO: It is a bit awkward that the mapper configuration allow
280 # directly setting the numeric _id columns. We should get rid of it
282 if hasattr(self, property + '_id') and \
283 getattr(self, property + '_id') is not None:
285 if not hasattr(self, property) or getattr(self, property) is None:
286 # str() might lead to races due to a 2nd flush
287 self.in_validation = True
288 message = self.validation_message % (property, str(self))
289 self.in_validation = False
290 raise DBUpdateError(message)
294 def get(cls, primary_key, session = None):
296 This is a support function that allows getting an object by its primary
299 Architecture.get(3[, session])
301 instead of the more verbose
303 session.query(Architecture).get(3)
305 return session.query(cls).get(primary_key)
307 def session(self, replace = False):
309 Returns the current session that is associated with the object. May
310 return None is object is in detached state.
313 return object_session(self)
315 def clone(self, session = None):
317 Clones the current object in a new session and returns the new clone. A
318 fresh session is created if the optional session parameter is not
319 provided. The function will fail if a session is provided and has
322 RATIONALE: SQLAlchemy's session is not thread safe. This method clones
323 an existing object to allow several threads to work with their own
324 instances of an ORMObject.
326 WARNING: Only persistent (committed) objects can be cloned. Changes
327 made to the original object that are not committed yet will get lost.
328 The session of the new object will always be rolled back to avoid
332 if self.session() is None:
333 raise RuntimeError( \
334 'Method clone() failed for detached object:\n%s' % self)
335 self.session().flush()
336 mapper = object_mapper(self)
337 primary_key = mapper.primary_key_from_instance(self)
338 object_class = self.__class__
340 session = DBConn().session()
341 elif len(session.new) + len(session.dirty) + len(session.deleted) > 0:
342 raise RuntimeError( \
343 'Method clone() failed due to unflushed changes in session.')
344 new_object = session.query(object_class).get(primary_key)
346 if new_object is None:
347 raise RuntimeError( \
348 'Method clone() failed for non-persistent object:\n%s' % self)
351 __all__.append('ORMObject')
353 ################################################################################
355 class Validator(MapperExtension):
357 This class calls the validate() method for each instance for the
358 'before_update' and 'before_insert' events. A global object validator is
359 used for configuring the individual mappers.
362 def before_update(self, mapper, connection, instance):
366 def before_insert(self, mapper, connection, instance):
370 validator = Validator()
372 ################################################################################
374 class Architecture(ORMObject):
375 def __init__(self, arch_string = None, description = None):
376 self.arch_string = arch_string
377 self.description = description
379 def __eq__(self, val):
380 if isinstance(val, str):
381 return (self.arch_string== val)
382 # This signals to use the normal comparison operator
383 return NotImplemented
385 def __ne__(self, val):
386 if isinstance(val, str):
387 return (self.arch_string != val)
388 # This signals to use the normal comparison operator
389 return NotImplemented
391 def properties(self):
392 return ['arch_string', 'arch_id', 'suites_count']
394 def not_null_constraints(self):
395 return ['arch_string']
397 __all__.append('Architecture')
400 def get_architecture(architecture, session=None):
402 Returns database id for given C{architecture}.
404 @type architecture: string
405 @param architecture: The name of the architecture
407 @type session: Session
408 @param session: Optional SQLA session object (a temporary one will be
409 generated if not supplied)
412 @return: Architecture object for the given arch (None if not present)
415 q = session.query(Architecture).filter_by(arch_string=architecture)
419 except NoResultFound:
422 __all__.append('get_architecture')
424 # TODO: should be removed because the implementation is too trivial
426 def get_architecture_suites(architecture, session=None):
428 Returns list of Suite objects for given C{architecture} name
430 @type architecture: str
431 @param architecture: Architecture name to search for
433 @type session: Session
434 @param session: Optional SQL session object (a temporary one will be
435 generated if not supplied)
438 @return: list of Suite objects for the given name (may be empty)
441 return get_architecture(architecture, session).suites
443 __all__.append('get_architecture_suites')
445 ################################################################################
447 class Archive(object):
448 def __init__(self, *args, **kwargs):
452 return '<Archive %s>' % self.archive_name
454 __all__.append('Archive')
457 def get_archive(archive, session=None):
459 returns database id for given C{archive}.
461 @type archive: string
462 @param archive: the name of the arhive
464 @type session: Session
465 @param session: Optional SQLA session object (a temporary one will be
466 generated if not supplied)
469 @return: Archive object for the given name (None if not present)
472 archive = archive.lower()
474 q = session.query(Archive).filter_by(archive_name=archive)
478 except NoResultFound:
481 __all__.append('get_archive')
483 ################################################################################
485 class ArchiveFile(object):
486 def __init__(self, archive=None, component=None, file=None):
487 self.archive = archive
488 self.component = component
492 return os.path.join(self.archive.path, 'pool', self.component.component_name, self.file.filename)
494 __all__.append('ArchiveFile')
496 ################################################################################
498 class BinContents(ORMObject):
499 def __init__(self, file = None, binary = None):
503 def properties(self):
504 return ['file', 'binary']
506 __all__.append('BinContents')
508 ################################################################################
510 def subprocess_setup():
511 # Python installs a SIGPIPE handler by default. This is usually not what
512 # non-Python subprocesses expect.
513 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
515 class DBBinary(ORMObject):
516 def __init__(self, package = None, source = None, version = None, \
517 maintainer = None, architecture = None, poolfile = None, \
518 binarytype = 'deb', fingerprint=None):
519 self.package = package
521 self.version = version
522 self.maintainer = maintainer
523 self.architecture = architecture
524 self.poolfile = poolfile
525 self.binarytype = binarytype
526 self.fingerprint = fingerprint
530 return self.binary_id
532 def properties(self):
533 return ['package', 'version', 'maintainer', 'source', 'architecture', \
534 'poolfile', 'binarytype', 'fingerprint', 'install_date', \
535 'suites_count', 'binary_id', 'contents_count', 'extra_sources']
537 def not_null_constraints(self):
538 return ['package', 'version', 'maintainer', 'source', 'poolfile', \
541 metadata = association_proxy('key', 'value')
543 def scan_contents(self):
545 Yields the contents of the package. Only regular files are yielded and
546 the path names are normalized after converting them from either utf-8
547 or iso8859-1 encoding. It yields the string ' <EMPTY PACKAGE>' if the
548 package does not contain any regular file.
550 fullpath = self.poolfile.fullpath
551 dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE,
552 preexec_fn = subprocess_setup)
553 tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
554 for member in tar.getmembers():
555 if not member.isdir():
556 name = normpath(member.name)
557 # enforce proper utf-8 encoding
560 except UnicodeDecodeError:
561 name = name.decode('iso8859-1').encode('utf-8')
567 def read_control(self):
569 Reads the control information from a binary.
572 @return: stanza text of the control section.
575 fullpath = self.poolfile.fullpath
576 deb_file = open(fullpath, 'r')
577 stanza = utils.deb_extract_control(deb_file)
582 def read_control_fields(self):
584 Reads the control information from a binary and return
588 @return: fields of the control section as a dictionary.
591 stanza = self.read_control()
592 return apt_pkg.TagSection(stanza)
594 __all__.append('DBBinary')
597 def get_suites_binary_in(package, session=None):
599 Returns list of Suite objects which given C{package} name is in
602 @param package: DBBinary package name to search for
605 @return: list of Suite objects for the given package
608 return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all()
610 __all__.append('get_suites_binary_in')
613 def get_component_by_package_suite(package, suite_list, arch_list=[], session=None):
615 Returns the component name of the newest binary package in suite_list or
616 None if no package is found. The result can be optionally filtered by a list
617 of architecture names.
620 @param package: DBBinary package name to search for
622 @type suite_list: list of str
623 @param suite_list: list of suite_name items
625 @type arch_list: list of str
626 @param arch_list: optional list of arch_string items that defaults to []
628 @rtype: str or NoneType
629 @return: name of component or None
632 q = session.query(DBBinary).filter_by(package = package). \
633 join(DBBinary.suites).filter(Suite.suite_name.in_(suite_list))
634 if len(arch_list) > 0:
635 q = q.join(DBBinary.architecture). \
636 filter(Architecture.arch_string.in_(arch_list))
637 binary = q.order_by(desc(DBBinary.version)).first()
641 return binary.get_component_name()
643 __all__.append('get_component_by_package_suite')
645 ################################################################################
647 class BinaryACL(object):
648 def __init__(self, *args, **kwargs):
652 return '<BinaryACL %s>' % self.binary_acl_id
654 __all__.append('BinaryACL')
656 ################################################################################
658 class BinaryACLMap(object):
659 def __init__(self, *args, **kwargs):
663 return '<BinaryACLMap %s>' % self.binary_acl_map_id
665 __all__.append('BinaryACLMap')
667 ################################################################################
669 class BuildQueue(object):
670 def __init__(self, *args, **kwargs):
674 return '<BuildQueue %s>' % self.queue_name
676 __all__.append('BuildQueue')
678 ################################################################################
680 class Component(ORMObject):
681 def __init__(self, component_name = None):
682 self.component_name = component_name
684 def __eq__(self, val):
685 if isinstance(val, str):
686 return (self.component_name == val)
687 # This signals to use the normal comparison operator
688 return NotImplemented
690 def __ne__(self, val):
691 if isinstance(val, str):
692 return (self.component_name != val)
693 # This signals to use the normal comparison operator
694 return NotImplemented
696 def properties(self):
697 return ['component_name', 'component_id', 'description', \
698 'meets_dfsg', 'overrides_count']
700 def not_null_constraints(self):
701 return ['component_name']
704 __all__.append('Component')
707 def get_component(component, session=None):
709 Returns database id for given C{component}.
711 @type component: string
712 @param component: The name of the override type
715 @return: the database id for the given component
718 component = component.lower()
720 q = session.query(Component).filter_by(component_name=component)
724 except NoResultFound:
727 __all__.append('get_component')
730 def get_mapped_component(component_name, session=None):
731 """get component after mappings
733 Evaluate component mappings from ComponentMappings in dak.conf for the
734 given component name.
736 @todo: ansgar wants to get rid of this. It's currently only used for
739 @type component_name: str
740 @param component_name: component name
742 @param session: database session
744 @rtype: L{daklib.dbconn.Component} or C{None}
745 @return: component after applying maps or C{None}
748 for m in cnf.value_list("ComponentMappings"):
749 (src, dst) = m.split()
750 if component_name == src:
752 component = session.query(Component).filter_by(component_name=component_name).first()
755 __all__.append('get_mapped_component')
758 def get_component_names(session=None):
760 Returns list of strings of component names.
763 @return: list of strings of component names
766 return [ x.component_name for x in session.query(Component).all() ]
768 __all__.append('get_component_names')
770 ################################################################################
772 class DBConfig(object):
773 def __init__(self, *args, **kwargs):
777 return '<DBConfig %s>' % self.name
779 __all__.append('DBConfig')
781 ################################################################################
784 def get_or_set_contents_file_id(filename, session=None):
786 Returns database id for given filename.
788 If no matching file is found, a row is inserted.
790 @type filename: string
791 @param filename: The filename
792 @type session: SQLAlchemy
793 @param session: Optional SQL session object (a temporary one will be
794 generated if not supplied). If not passed, a commit will be performed at
795 the end of the function, otherwise the caller is responsible for commiting.
798 @return: the database id for the given component
801 q = session.query(ContentFilename).filter_by(filename=filename)
804 ret = q.one().cafilename_id
805 except NoResultFound:
806 cf = ContentFilename()
807 cf.filename = filename
809 session.commit_or_flush()
810 ret = cf.cafilename_id
814 __all__.append('get_or_set_contents_file_id')
817 def get_contents(suite, overridetype, section=None, session=None):
819 Returns contents for a suite / overridetype combination, limiting
820 to a section if not None.
823 @param suite: Suite object
825 @type overridetype: OverrideType
826 @param overridetype: OverrideType object
828 @type section: Section
829 @param section: Optional section object to limit results to
831 @type session: SQLAlchemy
832 @param session: Optional SQL session object (a temporary one will be
833 generated if not supplied)
836 @return: ResultsProxy object set up to return tuples of (filename, section,
840 # find me all of the contents for a given suite
841 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
845 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
846 JOIN content_file_names n ON (c.filename=n.id)
847 JOIN binaries b ON (b.id=c.binary_pkg)
848 JOIN override o ON (o.package=b.package)
849 JOIN section s ON (s.id=o.section)
850 WHERE o.suite = :suiteid AND o.type = :overridetypeid
851 AND b.type=:overridetypename"""
853 vals = {'suiteid': suite.suite_id,
854 'overridetypeid': overridetype.overridetype_id,
855 'overridetypename': overridetype.overridetype}
857 if section is not None:
858 contents_q += " AND s.id = :sectionid"
859 vals['sectionid'] = section.section_id
861 contents_q += " ORDER BY fn"
863 return session.execute(contents_q, vals)
865 __all__.append('get_contents')
867 ################################################################################
869 class ContentFilepath(object):
870 def __init__(self, *args, **kwargs):
874 return '<ContentFilepath %s>' % self.filepath
876 __all__.append('ContentFilepath')
879 def get_or_set_contents_path_id(filepath, session=None):
881 Returns database id for given path.
883 If no matching file is found, a row is inserted.
885 @type filepath: string
886 @param filepath: The filepath
888 @type session: SQLAlchemy
889 @param session: Optional SQL session object (a temporary one will be
890 generated if not supplied). If not passed, a commit will be performed at
891 the end of the function, otherwise the caller is responsible for commiting.
894 @return: the database id for the given path
897 q = session.query(ContentFilepath).filter_by(filepath=filepath)
900 ret = q.one().cafilepath_id
901 except NoResultFound:
902 cf = ContentFilepath()
903 cf.filepath = filepath
905 session.commit_or_flush()
906 ret = cf.cafilepath_id
910 __all__.append('get_or_set_contents_path_id')
912 ################################################################################
914 class ContentAssociation(object):
915 def __init__(self, *args, **kwargs):
919 return '<ContentAssociation %s>' % self.ca_id
921 __all__.append('ContentAssociation')
923 def insert_content_paths(binary_id, fullpaths, session=None):
925 Make sure given path is associated with given binary id
928 @param binary_id: the id of the binary
929 @type fullpaths: list
930 @param fullpaths: the list of paths of the file being associated with the binary
931 @type session: SQLAlchemy session
932 @param session: Optional SQLAlchemy session. If this is passed, the caller
933 is responsible for ensuring a transaction has begun and committing the
934 results or rolling back based on the result code. If not passed, a commit
935 will be performed at the end of the function, otherwise the caller is
936 responsible for commiting.
938 @return: True upon success
943 session = DBConn().session()
948 def generate_path_dicts():
949 for fullpath in fullpaths:
950 if fullpath.startswith( './' ):
951 fullpath = fullpath[2:]
953 yield {'filename':fullpath, 'id': binary_id }
955 for d in generate_path_dicts():
956 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
965 traceback.print_exc()
967 # Only rollback if we set up the session ourself
974 __all__.append('insert_content_paths')
976 ################################################################################
978 class DSCFile(object):
979 def __init__(self, *args, **kwargs):
983 return '<DSCFile %s>' % self.dscfile_id
985 __all__.append('DSCFile')
988 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
990 Returns a list of DSCFiles which may be empty
992 @type dscfile_id: int (optional)
993 @param dscfile_id: the dscfile_id of the DSCFiles to find
995 @type source_id: int (optional)
996 @param source_id: the source id related to the DSCFiles to find
998 @type poolfile_id: int (optional)
999 @param poolfile_id: the poolfile id related to the DSCFiles to find
1002 @return: Possibly empty list of DSCFiles
1005 q = session.query(DSCFile)
1007 if dscfile_id is not None:
1008 q = q.filter_by(dscfile_id=dscfile_id)
1010 if source_id is not None:
1011 q = q.filter_by(source_id=source_id)
1013 if poolfile_id is not None:
1014 q = q.filter_by(poolfile_id=poolfile_id)
1018 __all__.append('get_dscfiles')
1020 ################################################################################
1022 class ExternalOverride(ORMObject):
1023 def __init__(self, *args, **kwargs):
1027 return '<ExternalOverride %s = %s: %s>' % (self.package, self.key, self.value)
1029 __all__.append('ExternalOverride')
1031 ################################################################################
1033 class PoolFile(ORMObject):
1034 def __init__(self, filename = None, filesize = -1, \
1036 self.filename = filename
1037 self.filesize = filesize
1038 self.md5sum = md5sum
1042 session = DBConn().session().object_session(self)
1043 af = session.query(ArchiveFile).join(Archive).filter(ArchiveFile.file == self).first()
1047 def component(self):
1048 session = DBConn().session().object_session(self)
1049 component_id = session.query(ArchiveFile.component_id).filter(ArchiveFile.file == self) \
1050 .group_by(ArchiveFile.component_id).one()
1051 return session.query(Component).get(component_id)
1055 return os.path.basename(self.filename)
1057 def is_valid(self, filesize = -1, md5sum = None):
1058 return self.filesize == long(filesize) and self.md5sum == md5sum
1060 def properties(self):
1061 return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \
1062 'sha256sum', 'source', 'binary', 'last_used']
1064 def not_null_constraints(self):
1065 return ['filename', 'md5sum']
1067 def identical_to(self, filename):
1069 compare size and hash with the given file
1072 @return: true if the given file has the same size and hash as this object; false otherwise
1074 st = os.stat(filename)
1075 if self.filesize != st.st_size:
1078 f = open(filename, "r")
1079 sha256sum = apt_pkg.sha256sum(f)
1080 if sha256sum != self.sha256sum:
1085 __all__.append('PoolFile')
1088 def get_poolfile_like_name(filename, session=None):
1090 Returns an array of PoolFile objects which are like the given name
1092 @type filename: string
1093 @param filename: the filename of the file to check against the DB
1096 @return: array of PoolFile objects
1099 # TODO: There must be a way of properly using bind parameters with %FOO%
1100 q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
1104 __all__.append('get_poolfile_like_name')
1106 ################################################################################
1108 class Fingerprint(ORMObject):
1109 def __init__(self, fingerprint = None):
1110 self.fingerprint = fingerprint
1112 def properties(self):
1113 return ['fingerprint', 'fingerprint_id', 'keyring', 'uid', \
1116 def not_null_constraints(self):
1117 return ['fingerprint']
1119 __all__.append('Fingerprint')
1122 def get_fingerprint(fpr, session=None):
1124 Returns Fingerprint object for given fpr.
1127 @param fpr: The fpr to find / add
1129 @type session: SQLAlchemy
1130 @param session: Optional SQL session object (a temporary one will be
1131 generated if not supplied).
1134 @return: the Fingerprint object for the given fpr or None
1137 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1141 except NoResultFound:
1146 __all__.append('get_fingerprint')
1149 def get_or_set_fingerprint(fpr, session=None):
1151 Returns Fingerprint object for given fpr.
1153 If no matching fpr is found, a row is inserted.
1156 @param fpr: The fpr to find / add
1158 @type session: SQLAlchemy
1159 @param session: Optional SQL session object (a temporary one will be
1160 generated if not supplied). If not passed, a commit will be performed at
1161 the end of the function, otherwise the caller is responsible for commiting.
1162 A flush will be performed either way.
1165 @return: the Fingerprint object for the given fpr
1168 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1172 except NoResultFound:
1173 fingerprint = Fingerprint()
1174 fingerprint.fingerprint = fpr
1175 session.add(fingerprint)
1176 session.commit_or_flush()
1181 __all__.append('get_or_set_fingerprint')
1183 ################################################################################
1185 # Helper routine for Keyring class
1186 def get_ldap_name(entry):
1188 for k in ["cn", "mn", "sn"]:
1190 if ret and ret[0] != "" and ret[0] != "-":
1192 return " ".join(name)
1194 ################################################################################
1196 class Keyring(object):
1197 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1198 " --with-colons --fingerprint --fingerprint"
1203 def __init__(self, *args, **kwargs):
1207 return '<Keyring %s>' % self.keyring_name
1209 def de_escape_gpg_str(self, txt):
1210 esclist = re.split(r'(\\x..)', txt)
1211 for x in range(1,len(esclist),2):
1212 esclist[x] = "%c" % (int(esclist[x][2:],16))
1213 return "".join(esclist)
1215 def parse_address(self, uid):
1216 """parses uid and returns a tuple of real name and email address"""
1218 (name, address) = email.Utils.parseaddr(uid)
1219 name = re.sub(r"\s*[(].*[)]", "", name)
1220 name = self.de_escape_gpg_str(name)
1223 return (name, address)
1225 def load_keys(self, keyring):
1226 if not self.keyring_id:
1227 raise Exception('Must be initialized with database information')
1229 k = os.popen(self.gpg_invocation % keyring, "r")
1234 field = line.split(":")
1235 if field[0] == "pub":
1238 (name, addr) = self.parse_address(field[9])
1240 self.keys[key]["email"] = addr
1241 self.keys[key]["name"] = name
1242 self.keys[key]["fingerprints"] = []
1244 elif key and field[0] == "sub" and len(field) >= 12:
1245 signingkey = ("s" in field[11])
1246 elif key and field[0] == "uid":
1247 (name, addr) = self.parse_address(field[9])
1248 if "email" not in self.keys[key] and "@" in addr:
1249 self.keys[key]["email"] = addr
1250 self.keys[key]["name"] = name
1251 elif signingkey and field[0] == "fpr":
1252 self.keys[key]["fingerprints"].append(field[9])
1253 self.fpr_lookup[field[9]] = key
1255 def import_users_from_ldap(self, session):
1259 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1260 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1262 l = ldap.open(LDAPServer)
1263 l.simple_bind_s("","")
1264 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1265 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1266 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1268 ldap_fin_uid_id = {}
1275 uid = entry["uid"][0]
1276 name = get_ldap_name(entry)
1277 fingerprints = entry["keyFingerPrint"]
1279 for f in fingerprints:
1280 key = self.fpr_lookup.get(f, None)
1281 if key not in self.keys:
1283 self.keys[key]["uid"] = uid
1287 keyid = get_or_set_uid(uid, session).uid_id
1288 byuid[keyid] = (uid, name)
1289 byname[uid] = (keyid, name)
1291 return (byname, byuid)
1293 def generate_users_from_keyring(self, format, session):
1297 for x in self.keys.keys():
1298 if "email" not in self.keys[x]:
1300 self.keys[x]["uid"] = format % "invalid-uid"
1302 uid = format % self.keys[x]["email"]
1303 keyid = get_or_set_uid(uid, session).uid_id
1304 byuid[keyid] = (uid, self.keys[x]["name"])
1305 byname[uid] = (keyid, self.keys[x]["name"])
1306 self.keys[x]["uid"] = uid
1309 uid = format % "invalid-uid"
1310 keyid = get_or_set_uid(uid, session).uid_id
1311 byuid[keyid] = (uid, "ungeneratable user id")
1312 byname[uid] = (keyid, "ungeneratable user id")
1314 return (byname, byuid)
1316 __all__.append('Keyring')
1319 def get_keyring(keyring, session=None):
1321 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1322 If C{keyring} already has an entry, simply return the existing Keyring
1324 @type keyring: string
1325 @param keyring: the keyring name
1328 @return: the Keyring object for this keyring
1331 q = session.query(Keyring).filter_by(keyring_name=keyring)
1335 except NoResultFound:
1338 __all__.append('get_keyring')
1341 def get_active_keyring_paths(session=None):
1344 @return: list of active keyring paths
1346 return [ x.keyring_name for x in session.query(Keyring).filter(Keyring.active == True).order_by(desc(Keyring.priority)).all() ]
1348 __all__.append('get_active_keyring_paths')
1351 def get_primary_keyring_path(session=None):
1353 Get the full path to the highest priority active keyring
1356 @return: path to the active keyring with the highest priority or None if no
1357 keyring is configured
1359 keyrings = get_active_keyring_paths()
1361 if len(keyrings) > 0:
1366 __all__.append('get_primary_keyring_path')
1368 ################################################################################
1370 class KeyringACLMap(object):
1371 def __init__(self, *args, **kwargs):
1375 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1377 __all__.append('KeyringACLMap')
1379 ################################################################################
1381 class DBChange(object):
1382 def __init__(self, *args, **kwargs):
1386 return '<DBChange %s>' % self.changesname
1388 __all__.append('DBChange')
1391 def get_dbchange(filename, session=None):
1393 returns DBChange object for given C{filename}.
1395 @type filename: string
1396 @param filename: the name of the file
1398 @type session: Session
1399 @param session: Optional SQLA session object (a temporary one will be
1400 generated if not supplied)
1403 @return: DBChange object for the given filename (C{None} if not present)
1406 q = session.query(DBChange).filter_by(changesname=filename)
1410 except NoResultFound:
1413 __all__.append('get_dbchange')
1415 ################################################################################
1417 class Maintainer(ORMObject):
1418 def __init__(self, name = None):
1421 def properties(self):
1422 return ['name', 'maintainer_id']
1424 def not_null_constraints(self):
1427 def get_split_maintainer(self):
1428 if not hasattr(self, 'name') or self.name is None:
1429 return ('', '', '', '')
1431 return fix_maintainer(self.name.strip())
1433 __all__.append('Maintainer')
1436 def get_or_set_maintainer(name, session=None):
1438 Returns Maintainer object for given maintainer name.
1440 If no matching maintainer name is found, a row is inserted.
1443 @param name: The maintainer name to add
1445 @type session: SQLAlchemy
1446 @param session: Optional SQL session object (a temporary one will be
1447 generated if not supplied). If not passed, a commit will be performed at
1448 the end of the function, otherwise the caller is responsible for commiting.
1449 A flush will be performed either way.
1452 @return: the Maintainer object for the given maintainer
1455 q = session.query(Maintainer).filter_by(name=name)
1458 except NoResultFound:
1459 maintainer = Maintainer()
1460 maintainer.name = name
1461 session.add(maintainer)
1462 session.commit_or_flush()
1467 __all__.append('get_or_set_maintainer')
1470 def get_maintainer(maintainer_id, session=None):
1472 Return the name of the maintainer behind C{maintainer_id} or None if that
1473 maintainer_id is invalid.
1475 @type maintainer_id: int
1476 @param maintainer_id: the id of the maintainer
1479 @return: the Maintainer with this C{maintainer_id}
1482 return session.query(Maintainer).get(maintainer_id)
1484 __all__.append('get_maintainer')
1486 ################################################################################
1488 class NewComment(object):
1489 def __init__(self, *args, **kwargs):
1493 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1495 __all__.append('NewComment')
1498 def has_new_comment(package, version, session=None):
1500 Returns true if the given combination of C{package}, C{version} has a comment.
1502 @type package: string
1503 @param package: name of the package
1505 @type version: string
1506 @param version: package version
1508 @type session: Session
1509 @param session: Optional SQLA session object (a temporary one will be
1510 generated if not supplied)
1516 q = session.query(NewComment)
1517 q = q.filter_by(package=package)
1518 q = q.filter_by(version=version)
1520 return bool(q.count() > 0)
1522 __all__.append('has_new_comment')
1525 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1527 Returns (possibly empty) list of NewComment objects for the given
1530 @type package: string (optional)
1531 @param package: name of the package
1533 @type version: string (optional)
1534 @param version: package version
1536 @type comment_id: int (optional)
1537 @param comment_id: An id of a comment
1539 @type session: Session
1540 @param session: Optional SQLA session object (a temporary one will be
1541 generated if not supplied)
1544 @return: A (possibly empty) list of NewComment objects will be returned
1547 q = session.query(NewComment)
1548 if package is not None: q = q.filter_by(package=package)
1549 if version is not None: q = q.filter_by(version=version)
1550 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1554 __all__.append('get_new_comments')
1556 ################################################################################
1558 class Override(ORMObject):
1559 def __init__(self, package = None, suite = None, component = None, overridetype = None, \
1560 section = None, priority = None):
1561 self.package = package
1563 self.component = component
1564 self.overridetype = overridetype
1565 self.section = section
1566 self.priority = priority
1568 def properties(self):
1569 return ['package', 'suite', 'component', 'overridetype', 'section', \
1572 def not_null_constraints(self):
1573 return ['package', 'suite', 'component', 'overridetype', 'section']
1575 __all__.append('Override')
1578 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1580 Returns Override object for the given parameters
1582 @type package: string
1583 @param package: The name of the package
1585 @type suite: string, list or None
1586 @param suite: The name of the suite (or suites if a list) to limit to. If
1587 None, don't limit. Defaults to None.
1589 @type component: string, list or None
1590 @param component: The name of the component (or components if a list) to
1591 limit to. If None, don't limit. Defaults to None.
1593 @type overridetype: string, list or None
1594 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1595 limit to. If None, don't limit. Defaults to None.
1597 @type session: Session
1598 @param session: Optional SQLA session object (a temporary one will be
1599 generated if not supplied)
1602 @return: A (possibly empty) list of Override objects will be returned
1605 q = session.query(Override)
1606 q = q.filter_by(package=package)
1608 if suite is not None:
1609 if not isinstance(suite, list): suite = [suite]
1610 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1612 if component is not None:
1613 if not isinstance(component, list): component = [component]
1614 q = q.join(Component).filter(Component.component_name.in_(component))
1616 if overridetype is not None:
1617 if not isinstance(overridetype, list): overridetype = [overridetype]
1618 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1622 __all__.append('get_override')
1625 ################################################################################
1627 class OverrideType(ORMObject):
1628 def __init__(self, overridetype = None):
1629 self.overridetype = overridetype
1631 def properties(self):
1632 return ['overridetype', 'overridetype_id', 'overrides_count']
1634 def not_null_constraints(self):
1635 return ['overridetype']
1637 __all__.append('OverrideType')
1640 def get_override_type(override_type, session=None):
1642 Returns OverrideType object for given C{override type}.
1644 @type override_type: string
1645 @param override_type: The name of the override type
1647 @type session: Session
1648 @param session: Optional SQLA session object (a temporary one will be
1649 generated if not supplied)
1652 @return: the database id for the given override type
1655 q = session.query(OverrideType).filter_by(overridetype=override_type)
1659 except NoResultFound:
1662 __all__.append('get_override_type')
1664 ################################################################################
1666 class PolicyQueue(object):
1667 def __init__(self, *args, **kwargs):
1671 return '<PolicyQueue %s>' % self.queue_name
1673 __all__.append('PolicyQueue')
1676 def get_policy_queue(queuename, session=None):
1678 Returns PolicyQueue object for given C{queue name}
1680 @type queuename: string
1681 @param queuename: The name of the queue
1683 @type session: Session
1684 @param session: Optional SQLA session object (a temporary one will be
1685 generated if not supplied)
1688 @return: PolicyQueue object for the given queue
1691 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
1695 except NoResultFound:
1698 __all__.append('get_policy_queue')
1700 ################################################################################
1702 class PolicyQueueUpload(object):
1703 def __cmp__(self, other):
1704 ret = cmp(self.changes.source, other.changes.source)
1706 ret = apt_pkg.version_compare(self.changes.version, other.changes.version)
1708 if self.source is not None and other.source is None:
1710 elif self.source is None and other.source is not None:
1713 ret = cmp(self.changes.changesname, other.changes.changesname)
1716 __all__.append('PolicyQueueUpload')
1718 ################################################################################
1720 class PolicyQueueByhandFile(object):
1723 __all__.append('PolicyQueueByhandFile')
1725 ################################################################################
1727 class Priority(ORMObject):
1728 def __init__(self, priority = None, level = None):
1729 self.priority = priority
1732 def properties(self):
1733 return ['priority', 'priority_id', 'level', 'overrides_count']
1735 def not_null_constraints(self):
1736 return ['priority', 'level']
1738 def __eq__(self, val):
1739 if isinstance(val, str):
1740 return (self.priority == val)
1741 # This signals to use the normal comparison operator
1742 return NotImplemented
1744 def __ne__(self, val):
1745 if isinstance(val, str):
1746 return (self.priority != val)
1747 # This signals to use the normal comparison operator
1748 return NotImplemented
1750 __all__.append('Priority')
1753 def get_priority(priority, session=None):
1755 Returns Priority object for given C{priority name}.
1757 @type priority: string
1758 @param priority: The name of the priority
1760 @type session: Session
1761 @param session: Optional SQLA session object (a temporary one will be
1762 generated if not supplied)
1765 @return: Priority object for the given priority
1768 q = session.query(Priority).filter_by(priority=priority)
1772 except NoResultFound:
1775 __all__.append('get_priority')
1778 def get_priorities(session=None):
1780 Returns dictionary of priority names -> id mappings
1782 @type session: Session
1783 @param session: Optional SQL session object (a temporary one will be
1784 generated if not supplied)
1787 @return: dictionary of priority names -> id mappings
1791 q = session.query(Priority)
1793 ret[x.priority] = x.priority_id
1797 __all__.append('get_priorities')
1799 ################################################################################
1801 class Section(ORMObject):
1802 def __init__(self, section = None):
1803 self.section = section
1805 def properties(self):
1806 return ['section', 'section_id', 'overrides_count']
1808 def not_null_constraints(self):
1811 def __eq__(self, val):
1812 if isinstance(val, str):
1813 return (self.section == val)
1814 # This signals to use the normal comparison operator
1815 return NotImplemented
1817 def __ne__(self, val):
1818 if isinstance(val, str):
1819 return (self.section != val)
1820 # This signals to use the normal comparison operator
1821 return NotImplemented
1823 __all__.append('Section')
1826 def get_section(section, session=None):
1828 Returns Section object for given C{section name}.
1830 @type section: string
1831 @param section: The name of the section
1833 @type session: Session
1834 @param session: Optional SQLA session object (a temporary one will be
1835 generated if not supplied)
1838 @return: Section object for the given section name
1841 q = session.query(Section).filter_by(section=section)
1845 except NoResultFound:
1848 __all__.append('get_section')
1851 def get_sections(session=None):
1853 Returns dictionary of section names -> id mappings
1855 @type session: Session
1856 @param session: Optional SQL session object (a temporary one will be
1857 generated if not supplied)
1860 @return: dictionary of section names -> id mappings
1864 q = session.query(Section)
1866 ret[x.section] = x.section_id
1870 __all__.append('get_sections')
1872 ################################################################################
1874 class SrcContents(ORMObject):
1875 def __init__(self, file = None, source = None):
1877 self.source = source
1879 def properties(self):
1880 return ['file', 'source']
1882 __all__.append('SrcContents')
1884 ################################################################################
1886 class DBSource(ORMObject):
1887 def __init__(self, source = None, version = None, maintainer = None, \
1888 changedby = None, poolfile = None, install_date = None, fingerprint = None):
1889 self.source = source
1890 self.version = version
1891 self.maintainer = maintainer
1892 self.changedby = changedby
1893 self.poolfile = poolfile
1894 self.install_date = install_date
1895 self.fingerprint = fingerprint
1899 return self.source_id
1901 def properties(self):
1902 return ['source', 'source_id', 'maintainer', 'changedby', \
1903 'fingerprint', 'poolfile', 'version', 'suites_count', \
1904 'install_date', 'binaries_count', 'uploaders_count']
1906 def not_null_constraints(self):
1907 return ['source', 'version', 'install_date', 'maintainer', \
1908 'changedby', 'poolfile']
1910 def read_control_fields(self):
1912 Reads the control information from a dsc
1915 @return: fields is the dsc information in a dictionary form
1917 fullpath = self.poolfile.fullpath
1918 contents = open(fullpath, 'r').read()
1919 signed_file = SignedFile(contents, keyrings=[], require_signature=False)
1920 fields = apt_pkg.TagSection(signed_file.contents)
1923 metadata = association_proxy('key', 'value')
1925 def scan_contents(self):
1927 Returns a set of names for non directories. The path names are
1928 normalized after converting them from either utf-8 or iso8859-1
1931 fullpath = self.poolfile.fullpath
1932 from daklib.contents import UnpackedSource
1933 unpacked = UnpackedSource(fullpath)
1935 for name in unpacked.get_all_filenames():
1936 # enforce proper utf-8 encoding
1938 name.decode('utf-8')
1939 except UnicodeDecodeError:
1940 name = name.decode('iso8859-1').encode('utf-8')
1944 __all__.append('DBSource')
1947 def source_exists(source, source_version, suites = ["any"], session=None):
1949 Ensure that source exists somewhere in the archive for the binary
1950 upload being processed.
1951 1. exact match => 1.0-3
1952 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1954 @type source: string
1955 @param source: source name
1957 @type source_version: string
1958 @param source_version: expected source version
1961 @param suites: list of suites to check in, default I{any}
1963 @type session: Session
1964 @param session: Optional SQLA session object (a temporary one will be
1965 generated if not supplied)
1968 @return: returns 1 if a source with expected version is found, otherwise 0
1975 from daklib.regexes import re_bin_only_nmu
1976 orig_source_version = re_bin_only_nmu.sub('', source_version)
1978 for suite in suites:
1979 q = session.query(DBSource).filter_by(source=source). \
1980 filter(DBSource.version.in_([source_version, orig_source_version]))
1982 # source must exist in 'suite' or a suite that is enhanced by 'suite'
1983 s = get_suite(suite, session)
1985 enhances_vcs = session.query(VersionCheck).filter(VersionCheck.suite==s).filter_by(check='Enhances')
1986 considered_suites = [ vc.reference for vc in enhances_vcs ]
1987 considered_suites.append(s)
1989 q = q.filter(DBSource.suites.any(Suite.suite_id.in_([s.suite_id for s in considered_suites])))
1994 # No source found so return not ok
1999 __all__.append('source_exists')
2002 def get_suites_source_in(source, session=None):
2004 Returns list of Suite objects which given C{source} name is in
2007 @param source: DBSource package name to search for
2010 @return: list of Suite objects for the given source
2013 return session.query(Suite).filter(Suite.sources.any(source=source)).all()
2015 __all__.append('get_suites_source_in')
2018 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
2020 Returns list of DBSource objects for given C{source} name and other parameters
2023 @param source: DBSource package name to search for
2025 @type version: str or None
2026 @param version: DBSource version name to search for or None if not applicable
2028 @type dm_upload_allowed: bool
2029 @param dm_upload_allowed: If None, no effect. If True or False, only
2030 return packages with that dm_upload_allowed setting
2032 @type session: Session
2033 @param session: Optional SQL session object (a temporary one will be
2034 generated if not supplied)
2037 @return: list of DBSource objects for the given name (may be empty)
2040 q = session.query(DBSource).filter_by(source=source)
2042 if version is not None:
2043 q = q.filter_by(version=version)
2045 if dm_upload_allowed is not None:
2046 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
2050 __all__.append('get_sources_from_name')
2052 # FIXME: This function fails badly if it finds more than 1 source package and
2053 # its implementation is trivial enough to be inlined.
2055 def get_source_in_suite(source, suite, session=None):
2057 Returns a DBSource object for a combination of C{source} and C{suite}.
2059 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2060 - B{suite} - a suite name, eg. I{unstable}
2062 @type source: string
2063 @param source: source package name
2066 @param suite: the suite name
2069 @return: the version for I{source} in I{suite}
2073 q = get_suite(suite, session).get_sources(source)
2076 except NoResultFound:
2079 __all__.append('get_source_in_suite')
2082 def import_metadata_into_db(obj, session=None):
2084 This routine works on either DBBinary or DBSource objects and imports
2085 their metadata into the database
2087 fields = obj.read_control_fields()
2088 for k in fields.keys():
2091 val = str(fields[k])
2092 except UnicodeEncodeError:
2093 # Fall back to UTF-8
2095 val = fields[k].encode('utf-8')
2096 except UnicodeEncodeError:
2097 # Finally try iso8859-1
2098 val = fields[k].encode('iso8859-1')
2099 # Otherwise we allow the exception to percolate up and we cause
2100 # a reject as someone is playing silly buggers
2102 obj.metadata[get_or_set_metadatakey(k, session)] = val
2104 session.commit_or_flush()
2106 __all__.append('import_metadata_into_db')
2108 ################################################################################
2110 class SourceACL(object):
2111 def __init__(self, *args, **kwargs):
2115 return '<SourceACL %s>' % self.source_acl_id
2117 __all__.append('SourceACL')
2119 ################################################################################
2121 class SrcFormat(object):
2122 def __init__(self, *args, **kwargs):
2126 return '<SrcFormat %s>' % (self.format_name)
2128 __all__.append('SrcFormat')
2130 ################################################################################
2132 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2133 ('SuiteID', 'suite_id'),
2134 ('Version', 'version'),
2135 ('Origin', 'origin'),
2137 ('Description', 'description'),
2138 ('Untouchable', 'untouchable'),
2139 ('Announce', 'announce'),
2140 ('Codename', 'codename'),
2141 ('OverrideCodename', 'overridecodename'),
2142 ('ValidTime', 'validtime'),
2143 ('Priority', 'priority'),
2144 ('NotAutomatic', 'notautomatic'),
2145 ('CopyChanges', 'copychanges'),
2146 ('OverrideSuite', 'overridesuite')]
2148 # Why the heck don't we have any UNIQUE constraints in table suite?
2149 # TODO: Add UNIQUE constraints for appropriate columns.
2150 class Suite(ORMObject):
2151 def __init__(self, suite_name = None, version = None):
2152 self.suite_name = suite_name
2153 self.version = version
2155 def properties(self):
2156 return ['suite_name', 'version', 'sources_count', 'binaries_count', \
2159 def not_null_constraints(self):
2160 return ['suite_name']
2162 def __eq__(self, val):
2163 if isinstance(val, str):
2164 return (self.suite_name == val)
2165 # This signals to use the normal comparison operator
2166 return NotImplemented
2168 def __ne__(self, val):
2169 if isinstance(val, str):
2170 return (self.suite_name != val)
2171 # This signals to use the normal comparison operator
2172 return NotImplemented
2176 for disp, field in SUITE_FIELDS:
2177 val = getattr(self, field, None)
2179 ret.append("%s: %s" % (disp, val))
2181 return "\n".join(ret)
2183 def get_architectures(self, skipsrc=False, skipall=False):
2185 Returns list of Architecture objects
2187 @type skipsrc: boolean
2188 @param skipsrc: Whether to skip returning the 'source' architecture entry
2191 @type skipall: boolean
2192 @param skipall: Whether to skip returning the 'all' architecture entry
2196 @return: list of Architecture objects for the given name (may be empty)
2199 q = object_session(self).query(Architecture).with_parent(self)
2201 q = q.filter(Architecture.arch_string != 'source')
2203 q = q.filter(Architecture.arch_string != 'all')
2204 return q.order_by(Architecture.arch_string).all()
2206 def get_sources(self, source):
2208 Returns a query object representing DBSource that is part of C{suite}.
2210 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2212 @type source: string
2213 @param source: source package name
2215 @rtype: sqlalchemy.orm.query.Query
2216 @return: a query of DBSource
2220 session = object_session(self)
2221 return session.query(DBSource).filter_by(source = source). \
2224 def get_overridesuite(self):
2225 if self.overridesuite is None:
2228 return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one()
2232 return os.path.join(self.archive.path, 'dists', self.suite_name)
2234 __all__.append('Suite')
2237 def get_suite(suite, session=None):
2239 Returns Suite object for given C{suite name}.
2242 @param suite: The name of the suite
2244 @type session: Session
2245 @param session: Optional SQLA session object (a temporary one will be
2246 generated if not supplied)
2249 @return: Suite object for the requested suite name (None if not present)
2252 q = session.query(Suite).filter_by(suite_name=suite)
2256 except NoResultFound:
2259 __all__.append('get_suite')
2261 ################################################################################
2264 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2266 Returns list of Architecture objects for given C{suite} name. The list is
2267 empty if suite does not exist.
2270 @param suite: Suite name to search for
2272 @type skipsrc: boolean
2273 @param skipsrc: Whether to skip returning the 'source' architecture entry
2276 @type skipall: boolean
2277 @param skipall: Whether to skip returning the 'all' architecture entry
2280 @type session: Session
2281 @param session: Optional SQL session object (a temporary one will be
2282 generated if not supplied)
2285 @return: list of Architecture objects for the given name (may be empty)
2289 return get_suite(suite, session).get_architectures(skipsrc, skipall)
2290 except AttributeError:
2293 __all__.append('get_suite_architectures')
2295 ################################################################################
2297 class Uid(ORMObject):
2298 def __init__(self, uid = None, name = None):
2302 def __eq__(self, val):
2303 if isinstance(val, str):
2304 return (self.uid == val)
2305 # This signals to use the normal comparison operator
2306 return NotImplemented
2308 def __ne__(self, val):
2309 if isinstance(val, str):
2310 return (self.uid != val)
2311 # This signals to use the normal comparison operator
2312 return NotImplemented
2314 def properties(self):
2315 return ['uid', 'name', 'fingerprint']
2317 def not_null_constraints(self):
2320 __all__.append('Uid')
2323 def get_or_set_uid(uidname, session=None):
2325 Returns uid object for given uidname.
2327 If no matching uidname is found, a row is inserted.
2329 @type uidname: string
2330 @param uidname: The uid to add
2332 @type session: SQLAlchemy
2333 @param session: Optional SQL session object (a temporary one will be
2334 generated if not supplied). If not passed, a commit will be performed at
2335 the end of the function, otherwise the caller is responsible for commiting.
2338 @return: the uid object for the given uidname
2341 q = session.query(Uid).filter_by(uid=uidname)
2345 except NoResultFound:
2349 session.commit_or_flush()
2354 __all__.append('get_or_set_uid')
2357 def get_uid_from_fingerprint(fpr, session=None):
2358 q = session.query(Uid)
2359 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2363 except NoResultFound:
2366 __all__.append('get_uid_from_fingerprint')
2368 ################################################################################
2370 class UploadBlock(object):
2371 def __init__(self, *args, **kwargs):
2375 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2377 __all__.append('UploadBlock')
2379 ################################################################################
2381 class MetadataKey(ORMObject):
2382 def __init__(self, key = None):
2385 def properties(self):
2388 def not_null_constraints(self):
2391 __all__.append('MetadataKey')
2394 def get_or_set_metadatakey(keyname, session=None):
2396 Returns MetadataKey object for given uidname.
2398 If no matching keyname is found, a row is inserted.
2400 @type uidname: string
2401 @param uidname: The keyname to add
2403 @type session: SQLAlchemy
2404 @param session: Optional SQL session object (a temporary one will be
2405 generated if not supplied). If not passed, a commit will be performed at
2406 the end of the function, otherwise the caller is responsible for commiting.
2409 @return: the metadatakey object for the given keyname
2412 q = session.query(MetadataKey).filter_by(key=keyname)
2416 except NoResultFound:
2417 ret = MetadataKey(keyname)
2419 session.commit_or_flush()
2423 __all__.append('get_or_set_metadatakey')
2425 ################################################################################
2427 class BinaryMetadata(ORMObject):
2428 def __init__(self, key = None, value = None, binary = None):
2431 self.binary = binary
2433 def properties(self):
2434 return ['binary', 'key', 'value']
2436 def not_null_constraints(self):
2439 __all__.append('BinaryMetadata')
2441 ################################################################################
2443 class SourceMetadata(ORMObject):
2444 def __init__(self, key = None, value = None, source = None):
2447 self.source = source
2449 def properties(self):
2450 return ['source', 'key', 'value']
2452 def not_null_constraints(self):
2455 __all__.append('SourceMetadata')
2457 ################################################################################
2459 class VersionCheck(ORMObject):
2460 def __init__(self, *args, **kwargs):
2463 def properties(self):
2464 #return ['suite_id', 'check', 'reference_id']
2467 def not_null_constraints(self):
2468 return ['suite', 'check', 'reference']
2470 __all__.append('VersionCheck')
2473 def get_version_checks(suite_name, check = None, session = None):
2474 suite = get_suite(suite_name, session)
2476 # Make sure that what we return is iterable so that list comprehensions
2477 # involving this don't cause a traceback
2479 q = session.query(VersionCheck).filter_by(suite=suite)
2481 q = q.filter_by(check=check)
2484 __all__.append('get_version_checks')
2486 ################################################################################
2488 class DBConn(object):
2490 database module init.
2494 def __init__(self, *args, **kwargs):
2495 self.__dict__ = self.__shared_state
2497 if not getattr(self, 'initialised', False):
2498 self.initialised = True
2499 self.debug = kwargs.has_key('debug')
2502 def __setuptables(self):
2509 'binaries_metadata',
2518 'external_overrides',
2519 'extra_src_references',
2521 'files_archive_map',
2528 # TODO: the maintainer column in table override should be removed.
2532 'policy_queue_upload',
2533 'policy_queue_upload_binaries_map',
2534 'policy_queue_byhand_file',
2545 'suite_architectures',
2546 'suite_build_queue_copy',
2547 'suite_src_formats',
2554 'almost_obsolete_all_associations',
2555 'almost_obsolete_src_associations',
2556 'any_associations_source',
2557 'bin_associations_binaries',
2558 'binaries_suite_arch',
2561 'newest_all_associations',
2562 'newest_any_associations',
2564 'newest_src_association',
2565 'obsolete_all_associations',
2566 'obsolete_any_associations',
2567 'obsolete_any_by_all_associations',
2568 'obsolete_src_associations',
2570 'src_associations_bin',
2571 'src_associations_src',
2572 'suite_arch_by_name',
2575 for table_name in tables:
2576 table = Table(table_name, self.db_meta, \
2577 autoload=True, useexisting=True)
2578 setattr(self, 'tbl_%s' % table_name, table)
2580 for view_name in views:
2581 view = Table(view_name, self.db_meta, autoload=True)
2582 setattr(self, 'view_%s' % view_name, view)
2584 def __setupmappers(self):
2585 mapper(Architecture, self.tbl_architecture,
2586 properties = dict(arch_id = self.tbl_architecture.c.id,
2587 suites = relation(Suite, secondary=self.tbl_suite_architectures,
2588 order_by=self.tbl_suite.c.suite_name,
2589 backref=backref('architectures', order_by=self.tbl_architecture.c.arch_string))),
2590 extension = validator)
2592 mapper(Archive, self.tbl_archive,
2593 properties = dict(archive_id = self.tbl_archive.c.id,
2594 archive_name = self.tbl_archive.c.name))
2596 mapper(ArchiveFile, self.tbl_files_archive_map,
2597 properties = dict(archive = relation(Archive, backref='files'),
2598 component = relation(Component),
2599 file = relation(PoolFile, backref='archives')))
2601 mapper(BuildQueue, self.tbl_build_queue,
2602 properties = dict(queue_id = self.tbl_build_queue.c.id,
2603 suite = relation(Suite, primaryjoin=(self.tbl_build_queue.c.suite_id==self.tbl_suite.c.id))))
2605 mapper(DBBinary, self.tbl_binaries,
2606 properties = dict(binary_id = self.tbl_binaries.c.id,
2607 package = self.tbl_binaries.c.package,
2608 version = self.tbl_binaries.c.version,
2609 maintainer_id = self.tbl_binaries.c.maintainer,
2610 maintainer = relation(Maintainer),
2611 source_id = self.tbl_binaries.c.source,
2612 source = relation(DBSource, backref='binaries'),
2613 arch_id = self.tbl_binaries.c.architecture,
2614 architecture = relation(Architecture),
2615 poolfile_id = self.tbl_binaries.c.file,
2616 poolfile = relation(PoolFile),
2617 binarytype = self.tbl_binaries.c.type,
2618 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2619 fingerprint = relation(Fingerprint),
2620 install_date = self.tbl_binaries.c.install_date,
2621 suites = relation(Suite, secondary=self.tbl_bin_associations,
2622 backref=backref('binaries', lazy='dynamic')),
2623 extra_sources = relation(DBSource, secondary=self.tbl_extra_src_references,
2624 backref=backref('extra_binary_references', lazy='dynamic')),
2625 key = relation(BinaryMetadata, cascade='all',
2626 collection_class=attribute_mapped_collection('key'))),
2627 extension = validator)
2629 mapper(BinaryACL, self.tbl_binary_acl,
2630 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2632 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2633 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2634 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2635 architecture = relation(Architecture)))
2637 mapper(Component, self.tbl_component,
2638 properties = dict(component_id = self.tbl_component.c.id,
2639 component_name = self.tbl_component.c.name),
2640 extension = validator)
2642 mapper(DBConfig, self.tbl_config,
2643 properties = dict(config_id = self.tbl_config.c.id))
2645 mapper(DSCFile, self.tbl_dsc_files,
2646 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2647 source_id = self.tbl_dsc_files.c.source,
2648 source = relation(DBSource),
2649 poolfile_id = self.tbl_dsc_files.c.file,
2650 poolfile = relation(PoolFile)))
2652 mapper(ExternalOverride, self.tbl_external_overrides,
2654 suite_id = self.tbl_external_overrides.c.suite,
2655 suite = relation(Suite),
2656 component_id = self.tbl_external_overrides.c.component,
2657 component = relation(Component)))
2659 mapper(PoolFile, self.tbl_files,
2660 properties = dict(file_id = self.tbl_files.c.id,
2661 filesize = self.tbl_files.c.size),
2662 extension = validator)
2664 mapper(Fingerprint, self.tbl_fingerprint,
2665 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2666 uid_id = self.tbl_fingerprint.c.uid,
2667 uid = relation(Uid),
2668 keyring_id = self.tbl_fingerprint.c.keyring,
2669 keyring = relation(Keyring),
2670 source_acl = relation(SourceACL),
2671 binary_acl = relation(BinaryACL)),
2672 extension = validator)
2674 mapper(Keyring, self.tbl_keyrings,
2675 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2676 keyring_id = self.tbl_keyrings.c.id))
2678 mapper(DBChange, self.tbl_changes,
2679 properties = dict(change_id = self.tbl_changes.c.id,
2680 seen = self.tbl_changes.c.seen,
2681 source = self.tbl_changes.c.source,
2682 binaries = self.tbl_changes.c.binaries,
2683 architecture = self.tbl_changes.c.architecture,
2684 distribution = self.tbl_changes.c.distribution,
2685 urgency = self.tbl_changes.c.urgency,
2686 maintainer = self.tbl_changes.c.maintainer,
2687 changedby = self.tbl_changes.c.changedby,
2688 date = self.tbl_changes.c.date,
2689 version = self.tbl_changes.c.version))
2691 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
2692 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
2693 keyring = relation(Keyring, backref="keyring_acl_map"),
2694 architecture = relation(Architecture)))
2696 mapper(Maintainer, self.tbl_maintainer,
2697 properties = dict(maintainer_id = self.tbl_maintainer.c.id,
2698 maintains_sources = relation(DBSource, backref='maintainer',
2699 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.maintainer)),
2700 changed_sources = relation(DBSource, backref='changedby',
2701 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.changedby))),
2702 extension = validator)
2704 mapper(NewComment, self.tbl_new_comments,
2705 properties = dict(comment_id = self.tbl_new_comments.c.id))
2707 mapper(Override, self.tbl_override,
2708 properties = dict(suite_id = self.tbl_override.c.suite,
2709 suite = relation(Suite, \
2710 backref=backref('overrides', lazy='dynamic')),
2711 package = self.tbl_override.c.package,
2712 component_id = self.tbl_override.c.component,
2713 component = relation(Component, \
2714 backref=backref('overrides', lazy='dynamic')),
2715 priority_id = self.tbl_override.c.priority,
2716 priority = relation(Priority, \
2717 backref=backref('overrides', lazy='dynamic')),
2718 section_id = self.tbl_override.c.section,
2719 section = relation(Section, \
2720 backref=backref('overrides', lazy='dynamic')),
2721 overridetype_id = self.tbl_override.c.type,
2722 overridetype = relation(OverrideType, \
2723 backref=backref('overrides', lazy='dynamic'))))
2725 mapper(OverrideType, self.tbl_override_type,
2726 properties = dict(overridetype = self.tbl_override_type.c.type,
2727 overridetype_id = self.tbl_override_type.c.id))
2729 mapper(PolicyQueue, self.tbl_policy_queue,
2730 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id,
2731 suite = relation(Suite, primaryjoin=(self.tbl_policy_queue.c.suite_id == self.tbl_suite.c.id))))
2733 mapper(PolicyQueueUpload, self.tbl_policy_queue_upload,
2735 changes = relation(DBChange),
2736 policy_queue = relation(PolicyQueue, backref='uploads'),
2737 target_suite = relation(Suite),
2738 source = relation(DBSource),
2739 binaries = relation(DBBinary, secondary=self.tbl_policy_queue_upload_binaries_map),
2742 mapper(PolicyQueueByhandFile, self.tbl_policy_queue_byhand_file,
2744 upload = relation(PolicyQueueUpload, backref='byhand'),
2748 mapper(Priority, self.tbl_priority,
2749 properties = dict(priority_id = self.tbl_priority.c.id))
2751 mapper(Section, self.tbl_section,
2752 properties = dict(section_id = self.tbl_section.c.id,
2753 section=self.tbl_section.c.section))
2755 mapper(DBSource, self.tbl_source,
2756 properties = dict(source_id = self.tbl_source.c.id,
2757 version = self.tbl_source.c.version,
2758 maintainer_id = self.tbl_source.c.maintainer,
2759 poolfile_id = self.tbl_source.c.file,
2760 poolfile = relation(PoolFile),
2761 fingerprint_id = self.tbl_source.c.sig_fpr,
2762 fingerprint = relation(Fingerprint),
2763 changedby_id = self.tbl_source.c.changedby,
2764 srcfiles = relation(DSCFile,
2765 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
2766 suites = relation(Suite, secondary=self.tbl_src_associations,
2767 backref=backref('sources', lazy='dynamic')),
2768 uploaders = relation(Maintainer,
2769 secondary=self.tbl_src_uploaders),
2770 key = relation(SourceMetadata, cascade='all',
2771 collection_class=attribute_mapped_collection('key'))),
2772 extension = validator)
2774 mapper(SourceACL, self.tbl_source_acl,
2775 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
2777 mapper(SrcFormat, self.tbl_src_format,
2778 properties = dict(src_format_id = self.tbl_src_format.c.id,
2779 format_name = self.tbl_src_format.c.format_name))
2781 mapper(Suite, self.tbl_suite,
2782 properties = dict(suite_id = self.tbl_suite.c.id,
2783 policy_queue = relation(PolicyQueue, primaryjoin=(self.tbl_suite.c.policy_queue_id == self.tbl_policy_queue.c.id)),
2784 copy_queues = relation(BuildQueue,
2785 secondary=self.tbl_suite_build_queue_copy),
2786 srcformats = relation(SrcFormat, secondary=self.tbl_suite_src_formats,
2787 backref=backref('suites', lazy='dynamic')),
2788 archive = relation(Archive, backref='suites')),
2789 extension = validator)
2791 mapper(Uid, self.tbl_uid,
2792 properties = dict(uid_id = self.tbl_uid.c.id,
2793 fingerprint = relation(Fingerprint)),
2794 extension = validator)
2796 mapper(UploadBlock, self.tbl_upload_blocks,
2797 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
2798 fingerprint = relation(Fingerprint, backref="uploadblocks"),
2799 uid = relation(Uid, backref="uploadblocks")))
2801 mapper(BinContents, self.tbl_bin_contents,
2803 binary = relation(DBBinary,
2804 backref=backref('contents', lazy='dynamic', cascade='all')),
2805 file = self.tbl_bin_contents.c.file))
2807 mapper(SrcContents, self.tbl_src_contents,
2809 source = relation(DBSource,
2810 backref=backref('contents', lazy='dynamic', cascade='all')),
2811 file = self.tbl_src_contents.c.file))
2813 mapper(MetadataKey, self.tbl_metadata_keys,
2815 key_id = self.tbl_metadata_keys.c.key_id,
2816 key = self.tbl_metadata_keys.c.key))
2818 mapper(BinaryMetadata, self.tbl_binaries_metadata,
2820 binary_id = self.tbl_binaries_metadata.c.bin_id,
2821 binary = relation(DBBinary),
2822 key_id = self.tbl_binaries_metadata.c.key_id,
2823 key = relation(MetadataKey),
2824 value = self.tbl_binaries_metadata.c.value))
2826 mapper(SourceMetadata, self.tbl_source_metadata,
2828 source_id = self.tbl_source_metadata.c.src_id,
2829 source = relation(DBSource),
2830 key_id = self.tbl_source_metadata.c.key_id,
2831 key = relation(MetadataKey),
2832 value = self.tbl_source_metadata.c.value))
2834 mapper(VersionCheck, self.tbl_version_check,
2836 suite_id = self.tbl_version_check.c.suite,
2837 suite = relation(Suite, primaryjoin=self.tbl_version_check.c.suite==self.tbl_suite.c.id),
2838 reference_id = self.tbl_version_check.c.reference,
2839 reference = relation(Suite, primaryjoin=self.tbl_version_check.c.reference==self.tbl_suite.c.id, lazy='joined')))
2841 ## Connection functions
2842 def __createconn(self):
2843 from config import Config
2845 if cnf.has_key("DB::Service"):
2846 connstr = "postgresql://service=%s" % cnf["DB::Service"]
2847 elif cnf.has_key("DB::Host"):
2849 connstr = "postgresql://%s" % cnf["DB::Host"]
2850 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
2851 connstr += ":%s" % cnf["DB::Port"]
2852 connstr += "/%s" % cnf["DB::Name"]
2855 connstr = "postgresql:///%s" % cnf["DB::Name"]
2856 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
2857 connstr += "?port=%s" % cnf["DB::Port"]
2859 engine_args = { 'echo': self.debug }
2860 if cnf.has_key('DB::PoolSize'):
2861 engine_args['pool_size'] = int(cnf['DB::PoolSize'])
2862 if cnf.has_key('DB::MaxOverflow'):
2863 engine_args['max_overflow'] = int(cnf['DB::MaxOverflow'])
2864 if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \
2865 cnf['DB::Unicode'] == 'false':
2866 engine_args['use_native_unicode'] = False
2868 # Monkey patch a new dialect in in order to support service= syntax
2869 import sqlalchemy.dialects.postgresql
2870 from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
2871 class PGDialect_psycopg2_dak(PGDialect_psycopg2):
2872 def create_connect_args(self, url):
2873 if str(url).startswith('postgresql://service='):
2875 servicename = str(url)[21:]
2876 return (['service=%s' % servicename], {})
2878 return PGDialect_psycopg2.create_connect_args(self, url)
2880 sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak
2883 self.db_pg = create_engine(connstr, **engine_args)
2884 self.db_meta = MetaData()
2885 self.db_meta.bind = self.db_pg
2886 self.db_smaker = sessionmaker(bind=self.db_pg,
2890 self.__setuptables()
2891 self.__setupmappers()
2893 except OperationalError as e:
2895 utils.fubar("Cannot connect to database (%s)" % str(e))
2897 self.pid = os.getpid()
2899 def session(self, work_mem = 0):
2901 Returns a new session object. If a work_mem parameter is provided a new
2902 transaction is started and the work_mem parameter is set for this
2903 transaction. The work_mem parameter is measured in MB. A default value
2904 will be used if the parameter is not set.
2906 # reinitialize DBConn in new processes
2907 if self.pid != os.getpid():
2910 session = self.db_smaker()
2912 session.execute("SET LOCAL work_mem TO '%d MB'" % work_mem)
2915 __all__.append('DBConn')