5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
37 from os.path import normpath
48 import simplejson as json
50 from datetime import datetime, timedelta
51 from errno import ENOENT
52 from tempfile import mkstemp, mkdtemp
53 from subprocess import Popen, PIPE
54 from tarfile import TarFile
56 from inspect import getargspec
59 from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \
61 from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
62 backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers
63 from sqlalchemy import types as sqltypes
64 from sqlalchemy.orm.collections import attribute_mapped_collection
65 from sqlalchemy.ext.associationproxy import association_proxy
67 # Don't remove this, we re-export the exceptions to scripts which import us
68 from sqlalchemy.exc import *
69 from sqlalchemy.orm.exc import NoResultFound
71 # Only import Config until Queue stuff is changed to store its config
73 from config import Config
74 from textutils import fix_maintainer
75 from dak_exceptions import DBUpdateError, NoSourceFieldError
77 # suppress some deprecation warnings in squeeze related to sqlalchemy
79 warnings.filterwarnings('ignore', \
80 "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
84 ################################################################################
86 # Patch in support for the debversion field type so that it works during
90 # that is for sqlalchemy 0.6
91 UserDefinedType = sqltypes.UserDefinedType
93 # this one for sqlalchemy 0.5
94 UserDefinedType = sqltypes.TypeEngine
96 class DebVersion(UserDefinedType):
97 def get_col_spec(self):
100 def bind_processor(self, dialect):
103 # ' = None' is needed for sqlalchemy 0.5:
104 def result_processor(self, dialect, coltype = None):
107 sa_major_version = sqlalchemy.__version__[0:3]
108 if sa_major_version in ["0.5", "0.6"]:
109 from sqlalchemy.databases import postgres
110 postgres.ischema_names['debversion'] = DebVersion
112 raise Exception("dak only ported to SQLA versions 0.5 and 0.6. See daklib/dbconn.py")
114 ################################################################################
116 __all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion']
118 ################################################################################
120 def session_wrapper(fn):
122 Wrapper around common ".., session=None):" handling. If the wrapped
123 function is called without passing 'session', we create a local one
124 and destroy it when the function ends.
126 Also attaches a commit_or_flush method to the session; if we created a
127 local session, this is a synonym for session.commit(), otherwise it is a
128 synonym for session.flush().
131 def wrapped(*args, **kwargs):
132 private_transaction = False
134 # Find the session object
135 session = kwargs.get('session')
138 if len(args) <= len(getargspec(fn)[0]) - 1:
139 # No session specified as last argument or in kwargs
140 private_transaction = True
141 session = kwargs['session'] = DBConn().session()
143 # Session is last argument in args
147 session = args[-1] = DBConn().session()
148 private_transaction = True
150 if private_transaction:
151 session.commit_or_flush = session.commit
153 session.commit_or_flush = session.flush
156 return fn(*args, **kwargs)
158 if private_transaction:
159 # We created a session; close it.
162 wrapped.__doc__ = fn.__doc__
163 wrapped.func_name = fn.func_name
167 __all__.append('session_wrapper')
169 ################################################################################
171 class ORMObject(object):
173 ORMObject is a base class for all ORM classes mapped by SQLalchemy. All
174 derived classes must implement the properties() method.
177 def properties(self):
179 This method should be implemented by all derived classes and returns a
180 list of the important properties. The properties 'created' and
181 'modified' will be added automatically. A suffix '_count' should be
182 added to properties that are lists or query objects. The most important
183 property name should be returned as the first element in the list
184 because it is used by repr().
190 Returns a JSON representation of the object based on the properties
191 returned from the properties() method.
194 # add created and modified
195 all_properties = self.properties() + ['created', 'modified']
196 for property in all_properties:
197 # check for list or query
198 if property[-6:] == '_count':
199 real_property = property[:-6]
200 if not hasattr(self, real_property):
202 value = getattr(self, real_property)
203 if hasattr(value, '__len__'):
206 elif hasattr(value, 'count'):
207 # query (but not during validation)
208 if self.in_validation:
210 value = value.count()
212 raise KeyError('Do not understand property %s.' % property)
214 if not hasattr(self, property):
217 value = getattr(self, property)
221 elif isinstance(value, ORMObject):
222 # use repr() for ORMObject types
225 # we want a string for all other types because json cannot
228 data[property] = value
229 return json.dumps(data)
233 Returns the name of the class.
235 return type(self).__name__
239 Returns a short string representation of the object using the first
240 element from the properties() method.
242 primary_property = self.properties()[0]
243 value = getattr(self, primary_property)
244 return '<%s %s>' % (self.classname(), str(value))
248 Returns a human readable form of the object using the properties()
251 return '<%s %s>' % (self.classname(), self.json())
253 def not_null_constraints(self):
255 Returns a list of properties that must be not NULL. Derived classes
256 should override this method if needed.
260 validation_message = \
261 "Validation failed because property '%s' must not be empty in object\n%s"
263 in_validation = False
267 This function validates the not NULL constraints as returned by
268 not_null_constraints(). It raises the DBUpdateError exception if
271 for property in self.not_null_constraints():
272 # TODO: It is a bit awkward that the mapper configuration allow
273 # directly setting the numeric _id columns. We should get rid of it
275 if hasattr(self, property + '_id') and \
276 getattr(self, property + '_id') is not None:
278 if not hasattr(self, property) or getattr(self, property) is None:
279 # str() might lead to races due to a 2nd flush
280 self.in_validation = True
281 message = self.validation_message % (property, str(self))
282 self.in_validation = False
283 raise DBUpdateError(message)
287 def get(cls, primary_key, session = None):
289 This is a support function that allows getting an object by its primary
292 Architecture.get(3[, session])
294 instead of the more verbose
296 session.query(Architecture).get(3)
298 return session.query(cls).get(primary_key)
300 def session(self, replace = False):
302 Returns the current session that is associated with the object. May
303 return None is object is in detached state.
306 return object_session(self)
308 def clone(self, session = None):
310 Clones the current object in a new session and returns the new clone. A
311 fresh session is created if the optional session parameter is not
312 provided. The function will fail if a session is provided and has
315 RATIONALE: SQLAlchemy's session is not thread safe. This method clones
316 an existing object to allow several threads to work with their own
317 instances of an ORMObject.
319 WARNING: Only persistent (committed) objects can be cloned. Changes
320 made to the original object that are not committed yet will get lost.
321 The session of the new object will always be rolled back to avoid
325 if self.session() is None:
326 raise RuntimeError( \
327 'Method clone() failed for detached object:\n%s' % self)
328 self.session().flush()
329 mapper = object_mapper(self)
330 primary_key = mapper.primary_key_from_instance(self)
331 object_class = self.__class__
333 session = DBConn().session()
334 elif len(session.new) + len(session.dirty) + len(session.deleted) > 0:
335 raise RuntimeError( \
336 'Method clone() failed due to unflushed changes in session.')
337 new_object = session.query(object_class).get(primary_key)
339 if new_object is None:
340 raise RuntimeError( \
341 'Method clone() failed for non-persistent object:\n%s' % self)
344 __all__.append('ORMObject')
346 ################################################################################
348 class Validator(MapperExtension):
350 This class calls the validate() method for each instance for the
351 'before_update' and 'before_insert' events. A global object validator is
352 used for configuring the individual mappers.
355 def before_update(self, mapper, connection, instance):
359 def before_insert(self, mapper, connection, instance):
363 validator = Validator()
365 ################################################################################
367 class Architecture(ORMObject):
368 def __init__(self, arch_string = None, description = None):
369 self.arch_string = arch_string
370 self.description = description
372 def __eq__(self, val):
373 if isinstance(val, str):
374 return (self.arch_string== val)
375 # This signals to use the normal comparison operator
376 return NotImplemented
378 def __ne__(self, val):
379 if isinstance(val, str):
380 return (self.arch_string != val)
381 # This signals to use the normal comparison operator
382 return NotImplemented
384 def properties(self):
385 return ['arch_string', 'arch_id', 'suites_count']
387 def not_null_constraints(self):
388 return ['arch_string']
390 __all__.append('Architecture')
393 def get_architecture(architecture, session=None):
395 Returns database id for given C{architecture}.
397 @type architecture: string
398 @param architecture: The name of the architecture
400 @type session: Session
401 @param session: Optional SQLA session object (a temporary one will be
402 generated if not supplied)
405 @return: Architecture object for the given arch (None if not present)
408 q = session.query(Architecture).filter_by(arch_string=architecture)
412 except NoResultFound:
415 __all__.append('get_architecture')
417 # TODO: should be removed because the implementation is too trivial
419 def get_architecture_suites(architecture, session=None):
421 Returns list of Suite objects for given C{architecture} name
423 @type architecture: str
424 @param architecture: Architecture name to search for
426 @type session: Session
427 @param session: Optional SQL session object (a temporary one will be
428 generated if not supplied)
431 @return: list of Suite objects for the given name (may be empty)
434 return get_architecture(architecture, session).suites
436 __all__.append('get_architecture_suites')
438 ################################################################################
440 class Archive(object):
441 def __init__(self, *args, **kwargs):
445 return '<Archive %s>' % self.archive_name
447 __all__.append('Archive')
450 def get_archive(archive, session=None):
452 returns database id for given C{archive}.
454 @type archive: string
455 @param archive: the name of the arhive
457 @type session: Session
458 @param session: Optional SQLA session object (a temporary one will be
459 generated if not supplied)
462 @return: Archive object for the given name (None if not present)
465 archive = archive.lower()
467 q = session.query(Archive).filter_by(archive_name=archive)
471 except NoResultFound:
474 __all__.append('get_archive')
476 ################################################################################
478 class BinContents(ORMObject):
479 def __init__(self, file = None, binary = None):
483 def properties(self):
484 return ['file', 'binary']
486 __all__.append('BinContents')
488 ################################################################################
490 class DBBinary(ORMObject):
491 def __init__(self, package = None, source = None, version = None, \
492 maintainer = None, architecture = None, poolfile = None, \
494 self.package = package
496 self.version = version
497 self.maintainer = maintainer
498 self.architecture = architecture
499 self.poolfile = poolfile
500 self.binarytype = binarytype
504 return self.binary_id
506 def properties(self):
507 return ['package', 'version', 'maintainer', 'source', 'architecture', \
508 'poolfile', 'binarytype', 'fingerprint', 'install_date', \
509 'suites_count', 'binary_id', 'contents_count', 'extra_sources']
511 def not_null_constraints(self):
512 return ['package', 'version', 'maintainer', 'source', 'poolfile', \
515 metadata = association_proxy('key', 'value')
517 def get_component_name(self):
518 return self.poolfile.location.component.component_name
520 def scan_contents(self):
522 Yields the contents of the package. Only regular files are yielded and
523 the path names are normalized after converting them from either utf-8
524 or iso8859-1 encoding. It yields the string ' <EMPTY PACKAGE>' if the
525 package does not contain any regular file.
527 fullpath = self.poolfile.fullpath
528 dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE)
529 tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
530 for member in tar.getmembers():
531 if not member.isdir():
532 name = normpath(member.name)
533 # enforce proper utf-8 encoding
536 except UnicodeDecodeError:
537 name = name.decode('iso8859-1').encode('utf-8')
543 def read_control(self):
545 Reads the control information from a binary.
548 @return: stanza text of the control section.
551 fullpath = self.poolfile.fullpath
552 deb_file = open(fullpath, 'r')
553 stanza = apt_inst.debExtractControl(deb_file)
558 def read_control_fields(self):
560 Reads the control information from a binary and return
564 @return: fields of the control section as a dictionary.
567 stanza = self.read_control()
568 return apt_pkg.TagSection(stanza)
570 __all__.append('DBBinary')
573 def get_suites_binary_in(package, session=None):
575 Returns list of Suite objects which given C{package} name is in
578 @param package: DBBinary package name to search for
581 @return: list of Suite objects for the given package
584 return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all()
586 __all__.append('get_suites_binary_in')
589 def get_component_by_package_suite(package, suite_list, arch_list=[], session=None):
591 Returns the component name of the newest binary package in suite_list or
592 None if no package is found. The result can be optionally filtered by a list
593 of architecture names.
596 @param package: DBBinary package name to search for
598 @type suite_list: list of str
599 @param suite_list: list of suite_name items
601 @type arch_list: list of str
602 @param arch_list: optional list of arch_string items that defaults to []
604 @rtype: str or NoneType
605 @return: name of component or None
608 q = session.query(DBBinary).filter_by(package = package). \
609 join(DBBinary.suites).filter(Suite.suite_name.in_(suite_list))
610 if len(arch_list) > 0:
611 q = q.join(DBBinary.architecture). \
612 filter(Architecture.arch_string.in_(arch_list))
613 binary = q.order_by(desc(DBBinary.version)).first()
617 return binary.get_component_name()
619 __all__.append('get_component_by_package_suite')
621 ################################################################################
623 class BinaryACL(object):
624 def __init__(self, *args, **kwargs):
628 return '<BinaryACL %s>' % self.binary_acl_id
630 __all__.append('BinaryACL')
632 ################################################################################
634 class BinaryACLMap(object):
635 def __init__(self, *args, **kwargs):
639 return '<BinaryACLMap %s>' % self.binary_acl_map_id
641 __all__.append('BinaryACLMap')
643 ################################################################################
648 ArchiveDir "%(archivepath)s";
649 OverrideDir "%(overridedir)s";
650 CacheDir "%(cachedir)s";
655 Packages::Compress ". bzip2 gzip";
656 Sources::Compress ". bzip2 gzip";
661 bindirectory "incoming"
666 BinOverride "override.sid.all3";
667 BinCacheDB "packages-accepted.db";
669 FileList "%(filelist)s";
672 Packages::Extensions ".deb .udeb";
675 bindirectory "incoming/"
678 BinOverride "override.sid.all3";
679 SrcOverride "override.sid.all3.src";
680 FileList "%(filelist)s";
684 class BuildQueue(object):
685 def __init__(self, *args, **kwargs):
689 return '<BuildQueue %s>' % self.queue_name
691 def write_metadata(self, starttime, force=False):
692 # Do we write out metafiles?
693 if not (force or self.generate_metadata):
696 session = DBConn().session().object_session(self)
698 fl_fd = fl_name = ac_fd = ac_name = None
700 arches = " ".join([ a.arch_string for a in session.query(Architecture).all() if a.arch_string != 'source' ])
701 startdir = os.getcwd()
704 # Grab files we want to include
705 newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
706 newer += session.query(BuildQueuePolicyFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueuePolicyFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
707 # Write file list with newer files
708 (fl_fd, fl_name) = mkstemp()
710 os.write(fl_fd, '%s\n' % n.fullpath)
715 # Write minimal apt.conf
716 # TODO: Remove hardcoding from template
717 (ac_fd, ac_name) = mkstemp()
718 os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path,
720 'cachedir': cnf["Dir::Cache"],
721 'overridedir': cnf["Dir::Override"],
725 # Run apt-ftparchive generate
726 os.chdir(os.path.dirname(ac_name))
727 os.system('apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
729 # Run apt-ftparchive release
730 # TODO: Eww - fix this
731 bname = os.path.basename(self.path)
735 # We have to remove the Release file otherwise it'll be included in the
738 os.unlink(os.path.join(bname, 'Release'))
742 os.system("""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="%s" release %s > Release""" % (self.origin, self.label, self.releasedescription, arches, bname))
744 # Crude hack with open and append, but this whole section is and should be redone.
745 if self.notautomatic:
746 release=open("Release", "a")
747 release.write("NotAutomatic: yes")
752 keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
753 if cnf.has_key("Dinstall::SigningPubKeyring"):
754 keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
756 os.system("gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey))
758 # Move the files if we got this far
759 os.rename('Release', os.path.join(bname, 'Release'))
761 os.rename('Release.gpg', os.path.join(bname, 'Release.gpg'))
763 # Clean up any left behind files
790 def clean_and_update(self, starttime, Logger, dryrun=False):
791 """WARNING: This routine commits for you"""
792 session = DBConn().session().object_session(self)
794 if self.generate_metadata and not dryrun:
795 self.write_metadata(starttime)
797 # Grab files older than our execution time
798 older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
799 older += session.query(BuildQueuePolicyFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueuePolicyFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
805 Logger.log(["I: Would have removed %s from the queue" % o.fullpath])
807 Logger.log(["I: Removing %s from the queue" % o.fullpath])
808 os.unlink(o.fullpath)
811 # If it wasn't there, don't worry
812 if e.errno == ENOENT:
815 # TODO: Replace with proper logging call
816 Logger.log(["E: Could not remove %s" % o.fullpath])
823 for f in os.listdir(self.path):
824 if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release') or f.startswith('advisory'):
827 if not self.contains_filename(f):
828 fp = os.path.join(self.path, f)
830 Logger.log(["I: Would remove unused link %s" % fp])
832 Logger.log(["I: Removing unused link %s" % fp])
836 Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath])
838 def contains_filename(self, filename):
841 @returns True if filename is supposed to be in the queue; False otherwise
843 session = DBConn().session().object_session(self)
844 if session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id, filename = filename).count() > 0:
846 elif session.query(BuildQueuePolicyFile).filter_by(build_queue = self, filename = filename).count() > 0:
850 def add_file_from_pool(self, poolfile):
851 """Copies a file into the pool. Assumes that the PoolFile object is
852 attached to the same SQLAlchemy session as the Queue object is.
854 The caller is responsible for committing after calling this function."""
855 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
857 # Check if we have a file of this name or this ID already
858 for f in self.queuefiles:
859 if (f.fileid is not None and f.fileid == poolfile.file_id) or \
860 (f.poolfile is not None and f.poolfile.filename == poolfile_basename):
861 # In this case, update the BuildQueueFile entry so we
862 # don't remove it too early
863 f.lastused = datetime.now()
864 DBConn().session().object_session(poolfile).add(f)
867 # Prepare BuildQueueFile object
868 qf = BuildQueueFile()
869 qf.build_queue_id = self.queue_id
870 qf.lastused = datetime.now()
871 qf.filename = poolfile_basename
873 targetpath = poolfile.fullpath
874 queuepath = os.path.join(self.path, poolfile_basename)
878 # We need to copy instead of symlink
880 utils.copy(targetpath, queuepath)
881 # NULL in the fileid field implies a copy
884 os.symlink(targetpath, queuepath)
885 qf.fileid = poolfile.file_id
889 # Get the same session as the PoolFile is using and add the qf to it
890 DBConn().session().object_session(poolfile).add(qf)
894 def add_changes_from_policy_queue(self, policyqueue, changes):
896 Copies a changes from a policy queue together with its poolfiles.
898 @type policyqueue: PolicyQueue
899 @param policyqueue: policy queue to copy the changes from
901 @type changes: DBChange
902 @param changes: changes to copy to this build queue
904 for policyqueuefile in changes.files:
905 self.add_file_from_policy_queue(policyqueue, policyqueuefile)
906 for poolfile in changes.poolfiles:
907 self.add_file_from_pool(poolfile)
909 def add_file_from_policy_queue(self, policyqueue, policyqueuefile):
911 Copies a file from a policy queue.
912 Assumes that the policyqueuefile is attached to the same SQLAlchemy
913 session as the Queue object is. The caller is responsible for
914 committing after calling this function.
916 @type policyqueue: PolicyQueue
917 @param policyqueue: policy queue to copy the file from
919 @type policyqueuefile: ChangePendingFile
920 @param policyqueuefile: file to be added to the build queue
922 session = DBConn().session().object_session(policyqueuefile)
924 # Is the file already there?
926 f = session.query(BuildQueuePolicyFile).filter_by(build_queue=self, file=policyqueuefile).one()
927 f.lastused = datetime.now()
929 except NoResultFound:
930 pass # continue below
932 # We have to add the file.
933 f = BuildQueuePolicyFile()
935 f.file = policyqueuefile
936 f.filename = policyqueuefile.filename
938 source = os.path.join(policyqueue.path, policyqueuefile.filename)
941 # Always copy files from policy queues as they might move around.
943 utils.copy(source, target)
950 __all__.append('BuildQueue')
953 def get_build_queue(queuename, session=None):
955 Returns BuildQueue object for given C{queue name}, creating it if it does not
958 @type queuename: string
959 @param queuename: The name of the queue
961 @type session: Session
962 @param session: Optional SQLA session object (a temporary one will be
963 generated if not supplied)
966 @return: BuildQueue object for the given queue
969 q = session.query(BuildQueue).filter_by(queue_name=queuename)
973 except NoResultFound:
976 __all__.append('get_build_queue')
978 ################################################################################
980 class BuildQueueFile(object):
982 BuildQueueFile represents a file in a build queue coming from a pool.
985 def __init__(self, *args, **kwargs):
989 return '<BuildQueueFile %s (%s)>' % (self.filename, self.build_queue_id)
993 return os.path.join(self.buildqueue.path, self.filename)
996 __all__.append('BuildQueueFile')
998 ################################################################################
1000 class BuildQueuePolicyFile(object):
1002 BuildQueuePolicyFile represents a file in a build queue that comes from a
1003 policy queue (and not a pool).
1006 def __init__(self, *args, **kwargs):
1010 #def filename(self):
1011 # return self.file.filename
1015 return os.path.join(self.build_queue.path, self.filename)
1017 __all__.append('BuildQueuePolicyFile')
1019 ################################################################################
1021 class ChangePendingBinary(object):
1022 def __init__(self, *args, **kwargs):
1026 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
1028 __all__.append('ChangePendingBinary')
1030 ################################################################################
1032 class ChangePendingFile(object):
1033 def __init__(self, *args, **kwargs):
1037 return '<ChangePendingFile %s>' % self.change_pending_file_id
1039 __all__.append('ChangePendingFile')
1041 ################################################################################
1043 class ChangePendingSource(object):
1044 def __init__(self, *args, **kwargs):
1048 return '<ChangePendingSource %s>' % self.change_pending_source_id
1050 __all__.append('ChangePendingSource')
1052 ################################################################################
1054 class Component(ORMObject):
1055 def __init__(self, component_name = None):
1056 self.component_name = component_name
1058 def __eq__(self, val):
1059 if isinstance(val, str):
1060 return (self.component_name == val)
1061 # This signals to use the normal comparison operator
1062 return NotImplemented
1064 def __ne__(self, val):
1065 if isinstance(val, str):
1066 return (self.component_name != val)
1067 # This signals to use the normal comparison operator
1068 return NotImplemented
1070 def properties(self):
1071 return ['component_name', 'component_id', 'description', \
1072 'location_count', 'meets_dfsg', 'overrides_count']
1074 def not_null_constraints(self):
1075 return ['component_name']
1078 __all__.append('Component')
1081 def get_component(component, session=None):
1083 Returns database id for given C{component}.
1085 @type component: string
1086 @param component: The name of the override type
1089 @return: the database id for the given component
1092 component = component.lower()
1094 q = session.query(Component).filter_by(component_name=component)
1098 except NoResultFound:
1101 __all__.append('get_component')
1103 ################################################################################
1105 class DBConfig(object):
1106 def __init__(self, *args, **kwargs):
1110 return '<DBConfig %s>' % self.name
1112 __all__.append('DBConfig')
1114 ################################################################################
1117 def get_or_set_contents_file_id(filename, session=None):
1119 Returns database id for given filename.
1121 If no matching file is found, a row is inserted.
1123 @type filename: string
1124 @param filename: The filename
1125 @type session: SQLAlchemy
1126 @param session: Optional SQL session object (a temporary one will be
1127 generated if not supplied). If not passed, a commit will be performed at
1128 the end of the function, otherwise the caller is responsible for commiting.
1131 @return: the database id for the given component
1134 q = session.query(ContentFilename).filter_by(filename=filename)
1137 ret = q.one().cafilename_id
1138 except NoResultFound:
1139 cf = ContentFilename()
1140 cf.filename = filename
1142 session.commit_or_flush()
1143 ret = cf.cafilename_id
1147 __all__.append('get_or_set_contents_file_id')
1150 def get_contents(suite, overridetype, section=None, session=None):
1152 Returns contents for a suite / overridetype combination, limiting
1153 to a section if not None.
1156 @param suite: Suite object
1158 @type overridetype: OverrideType
1159 @param overridetype: OverrideType object
1161 @type section: Section
1162 @param section: Optional section object to limit results to
1164 @type session: SQLAlchemy
1165 @param session: Optional SQL session object (a temporary one will be
1166 generated if not supplied)
1168 @rtype: ResultsProxy
1169 @return: ResultsProxy object set up to return tuples of (filename, section,
1173 # find me all of the contents for a given suite
1174 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
1178 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
1179 JOIN content_file_names n ON (c.filename=n.id)
1180 JOIN binaries b ON (b.id=c.binary_pkg)
1181 JOIN override o ON (o.package=b.package)
1182 JOIN section s ON (s.id=o.section)
1183 WHERE o.suite = :suiteid AND o.type = :overridetypeid
1184 AND b.type=:overridetypename"""
1186 vals = {'suiteid': suite.suite_id,
1187 'overridetypeid': overridetype.overridetype_id,
1188 'overridetypename': overridetype.overridetype}
1190 if section is not None:
1191 contents_q += " AND s.id = :sectionid"
1192 vals['sectionid'] = section.section_id
1194 contents_q += " ORDER BY fn"
1196 return session.execute(contents_q, vals)
1198 __all__.append('get_contents')
1200 ################################################################################
1202 class ContentFilepath(object):
1203 def __init__(self, *args, **kwargs):
1207 return '<ContentFilepath %s>' % self.filepath
1209 __all__.append('ContentFilepath')
1212 def get_or_set_contents_path_id(filepath, session=None):
1214 Returns database id for given path.
1216 If no matching file is found, a row is inserted.
1218 @type filepath: string
1219 @param filepath: The filepath
1221 @type session: SQLAlchemy
1222 @param session: Optional SQL session object (a temporary one will be
1223 generated if not supplied). If not passed, a commit will be performed at
1224 the end of the function, otherwise the caller is responsible for commiting.
1227 @return: the database id for the given path
1230 q = session.query(ContentFilepath).filter_by(filepath=filepath)
1233 ret = q.one().cafilepath_id
1234 except NoResultFound:
1235 cf = ContentFilepath()
1236 cf.filepath = filepath
1238 session.commit_or_flush()
1239 ret = cf.cafilepath_id
1243 __all__.append('get_or_set_contents_path_id')
1245 ################################################################################
1247 class ContentAssociation(object):
1248 def __init__(self, *args, **kwargs):
1252 return '<ContentAssociation %s>' % self.ca_id
1254 __all__.append('ContentAssociation')
1256 def insert_content_paths(binary_id, fullpaths, session=None):
1258 Make sure given path is associated with given binary id
1260 @type binary_id: int
1261 @param binary_id: the id of the binary
1262 @type fullpaths: list
1263 @param fullpaths: the list of paths of the file being associated with the binary
1264 @type session: SQLAlchemy session
1265 @param session: Optional SQLAlchemy session. If this is passed, the caller
1266 is responsible for ensuring a transaction has begun and committing the
1267 results or rolling back based on the result code. If not passed, a commit
1268 will be performed at the end of the function, otherwise the caller is
1269 responsible for commiting.
1271 @return: True upon success
1274 privatetrans = False
1276 session = DBConn().session()
1281 def generate_path_dicts():
1282 for fullpath in fullpaths:
1283 if fullpath.startswith( './' ):
1284 fullpath = fullpath[2:]
1286 yield {'filename':fullpath, 'id': binary_id }
1288 for d in generate_path_dicts():
1289 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
1298 traceback.print_exc()
1300 # Only rollback if we set up the session ourself
1307 __all__.append('insert_content_paths')
1309 ################################################################################
1311 class DSCFile(object):
1312 def __init__(self, *args, **kwargs):
1316 return '<DSCFile %s>' % self.dscfile_id
1318 __all__.append('DSCFile')
1321 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
1323 Returns a list of DSCFiles which may be empty
1325 @type dscfile_id: int (optional)
1326 @param dscfile_id: the dscfile_id of the DSCFiles to find
1328 @type source_id: int (optional)
1329 @param source_id: the source id related to the DSCFiles to find
1331 @type poolfile_id: int (optional)
1332 @param poolfile_id: the poolfile id related to the DSCFiles to find
1335 @return: Possibly empty list of DSCFiles
1338 q = session.query(DSCFile)
1340 if dscfile_id is not None:
1341 q = q.filter_by(dscfile_id=dscfile_id)
1343 if source_id is not None:
1344 q = q.filter_by(source_id=source_id)
1346 if poolfile_id is not None:
1347 q = q.filter_by(poolfile_id=poolfile_id)
1351 __all__.append('get_dscfiles')
1353 ################################################################################
1355 class ExternalOverride(ORMObject):
1356 def __init__(self, *args, **kwargs):
1360 return '<ExternalOverride %s = %s: %s>' % (self.package, self.key, self.value)
1362 __all__.append('ExternalOverride')
1364 ################################################################################
1366 class PoolFile(ORMObject):
1367 def __init__(self, filename = None, location = None, filesize = -1, \
1369 self.filename = filename
1370 self.location = location
1371 self.filesize = filesize
1372 self.md5sum = md5sum
1376 return os.path.join(self.location.path, self.filename)
1378 def is_valid(self, filesize = -1, md5sum = None):
1379 return self.filesize == long(filesize) and self.md5sum == md5sum
1381 def properties(self):
1382 return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \
1383 'sha256sum', 'location', 'source', 'binary', 'last_used']
1385 def not_null_constraints(self):
1386 return ['filename', 'md5sum', 'location']
1388 __all__.append('PoolFile')
1391 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
1394 (ValidFileFound [boolean], PoolFile object or None)
1396 @type filename: string
1397 @param filename: the filename of the file to check against the DB
1400 @param filesize: the size of the file to check against the DB
1402 @type md5sum: string
1403 @param md5sum: the md5sum of the file to check against the DB
1405 @type location_id: int
1406 @param location_id: the id of the location to look in
1409 @return: Tuple of length 2.
1410 - If valid pool file found: (C{True}, C{PoolFile object})
1411 - If valid pool file not found:
1412 - (C{False}, C{None}) if no file found
1413 - (C{False}, C{PoolFile object}) if file found with size/md5sum mismatch
1416 poolfile = session.query(Location).get(location_id). \
1417 files.filter_by(filename=filename).first()
1419 if poolfile and poolfile.is_valid(filesize = filesize, md5sum = md5sum):
1422 return (valid, poolfile)
1424 __all__.append('check_poolfile')
1426 # TODO: the implementation can trivially be inlined at the place where the
1427 # function is called
1429 def get_poolfile_by_id(file_id, session=None):
1431 Returns a PoolFile objects or None for the given id
1434 @param file_id: the id of the file to look for
1436 @rtype: PoolFile or None
1437 @return: either the PoolFile object or None
1440 return session.query(PoolFile).get(file_id)
1442 __all__.append('get_poolfile_by_id')
1445 def get_poolfile_like_name(filename, session=None):
1447 Returns an array of PoolFile objects which are like the given name
1449 @type filename: string
1450 @param filename: the filename of the file to check against the DB
1453 @return: array of PoolFile objects
1456 # TODO: There must be a way of properly using bind parameters with %FOO%
1457 q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
1461 __all__.append('get_poolfile_like_name')
1464 def add_poolfile(filename, datadict, location_id, session=None):
1466 Add a new file to the pool
1468 @type filename: string
1469 @param filename: filename
1471 @type datadict: dict
1472 @param datadict: dict with needed data
1474 @type location_id: int
1475 @param location_id: database id of the location
1478 @return: the PoolFile object created
1480 poolfile = PoolFile()
1481 poolfile.filename = filename
1482 poolfile.filesize = datadict["size"]
1483 poolfile.md5sum = datadict["md5sum"]
1484 poolfile.sha1sum = datadict["sha1sum"]
1485 poolfile.sha256sum = datadict["sha256sum"]
1486 poolfile.location_id = location_id
1488 session.add(poolfile)
1489 # Flush to get a file id (NB: This is not a commit)
1494 __all__.append('add_poolfile')
1496 ################################################################################
1498 class Fingerprint(ORMObject):
1499 def __init__(self, fingerprint = None):
1500 self.fingerprint = fingerprint
1502 def properties(self):
1503 return ['fingerprint', 'fingerprint_id', 'keyring', 'uid', \
1506 def not_null_constraints(self):
1507 return ['fingerprint']
1509 __all__.append('Fingerprint')
1512 def get_fingerprint(fpr, session=None):
1514 Returns Fingerprint object for given fpr.
1517 @param fpr: The fpr to find / add
1519 @type session: SQLAlchemy
1520 @param session: Optional SQL session object (a temporary one will be
1521 generated if not supplied).
1524 @return: the Fingerprint object for the given fpr or None
1527 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1531 except NoResultFound:
1536 __all__.append('get_fingerprint')
1539 def get_or_set_fingerprint(fpr, session=None):
1541 Returns Fingerprint object for given fpr.
1543 If no matching fpr is found, a row is inserted.
1546 @param fpr: The fpr to find / add
1548 @type session: SQLAlchemy
1549 @param session: Optional SQL session object (a temporary one will be
1550 generated if not supplied). If not passed, a commit will be performed at
1551 the end of the function, otherwise the caller is responsible for commiting.
1552 A flush will be performed either way.
1555 @return: the Fingerprint object for the given fpr
1558 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1562 except NoResultFound:
1563 fingerprint = Fingerprint()
1564 fingerprint.fingerprint = fpr
1565 session.add(fingerprint)
1566 session.commit_or_flush()
1571 __all__.append('get_or_set_fingerprint')
1573 ################################################################################
1575 # Helper routine for Keyring class
1576 def get_ldap_name(entry):
1578 for k in ["cn", "mn", "sn"]:
1580 if ret and ret[0] != "" and ret[0] != "-":
1582 return " ".join(name)
1584 ################################################################################
1586 class Keyring(object):
1587 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1588 " --with-colons --fingerprint --fingerprint"
1593 def __init__(self, *args, **kwargs):
1597 return '<Keyring %s>' % self.keyring_name
1599 def de_escape_gpg_str(self, txt):
1600 esclist = re.split(r'(\\x..)', txt)
1601 for x in range(1,len(esclist),2):
1602 esclist[x] = "%c" % (int(esclist[x][2:],16))
1603 return "".join(esclist)
1605 def parse_address(self, uid):
1606 """parses uid and returns a tuple of real name and email address"""
1608 (name, address) = email.Utils.parseaddr(uid)
1609 name = re.sub(r"\s*[(].*[)]", "", name)
1610 name = self.de_escape_gpg_str(name)
1613 return (name, address)
1615 def load_keys(self, keyring):
1616 if not self.keyring_id:
1617 raise Exception('Must be initialized with database information')
1619 k = os.popen(self.gpg_invocation % keyring, "r")
1623 for line in k.xreadlines():
1624 field = line.split(":")
1625 if field[0] == "pub":
1628 (name, addr) = self.parse_address(field[9])
1630 self.keys[key]["email"] = addr
1631 self.keys[key]["name"] = name
1632 self.keys[key]["fingerprints"] = []
1634 elif key and field[0] == "sub" and len(field) >= 12:
1635 signingkey = ("s" in field[11])
1636 elif key and field[0] == "uid":
1637 (name, addr) = self.parse_address(field[9])
1638 if "email" not in self.keys[key] and "@" in addr:
1639 self.keys[key]["email"] = addr
1640 self.keys[key]["name"] = name
1641 elif signingkey and field[0] == "fpr":
1642 self.keys[key]["fingerprints"].append(field[9])
1643 self.fpr_lookup[field[9]] = key
1645 def import_users_from_ldap(self, session):
1649 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1650 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1652 l = ldap.open(LDAPServer)
1653 l.simple_bind_s("","")
1654 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1655 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1656 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1658 ldap_fin_uid_id = {}
1665 uid = entry["uid"][0]
1666 name = get_ldap_name(entry)
1667 fingerprints = entry["keyFingerPrint"]
1669 for f in fingerprints:
1670 key = self.fpr_lookup.get(f, None)
1671 if key not in self.keys:
1673 self.keys[key]["uid"] = uid
1677 keyid = get_or_set_uid(uid, session).uid_id
1678 byuid[keyid] = (uid, name)
1679 byname[uid] = (keyid, name)
1681 return (byname, byuid)
1683 def generate_users_from_keyring(self, format, session):
1687 for x in self.keys.keys():
1688 if "email" not in self.keys[x]:
1690 self.keys[x]["uid"] = format % "invalid-uid"
1692 uid = format % self.keys[x]["email"]
1693 keyid = get_or_set_uid(uid, session).uid_id
1694 byuid[keyid] = (uid, self.keys[x]["name"])
1695 byname[uid] = (keyid, self.keys[x]["name"])
1696 self.keys[x]["uid"] = uid
1699 uid = format % "invalid-uid"
1700 keyid = get_or_set_uid(uid, session).uid_id
1701 byuid[keyid] = (uid, "ungeneratable user id")
1702 byname[uid] = (keyid, "ungeneratable user id")
1704 return (byname, byuid)
1706 __all__.append('Keyring')
1709 def get_keyring(keyring, session=None):
1711 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1712 If C{keyring} already has an entry, simply return the existing Keyring
1714 @type keyring: string
1715 @param keyring: the keyring name
1718 @return: the Keyring object for this keyring
1721 q = session.query(Keyring).filter_by(keyring_name=keyring)
1725 except NoResultFound:
1728 __all__.append('get_keyring')
1730 ################################################################################
1732 class KeyringACLMap(object):
1733 def __init__(self, *args, **kwargs):
1737 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1739 __all__.append('KeyringACLMap')
1741 ################################################################################
1743 class DBChange(object):
1744 def __init__(self, *args, **kwargs):
1748 return '<DBChange %s>' % self.changesname
1750 def clean_from_queue(self):
1751 session = DBConn().session().object_session(self)
1753 # Remove changes_pool_files entries
1756 # Remove changes_pending_files references
1759 # Clear out of queue
1760 self.in_queue = None
1761 self.approved_for_id = None
1763 __all__.append('DBChange')
1766 def get_dbchange(filename, session=None):
1768 returns DBChange object for given C{filename}.
1770 @type filename: string
1771 @param filename: the name of the file
1773 @type session: Session
1774 @param session: Optional SQLA session object (a temporary one will be
1775 generated if not supplied)
1778 @return: DBChange object for the given filename (C{None} if not present)
1781 q = session.query(DBChange).filter_by(changesname=filename)
1785 except NoResultFound:
1788 __all__.append('get_dbchange')
1790 ################################################################################
1792 class Location(ORMObject):
1793 def __init__(self, path = None, component = None):
1795 self.component = component
1796 # the column 'type' should go away, see comment at mapper
1797 self.archive_type = 'pool'
1799 def properties(self):
1800 return ['path', 'location_id', 'archive_type', 'component', \
1803 def not_null_constraints(self):
1804 return ['path', 'archive_type']
1806 __all__.append('Location')
1809 def get_location(location, component=None, archive=None, session=None):
1811 Returns Location object for the given combination of location, component
1814 @type location: string
1815 @param location: the path of the location, e.g. I{/srv/ftp-master.debian.org/ftp/pool/}
1817 @type component: string
1818 @param component: the component name (if None, no restriction applied)
1820 @type archive: string
1821 @param archive: the archive name (if None, no restriction applied)
1823 @rtype: Location / None
1824 @return: Either a Location object or None if one can't be found
1827 q = session.query(Location).filter_by(path=location)
1829 if archive is not None:
1830 q = q.join(Archive).filter_by(archive_name=archive)
1832 if component is not None:
1833 q = q.join(Component).filter_by(component_name=component)
1837 except NoResultFound:
1840 __all__.append('get_location')
1842 ################################################################################
1844 class Maintainer(ORMObject):
1845 def __init__(self, name = None):
1848 def properties(self):
1849 return ['name', 'maintainer_id']
1851 def not_null_constraints(self):
1854 def get_split_maintainer(self):
1855 if not hasattr(self, 'name') or self.name is None:
1856 return ('', '', '', '')
1858 return fix_maintainer(self.name.strip())
1860 __all__.append('Maintainer')
1863 def get_or_set_maintainer(name, session=None):
1865 Returns Maintainer object for given maintainer name.
1867 If no matching maintainer name is found, a row is inserted.
1870 @param name: The maintainer name to add
1872 @type session: SQLAlchemy
1873 @param session: Optional SQL session object (a temporary one will be
1874 generated if not supplied). If not passed, a commit will be performed at
1875 the end of the function, otherwise the caller is responsible for commiting.
1876 A flush will be performed either way.
1879 @return: the Maintainer object for the given maintainer
1882 q = session.query(Maintainer).filter_by(name=name)
1885 except NoResultFound:
1886 maintainer = Maintainer()
1887 maintainer.name = name
1888 session.add(maintainer)
1889 session.commit_or_flush()
1894 __all__.append('get_or_set_maintainer')
1897 def get_maintainer(maintainer_id, session=None):
1899 Return the name of the maintainer behind C{maintainer_id} or None if that
1900 maintainer_id is invalid.
1902 @type maintainer_id: int
1903 @param maintainer_id: the id of the maintainer
1906 @return: the Maintainer with this C{maintainer_id}
1909 return session.query(Maintainer).get(maintainer_id)
1911 __all__.append('get_maintainer')
1913 ################################################################################
1915 class NewComment(object):
1916 def __init__(self, *args, **kwargs):
1920 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1922 __all__.append('NewComment')
1925 def has_new_comment(package, version, session=None):
1927 Returns true if the given combination of C{package}, C{version} has a comment.
1929 @type package: string
1930 @param package: name of the package
1932 @type version: string
1933 @param version: package version
1935 @type session: Session
1936 @param session: Optional SQLA session object (a temporary one will be
1937 generated if not supplied)
1943 q = session.query(NewComment)
1944 q = q.filter_by(package=package)
1945 q = q.filter_by(version=version)
1947 return bool(q.count() > 0)
1949 __all__.append('has_new_comment')
1952 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1954 Returns (possibly empty) list of NewComment objects for the given
1957 @type package: string (optional)
1958 @param package: name of the package
1960 @type version: string (optional)
1961 @param version: package version
1963 @type comment_id: int (optional)
1964 @param comment_id: An id of a comment
1966 @type session: Session
1967 @param session: Optional SQLA session object (a temporary one will be
1968 generated if not supplied)
1971 @return: A (possibly empty) list of NewComment objects will be returned
1974 q = session.query(NewComment)
1975 if package is not None: q = q.filter_by(package=package)
1976 if version is not None: q = q.filter_by(version=version)
1977 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1981 __all__.append('get_new_comments')
1983 ################################################################################
1985 class Override(ORMObject):
1986 def __init__(self, package = None, suite = None, component = None, overridetype = None, \
1987 section = None, priority = None):
1988 self.package = package
1990 self.component = component
1991 self.overridetype = overridetype
1992 self.section = section
1993 self.priority = priority
1995 def properties(self):
1996 return ['package', 'suite', 'component', 'overridetype', 'section', \
1999 def not_null_constraints(self):
2000 return ['package', 'suite', 'component', 'overridetype', 'section']
2002 __all__.append('Override')
2005 def get_override(package, suite=None, component=None, overridetype=None, session=None):
2007 Returns Override object for the given parameters
2009 @type package: string
2010 @param package: The name of the package
2012 @type suite: string, list or None
2013 @param suite: The name of the suite (or suites if a list) to limit to. If
2014 None, don't limit. Defaults to None.
2016 @type component: string, list or None
2017 @param component: The name of the component (or components if a list) to
2018 limit to. If None, don't limit. Defaults to None.
2020 @type overridetype: string, list or None
2021 @param overridetype: The name of the overridetype (or overridetypes if a list) to
2022 limit to. If None, don't limit. Defaults to None.
2024 @type session: Session
2025 @param session: Optional SQLA session object (a temporary one will be
2026 generated if not supplied)
2029 @return: A (possibly empty) list of Override objects will be returned
2032 q = session.query(Override)
2033 q = q.filter_by(package=package)
2035 if suite is not None:
2036 if not isinstance(suite, list): suite = [suite]
2037 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
2039 if component is not None:
2040 if not isinstance(component, list): component = [component]
2041 q = q.join(Component).filter(Component.component_name.in_(component))
2043 if overridetype is not None:
2044 if not isinstance(overridetype, list): overridetype = [overridetype]
2045 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
2049 __all__.append('get_override')
2052 ################################################################################
2054 class OverrideType(ORMObject):
2055 def __init__(self, overridetype = None):
2056 self.overridetype = overridetype
2058 def properties(self):
2059 return ['overridetype', 'overridetype_id', 'overrides_count']
2061 def not_null_constraints(self):
2062 return ['overridetype']
2064 __all__.append('OverrideType')
2067 def get_override_type(override_type, session=None):
2069 Returns OverrideType object for given C{override type}.
2071 @type override_type: string
2072 @param override_type: The name of the override type
2074 @type session: Session
2075 @param session: Optional SQLA session object (a temporary one will be
2076 generated if not supplied)
2079 @return: the database id for the given override type
2082 q = session.query(OverrideType).filter_by(overridetype=override_type)
2086 except NoResultFound:
2089 __all__.append('get_override_type')
2091 ################################################################################
2093 class PolicyQueue(object):
2094 def __init__(self, *args, **kwargs):
2098 return '<PolicyQueue %s>' % self.queue_name
2100 __all__.append('PolicyQueue')
2103 def get_policy_queue(queuename, session=None):
2105 Returns PolicyQueue object for given C{queue name}
2107 @type queuename: string
2108 @param queuename: The name of the queue
2110 @type session: Session
2111 @param session: Optional SQLA session object (a temporary one will be
2112 generated if not supplied)
2115 @return: PolicyQueue object for the given queue
2118 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
2122 except NoResultFound:
2125 __all__.append('get_policy_queue')
2128 def get_policy_queue_from_path(pathname, session=None):
2130 Returns PolicyQueue object for given C{path name}
2132 @type queuename: string
2133 @param queuename: The path
2135 @type session: Session
2136 @param session: Optional SQLA session object (a temporary one will be
2137 generated if not supplied)
2140 @return: PolicyQueue object for the given queue
2143 q = session.query(PolicyQueue).filter_by(path=pathname)
2147 except NoResultFound:
2150 __all__.append('get_policy_queue_from_path')
2152 ################################################################################
2154 class Priority(ORMObject):
2155 def __init__(self, priority = None, level = None):
2156 self.priority = priority
2159 def properties(self):
2160 return ['priority', 'priority_id', 'level', 'overrides_count']
2162 def not_null_constraints(self):
2163 return ['priority', 'level']
2165 def __eq__(self, val):
2166 if isinstance(val, str):
2167 return (self.priority == val)
2168 # This signals to use the normal comparison operator
2169 return NotImplemented
2171 def __ne__(self, val):
2172 if isinstance(val, str):
2173 return (self.priority != val)
2174 # This signals to use the normal comparison operator
2175 return NotImplemented
2177 __all__.append('Priority')
2180 def get_priority(priority, session=None):
2182 Returns Priority object for given C{priority name}.
2184 @type priority: string
2185 @param priority: The name of the priority
2187 @type session: Session
2188 @param session: Optional SQLA session object (a temporary one will be
2189 generated if not supplied)
2192 @return: Priority object for the given priority
2195 q = session.query(Priority).filter_by(priority=priority)
2199 except NoResultFound:
2202 __all__.append('get_priority')
2205 def get_priorities(session=None):
2207 Returns dictionary of priority names -> id mappings
2209 @type session: Session
2210 @param session: Optional SQL session object (a temporary one will be
2211 generated if not supplied)
2214 @return: dictionary of priority names -> id mappings
2218 q = session.query(Priority)
2220 ret[x.priority] = x.priority_id
2224 __all__.append('get_priorities')
2226 ################################################################################
2228 class Section(ORMObject):
2229 def __init__(self, section = None):
2230 self.section = section
2232 def properties(self):
2233 return ['section', 'section_id', 'overrides_count']
2235 def not_null_constraints(self):
2238 def __eq__(self, val):
2239 if isinstance(val, str):
2240 return (self.section == val)
2241 # This signals to use the normal comparison operator
2242 return NotImplemented
2244 def __ne__(self, val):
2245 if isinstance(val, str):
2246 return (self.section != val)
2247 # This signals to use the normal comparison operator
2248 return NotImplemented
2250 __all__.append('Section')
2253 def get_section(section, session=None):
2255 Returns Section object for given C{section name}.
2257 @type section: string
2258 @param section: The name of the section
2260 @type session: Session
2261 @param session: Optional SQLA session object (a temporary one will be
2262 generated if not supplied)
2265 @return: Section object for the given section name
2268 q = session.query(Section).filter_by(section=section)
2272 except NoResultFound:
2275 __all__.append('get_section')
2278 def get_sections(session=None):
2280 Returns dictionary of section names -> id mappings
2282 @type session: Session
2283 @param session: Optional SQL session object (a temporary one will be
2284 generated if not supplied)
2287 @return: dictionary of section names -> id mappings
2291 q = session.query(Section)
2293 ret[x.section] = x.section_id
2297 __all__.append('get_sections')
2299 ################################################################################
2301 class SrcContents(ORMObject):
2302 def __init__(self, file = None, source = None):
2304 self.source = source
2306 def properties(self):
2307 return ['file', 'source']
2309 __all__.append('SrcContents')
2311 ################################################################################
2313 from debian.debfile import Deb822
2315 # Temporary Deb822 subclass to fix bugs with : handling; see #597249
2316 class Dak822(Deb822):
2317 def _internal_parser(self, sequence, fields=None):
2318 # The key is non-whitespace, non-colon characters before any colon.
2319 key_part = r"^(?P<key>[^: \t\n\r\f\v]+)\s*:\s*"
2320 single = re.compile(key_part + r"(?P<data>\S.*?)\s*$")
2321 multi = re.compile(key_part + r"$")
2322 multidata = re.compile(r"^\s(?P<data>.+?)\s*$")
2324 wanted_field = lambda f: fields is None or f in fields
2326 if isinstance(sequence, basestring):
2327 sequence = sequence.splitlines()
2331 for line in self.gpg_stripped_paragraph(sequence):
2332 m = single.match(line)
2335 self[curkey] = content
2337 if not wanted_field(m.group('key')):
2341 curkey = m.group('key')
2342 content = m.group('data')
2345 m = multi.match(line)
2348 self[curkey] = content
2350 if not wanted_field(m.group('key')):
2354 curkey = m.group('key')
2358 m = multidata.match(line)
2360 content += '\n' + line # XXX not m.group('data')?
2364 self[curkey] = content
2367 class DBSource(ORMObject):
2368 def __init__(self, source = None, version = None, maintainer = None, \
2369 changedby = None, poolfile = None, install_date = None):
2370 self.source = source
2371 self.version = version
2372 self.maintainer = maintainer
2373 self.changedby = changedby
2374 self.poolfile = poolfile
2375 self.install_date = install_date
2379 return self.source_id
2381 def properties(self):
2382 return ['source', 'source_id', 'maintainer', 'changedby', \
2383 'fingerprint', 'poolfile', 'version', 'suites_count', \
2384 'install_date', 'binaries_count', 'uploaders_count']
2386 def not_null_constraints(self):
2387 return ['source', 'version', 'install_date', 'maintainer', \
2388 'changedby', 'poolfile', 'install_date']
2390 def read_control_fields(self):
2392 Reads the control information from a dsc
2395 @return: fields is the dsc information in a dictionary form
2397 fullpath = self.poolfile.fullpath
2398 fields = Dak822(open(self.poolfile.fullpath, 'r'))
2401 metadata = association_proxy('key', 'value')
2403 def scan_contents(self):
2405 Returns a set of names for non directories. The path names are
2406 normalized after converting them from either utf-8 or iso8859-1
2409 fullpath = self.poolfile.fullpath
2410 from daklib.contents import UnpackedSource
2411 unpacked = UnpackedSource(fullpath)
2413 for name in unpacked.get_all_filenames():
2414 # enforce proper utf-8 encoding
2416 name.decode('utf-8')
2417 except UnicodeDecodeError:
2418 name = name.decode('iso8859-1').encode('utf-8')
2422 __all__.append('DBSource')
2425 def source_exists(source, source_version, suites = ["any"], session=None):
2427 Ensure that source exists somewhere in the archive for the binary
2428 upload being processed.
2429 1. exact match => 1.0-3
2430 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
2432 @type source: string
2433 @param source: source name
2435 @type source_version: string
2436 @param source_version: expected source version
2439 @param suites: list of suites to check in, default I{any}
2441 @type session: Session
2442 @param session: Optional SQLA session object (a temporary one will be
2443 generated if not supplied)
2446 @return: returns 1 if a source with expected version is found, otherwise 0
2453 from daklib.regexes import re_bin_only_nmu
2454 orig_source_version = re_bin_only_nmu.sub('', source_version)
2456 for suite in suites:
2457 q = session.query(DBSource).filter_by(source=source). \
2458 filter(DBSource.version.in_([source_version, orig_source_version]))
2460 # source must exist in suite X, or in some other suite that's
2461 # mapped to X, recursively... silent-maps are counted too,
2462 # unreleased-maps aren't.
2463 maps = cnf.ValueList("SuiteMappings")[:]
2465 maps = [ m.split() for m in maps ]
2466 maps = [ (x[1], x[2]) for x in maps
2467 if x[0] == "map" or x[0] == "silent-map" ]
2469 for (from_, to) in maps:
2470 if from_ in s and to not in s:
2473 q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
2478 # No source found so return not ok
2483 __all__.append('source_exists')
2486 def get_suites_source_in(source, session=None):
2488 Returns list of Suite objects which given C{source} name is in
2491 @param source: DBSource package name to search for
2494 @return: list of Suite objects for the given source
2497 return session.query(Suite).filter(Suite.sources.any(source=source)).all()
2499 __all__.append('get_suites_source_in')
2502 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
2504 Returns list of DBSource objects for given C{source} name and other parameters
2507 @param source: DBSource package name to search for
2509 @type version: str or None
2510 @param version: DBSource version name to search for or None if not applicable
2512 @type dm_upload_allowed: bool
2513 @param dm_upload_allowed: If None, no effect. If True or False, only
2514 return packages with that dm_upload_allowed setting
2516 @type session: Session
2517 @param session: Optional SQL session object (a temporary one will be
2518 generated if not supplied)
2521 @return: list of DBSource objects for the given name (may be empty)
2524 q = session.query(DBSource).filter_by(source=source)
2526 if version is not None:
2527 q = q.filter_by(version=version)
2529 if dm_upload_allowed is not None:
2530 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
2534 __all__.append('get_sources_from_name')
2536 # FIXME: This function fails badly if it finds more than 1 source package and
2537 # its implementation is trivial enough to be inlined.
2539 def get_source_in_suite(source, suite, session=None):
2541 Returns a DBSource object for a combination of C{source} and C{suite}.
2543 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2544 - B{suite} - a suite name, eg. I{unstable}
2546 @type source: string
2547 @param source: source package name
2550 @param suite: the suite name
2553 @return: the version for I{source} in I{suite}
2557 q = get_suite(suite, session).get_sources(source)
2560 except NoResultFound:
2563 __all__.append('get_source_in_suite')
2566 def import_metadata_into_db(obj, session=None):
2568 This routine works on either DBBinary or DBSource objects and imports
2569 their metadata into the database
2571 fields = obj.read_control_fields()
2572 for k in fields.keys():
2575 val = str(fields[k])
2576 except UnicodeEncodeError:
2577 # Fall back to UTF-8
2579 val = fields[k].encode('utf-8')
2580 except UnicodeEncodeError:
2581 # Finally try iso8859-1
2582 val = fields[k].encode('iso8859-1')
2583 # Otherwise we allow the exception to percolate up and we cause
2584 # a reject as someone is playing silly buggers
2586 obj.metadata[get_or_set_metadatakey(k, session)] = val
2588 session.commit_or_flush()
2590 __all__.append('import_metadata_into_db')
2593 ################################################################################
2596 def add_dsc_to_db(u, filename, session=None):
2597 entry = u.pkg.files[filename]
2601 source.source = u.pkg.dsc["source"]
2602 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2603 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2604 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2605 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2606 source.install_date = datetime.now().date()
2608 dsc_component = entry["component"]
2609 dsc_location_id = entry["location id"]
2611 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2613 # Set up a new poolfile if necessary
2614 if not entry.has_key("files id") or not entry["files id"]:
2615 filename = entry["pool name"] + filename
2616 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2618 pfs.append(poolfile)
2619 entry["files id"] = poolfile.file_id
2621 source.poolfile_id = entry["files id"]
2624 suite_names = u.pkg.changes["distribution"].keys()
2625 source.suites = session.query(Suite). \
2626 filter(Suite.suite_name.in_(suite_names)).all()
2628 # Add the source files to the DB (files and dsc_files)
2630 dscfile.source_id = source.source_id
2631 dscfile.poolfile_id = entry["files id"]
2632 session.add(dscfile)
2634 for dsc_file, dentry in u.pkg.dsc_files.items():
2636 df.source_id = source.source_id
2638 # If the .orig tarball is already in the pool, it's
2639 # files id is stored in dsc_files by check_dsc().
2640 files_id = dentry.get("files id", None)
2642 # Find the entry in the files hash
2643 # TODO: Bail out here properly
2645 for f, e in u.pkg.files.items():
2650 if files_id is None:
2651 filename = dfentry["pool name"] + dsc_file
2653 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2654 # FIXME: needs to check for -1/-2 and or handle exception
2655 if found and obj is not None:
2656 files_id = obj.file_id
2659 # If still not found, add it
2660 if files_id is None:
2661 # HACK: Force sha1sum etc into dentry
2662 dentry["sha1sum"] = dfentry["sha1sum"]
2663 dentry["sha256sum"] = dfentry["sha256sum"]
2664 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2665 pfs.append(poolfile)
2666 files_id = poolfile.file_id
2668 poolfile = get_poolfile_by_id(files_id, session)
2669 if poolfile is None:
2670 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2671 pfs.append(poolfile)
2673 df.poolfile_id = files_id
2676 # Add the src_uploaders to the DB
2677 source.uploaders = [source.maintainer]
2678 if u.pkg.dsc.has_key("uploaders"):
2679 for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
2681 source.uploaders.append(get_or_set_maintainer(up, session))
2685 return source, dsc_component, dsc_location_id, pfs
2687 __all__.append('add_dsc_to_db')
2690 def add_deb_to_db(u, filename, session=None):
2692 Contrary to what you might expect, this routine deals with both
2693 debs and udebs. That info is in 'dbtype', whilst 'type' is
2694 'deb' for both of them
2697 entry = u.pkg.files[filename]
2700 bin.package = entry["package"]
2701 bin.version = entry["version"]
2702 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2703 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2704 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2705 bin.binarytype = entry["dbtype"]
2708 filename = entry["pool name"] + filename
2709 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2710 if not entry.get("location id", None):
2711 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2713 if entry.get("files id", None):
2714 poolfile = get_poolfile_by_id(bin.poolfile_id)
2715 bin.poolfile_id = entry["files id"]
2717 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2718 bin.poolfile_id = entry["files id"] = poolfile.file_id
2721 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2722 if len(bin_sources) != 1:
2723 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2724 (bin.package, bin.version, entry["architecture"],
2725 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2727 bin.source_id = bin_sources[0].source_id
2729 if entry.has_key("built-using"):
2730 for srcname, version in entry["built-using"]:
2731 exsources = get_sources_from_name(srcname, version, session=session)
2732 if len(exsources) != 1:
2733 raise NoSourceFieldError, "Unable to find source package (%s = %s) in Built-Using for %s (%s), %s, file %s, type %s, signed by %s" % \
2734 (srcname, version, bin.package, bin.version, entry["architecture"],
2735 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2737 bin.extra_sources.append(exsources[0])
2739 # Add and flush object so it has an ID
2742 suite_names = u.pkg.changes["distribution"].keys()
2743 bin.suites = session.query(Suite). \
2744 filter(Suite.suite_name.in_(suite_names)).all()
2748 # Deal with contents - disabled for now
2749 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2751 # print "REJECT\nCould not determine contents of package %s" % bin.package
2752 # session.rollback()
2753 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2755 return bin, poolfile
2757 __all__.append('add_deb_to_db')
2759 ################################################################################
2761 class SourceACL(object):
2762 def __init__(self, *args, **kwargs):
2766 return '<SourceACL %s>' % self.source_acl_id
2768 __all__.append('SourceACL')
2770 ################################################################################
2772 class SrcFormat(object):
2773 def __init__(self, *args, **kwargs):
2777 return '<SrcFormat %s>' % (self.format_name)
2779 __all__.append('SrcFormat')
2781 ################################################################################
2783 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2784 ('SuiteID', 'suite_id'),
2785 ('Version', 'version'),
2786 ('Origin', 'origin'),
2788 ('Description', 'description'),
2789 ('Untouchable', 'untouchable'),
2790 ('Announce', 'announce'),
2791 ('Codename', 'codename'),
2792 ('OverrideCodename', 'overridecodename'),
2793 ('ValidTime', 'validtime'),
2794 ('Priority', 'priority'),
2795 ('NotAutomatic', 'notautomatic'),
2796 ('CopyChanges', 'copychanges'),
2797 ('OverrideSuite', 'overridesuite')]
2799 # Why the heck don't we have any UNIQUE constraints in table suite?
2800 # TODO: Add UNIQUE constraints for appropriate columns.
2801 class Suite(ORMObject):
2802 def __init__(self, suite_name = None, version = None):
2803 self.suite_name = suite_name
2804 self.version = version
2806 def properties(self):
2807 return ['suite_name', 'version', 'sources_count', 'binaries_count', \
2810 def not_null_constraints(self):
2811 return ['suite_name']
2813 def __eq__(self, val):
2814 if isinstance(val, str):
2815 return (self.suite_name == val)
2816 # This signals to use the normal comparison operator
2817 return NotImplemented
2819 def __ne__(self, val):
2820 if isinstance(val, str):
2821 return (self.suite_name != val)
2822 # This signals to use the normal comparison operator
2823 return NotImplemented
2827 for disp, field in SUITE_FIELDS:
2828 val = getattr(self, field, None)
2830 ret.append("%s: %s" % (disp, val))
2832 return "\n".join(ret)
2834 def get_architectures(self, skipsrc=False, skipall=False):
2836 Returns list of Architecture objects
2838 @type skipsrc: boolean
2839 @param skipsrc: Whether to skip returning the 'source' architecture entry
2842 @type skipall: boolean
2843 @param skipall: Whether to skip returning the 'all' architecture entry
2847 @return: list of Architecture objects for the given name (may be empty)
2850 q = object_session(self).query(Architecture).with_parent(self)
2852 q = q.filter(Architecture.arch_string != 'source')
2854 q = q.filter(Architecture.arch_string != 'all')
2855 return q.order_by(Architecture.arch_string).all()
2857 def get_sources(self, source):
2859 Returns a query object representing DBSource that is part of C{suite}.
2861 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2863 @type source: string
2864 @param source: source package name
2866 @rtype: sqlalchemy.orm.query.Query
2867 @return: a query of DBSource
2871 session = object_session(self)
2872 return session.query(DBSource).filter_by(source = source). \
2875 def get_overridesuite(self):
2876 if self.overridesuite is None:
2879 return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one()
2881 __all__.append('Suite')
2884 def get_suite(suite, session=None):
2886 Returns Suite object for given C{suite name}.
2889 @param suite: The name of the suite
2891 @type session: Session
2892 @param session: Optional SQLA session object (a temporary one will be
2893 generated if not supplied)
2896 @return: Suite object for the requested suite name (None if not present)
2899 q = session.query(Suite).filter_by(suite_name=suite)
2903 except NoResultFound:
2906 __all__.append('get_suite')
2908 ################################################################################
2910 # TODO: should be removed because the implementation is too trivial
2912 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2914 Returns list of Architecture objects for given C{suite} name
2917 @param suite: Suite name to search for
2919 @type skipsrc: boolean
2920 @param skipsrc: Whether to skip returning the 'source' architecture entry
2923 @type skipall: boolean
2924 @param skipall: Whether to skip returning the 'all' architecture entry
2927 @type session: Session
2928 @param session: Optional SQL session object (a temporary one will be
2929 generated if not supplied)
2932 @return: list of Architecture objects for the given name (may be empty)
2935 return get_suite(suite, session).get_architectures(skipsrc, skipall)
2937 __all__.append('get_suite_architectures')
2939 ################################################################################
2941 class SuiteSrcFormat(object):
2942 def __init__(self, *args, **kwargs):
2946 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2948 __all__.append('SuiteSrcFormat')
2951 def get_suite_src_formats(suite, session=None):
2953 Returns list of allowed SrcFormat for C{suite}.
2956 @param suite: Suite name to search for
2958 @type session: Session
2959 @param session: Optional SQL session object (a temporary one will be
2960 generated if not supplied)
2963 @return: the list of allowed source formats for I{suite}
2966 q = session.query(SrcFormat)
2967 q = q.join(SuiteSrcFormat)
2968 q = q.join(Suite).filter_by(suite_name=suite)
2969 q = q.order_by('format_name')
2973 __all__.append('get_suite_src_formats')
2975 ################################################################################
2977 class Uid(ORMObject):
2978 def __init__(self, uid = None, name = None):
2982 def __eq__(self, val):
2983 if isinstance(val, str):
2984 return (self.uid == val)
2985 # This signals to use the normal comparison operator
2986 return NotImplemented
2988 def __ne__(self, val):
2989 if isinstance(val, str):
2990 return (self.uid != val)
2991 # This signals to use the normal comparison operator
2992 return NotImplemented
2994 def properties(self):
2995 return ['uid', 'name', 'fingerprint']
2997 def not_null_constraints(self):
3000 __all__.append('Uid')
3003 def get_or_set_uid(uidname, session=None):
3005 Returns uid object for given uidname.
3007 If no matching uidname is found, a row is inserted.
3009 @type uidname: string
3010 @param uidname: The uid to add
3012 @type session: SQLAlchemy
3013 @param session: Optional SQL session object (a temporary one will be
3014 generated if not supplied). If not passed, a commit will be performed at
3015 the end of the function, otherwise the caller is responsible for commiting.
3018 @return: the uid object for the given uidname
3021 q = session.query(Uid).filter_by(uid=uidname)
3025 except NoResultFound:
3029 session.commit_or_flush()
3034 __all__.append('get_or_set_uid')
3037 def get_uid_from_fingerprint(fpr, session=None):
3038 q = session.query(Uid)
3039 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
3043 except NoResultFound:
3046 __all__.append('get_uid_from_fingerprint')
3048 ################################################################################
3050 class UploadBlock(object):
3051 def __init__(self, *args, **kwargs):
3055 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
3057 __all__.append('UploadBlock')
3059 ################################################################################
3061 class MetadataKey(ORMObject):
3062 def __init__(self, key = None):
3065 def properties(self):
3068 def not_null_constraints(self):
3071 __all__.append('MetadataKey')
3074 def get_or_set_metadatakey(keyname, session=None):
3076 Returns MetadataKey object for given uidname.
3078 If no matching keyname is found, a row is inserted.
3080 @type uidname: string
3081 @param uidname: The keyname to add
3083 @type session: SQLAlchemy
3084 @param session: Optional SQL session object (a temporary one will be
3085 generated if not supplied). If not passed, a commit will be performed at
3086 the end of the function, otherwise the caller is responsible for commiting.
3089 @return: the metadatakey object for the given keyname
3092 q = session.query(MetadataKey).filter_by(key=keyname)
3096 except NoResultFound:
3097 ret = MetadataKey(keyname)
3099 session.commit_or_flush()
3103 __all__.append('get_or_set_metadatakey')
3105 ################################################################################
3107 class BinaryMetadata(ORMObject):
3108 def __init__(self, key = None, value = None, binary = None):
3111 self.binary = binary
3113 def properties(self):
3114 return ['binary', 'key', 'value']
3116 def not_null_constraints(self):
3119 __all__.append('BinaryMetadata')
3121 ################################################################################
3123 class SourceMetadata(ORMObject):
3124 def __init__(self, key = None, value = None, source = None):
3127 self.source = source
3129 def properties(self):
3130 return ['source', 'key', 'value']
3132 def not_null_constraints(self):
3135 __all__.append('SourceMetadata')
3137 ################################################################################
3139 class VersionCheck(ORMObject):
3140 def __init__(self, *args, **kwargs):
3143 def properties(self):
3144 #return ['suite_id', 'check', 'reference_id']
3147 def not_null_constraints(self):
3148 return ['suite', 'check', 'reference']
3150 __all__.append('VersionCheck')
3153 def get_version_checks(suite_name, check = None, session = None):
3154 suite = get_suite(suite_name, session)
3156 # Make sure that what we return is iterable so that list comprehensions
3157 # involving this don't cause a traceback
3159 q = session.query(VersionCheck).filter_by(suite=suite)
3161 q = q.filter_by(check=check)
3164 __all__.append('get_version_checks')
3166 ################################################################################
3168 class DBConn(object):
3170 database module init.
3174 def __init__(self, *args, **kwargs):
3175 self.__dict__ = self.__shared_state
3177 if not getattr(self, 'initialised', False):
3178 self.initialised = True
3179 self.debug = kwargs.has_key('debug')
3182 def __setuptables(self):
3189 'binaries_metadata',
3193 'build_queue_files',
3194 'build_queue_policy_files',
3199 'changes_pending_binaries',
3200 'changes_pending_files',
3201 'changes_pending_source',
3202 'changes_pending_files_map',
3203 'changes_pending_source_files',
3204 'changes_pool_files',
3206 'external_overrides',
3207 'extra_src_references',
3216 # TODO: the maintainer column in table override should be removed.
3230 'suite_architectures',
3231 'suite_build_queue_copy',
3232 'suite_src_formats',
3239 'almost_obsolete_all_associations',
3240 'almost_obsolete_src_associations',
3241 'any_associations_source',
3242 'bin_associations_binaries',
3243 'binaries_suite_arch',
3244 'binfiles_suite_component_arch',
3247 'newest_all_associations',
3248 'newest_any_associations',
3250 'newest_src_association',
3251 'obsolete_all_associations',
3252 'obsolete_any_associations',
3253 'obsolete_any_by_all_associations',
3254 'obsolete_src_associations',
3256 'src_associations_bin',
3257 'src_associations_src',
3258 'suite_arch_by_name',
3261 for table_name in tables:
3262 table = Table(table_name, self.db_meta, \
3263 autoload=True, useexisting=True)
3264 setattr(self, 'tbl_%s' % table_name, table)
3266 for view_name in views:
3267 view = Table(view_name, self.db_meta, autoload=True)
3268 setattr(self, 'view_%s' % view_name, view)
3270 def __setupmappers(self):
3271 mapper(Architecture, self.tbl_architecture,
3272 properties = dict(arch_id = self.tbl_architecture.c.id,
3273 suites = relation(Suite, secondary=self.tbl_suite_architectures,
3274 order_by='suite_name',
3275 backref=backref('architectures', order_by='arch_string'))),
3276 extension = validator)
3278 mapper(Archive, self.tbl_archive,
3279 properties = dict(archive_id = self.tbl_archive.c.id,
3280 archive_name = self.tbl_archive.c.name))
3282 mapper(BuildQueue, self.tbl_build_queue,
3283 properties = dict(queue_id = self.tbl_build_queue.c.id))
3285 mapper(BuildQueueFile, self.tbl_build_queue_files,
3286 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
3287 poolfile = relation(PoolFile, backref='buildqueueinstances')))
3289 mapper(BuildQueuePolicyFile, self.tbl_build_queue_policy_files,
3291 build_queue = relation(BuildQueue, backref='policy_queue_files'),
3292 file = relation(ChangePendingFile, lazy='joined')))
3294 mapper(DBBinary, self.tbl_binaries,
3295 properties = dict(binary_id = self.tbl_binaries.c.id,
3296 package = self.tbl_binaries.c.package,
3297 version = self.tbl_binaries.c.version,
3298 maintainer_id = self.tbl_binaries.c.maintainer,
3299 maintainer = relation(Maintainer),
3300 source_id = self.tbl_binaries.c.source,
3301 source = relation(DBSource, backref='binaries'),
3302 arch_id = self.tbl_binaries.c.architecture,
3303 architecture = relation(Architecture),
3304 poolfile_id = self.tbl_binaries.c.file,
3305 poolfile = relation(PoolFile, backref=backref('binary', uselist = False)),
3306 binarytype = self.tbl_binaries.c.type,
3307 fingerprint_id = self.tbl_binaries.c.sig_fpr,
3308 fingerprint = relation(Fingerprint),
3309 install_date = self.tbl_binaries.c.install_date,
3310 suites = relation(Suite, secondary=self.tbl_bin_associations,
3311 backref=backref('binaries', lazy='dynamic')),
3312 extra_sources = relation(DBSource, secondary=self.tbl_extra_src_references,
3313 backref=backref('extra_binary_references', lazy='dynamic')),
3314 key = relation(BinaryMetadata, cascade='all',
3315 collection_class=attribute_mapped_collection('key'))),
3316 extension = validator)
3318 mapper(BinaryACL, self.tbl_binary_acl,
3319 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
3321 mapper(BinaryACLMap, self.tbl_binary_acl_map,
3322 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
3323 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
3324 architecture = relation(Architecture)))
3326 mapper(Component, self.tbl_component,
3327 properties = dict(component_id = self.tbl_component.c.id,
3328 component_name = self.tbl_component.c.name),
3329 extension = validator)
3331 mapper(DBConfig, self.tbl_config,
3332 properties = dict(config_id = self.tbl_config.c.id))
3334 mapper(DSCFile, self.tbl_dsc_files,
3335 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
3336 source_id = self.tbl_dsc_files.c.source,
3337 source = relation(DBSource),
3338 poolfile_id = self.tbl_dsc_files.c.file,
3339 poolfile = relation(PoolFile)))
3341 mapper(ExternalOverride, self.tbl_external_overrides,
3343 suite_id = self.tbl_external_overrides.c.suite,
3344 suite = relation(Suite),
3345 component_id = self.tbl_external_overrides.c.component,
3346 component = relation(Component)))
3348 mapper(PoolFile, self.tbl_files,
3349 properties = dict(file_id = self.tbl_files.c.id,
3350 filesize = self.tbl_files.c.size,
3351 location_id = self.tbl_files.c.location,
3352 location = relation(Location,
3353 # using lazy='dynamic' in the back
3354 # reference because we have A LOT of
3355 # files in one location
3356 backref=backref('files', lazy='dynamic'))),
3357 extension = validator)
3359 mapper(Fingerprint, self.tbl_fingerprint,
3360 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
3361 uid_id = self.tbl_fingerprint.c.uid,
3362 uid = relation(Uid),
3363 keyring_id = self.tbl_fingerprint.c.keyring,
3364 keyring = relation(Keyring),
3365 source_acl = relation(SourceACL),
3366 binary_acl = relation(BinaryACL)),
3367 extension = validator)
3369 mapper(Keyring, self.tbl_keyrings,
3370 properties = dict(keyring_name = self.tbl_keyrings.c.name,
3371 keyring_id = self.tbl_keyrings.c.id))
3373 mapper(DBChange, self.tbl_changes,
3374 properties = dict(change_id = self.tbl_changes.c.id,
3375 poolfiles = relation(PoolFile,
3376 secondary=self.tbl_changes_pool_files,
3377 backref="changeslinks"),
3378 seen = self.tbl_changes.c.seen,
3379 source = self.tbl_changes.c.source,
3380 binaries = self.tbl_changes.c.binaries,
3381 architecture = self.tbl_changes.c.architecture,
3382 distribution = self.tbl_changes.c.distribution,
3383 urgency = self.tbl_changes.c.urgency,
3384 maintainer = self.tbl_changes.c.maintainer,
3385 changedby = self.tbl_changes.c.changedby,
3386 date = self.tbl_changes.c.date,
3387 version = self.tbl_changes.c.version,
3388 files = relation(ChangePendingFile,
3389 secondary=self.tbl_changes_pending_files_map,
3390 backref="changesfile"),
3391 in_queue_id = self.tbl_changes.c.in_queue,
3392 in_queue = relation(PolicyQueue,
3393 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
3394 approved_for_id = self.tbl_changes.c.approved_for))
3396 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
3397 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
3399 mapper(ChangePendingFile, self.tbl_changes_pending_files,
3400 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id,
3401 filename = self.tbl_changes_pending_files.c.filename,
3402 size = self.tbl_changes_pending_files.c.size,
3403 md5sum = self.tbl_changes_pending_files.c.md5sum,
3404 sha1sum = self.tbl_changes_pending_files.c.sha1sum,
3405 sha256sum = self.tbl_changes_pending_files.c.sha256sum))
3407 mapper(ChangePendingSource, self.tbl_changes_pending_source,
3408 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
3409 change = relation(DBChange),
3410 maintainer = relation(Maintainer,
3411 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
3412 changedby = relation(Maintainer,
3413 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
3414 fingerprint = relation(Fingerprint),
3415 source_files = relation(ChangePendingFile,
3416 secondary=self.tbl_changes_pending_source_files,
3417 backref="pending_sources")))
3420 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
3421 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
3422 keyring = relation(Keyring, backref="keyring_acl_map"),
3423 architecture = relation(Architecture)))
3425 mapper(Location, self.tbl_location,
3426 properties = dict(location_id = self.tbl_location.c.id,
3427 component_id = self.tbl_location.c.component,
3428 component = relation(Component, backref='location'),
3429 archive_id = self.tbl_location.c.archive,
3430 archive = relation(Archive),
3431 # FIXME: the 'type' column is old cruft and
3432 # should be removed in the future.
3433 archive_type = self.tbl_location.c.type),
3434 extension = validator)
3436 mapper(Maintainer, self.tbl_maintainer,
3437 properties = dict(maintainer_id = self.tbl_maintainer.c.id,
3438 maintains_sources = relation(DBSource, backref='maintainer',
3439 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.maintainer)),
3440 changed_sources = relation(DBSource, backref='changedby',
3441 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.changedby))),
3442 extension = validator)
3444 mapper(NewComment, self.tbl_new_comments,
3445 properties = dict(comment_id = self.tbl_new_comments.c.id))
3447 mapper(Override, self.tbl_override,
3448 properties = dict(suite_id = self.tbl_override.c.suite,
3449 suite = relation(Suite, \
3450 backref=backref('overrides', lazy='dynamic')),
3451 package = self.tbl_override.c.package,
3452 component_id = self.tbl_override.c.component,
3453 component = relation(Component, \
3454 backref=backref('overrides', lazy='dynamic')),
3455 priority_id = self.tbl_override.c.priority,
3456 priority = relation(Priority, \
3457 backref=backref('overrides', lazy='dynamic')),
3458 section_id = self.tbl_override.c.section,
3459 section = relation(Section, \
3460 backref=backref('overrides', lazy='dynamic')),
3461 overridetype_id = self.tbl_override.c.type,
3462 overridetype = relation(OverrideType, \
3463 backref=backref('overrides', lazy='dynamic'))))
3465 mapper(OverrideType, self.tbl_override_type,
3466 properties = dict(overridetype = self.tbl_override_type.c.type,
3467 overridetype_id = self.tbl_override_type.c.id))
3469 mapper(PolicyQueue, self.tbl_policy_queue,
3470 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
3472 mapper(Priority, self.tbl_priority,
3473 properties = dict(priority_id = self.tbl_priority.c.id))
3475 mapper(Section, self.tbl_section,
3476 properties = dict(section_id = self.tbl_section.c.id,
3477 section=self.tbl_section.c.section))
3479 mapper(DBSource, self.tbl_source,
3480 properties = dict(source_id = self.tbl_source.c.id,
3481 version = self.tbl_source.c.version,
3482 maintainer_id = self.tbl_source.c.maintainer,
3483 poolfile_id = self.tbl_source.c.file,
3484 poolfile = relation(PoolFile, backref=backref('source', uselist = False)),
3485 fingerprint_id = self.tbl_source.c.sig_fpr,
3486 fingerprint = relation(Fingerprint),
3487 changedby_id = self.tbl_source.c.changedby,
3488 srcfiles = relation(DSCFile,
3489 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
3490 suites = relation(Suite, secondary=self.tbl_src_associations,
3491 backref=backref('sources', lazy='dynamic')),
3492 uploaders = relation(Maintainer,
3493 secondary=self.tbl_src_uploaders),
3494 key = relation(SourceMetadata, cascade='all',
3495 collection_class=attribute_mapped_collection('key'))),
3496 extension = validator)
3498 mapper(SourceACL, self.tbl_source_acl,
3499 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
3501 mapper(SrcFormat, self.tbl_src_format,
3502 properties = dict(src_format_id = self.tbl_src_format.c.id,
3503 format_name = self.tbl_src_format.c.format_name))
3505 mapper(Suite, self.tbl_suite,
3506 properties = dict(suite_id = self.tbl_suite.c.id,
3507 policy_queue = relation(PolicyQueue),
3508 copy_queues = relation(BuildQueue,
3509 secondary=self.tbl_suite_build_queue_copy)),
3510 extension = validator)
3512 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
3513 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
3514 suite = relation(Suite, backref='suitesrcformats'),
3515 src_format_id = self.tbl_suite_src_formats.c.src_format,
3516 src_format = relation(SrcFormat)))
3518 mapper(Uid, self.tbl_uid,
3519 properties = dict(uid_id = self.tbl_uid.c.id,
3520 fingerprint = relation(Fingerprint)),
3521 extension = validator)
3523 mapper(UploadBlock, self.tbl_upload_blocks,
3524 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
3525 fingerprint = relation(Fingerprint, backref="uploadblocks"),
3526 uid = relation(Uid, backref="uploadblocks")))
3528 mapper(BinContents, self.tbl_bin_contents,
3530 binary = relation(DBBinary,
3531 backref=backref('contents', lazy='dynamic', cascade='all')),
3532 file = self.tbl_bin_contents.c.file))
3534 mapper(SrcContents, self.tbl_src_contents,
3536 source = relation(DBSource,
3537 backref=backref('contents', lazy='dynamic', cascade='all')),
3538 file = self.tbl_src_contents.c.file))
3540 mapper(MetadataKey, self.tbl_metadata_keys,
3542 key_id = self.tbl_metadata_keys.c.key_id,
3543 key = self.tbl_metadata_keys.c.key))
3545 mapper(BinaryMetadata, self.tbl_binaries_metadata,
3547 binary_id = self.tbl_binaries_metadata.c.bin_id,
3548 binary = relation(DBBinary),
3549 key_id = self.tbl_binaries_metadata.c.key_id,
3550 key = relation(MetadataKey),
3551 value = self.tbl_binaries_metadata.c.value))
3553 mapper(SourceMetadata, self.tbl_source_metadata,
3555 source_id = self.tbl_source_metadata.c.src_id,
3556 source = relation(DBSource),
3557 key_id = self.tbl_source_metadata.c.key_id,
3558 key = relation(MetadataKey),
3559 value = self.tbl_source_metadata.c.value))
3561 mapper(VersionCheck, self.tbl_version_check,
3563 suite_id = self.tbl_version_check.c.suite,
3564 suite = relation(Suite, primaryjoin=self.tbl_version_check.c.suite==self.tbl_suite.c.id),
3565 reference_id = self.tbl_version_check.c.reference,
3566 reference = relation(Suite, primaryjoin=self.tbl_version_check.c.reference==self.tbl_suite.c.id, lazy='joined')))
3568 ## Connection functions
3569 def __createconn(self):
3570 from config import Config
3572 if cnf.has_key("DB::Service"):
3573 connstr = "postgresql://service=%s" % cnf["DB::Service"]
3574 elif cnf.has_key("DB::Host"):
3576 connstr = "postgresql://%s" % cnf["DB::Host"]
3577 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
3578 connstr += ":%s" % cnf["DB::Port"]
3579 connstr += "/%s" % cnf["DB::Name"]
3582 connstr = "postgresql:///%s" % cnf["DB::Name"]
3583 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
3584 connstr += "?port=%s" % cnf["DB::Port"]
3586 engine_args = { 'echo': self.debug }
3587 if cnf.has_key('DB::PoolSize'):
3588 engine_args['pool_size'] = int(cnf['DB::PoolSize'])
3589 if cnf.has_key('DB::MaxOverflow'):
3590 engine_args['max_overflow'] = int(cnf['DB::MaxOverflow'])
3591 if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \
3592 cnf['DB::Unicode'] == 'false':
3593 engine_args['use_native_unicode'] = False
3595 # Monkey patch a new dialect in in order to support service= syntax
3596 import sqlalchemy.dialects.postgresql
3597 from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
3598 class PGDialect_psycopg2_dak(PGDialect_psycopg2):
3599 def create_connect_args(self, url):
3600 if str(url).startswith('postgresql://service='):
3602 servicename = str(url)[21:]
3603 return (['service=%s' % servicename], {})
3605 return PGDialect_psycopg2.create_connect_args(self, url)
3607 sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak
3609 self.db_pg = create_engine(connstr, **engine_args)
3610 self.db_meta = MetaData()
3611 self.db_meta.bind = self.db_pg
3612 self.db_smaker = sessionmaker(bind=self.db_pg,
3616 self.__setuptables()
3617 self.__setupmappers()
3618 self.pid = os.getpid()
3620 def session(self, work_mem = 0):
3622 Returns a new session object. If a work_mem parameter is provided a new
3623 transaction is started and the work_mem parameter is set for this
3624 transaction. The work_mem parameter is measured in MB. A default value
3625 will be used if the parameter is not set.
3627 # reinitialize DBConn in new processes
3628 if self.pid != os.getpid():
3631 session = self.db_smaker()
3633 session.execute("SET LOCAL work_mem TO '%d MB'" % work_mem)
3636 __all__.append('DBConn')