5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
37 from os.path import normpath
49 import simplejson as json
51 from datetime import datetime, timedelta
52 from errno import ENOENT
53 from tempfile import mkstemp, mkdtemp
54 from subprocess import Popen, PIPE
55 from tarfile import TarFile
57 from inspect import getargspec
60 from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \
62 from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
63 backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers
64 from sqlalchemy import types as sqltypes
65 from sqlalchemy.orm.collections import attribute_mapped_collection
66 from sqlalchemy.ext.associationproxy import association_proxy
68 # Don't remove this, we re-export the exceptions to scripts which import us
69 from sqlalchemy.exc import *
70 from sqlalchemy.orm.exc import NoResultFound
72 # Only import Config until Queue stuff is changed to store its config
74 from config import Config
75 from textutils import fix_maintainer
76 from dak_exceptions import DBUpdateError, NoSourceFieldError
78 # suppress some deprecation warnings in squeeze related to sqlalchemy
80 warnings.filterwarnings('ignore', \
81 "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
85 ################################################################################
87 # Patch in support for the debversion field type so that it works during
91 # that is for sqlalchemy 0.6
92 UserDefinedType = sqltypes.UserDefinedType
94 # this one for sqlalchemy 0.5
95 UserDefinedType = sqltypes.TypeEngine
97 class DebVersion(UserDefinedType):
98 def get_col_spec(self):
101 def bind_processor(self, dialect):
104 # ' = None' is needed for sqlalchemy 0.5:
105 def result_processor(self, dialect, coltype = None):
108 sa_major_version = sqlalchemy.__version__[0:3]
109 if sa_major_version in ["0.5", "0.6"]:
110 from sqlalchemy.databases import postgres
111 postgres.ischema_names['debversion'] = DebVersion
113 raise Exception("dak only ported to SQLA versions 0.5 and 0.6. See daklib/dbconn.py")
115 ################################################################################
117 __all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion']
119 ################################################################################
121 def session_wrapper(fn):
123 Wrapper around common ".., session=None):" handling. If the wrapped
124 function is called without passing 'session', we create a local one
125 and destroy it when the function ends.
127 Also attaches a commit_or_flush method to the session; if we created a
128 local session, this is a synonym for session.commit(), otherwise it is a
129 synonym for session.flush().
132 def wrapped(*args, **kwargs):
133 private_transaction = False
135 # Find the session object
136 session = kwargs.get('session')
139 if len(args) <= len(getargspec(fn)[0]) - 1:
140 # No session specified as last argument or in kwargs
141 private_transaction = True
142 session = kwargs['session'] = DBConn().session()
144 # Session is last argument in args
148 session = args[-1] = DBConn().session()
149 private_transaction = True
151 if private_transaction:
152 session.commit_or_flush = session.commit
154 session.commit_or_flush = session.flush
157 return fn(*args, **kwargs)
159 if private_transaction:
160 # We created a session; close it.
163 wrapped.__doc__ = fn.__doc__
164 wrapped.func_name = fn.func_name
168 __all__.append('session_wrapper')
170 ################################################################################
172 class ORMObject(object):
174 ORMObject is a base class for all ORM classes mapped by SQLalchemy. All
175 derived classes must implement the properties() method.
178 def properties(self):
180 This method should be implemented by all derived classes and returns a
181 list of the important properties. The properties 'created' and
182 'modified' will be added automatically. A suffix '_count' should be
183 added to properties that are lists or query objects. The most important
184 property name should be returned as the first element in the list
185 because it is used by repr().
191 Returns a JSON representation of the object based on the properties
192 returned from the properties() method.
195 # add created and modified
196 all_properties = self.properties() + ['created', 'modified']
197 for property in all_properties:
198 # check for list or query
199 if property[-6:] == '_count':
200 real_property = property[:-6]
201 if not hasattr(self, real_property):
203 value = getattr(self, real_property)
204 if hasattr(value, '__len__'):
207 elif hasattr(value, 'count'):
208 # query (but not during validation)
209 if self.in_validation:
211 value = value.count()
213 raise KeyError('Do not understand property %s.' % property)
215 if not hasattr(self, property):
218 value = getattr(self, property)
222 elif isinstance(value, ORMObject):
223 # use repr() for ORMObject types
226 # we want a string for all other types because json cannot
229 data[property] = value
230 return json.dumps(data)
234 Returns the name of the class.
236 return type(self).__name__
240 Returns a short string representation of the object using the first
241 element from the properties() method.
243 primary_property = self.properties()[0]
244 value = getattr(self, primary_property)
245 return '<%s %s>' % (self.classname(), str(value))
249 Returns a human readable form of the object using the properties()
252 return '<%s %s>' % (self.classname(), self.json())
254 def not_null_constraints(self):
256 Returns a list of properties that must be not NULL. Derived classes
257 should override this method if needed.
261 validation_message = \
262 "Validation failed because property '%s' must not be empty in object\n%s"
264 in_validation = False
268 This function validates the not NULL constraints as returned by
269 not_null_constraints(). It raises the DBUpdateError exception if
272 for property in self.not_null_constraints():
273 # TODO: It is a bit awkward that the mapper configuration allow
274 # directly setting the numeric _id columns. We should get rid of it
276 if hasattr(self, property + '_id') and \
277 getattr(self, property + '_id') is not None:
279 if not hasattr(self, property) or getattr(self, property) is None:
280 # str() might lead to races due to a 2nd flush
281 self.in_validation = True
282 message = self.validation_message % (property, str(self))
283 self.in_validation = False
284 raise DBUpdateError(message)
288 def get(cls, primary_key, session = None):
290 This is a support function that allows getting an object by its primary
293 Architecture.get(3[, session])
295 instead of the more verbose
297 session.query(Architecture).get(3)
299 return session.query(cls).get(primary_key)
301 def session(self, replace = False):
303 Returns the current session that is associated with the object. May
304 return None is object is in detached state.
307 return object_session(self)
309 def clone(self, session = None):
311 Clones the current object in a new session and returns the new clone. A
312 fresh session is created if the optional session parameter is not
313 provided. The function will fail if a session is provided and has
316 RATIONALE: SQLAlchemy's session is not thread safe. This method clones
317 an existing object to allow several threads to work with their own
318 instances of an ORMObject.
320 WARNING: Only persistent (committed) objects can be cloned. Changes
321 made to the original object that are not committed yet will get lost.
322 The session of the new object will always be rolled back to avoid
326 if self.session() is None:
327 raise RuntimeError( \
328 'Method clone() failed for detached object:\n%s' % self)
329 self.session().flush()
330 mapper = object_mapper(self)
331 primary_key = mapper.primary_key_from_instance(self)
332 object_class = self.__class__
334 session = DBConn().session()
335 elif len(session.new) + len(session.dirty) + len(session.deleted) > 0:
336 raise RuntimeError( \
337 'Method clone() failed due to unflushed changes in session.')
338 new_object = session.query(object_class).get(primary_key)
340 if new_object is None:
341 raise RuntimeError( \
342 'Method clone() failed for non-persistent object:\n%s' % self)
345 __all__.append('ORMObject')
347 ################################################################################
349 class Validator(MapperExtension):
351 This class calls the validate() method for each instance for the
352 'before_update' and 'before_insert' events. A global object validator is
353 used for configuring the individual mappers.
356 def before_update(self, mapper, connection, instance):
360 def before_insert(self, mapper, connection, instance):
364 validator = Validator()
366 ################################################################################
368 class Architecture(ORMObject):
369 def __init__(self, arch_string = None, description = None):
370 self.arch_string = arch_string
371 self.description = description
373 def __eq__(self, val):
374 if isinstance(val, str):
375 return (self.arch_string== val)
376 # This signals to use the normal comparison operator
377 return NotImplemented
379 def __ne__(self, val):
380 if isinstance(val, str):
381 return (self.arch_string != val)
382 # This signals to use the normal comparison operator
383 return NotImplemented
385 def properties(self):
386 return ['arch_string', 'arch_id', 'suites_count']
388 def not_null_constraints(self):
389 return ['arch_string']
391 __all__.append('Architecture')
394 def get_architecture(architecture, session=None):
396 Returns database id for given C{architecture}.
398 @type architecture: string
399 @param architecture: The name of the architecture
401 @type session: Session
402 @param session: Optional SQLA session object (a temporary one will be
403 generated if not supplied)
406 @return: Architecture object for the given arch (None if not present)
409 q = session.query(Architecture).filter_by(arch_string=architecture)
413 except NoResultFound:
416 __all__.append('get_architecture')
418 # TODO: should be removed because the implementation is too trivial
420 def get_architecture_suites(architecture, session=None):
422 Returns list of Suite objects for given C{architecture} name
424 @type architecture: str
425 @param architecture: Architecture name to search for
427 @type session: Session
428 @param session: Optional SQL session object (a temporary one will be
429 generated if not supplied)
432 @return: list of Suite objects for the given name (may be empty)
435 return get_architecture(architecture, session).suites
437 __all__.append('get_architecture_suites')
439 ################################################################################
441 class Archive(object):
442 def __init__(self, *args, **kwargs):
446 return '<Archive %s>' % self.archive_name
448 __all__.append('Archive')
451 def get_archive(archive, session=None):
453 returns database id for given C{archive}.
455 @type archive: string
456 @param archive: the name of the arhive
458 @type session: Session
459 @param session: Optional SQLA session object (a temporary one will be
460 generated if not supplied)
463 @return: Archive object for the given name (None if not present)
466 archive = archive.lower()
468 q = session.query(Archive).filter_by(archive_name=archive)
472 except NoResultFound:
475 __all__.append('get_archive')
477 ################################################################################
479 class BinContents(ORMObject):
480 def __init__(self, file = None, binary = None):
484 def properties(self):
485 return ['file', 'binary']
487 __all__.append('BinContents')
489 ################################################################################
491 def subprocess_setup():
492 # Python installs a SIGPIPE handler by default. This is usually not what
493 # non-Python subprocesses expect.
494 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
496 class DBBinary(ORMObject):
497 def __init__(self, package = None, source = None, version = None, \
498 maintainer = None, architecture = None, poolfile = None, \
500 self.package = package
502 self.version = version
503 self.maintainer = maintainer
504 self.architecture = architecture
505 self.poolfile = poolfile
506 self.binarytype = binarytype
510 return self.binary_id
512 def properties(self):
513 return ['package', 'version', 'maintainer', 'source', 'architecture', \
514 'poolfile', 'binarytype', 'fingerprint', 'install_date', \
515 'suites_count', 'binary_id', 'contents_count', 'extra_sources']
517 def not_null_constraints(self):
518 return ['package', 'version', 'maintainer', 'source', 'poolfile', \
521 metadata = association_proxy('key', 'value')
523 def get_component_name(self):
524 return self.poolfile.location.component.component_name
526 def scan_contents(self):
528 Yields the contents of the package. Only regular files are yielded and
529 the path names are normalized after converting them from either utf-8
530 or iso8859-1 encoding. It yields the string ' <EMPTY PACKAGE>' if the
531 package does not contain any regular file.
533 fullpath = self.poolfile.fullpath
534 dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE,
535 preexec_fn = subprocess_setup)
536 tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
537 for member in tar.getmembers():
538 if not member.isdir():
539 name = normpath(member.name)
540 # enforce proper utf-8 encoding
543 except UnicodeDecodeError:
544 name = name.decode('iso8859-1').encode('utf-8')
550 def read_control(self):
552 Reads the control information from a binary.
555 @return: stanza text of the control section.
558 fullpath = self.poolfile.fullpath
559 deb_file = open(fullpath, 'r')
560 stanza = apt_inst.debExtractControl(deb_file)
565 def read_control_fields(self):
567 Reads the control information from a binary and return
571 @return: fields of the control section as a dictionary.
574 stanza = self.read_control()
575 return apt_pkg.TagSection(stanza)
577 __all__.append('DBBinary')
580 def get_suites_binary_in(package, session=None):
582 Returns list of Suite objects which given C{package} name is in
585 @param package: DBBinary package name to search for
588 @return: list of Suite objects for the given package
591 return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all()
593 __all__.append('get_suites_binary_in')
596 def get_component_by_package_suite(package, suite_list, arch_list=[], session=None):
598 Returns the component name of the newest binary package in suite_list or
599 None if no package is found. The result can be optionally filtered by a list
600 of architecture names.
603 @param package: DBBinary package name to search for
605 @type suite_list: list of str
606 @param suite_list: list of suite_name items
608 @type arch_list: list of str
609 @param arch_list: optional list of arch_string items that defaults to []
611 @rtype: str or NoneType
612 @return: name of component or None
615 q = session.query(DBBinary).filter_by(package = package). \
616 join(DBBinary.suites).filter(Suite.suite_name.in_(suite_list))
617 if len(arch_list) > 0:
618 q = q.join(DBBinary.architecture). \
619 filter(Architecture.arch_string.in_(arch_list))
620 binary = q.order_by(desc(DBBinary.version)).first()
624 return binary.get_component_name()
626 __all__.append('get_component_by_package_suite')
628 ################################################################################
630 class BinaryACL(object):
631 def __init__(self, *args, **kwargs):
635 return '<BinaryACL %s>' % self.binary_acl_id
637 __all__.append('BinaryACL')
639 ################################################################################
641 class BinaryACLMap(object):
642 def __init__(self, *args, **kwargs):
646 return '<BinaryACLMap %s>' % self.binary_acl_map_id
648 __all__.append('BinaryACLMap')
650 ################################################################################
655 ArchiveDir "%(archivepath)s";
656 OverrideDir "%(overridedir)s";
657 CacheDir "%(cachedir)s";
662 Packages::Compress ". bzip2 gzip";
663 Sources::Compress ". bzip2 gzip";
668 bindirectory "incoming"
673 BinOverride "override.sid.all3";
674 BinCacheDB "packages-accepted.db";
676 FileList "%(filelist)s";
679 Packages::Extensions ".deb .udeb";
682 bindirectory "incoming/"
685 BinOverride "override.sid.all3";
686 SrcOverride "override.sid.all3.src";
687 FileList "%(filelist)s";
691 class BuildQueue(object):
692 def __init__(self, *args, **kwargs):
696 return '<BuildQueue %s>' % self.queue_name
698 def write_metadata(self, starttime, force=False):
699 # Do we write out metafiles?
700 if not (force or self.generate_metadata):
703 session = DBConn().session().object_session(self)
705 fl_fd = fl_name = ac_fd = ac_name = None
707 arches = " ".join([ a.arch_string for a in session.query(Architecture).all() if a.arch_string != 'source' ])
708 startdir = os.getcwd()
711 # Grab files we want to include
712 newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
713 newer += session.query(BuildQueuePolicyFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueuePolicyFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
714 # Write file list with newer files
715 (fl_fd, fl_name) = mkstemp()
717 os.write(fl_fd, '%s\n' % n.fullpath)
722 # Write minimal apt.conf
723 # TODO: Remove hardcoding from template
724 (ac_fd, ac_name) = mkstemp()
725 os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path,
727 'cachedir': cnf["Dir::Cache"],
728 'overridedir': cnf["Dir::Override"],
732 # Run apt-ftparchive generate
733 os.chdir(os.path.dirname(ac_name))
734 os.system('apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
736 # Run apt-ftparchive release
737 # TODO: Eww - fix this
738 bname = os.path.basename(self.path)
742 # We have to remove the Release file otherwise it'll be included in the
745 os.unlink(os.path.join(bname, 'Release'))
749 os.system("""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="%s" release %s > Release""" % (self.origin, self.label, self.releasedescription, arches, bname))
751 # Crude hack with open and append, but this whole section is and should be redone.
752 if self.notautomatic:
753 release=open("Release", "a")
754 release.write("NotAutomatic: yes")
759 keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
760 if cnf.has_key("Dinstall::SigningPubKeyring"):
761 keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
763 os.system("gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey))
765 # Move the files if we got this far
766 os.rename('Release', os.path.join(bname, 'Release'))
768 os.rename('Release.gpg', os.path.join(bname, 'Release.gpg'))
770 # Clean up any left behind files
797 def clean_and_update(self, starttime, Logger, dryrun=False):
798 """WARNING: This routine commits for you"""
799 session = DBConn().session().object_session(self)
801 if self.generate_metadata and not dryrun:
802 self.write_metadata(starttime)
804 # Grab files older than our execution time
805 older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
806 older += session.query(BuildQueuePolicyFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueuePolicyFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
812 Logger.log(["I: Would have removed %s from the queue" % o.fullpath])
814 Logger.log(["I: Removing %s from the queue" % o.fullpath])
815 os.unlink(o.fullpath)
818 # If it wasn't there, don't worry
819 if e.errno == ENOENT:
822 # TODO: Replace with proper logging call
823 Logger.log(["E: Could not remove %s" % o.fullpath])
830 for f in os.listdir(self.path):
831 if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release') or f.startswith('advisory'):
834 if not self.contains_filename(f):
835 fp = os.path.join(self.path, f)
837 Logger.log(["I: Would remove unused link %s" % fp])
839 Logger.log(["I: Removing unused link %s" % fp])
843 Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath])
845 def contains_filename(self, filename):
848 @returns True if filename is supposed to be in the queue; False otherwise
850 session = DBConn().session().object_session(self)
851 if session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id, filename = filename).count() > 0:
853 elif session.query(BuildQueuePolicyFile).filter_by(build_queue = self, filename = filename).count() > 0:
857 def add_file_from_pool(self, poolfile):
858 """Copies a file into the pool. Assumes that the PoolFile object is
859 attached to the same SQLAlchemy session as the Queue object is.
861 The caller is responsible for committing after calling this function."""
862 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
864 # Check if we have a file of this name or this ID already
865 for f in self.queuefiles:
866 if (f.fileid is not None and f.fileid == poolfile.file_id) or \
867 (f.poolfile is not None and f.poolfile.filename == poolfile_basename):
868 # In this case, update the BuildQueueFile entry so we
869 # don't remove it too early
870 f.lastused = datetime.now()
871 DBConn().session().object_session(poolfile).add(f)
874 # Prepare BuildQueueFile object
875 qf = BuildQueueFile()
876 qf.build_queue_id = self.queue_id
877 qf.lastused = datetime.now()
878 qf.filename = poolfile_basename
880 targetpath = poolfile.fullpath
881 queuepath = os.path.join(self.path, poolfile_basename)
885 # We need to copy instead of symlink
887 utils.copy(targetpath, queuepath)
888 # NULL in the fileid field implies a copy
891 os.symlink(targetpath, queuepath)
892 qf.fileid = poolfile.file_id
896 # Get the same session as the PoolFile is using and add the qf to it
897 DBConn().session().object_session(poolfile).add(qf)
901 def add_changes_from_policy_queue(self, policyqueue, changes):
903 Copies a changes from a policy queue together with its poolfiles.
905 @type policyqueue: PolicyQueue
906 @param policyqueue: policy queue to copy the changes from
908 @type changes: DBChange
909 @param changes: changes to copy to this build queue
911 for policyqueuefile in changes.files:
912 self.add_file_from_policy_queue(policyqueue, policyqueuefile)
913 for poolfile in changes.poolfiles:
914 self.add_file_from_pool(poolfile)
916 def add_file_from_policy_queue(self, policyqueue, policyqueuefile):
918 Copies a file from a policy queue.
919 Assumes that the policyqueuefile is attached to the same SQLAlchemy
920 session as the Queue object is. The caller is responsible for
921 committing after calling this function.
923 @type policyqueue: PolicyQueue
924 @param policyqueue: policy queue to copy the file from
926 @type policyqueuefile: ChangePendingFile
927 @param policyqueuefile: file to be added to the build queue
929 session = DBConn().session().object_session(policyqueuefile)
931 # Is the file already there?
933 f = session.query(BuildQueuePolicyFile).filter_by(build_queue=self, file=policyqueuefile).one()
934 f.lastused = datetime.now()
936 except NoResultFound:
937 pass # continue below
939 # We have to add the file.
940 f = BuildQueuePolicyFile()
942 f.file = policyqueuefile
943 f.filename = policyqueuefile.filename
945 source = os.path.join(policyqueue.path, policyqueuefile.filename)
948 # Always copy files from policy queues as they might move around.
950 utils.copy(source, target)
957 __all__.append('BuildQueue')
960 def get_build_queue(queuename, session=None):
962 Returns BuildQueue object for given C{queue name}, creating it if it does not
965 @type queuename: string
966 @param queuename: The name of the queue
968 @type session: Session
969 @param session: Optional SQLA session object (a temporary one will be
970 generated if not supplied)
973 @return: BuildQueue object for the given queue
976 q = session.query(BuildQueue).filter_by(queue_name=queuename)
980 except NoResultFound:
983 __all__.append('get_build_queue')
985 ################################################################################
987 class BuildQueueFile(object):
989 BuildQueueFile represents a file in a build queue coming from a pool.
992 def __init__(self, *args, **kwargs):
996 return '<BuildQueueFile %s (%s)>' % (self.filename, self.build_queue_id)
1000 return os.path.join(self.buildqueue.path, self.filename)
1003 __all__.append('BuildQueueFile')
1005 ################################################################################
1007 class BuildQueuePolicyFile(object):
1009 BuildQueuePolicyFile represents a file in a build queue that comes from a
1010 policy queue (and not a pool).
1013 def __init__(self, *args, **kwargs):
1017 #def filename(self):
1018 # return self.file.filename
1022 return os.path.join(self.build_queue.path, self.filename)
1024 __all__.append('BuildQueuePolicyFile')
1026 ################################################################################
1028 class ChangePendingBinary(object):
1029 def __init__(self, *args, **kwargs):
1033 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
1035 __all__.append('ChangePendingBinary')
1037 ################################################################################
1039 class ChangePendingFile(object):
1040 def __init__(self, *args, **kwargs):
1044 return '<ChangePendingFile %s>' % self.change_pending_file_id
1046 __all__.append('ChangePendingFile')
1048 ################################################################################
1050 class ChangePendingSource(object):
1051 def __init__(self, *args, **kwargs):
1055 return '<ChangePendingSource %s>' % self.change_pending_source_id
1057 __all__.append('ChangePendingSource')
1059 ################################################################################
1061 class Component(ORMObject):
1062 def __init__(self, component_name = None):
1063 self.component_name = component_name
1065 def __eq__(self, val):
1066 if isinstance(val, str):
1067 return (self.component_name == val)
1068 # This signals to use the normal comparison operator
1069 return NotImplemented
1071 def __ne__(self, val):
1072 if isinstance(val, str):
1073 return (self.component_name != val)
1074 # This signals to use the normal comparison operator
1075 return NotImplemented
1077 def properties(self):
1078 return ['component_name', 'component_id', 'description', \
1079 'location_count', 'meets_dfsg', 'overrides_count']
1081 def not_null_constraints(self):
1082 return ['component_name']
1085 __all__.append('Component')
1088 def get_component(component, session=None):
1090 Returns database id for given C{component}.
1092 @type component: string
1093 @param component: The name of the override type
1096 @return: the database id for the given component
1099 component = component.lower()
1101 q = session.query(Component).filter_by(component_name=component)
1105 except NoResultFound:
1108 __all__.append('get_component')
1110 ################################################################################
1112 class DBConfig(object):
1113 def __init__(self, *args, **kwargs):
1117 return '<DBConfig %s>' % self.name
1119 __all__.append('DBConfig')
1121 ################################################################################
1124 def get_or_set_contents_file_id(filename, session=None):
1126 Returns database id for given filename.
1128 If no matching file is found, a row is inserted.
1130 @type filename: string
1131 @param filename: The filename
1132 @type session: SQLAlchemy
1133 @param session: Optional SQL session object (a temporary one will be
1134 generated if not supplied). If not passed, a commit will be performed at
1135 the end of the function, otherwise the caller is responsible for commiting.
1138 @return: the database id for the given component
1141 q = session.query(ContentFilename).filter_by(filename=filename)
1144 ret = q.one().cafilename_id
1145 except NoResultFound:
1146 cf = ContentFilename()
1147 cf.filename = filename
1149 session.commit_or_flush()
1150 ret = cf.cafilename_id
1154 __all__.append('get_or_set_contents_file_id')
1157 def get_contents(suite, overridetype, section=None, session=None):
1159 Returns contents for a suite / overridetype combination, limiting
1160 to a section if not None.
1163 @param suite: Suite object
1165 @type overridetype: OverrideType
1166 @param overridetype: OverrideType object
1168 @type section: Section
1169 @param section: Optional section object to limit results to
1171 @type session: SQLAlchemy
1172 @param session: Optional SQL session object (a temporary one will be
1173 generated if not supplied)
1175 @rtype: ResultsProxy
1176 @return: ResultsProxy object set up to return tuples of (filename, section,
1180 # find me all of the contents for a given suite
1181 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
1185 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
1186 JOIN content_file_names n ON (c.filename=n.id)
1187 JOIN binaries b ON (b.id=c.binary_pkg)
1188 JOIN override o ON (o.package=b.package)
1189 JOIN section s ON (s.id=o.section)
1190 WHERE o.suite = :suiteid AND o.type = :overridetypeid
1191 AND b.type=:overridetypename"""
1193 vals = {'suiteid': suite.suite_id,
1194 'overridetypeid': overridetype.overridetype_id,
1195 'overridetypename': overridetype.overridetype}
1197 if section is not None:
1198 contents_q += " AND s.id = :sectionid"
1199 vals['sectionid'] = section.section_id
1201 contents_q += " ORDER BY fn"
1203 return session.execute(contents_q, vals)
1205 __all__.append('get_contents')
1207 ################################################################################
1209 class ContentFilepath(object):
1210 def __init__(self, *args, **kwargs):
1214 return '<ContentFilepath %s>' % self.filepath
1216 __all__.append('ContentFilepath')
1219 def get_or_set_contents_path_id(filepath, session=None):
1221 Returns database id for given path.
1223 If no matching file is found, a row is inserted.
1225 @type filepath: string
1226 @param filepath: The filepath
1228 @type session: SQLAlchemy
1229 @param session: Optional SQL session object (a temporary one will be
1230 generated if not supplied). If not passed, a commit will be performed at
1231 the end of the function, otherwise the caller is responsible for commiting.
1234 @return: the database id for the given path
1237 q = session.query(ContentFilepath).filter_by(filepath=filepath)
1240 ret = q.one().cafilepath_id
1241 except NoResultFound:
1242 cf = ContentFilepath()
1243 cf.filepath = filepath
1245 session.commit_or_flush()
1246 ret = cf.cafilepath_id
1250 __all__.append('get_or_set_contents_path_id')
1252 ################################################################################
1254 class ContentAssociation(object):
1255 def __init__(self, *args, **kwargs):
1259 return '<ContentAssociation %s>' % self.ca_id
1261 __all__.append('ContentAssociation')
1263 def insert_content_paths(binary_id, fullpaths, session=None):
1265 Make sure given path is associated with given binary id
1267 @type binary_id: int
1268 @param binary_id: the id of the binary
1269 @type fullpaths: list
1270 @param fullpaths: the list of paths of the file being associated with the binary
1271 @type session: SQLAlchemy session
1272 @param session: Optional SQLAlchemy session. If this is passed, the caller
1273 is responsible for ensuring a transaction has begun and committing the
1274 results or rolling back based on the result code. If not passed, a commit
1275 will be performed at the end of the function, otherwise the caller is
1276 responsible for commiting.
1278 @return: True upon success
1281 privatetrans = False
1283 session = DBConn().session()
1288 def generate_path_dicts():
1289 for fullpath in fullpaths:
1290 if fullpath.startswith( './' ):
1291 fullpath = fullpath[2:]
1293 yield {'filename':fullpath, 'id': binary_id }
1295 for d in generate_path_dicts():
1296 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
1305 traceback.print_exc()
1307 # Only rollback if we set up the session ourself
1314 __all__.append('insert_content_paths')
1316 ################################################################################
1318 class DSCFile(object):
1319 def __init__(self, *args, **kwargs):
1323 return '<DSCFile %s>' % self.dscfile_id
1325 __all__.append('DSCFile')
1328 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
1330 Returns a list of DSCFiles which may be empty
1332 @type dscfile_id: int (optional)
1333 @param dscfile_id: the dscfile_id of the DSCFiles to find
1335 @type source_id: int (optional)
1336 @param source_id: the source id related to the DSCFiles to find
1338 @type poolfile_id: int (optional)
1339 @param poolfile_id: the poolfile id related to the DSCFiles to find
1342 @return: Possibly empty list of DSCFiles
1345 q = session.query(DSCFile)
1347 if dscfile_id is not None:
1348 q = q.filter_by(dscfile_id=dscfile_id)
1350 if source_id is not None:
1351 q = q.filter_by(source_id=source_id)
1353 if poolfile_id is not None:
1354 q = q.filter_by(poolfile_id=poolfile_id)
1358 __all__.append('get_dscfiles')
1360 ################################################################################
1362 class ExternalOverride(ORMObject):
1363 def __init__(self, *args, **kwargs):
1367 return '<ExternalOverride %s = %s: %s>' % (self.package, self.key, self.value)
1369 __all__.append('ExternalOverride')
1371 ################################################################################
1373 class PoolFile(ORMObject):
1374 def __init__(self, filename = None, location = None, filesize = -1, \
1376 self.filename = filename
1377 self.location = location
1378 self.filesize = filesize
1379 self.md5sum = md5sum
1383 return os.path.join(self.location.path, self.filename)
1385 def is_valid(self, filesize = -1, md5sum = None):
1386 return self.filesize == long(filesize) and self.md5sum == md5sum
1388 def properties(self):
1389 return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \
1390 'sha256sum', 'location', 'source', 'binary', 'last_used']
1392 def not_null_constraints(self):
1393 return ['filename', 'md5sum', 'location']
1395 __all__.append('PoolFile')
1398 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
1401 (ValidFileFound [boolean], PoolFile object or None)
1403 @type filename: string
1404 @param filename: the filename of the file to check against the DB
1407 @param filesize: the size of the file to check against the DB
1409 @type md5sum: string
1410 @param md5sum: the md5sum of the file to check against the DB
1412 @type location_id: int
1413 @param location_id: the id of the location to look in
1416 @return: Tuple of length 2.
1417 - If valid pool file found: (C{True}, C{PoolFile object})
1418 - If valid pool file not found:
1419 - (C{False}, C{None}) if no file found
1420 - (C{False}, C{PoolFile object}) if file found with size/md5sum mismatch
1423 poolfile = session.query(Location).get(location_id). \
1424 files.filter_by(filename=filename).first()
1426 if poolfile and poolfile.is_valid(filesize = filesize, md5sum = md5sum):
1429 return (valid, poolfile)
1431 __all__.append('check_poolfile')
1433 # TODO: the implementation can trivially be inlined at the place where the
1434 # function is called
1436 def get_poolfile_by_id(file_id, session=None):
1438 Returns a PoolFile objects or None for the given id
1441 @param file_id: the id of the file to look for
1443 @rtype: PoolFile or None
1444 @return: either the PoolFile object or None
1447 return session.query(PoolFile).get(file_id)
1449 __all__.append('get_poolfile_by_id')
1452 def get_poolfile_like_name(filename, session=None):
1454 Returns an array of PoolFile objects which are like the given name
1456 @type filename: string
1457 @param filename: the filename of the file to check against the DB
1460 @return: array of PoolFile objects
1463 # TODO: There must be a way of properly using bind parameters with %FOO%
1464 q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
1468 __all__.append('get_poolfile_like_name')
1471 def add_poolfile(filename, datadict, location_id, session=None):
1473 Add a new file to the pool
1475 @type filename: string
1476 @param filename: filename
1478 @type datadict: dict
1479 @param datadict: dict with needed data
1481 @type location_id: int
1482 @param location_id: database id of the location
1485 @return: the PoolFile object created
1487 poolfile = PoolFile()
1488 poolfile.filename = filename
1489 poolfile.filesize = datadict["size"]
1490 poolfile.md5sum = datadict["md5sum"]
1491 poolfile.sha1sum = datadict["sha1sum"]
1492 poolfile.sha256sum = datadict["sha256sum"]
1493 poolfile.location_id = location_id
1495 session.add(poolfile)
1496 # Flush to get a file id (NB: This is not a commit)
1501 __all__.append('add_poolfile')
1503 ################################################################################
1505 class Fingerprint(ORMObject):
1506 def __init__(self, fingerprint = None):
1507 self.fingerprint = fingerprint
1509 def properties(self):
1510 return ['fingerprint', 'fingerprint_id', 'keyring', 'uid', \
1513 def not_null_constraints(self):
1514 return ['fingerprint']
1516 __all__.append('Fingerprint')
1519 def get_fingerprint(fpr, session=None):
1521 Returns Fingerprint object for given fpr.
1524 @param fpr: The fpr to find / add
1526 @type session: SQLAlchemy
1527 @param session: Optional SQL session object (a temporary one will be
1528 generated if not supplied).
1531 @return: the Fingerprint object for the given fpr or None
1534 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1538 except NoResultFound:
1543 __all__.append('get_fingerprint')
1546 def get_or_set_fingerprint(fpr, session=None):
1548 Returns Fingerprint object for given fpr.
1550 If no matching fpr is found, a row is inserted.
1553 @param fpr: The fpr to find / add
1555 @type session: SQLAlchemy
1556 @param session: Optional SQL session object (a temporary one will be
1557 generated if not supplied). If not passed, a commit will be performed at
1558 the end of the function, otherwise the caller is responsible for commiting.
1559 A flush will be performed either way.
1562 @return: the Fingerprint object for the given fpr
1565 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1569 except NoResultFound:
1570 fingerprint = Fingerprint()
1571 fingerprint.fingerprint = fpr
1572 session.add(fingerprint)
1573 session.commit_or_flush()
1578 __all__.append('get_or_set_fingerprint')
1580 ################################################################################
1582 # Helper routine for Keyring class
1583 def get_ldap_name(entry):
1585 for k in ["cn", "mn", "sn"]:
1587 if ret and ret[0] != "" and ret[0] != "-":
1589 return " ".join(name)
1591 ################################################################################
1593 class Keyring(object):
1594 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1595 " --with-colons --fingerprint --fingerprint"
1600 def __init__(self, *args, **kwargs):
1604 return '<Keyring %s>' % self.keyring_name
1606 def de_escape_gpg_str(self, txt):
1607 esclist = re.split(r'(\\x..)', txt)
1608 for x in range(1,len(esclist),2):
1609 esclist[x] = "%c" % (int(esclist[x][2:],16))
1610 return "".join(esclist)
1612 def parse_address(self, uid):
1613 """parses uid and returns a tuple of real name and email address"""
1615 (name, address) = email.Utils.parseaddr(uid)
1616 name = re.sub(r"\s*[(].*[)]", "", name)
1617 name = self.de_escape_gpg_str(name)
1620 return (name, address)
1622 def load_keys(self, keyring):
1623 if not self.keyring_id:
1624 raise Exception('Must be initialized with database information')
1626 k = os.popen(self.gpg_invocation % keyring, "r")
1630 for line in k.xreadlines():
1631 field = line.split(":")
1632 if field[0] == "pub":
1635 (name, addr) = self.parse_address(field[9])
1637 self.keys[key]["email"] = addr
1638 self.keys[key]["name"] = name
1639 self.keys[key]["fingerprints"] = []
1641 elif key and field[0] == "sub" and len(field) >= 12:
1642 signingkey = ("s" in field[11])
1643 elif key and field[0] == "uid":
1644 (name, addr) = self.parse_address(field[9])
1645 if "email" not in self.keys[key] and "@" in addr:
1646 self.keys[key]["email"] = addr
1647 self.keys[key]["name"] = name
1648 elif signingkey and field[0] == "fpr":
1649 self.keys[key]["fingerprints"].append(field[9])
1650 self.fpr_lookup[field[9]] = key
1652 def import_users_from_ldap(self, session):
1656 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1657 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1659 l = ldap.open(LDAPServer)
1660 l.simple_bind_s("","")
1661 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1662 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1663 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1665 ldap_fin_uid_id = {}
1672 uid = entry["uid"][0]
1673 name = get_ldap_name(entry)
1674 fingerprints = entry["keyFingerPrint"]
1676 for f in fingerprints:
1677 key = self.fpr_lookup.get(f, None)
1678 if key not in self.keys:
1680 self.keys[key]["uid"] = uid
1684 keyid = get_or_set_uid(uid, session).uid_id
1685 byuid[keyid] = (uid, name)
1686 byname[uid] = (keyid, name)
1688 return (byname, byuid)
1690 def generate_users_from_keyring(self, format, session):
1694 for x in self.keys.keys():
1695 if "email" not in self.keys[x]:
1697 self.keys[x]["uid"] = format % "invalid-uid"
1699 uid = format % self.keys[x]["email"]
1700 keyid = get_or_set_uid(uid, session).uid_id
1701 byuid[keyid] = (uid, self.keys[x]["name"])
1702 byname[uid] = (keyid, self.keys[x]["name"])
1703 self.keys[x]["uid"] = uid
1706 uid = format % "invalid-uid"
1707 keyid = get_or_set_uid(uid, session).uid_id
1708 byuid[keyid] = (uid, "ungeneratable user id")
1709 byname[uid] = (keyid, "ungeneratable user id")
1711 return (byname, byuid)
1713 __all__.append('Keyring')
1716 def get_keyring(keyring, session=None):
1718 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1719 If C{keyring} already has an entry, simply return the existing Keyring
1721 @type keyring: string
1722 @param keyring: the keyring name
1725 @return: the Keyring object for this keyring
1728 q = session.query(Keyring).filter_by(keyring_name=keyring)
1732 except NoResultFound:
1735 __all__.append('get_keyring')
1737 ################################################################################
1739 class KeyringACLMap(object):
1740 def __init__(self, *args, **kwargs):
1744 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1746 __all__.append('KeyringACLMap')
1748 ################################################################################
1750 class DBChange(object):
1751 def __init__(self, *args, **kwargs):
1755 return '<DBChange %s>' % self.changesname
1757 def clean_from_queue(self):
1758 session = DBConn().session().object_session(self)
1760 # Remove changes_pool_files entries
1763 # Remove changes_pending_files references
1766 # Clear out of queue
1767 self.in_queue = None
1768 self.approved_for_id = None
1770 __all__.append('DBChange')
1773 def get_dbchange(filename, session=None):
1775 returns DBChange object for given C{filename}.
1777 @type filename: string
1778 @param filename: the name of the file
1780 @type session: Session
1781 @param session: Optional SQLA session object (a temporary one will be
1782 generated if not supplied)
1785 @return: DBChange object for the given filename (C{None} if not present)
1788 q = session.query(DBChange).filter_by(changesname=filename)
1792 except NoResultFound:
1795 __all__.append('get_dbchange')
1797 ################################################################################
1799 class Location(ORMObject):
1800 def __init__(self, path = None, component = None):
1802 self.component = component
1803 # the column 'type' should go away, see comment at mapper
1804 self.archive_type = 'pool'
1806 def properties(self):
1807 return ['path', 'location_id', 'archive_type', 'component', \
1810 def not_null_constraints(self):
1811 return ['path', 'archive_type']
1813 __all__.append('Location')
1816 def get_location(location, component=None, archive=None, session=None):
1818 Returns Location object for the given combination of location, component
1821 @type location: string
1822 @param location: the path of the location, e.g. I{/srv/ftp-master.debian.org/ftp/pool/}
1824 @type component: string
1825 @param component: the component name (if None, no restriction applied)
1827 @type archive: string
1828 @param archive: the archive name (if None, no restriction applied)
1830 @rtype: Location / None
1831 @return: Either a Location object or None if one can't be found
1834 q = session.query(Location).filter_by(path=location)
1836 if archive is not None:
1837 q = q.join(Archive).filter_by(archive_name=archive)
1839 if component is not None:
1840 q = q.join(Component).filter_by(component_name=component)
1844 except NoResultFound:
1847 __all__.append('get_location')
1849 ################################################################################
1851 class Maintainer(ORMObject):
1852 def __init__(self, name = None):
1855 def properties(self):
1856 return ['name', 'maintainer_id']
1858 def not_null_constraints(self):
1861 def get_split_maintainer(self):
1862 if not hasattr(self, 'name') or self.name is None:
1863 return ('', '', '', '')
1865 return fix_maintainer(self.name.strip())
1867 __all__.append('Maintainer')
1870 def get_or_set_maintainer(name, session=None):
1872 Returns Maintainer object for given maintainer name.
1874 If no matching maintainer name is found, a row is inserted.
1877 @param name: The maintainer name to add
1879 @type session: SQLAlchemy
1880 @param session: Optional SQL session object (a temporary one will be
1881 generated if not supplied). If not passed, a commit will be performed at
1882 the end of the function, otherwise the caller is responsible for commiting.
1883 A flush will be performed either way.
1886 @return: the Maintainer object for the given maintainer
1889 q = session.query(Maintainer).filter_by(name=name)
1892 except NoResultFound:
1893 maintainer = Maintainer()
1894 maintainer.name = name
1895 session.add(maintainer)
1896 session.commit_or_flush()
1901 __all__.append('get_or_set_maintainer')
1904 def get_maintainer(maintainer_id, session=None):
1906 Return the name of the maintainer behind C{maintainer_id} or None if that
1907 maintainer_id is invalid.
1909 @type maintainer_id: int
1910 @param maintainer_id: the id of the maintainer
1913 @return: the Maintainer with this C{maintainer_id}
1916 return session.query(Maintainer).get(maintainer_id)
1918 __all__.append('get_maintainer')
1920 ################################################################################
1922 class NewComment(object):
1923 def __init__(self, *args, **kwargs):
1927 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1929 __all__.append('NewComment')
1932 def has_new_comment(package, version, session=None):
1934 Returns true if the given combination of C{package}, C{version} has a comment.
1936 @type package: string
1937 @param package: name of the package
1939 @type version: string
1940 @param version: package version
1942 @type session: Session
1943 @param session: Optional SQLA session object (a temporary one will be
1944 generated if not supplied)
1950 q = session.query(NewComment)
1951 q = q.filter_by(package=package)
1952 q = q.filter_by(version=version)
1954 return bool(q.count() > 0)
1956 __all__.append('has_new_comment')
1959 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1961 Returns (possibly empty) list of NewComment objects for the given
1964 @type package: string (optional)
1965 @param package: name of the package
1967 @type version: string (optional)
1968 @param version: package version
1970 @type comment_id: int (optional)
1971 @param comment_id: An id of a comment
1973 @type session: Session
1974 @param session: Optional SQLA session object (a temporary one will be
1975 generated if not supplied)
1978 @return: A (possibly empty) list of NewComment objects will be returned
1981 q = session.query(NewComment)
1982 if package is not None: q = q.filter_by(package=package)
1983 if version is not None: q = q.filter_by(version=version)
1984 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1988 __all__.append('get_new_comments')
1990 ################################################################################
1992 class Override(ORMObject):
1993 def __init__(self, package = None, suite = None, component = None, overridetype = None, \
1994 section = None, priority = None):
1995 self.package = package
1997 self.component = component
1998 self.overridetype = overridetype
1999 self.section = section
2000 self.priority = priority
2002 def properties(self):
2003 return ['package', 'suite', 'component', 'overridetype', 'section', \
2006 def not_null_constraints(self):
2007 return ['package', 'suite', 'component', 'overridetype', 'section']
2009 __all__.append('Override')
2012 def get_override(package, suite=None, component=None, overridetype=None, session=None):
2014 Returns Override object for the given parameters
2016 @type package: string
2017 @param package: The name of the package
2019 @type suite: string, list or None
2020 @param suite: The name of the suite (or suites if a list) to limit to. If
2021 None, don't limit. Defaults to None.
2023 @type component: string, list or None
2024 @param component: The name of the component (or components if a list) to
2025 limit to. If None, don't limit. Defaults to None.
2027 @type overridetype: string, list or None
2028 @param overridetype: The name of the overridetype (or overridetypes if a list) to
2029 limit to. If None, don't limit. Defaults to None.
2031 @type session: Session
2032 @param session: Optional SQLA session object (a temporary one will be
2033 generated if not supplied)
2036 @return: A (possibly empty) list of Override objects will be returned
2039 q = session.query(Override)
2040 q = q.filter_by(package=package)
2042 if suite is not None:
2043 if not isinstance(suite, list): suite = [suite]
2044 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
2046 if component is not None:
2047 if not isinstance(component, list): component = [component]
2048 q = q.join(Component).filter(Component.component_name.in_(component))
2050 if overridetype is not None:
2051 if not isinstance(overridetype, list): overridetype = [overridetype]
2052 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
2056 __all__.append('get_override')
2059 ################################################################################
2061 class OverrideType(ORMObject):
2062 def __init__(self, overridetype = None):
2063 self.overridetype = overridetype
2065 def properties(self):
2066 return ['overridetype', 'overridetype_id', 'overrides_count']
2068 def not_null_constraints(self):
2069 return ['overridetype']
2071 __all__.append('OverrideType')
2074 def get_override_type(override_type, session=None):
2076 Returns OverrideType object for given C{override type}.
2078 @type override_type: string
2079 @param override_type: The name of the override type
2081 @type session: Session
2082 @param session: Optional SQLA session object (a temporary one will be
2083 generated if not supplied)
2086 @return: the database id for the given override type
2089 q = session.query(OverrideType).filter_by(overridetype=override_type)
2093 except NoResultFound:
2096 __all__.append('get_override_type')
2098 ################################################################################
2100 class PolicyQueue(object):
2101 def __init__(self, *args, **kwargs):
2105 return '<PolicyQueue %s>' % self.queue_name
2107 __all__.append('PolicyQueue')
2110 def get_policy_queue(queuename, session=None):
2112 Returns PolicyQueue object for given C{queue name}
2114 @type queuename: string
2115 @param queuename: The name of the queue
2117 @type session: Session
2118 @param session: Optional SQLA session object (a temporary one will be
2119 generated if not supplied)
2122 @return: PolicyQueue object for the given queue
2125 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
2129 except NoResultFound:
2132 __all__.append('get_policy_queue')
2135 def get_policy_queue_from_path(pathname, session=None):
2137 Returns PolicyQueue object for given C{path name}
2139 @type queuename: string
2140 @param queuename: The path
2142 @type session: Session
2143 @param session: Optional SQLA session object (a temporary one will be
2144 generated if not supplied)
2147 @return: PolicyQueue object for the given queue
2150 q = session.query(PolicyQueue).filter_by(path=pathname)
2154 except NoResultFound:
2157 __all__.append('get_policy_queue_from_path')
2159 ################################################################################
2161 class Priority(ORMObject):
2162 def __init__(self, priority = None, level = None):
2163 self.priority = priority
2166 def properties(self):
2167 return ['priority', 'priority_id', 'level', 'overrides_count']
2169 def not_null_constraints(self):
2170 return ['priority', 'level']
2172 def __eq__(self, val):
2173 if isinstance(val, str):
2174 return (self.priority == val)
2175 # This signals to use the normal comparison operator
2176 return NotImplemented
2178 def __ne__(self, val):
2179 if isinstance(val, str):
2180 return (self.priority != val)
2181 # This signals to use the normal comparison operator
2182 return NotImplemented
2184 __all__.append('Priority')
2187 def get_priority(priority, session=None):
2189 Returns Priority object for given C{priority name}.
2191 @type priority: string
2192 @param priority: The name of the priority
2194 @type session: Session
2195 @param session: Optional SQLA session object (a temporary one will be
2196 generated if not supplied)
2199 @return: Priority object for the given priority
2202 q = session.query(Priority).filter_by(priority=priority)
2206 except NoResultFound:
2209 __all__.append('get_priority')
2212 def get_priorities(session=None):
2214 Returns dictionary of priority names -> id mappings
2216 @type session: Session
2217 @param session: Optional SQL session object (a temporary one will be
2218 generated if not supplied)
2221 @return: dictionary of priority names -> id mappings
2225 q = session.query(Priority)
2227 ret[x.priority] = x.priority_id
2231 __all__.append('get_priorities')
2233 ################################################################################
2235 class Section(ORMObject):
2236 def __init__(self, section = None):
2237 self.section = section
2239 def properties(self):
2240 return ['section', 'section_id', 'overrides_count']
2242 def not_null_constraints(self):
2245 def __eq__(self, val):
2246 if isinstance(val, str):
2247 return (self.section == val)
2248 # This signals to use the normal comparison operator
2249 return NotImplemented
2251 def __ne__(self, val):
2252 if isinstance(val, str):
2253 return (self.section != val)
2254 # This signals to use the normal comparison operator
2255 return NotImplemented
2257 __all__.append('Section')
2260 def get_section(section, session=None):
2262 Returns Section object for given C{section name}.
2264 @type section: string
2265 @param section: The name of the section
2267 @type session: Session
2268 @param session: Optional SQLA session object (a temporary one will be
2269 generated if not supplied)
2272 @return: Section object for the given section name
2275 q = session.query(Section).filter_by(section=section)
2279 except NoResultFound:
2282 __all__.append('get_section')
2285 def get_sections(session=None):
2287 Returns dictionary of section names -> id mappings
2289 @type session: Session
2290 @param session: Optional SQL session object (a temporary one will be
2291 generated if not supplied)
2294 @return: dictionary of section names -> id mappings
2298 q = session.query(Section)
2300 ret[x.section] = x.section_id
2304 __all__.append('get_sections')
2306 ################################################################################
2308 class SrcContents(ORMObject):
2309 def __init__(self, file = None, source = None):
2311 self.source = source
2313 def properties(self):
2314 return ['file', 'source']
2316 __all__.append('SrcContents')
2318 ################################################################################
2320 from debian.debfile import Deb822
2322 # Temporary Deb822 subclass to fix bugs with : handling; see #597249
2323 class Dak822(Deb822):
2324 def _internal_parser(self, sequence, fields=None):
2325 # The key is non-whitespace, non-colon characters before any colon.
2326 key_part = r"^(?P<key>[^: \t\n\r\f\v]+)\s*:\s*"
2327 single = re.compile(key_part + r"(?P<data>\S.*?)\s*$")
2328 multi = re.compile(key_part + r"$")
2329 multidata = re.compile(r"^\s(?P<data>.+?)\s*$")
2331 wanted_field = lambda f: fields is None or f in fields
2333 if isinstance(sequence, basestring):
2334 sequence = sequence.splitlines()
2338 for line in self.gpg_stripped_paragraph(sequence):
2339 m = single.match(line)
2342 self[curkey] = content
2344 if not wanted_field(m.group('key')):
2348 curkey = m.group('key')
2349 content = m.group('data')
2352 m = multi.match(line)
2355 self[curkey] = content
2357 if not wanted_field(m.group('key')):
2361 curkey = m.group('key')
2365 m = multidata.match(line)
2367 content += '\n' + line # XXX not m.group('data')?
2371 self[curkey] = content
2374 class DBSource(ORMObject):
2375 def __init__(self, source = None, version = None, maintainer = None, \
2376 changedby = None, poolfile = None, install_date = None):
2377 self.source = source
2378 self.version = version
2379 self.maintainer = maintainer
2380 self.changedby = changedby
2381 self.poolfile = poolfile
2382 self.install_date = install_date
2386 return self.source_id
2388 def properties(self):
2389 return ['source', 'source_id', 'maintainer', 'changedby', \
2390 'fingerprint', 'poolfile', 'version', 'suites_count', \
2391 'install_date', 'binaries_count', 'uploaders_count']
2393 def not_null_constraints(self):
2394 return ['source', 'version', 'install_date', 'maintainer', \
2395 'changedby', 'poolfile', 'install_date']
2397 def read_control_fields(self):
2399 Reads the control information from a dsc
2402 @return: fields is the dsc information in a dictionary form
2404 fullpath = self.poolfile.fullpath
2405 fields = Dak822(open(self.poolfile.fullpath, 'r'))
2408 metadata = association_proxy('key', 'value')
2410 def scan_contents(self):
2412 Returns a set of names for non directories. The path names are
2413 normalized after converting them from either utf-8 or iso8859-1
2416 fullpath = self.poolfile.fullpath
2417 from daklib.contents import UnpackedSource
2418 unpacked = UnpackedSource(fullpath)
2420 for name in unpacked.get_all_filenames():
2421 # enforce proper utf-8 encoding
2423 name.decode('utf-8')
2424 except UnicodeDecodeError:
2425 name = name.decode('iso8859-1').encode('utf-8')
2429 __all__.append('DBSource')
2432 def source_exists(source, source_version, suites = ["any"], session=None):
2434 Ensure that source exists somewhere in the archive for the binary
2435 upload being processed.
2436 1. exact match => 1.0-3
2437 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
2439 @type source: string
2440 @param source: source name
2442 @type source_version: string
2443 @param source_version: expected source version
2446 @param suites: list of suites to check in, default I{any}
2448 @type session: Session
2449 @param session: Optional SQLA session object (a temporary one will be
2450 generated if not supplied)
2453 @return: returns 1 if a source with expected version is found, otherwise 0
2460 from daklib.regexes import re_bin_only_nmu
2461 orig_source_version = re_bin_only_nmu.sub('', source_version)
2463 for suite in suites:
2464 q = session.query(DBSource).filter_by(source=source). \
2465 filter(DBSource.version.in_([source_version, orig_source_version]))
2467 # source must exist in suite X, or in some other suite that's
2468 # mapped to X, recursively... silent-maps are counted too,
2469 # unreleased-maps aren't.
2470 maps = cnf.ValueList("SuiteMappings")[:]
2472 maps = [ m.split() for m in maps ]
2473 maps = [ (x[1], x[2]) for x in maps
2474 if x[0] == "map" or x[0] == "silent-map" ]
2476 for (from_, to) in maps:
2477 if from_ in s and to not in s:
2480 q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
2485 # No source found so return not ok
2490 __all__.append('source_exists')
2493 def get_suites_source_in(source, session=None):
2495 Returns list of Suite objects which given C{source} name is in
2498 @param source: DBSource package name to search for
2501 @return: list of Suite objects for the given source
2504 return session.query(Suite).filter(Suite.sources.any(source=source)).all()
2506 __all__.append('get_suites_source_in')
2509 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
2511 Returns list of DBSource objects for given C{source} name and other parameters
2514 @param source: DBSource package name to search for
2516 @type version: str or None
2517 @param version: DBSource version name to search for or None if not applicable
2519 @type dm_upload_allowed: bool
2520 @param dm_upload_allowed: If None, no effect. If True or False, only
2521 return packages with that dm_upload_allowed setting
2523 @type session: Session
2524 @param session: Optional SQL session object (a temporary one will be
2525 generated if not supplied)
2528 @return: list of DBSource objects for the given name (may be empty)
2531 q = session.query(DBSource).filter_by(source=source)
2533 if version is not None:
2534 q = q.filter_by(version=version)
2536 if dm_upload_allowed is not None:
2537 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
2541 __all__.append('get_sources_from_name')
2543 # FIXME: This function fails badly if it finds more than 1 source package and
2544 # its implementation is trivial enough to be inlined.
2546 def get_source_in_suite(source, suite, session=None):
2548 Returns a DBSource object for a combination of C{source} and C{suite}.
2550 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2551 - B{suite} - a suite name, eg. I{unstable}
2553 @type source: string
2554 @param source: source package name
2557 @param suite: the suite name
2560 @return: the version for I{source} in I{suite}
2564 q = get_suite(suite, session).get_sources(source)
2567 except NoResultFound:
2570 __all__.append('get_source_in_suite')
2573 def import_metadata_into_db(obj, session=None):
2575 This routine works on either DBBinary or DBSource objects and imports
2576 their metadata into the database
2578 fields = obj.read_control_fields()
2579 for k in fields.keys():
2582 val = str(fields[k])
2583 except UnicodeEncodeError:
2584 # Fall back to UTF-8
2586 val = fields[k].encode('utf-8')
2587 except UnicodeEncodeError:
2588 # Finally try iso8859-1
2589 val = fields[k].encode('iso8859-1')
2590 # Otherwise we allow the exception to percolate up and we cause
2591 # a reject as someone is playing silly buggers
2593 obj.metadata[get_or_set_metadatakey(k, session)] = val
2595 session.commit_or_flush()
2597 __all__.append('import_metadata_into_db')
2600 ################################################################################
2603 def add_dsc_to_db(u, filename, session=None):
2604 entry = u.pkg.files[filename]
2608 source.source = u.pkg.dsc["source"]
2609 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2610 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2611 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2612 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2613 source.install_date = datetime.now().date()
2615 dsc_component = entry["component"]
2616 dsc_location_id = entry["location id"]
2618 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2620 # Set up a new poolfile if necessary
2621 if not entry.has_key("files id") or not entry["files id"]:
2622 filename = entry["pool name"] + filename
2623 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2625 pfs.append(poolfile)
2626 entry["files id"] = poolfile.file_id
2628 source.poolfile_id = entry["files id"]
2631 suite_names = u.pkg.changes["distribution"].keys()
2632 source.suites = session.query(Suite). \
2633 filter(Suite.suite_name.in_(suite_names)).all()
2635 # Add the source files to the DB (files and dsc_files)
2637 dscfile.source_id = source.source_id
2638 dscfile.poolfile_id = entry["files id"]
2639 session.add(dscfile)
2641 for dsc_file, dentry in u.pkg.dsc_files.items():
2643 df.source_id = source.source_id
2645 # If the .orig tarball is already in the pool, it's
2646 # files id is stored in dsc_files by check_dsc().
2647 files_id = dentry.get("files id", None)
2649 # Find the entry in the files hash
2650 # TODO: Bail out here properly
2652 for f, e in u.pkg.files.items():
2657 if files_id is None:
2658 filename = dfentry["pool name"] + dsc_file
2660 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2661 # FIXME: needs to check for -1/-2 and or handle exception
2662 if found and obj is not None:
2663 files_id = obj.file_id
2666 # If still not found, add it
2667 if files_id is None:
2668 # HACK: Force sha1sum etc into dentry
2669 dentry["sha1sum"] = dfentry["sha1sum"]
2670 dentry["sha256sum"] = dfentry["sha256sum"]
2671 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2672 pfs.append(poolfile)
2673 files_id = poolfile.file_id
2675 poolfile = get_poolfile_by_id(files_id, session)
2676 if poolfile is None:
2677 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2678 pfs.append(poolfile)
2680 df.poolfile_id = files_id
2683 # Add the src_uploaders to the DB
2685 session.refresh(source)
2686 source.uploaders = [source.maintainer]
2687 if u.pkg.dsc.has_key("uploaders"):
2688 for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
2690 source.uploaders.append(get_or_set_maintainer(up, session))
2694 return source, dsc_component, dsc_location_id, pfs
2696 __all__.append('add_dsc_to_db')
2699 def add_deb_to_db(u, filename, session=None):
2701 Contrary to what you might expect, this routine deals with both
2702 debs and udebs. That info is in 'dbtype', whilst 'type' is
2703 'deb' for both of them
2706 entry = u.pkg.files[filename]
2709 bin.package = entry["package"]
2710 bin.version = entry["version"]
2711 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2712 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2713 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2714 bin.binarytype = entry["dbtype"]
2717 filename = entry["pool name"] + filename
2718 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2719 if not entry.get("location id", None):
2720 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2722 if entry.get("files id", None):
2723 poolfile = get_poolfile_by_id(bin.poolfile_id)
2724 bin.poolfile_id = entry["files id"]
2726 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2727 bin.poolfile_id = entry["files id"] = poolfile.file_id
2730 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2731 if len(bin_sources) != 1:
2732 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2733 (bin.package, bin.version, entry["architecture"],
2734 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2736 bin.source_id = bin_sources[0].source_id
2738 if entry.has_key("built-using"):
2739 for srcname, version in entry["built-using"]:
2740 exsources = get_sources_from_name(srcname, version, session=session)
2741 if len(exsources) != 1:
2742 raise NoSourceFieldError, "Unable to find source package (%s = %s) in Built-Using for %s (%s), %s, file %s, type %s, signed by %s" % \
2743 (srcname, version, bin.package, bin.version, entry["architecture"],
2744 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2746 bin.extra_sources.append(exsources[0])
2748 # Add and flush object so it has an ID
2751 suite_names = u.pkg.changes["distribution"].keys()
2752 bin.suites = session.query(Suite). \
2753 filter(Suite.suite_name.in_(suite_names)).all()
2757 # Deal with contents - disabled for now
2758 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2760 # print "REJECT\nCould not determine contents of package %s" % bin.package
2761 # session.rollback()
2762 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2764 return bin, poolfile
2766 __all__.append('add_deb_to_db')
2768 ################################################################################
2770 class SourceACL(object):
2771 def __init__(self, *args, **kwargs):
2775 return '<SourceACL %s>' % self.source_acl_id
2777 __all__.append('SourceACL')
2779 ################################################################################
2781 class SrcFormat(object):
2782 def __init__(self, *args, **kwargs):
2786 return '<SrcFormat %s>' % (self.format_name)
2788 __all__.append('SrcFormat')
2790 ################################################################################
2792 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2793 ('SuiteID', 'suite_id'),
2794 ('Version', 'version'),
2795 ('Origin', 'origin'),
2797 ('Description', 'description'),
2798 ('Untouchable', 'untouchable'),
2799 ('Announce', 'announce'),
2800 ('Codename', 'codename'),
2801 ('OverrideCodename', 'overridecodename'),
2802 ('ValidTime', 'validtime'),
2803 ('Priority', 'priority'),
2804 ('NotAutomatic', 'notautomatic'),
2805 ('CopyChanges', 'copychanges'),
2806 ('OverrideSuite', 'overridesuite')]
2808 # Why the heck don't we have any UNIQUE constraints in table suite?
2809 # TODO: Add UNIQUE constraints for appropriate columns.
2810 class Suite(ORMObject):
2811 def __init__(self, suite_name = None, version = None):
2812 self.suite_name = suite_name
2813 self.version = version
2815 def properties(self):
2816 return ['suite_name', 'version', 'sources_count', 'binaries_count', \
2819 def not_null_constraints(self):
2820 return ['suite_name']
2822 def __eq__(self, val):
2823 if isinstance(val, str):
2824 return (self.suite_name == val)
2825 # This signals to use the normal comparison operator
2826 return NotImplemented
2828 def __ne__(self, val):
2829 if isinstance(val, str):
2830 return (self.suite_name != val)
2831 # This signals to use the normal comparison operator
2832 return NotImplemented
2836 for disp, field in SUITE_FIELDS:
2837 val = getattr(self, field, None)
2839 ret.append("%s: %s" % (disp, val))
2841 return "\n".join(ret)
2843 def get_architectures(self, skipsrc=False, skipall=False):
2845 Returns list of Architecture objects
2847 @type skipsrc: boolean
2848 @param skipsrc: Whether to skip returning the 'source' architecture entry
2851 @type skipall: boolean
2852 @param skipall: Whether to skip returning the 'all' architecture entry
2856 @return: list of Architecture objects for the given name (may be empty)
2859 q = object_session(self).query(Architecture).with_parent(self)
2861 q = q.filter(Architecture.arch_string != 'source')
2863 q = q.filter(Architecture.arch_string != 'all')
2864 return q.order_by(Architecture.arch_string).all()
2866 def get_sources(self, source):
2868 Returns a query object representing DBSource that is part of C{suite}.
2870 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2872 @type source: string
2873 @param source: source package name
2875 @rtype: sqlalchemy.orm.query.Query
2876 @return: a query of DBSource
2880 session = object_session(self)
2881 return session.query(DBSource).filter_by(source = source). \
2884 def get_overridesuite(self):
2885 if self.overridesuite is None:
2888 return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one()
2890 __all__.append('Suite')
2893 def get_suite(suite, session=None):
2895 Returns Suite object for given C{suite name}.
2898 @param suite: The name of the suite
2900 @type session: Session
2901 @param session: Optional SQLA session object (a temporary one will be
2902 generated if not supplied)
2905 @return: Suite object for the requested suite name (None if not present)
2908 q = session.query(Suite).filter_by(suite_name=suite)
2912 except NoResultFound:
2915 __all__.append('get_suite')
2917 ################################################################################
2919 # TODO: should be removed because the implementation is too trivial
2921 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2923 Returns list of Architecture objects for given C{suite} name
2926 @param suite: Suite name to search for
2928 @type skipsrc: boolean
2929 @param skipsrc: Whether to skip returning the 'source' architecture entry
2932 @type skipall: boolean
2933 @param skipall: Whether to skip returning the 'all' architecture entry
2936 @type session: Session
2937 @param session: Optional SQL session object (a temporary one will be
2938 generated if not supplied)
2941 @return: list of Architecture objects for the given name (may be empty)
2944 return get_suite(suite, session).get_architectures(skipsrc, skipall)
2946 __all__.append('get_suite_architectures')
2948 ################################################################################
2950 class SuiteSrcFormat(object):
2951 def __init__(self, *args, **kwargs):
2955 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2957 __all__.append('SuiteSrcFormat')
2960 def get_suite_src_formats(suite, session=None):
2962 Returns list of allowed SrcFormat for C{suite}.
2965 @param suite: Suite name to search for
2967 @type session: Session
2968 @param session: Optional SQL session object (a temporary one will be
2969 generated if not supplied)
2972 @return: the list of allowed source formats for I{suite}
2975 q = session.query(SrcFormat)
2976 q = q.join(SuiteSrcFormat)
2977 q = q.join(Suite).filter_by(suite_name=suite)
2978 q = q.order_by('format_name')
2982 __all__.append('get_suite_src_formats')
2984 ################################################################################
2986 class Uid(ORMObject):
2987 def __init__(self, uid = None, name = None):
2991 def __eq__(self, val):
2992 if isinstance(val, str):
2993 return (self.uid == val)
2994 # This signals to use the normal comparison operator
2995 return NotImplemented
2997 def __ne__(self, val):
2998 if isinstance(val, str):
2999 return (self.uid != val)
3000 # This signals to use the normal comparison operator
3001 return NotImplemented
3003 def properties(self):
3004 return ['uid', 'name', 'fingerprint']
3006 def not_null_constraints(self):
3009 __all__.append('Uid')
3012 def get_or_set_uid(uidname, session=None):
3014 Returns uid object for given uidname.
3016 If no matching uidname is found, a row is inserted.
3018 @type uidname: string
3019 @param uidname: The uid to add
3021 @type session: SQLAlchemy
3022 @param session: Optional SQL session object (a temporary one will be
3023 generated if not supplied). If not passed, a commit will be performed at
3024 the end of the function, otherwise the caller is responsible for commiting.
3027 @return: the uid object for the given uidname
3030 q = session.query(Uid).filter_by(uid=uidname)
3034 except NoResultFound:
3038 session.commit_or_flush()
3043 __all__.append('get_or_set_uid')
3046 def get_uid_from_fingerprint(fpr, session=None):
3047 q = session.query(Uid)
3048 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
3052 except NoResultFound:
3055 __all__.append('get_uid_from_fingerprint')
3057 ################################################################################
3059 class UploadBlock(object):
3060 def __init__(self, *args, **kwargs):
3064 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
3066 __all__.append('UploadBlock')
3068 ################################################################################
3070 class MetadataKey(ORMObject):
3071 def __init__(self, key = None):
3074 def properties(self):
3077 def not_null_constraints(self):
3080 __all__.append('MetadataKey')
3083 def get_or_set_metadatakey(keyname, session=None):
3085 Returns MetadataKey object for given uidname.
3087 If no matching keyname is found, a row is inserted.
3089 @type uidname: string
3090 @param uidname: The keyname to add
3092 @type session: SQLAlchemy
3093 @param session: Optional SQL session object (a temporary one will be
3094 generated if not supplied). If not passed, a commit will be performed at
3095 the end of the function, otherwise the caller is responsible for commiting.
3098 @return: the metadatakey object for the given keyname
3101 q = session.query(MetadataKey).filter_by(key=keyname)
3105 except NoResultFound:
3106 ret = MetadataKey(keyname)
3108 session.commit_or_flush()
3112 __all__.append('get_or_set_metadatakey')
3114 ################################################################################
3116 class BinaryMetadata(ORMObject):
3117 def __init__(self, key = None, value = None, binary = None):
3120 self.binary = binary
3122 def properties(self):
3123 return ['binary', 'key', 'value']
3125 def not_null_constraints(self):
3128 __all__.append('BinaryMetadata')
3130 ################################################################################
3132 class SourceMetadata(ORMObject):
3133 def __init__(self, key = None, value = None, source = None):
3136 self.source = source
3138 def properties(self):
3139 return ['source', 'key', 'value']
3141 def not_null_constraints(self):
3144 __all__.append('SourceMetadata')
3146 ################################################################################
3148 class VersionCheck(ORMObject):
3149 def __init__(self, *args, **kwargs):
3152 def properties(self):
3153 #return ['suite_id', 'check', 'reference_id']
3156 def not_null_constraints(self):
3157 return ['suite', 'check', 'reference']
3159 __all__.append('VersionCheck')
3162 def get_version_checks(suite_name, check = None, session = None):
3163 suite = get_suite(suite_name, session)
3165 # Make sure that what we return is iterable so that list comprehensions
3166 # involving this don't cause a traceback
3168 q = session.query(VersionCheck).filter_by(suite=suite)
3170 q = q.filter_by(check=check)
3173 __all__.append('get_version_checks')
3175 ################################################################################
3177 class DBConn(object):
3179 database module init.
3183 def __init__(self, *args, **kwargs):
3184 self.__dict__ = self.__shared_state
3186 if not getattr(self, 'initialised', False):
3187 self.initialised = True
3188 self.debug = kwargs.has_key('debug')
3191 def __setuptables(self):
3198 'binaries_metadata',
3202 'build_queue_files',
3203 'build_queue_policy_files',
3208 'changes_pending_binaries',
3209 'changes_pending_files',
3210 'changes_pending_source',
3211 'changes_pending_files_map',
3212 'changes_pending_source_files',
3213 'changes_pool_files',
3215 'external_overrides',
3216 'extra_src_references',
3225 # TODO: the maintainer column in table override should be removed.
3239 'suite_architectures',
3240 'suite_build_queue_copy',
3241 'suite_src_formats',
3248 'almost_obsolete_all_associations',
3249 'almost_obsolete_src_associations',
3250 'any_associations_source',
3251 'bin_associations_binaries',
3252 'binaries_suite_arch',
3253 'binfiles_suite_component_arch',
3256 'newest_all_associations',
3257 'newest_any_associations',
3259 'newest_src_association',
3260 'obsolete_all_associations',
3261 'obsolete_any_associations',
3262 'obsolete_any_by_all_associations',
3263 'obsolete_src_associations',
3265 'src_associations_bin',
3266 'src_associations_src',
3267 'suite_arch_by_name',
3270 for table_name in tables:
3271 table = Table(table_name, self.db_meta, \
3272 autoload=True, useexisting=True)
3273 setattr(self, 'tbl_%s' % table_name, table)
3275 for view_name in views:
3276 view = Table(view_name, self.db_meta, autoload=True)
3277 setattr(self, 'view_%s' % view_name, view)
3279 def __setupmappers(self):
3280 mapper(Architecture, self.tbl_architecture,
3281 properties = dict(arch_id = self.tbl_architecture.c.id,
3282 suites = relation(Suite, secondary=self.tbl_suite_architectures,
3283 order_by='suite_name',
3284 backref=backref('architectures', order_by='arch_string'))),
3285 extension = validator)
3287 mapper(Archive, self.tbl_archive,
3288 properties = dict(archive_id = self.tbl_archive.c.id,
3289 archive_name = self.tbl_archive.c.name))
3291 mapper(BuildQueue, self.tbl_build_queue,
3292 properties = dict(queue_id = self.tbl_build_queue.c.id))
3294 mapper(BuildQueueFile, self.tbl_build_queue_files,
3295 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
3296 poolfile = relation(PoolFile, backref='buildqueueinstances')))
3298 mapper(BuildQueuePolicyFile, self.tbl_build_queue_policy_files,
3300 build_queue = relation(BuildQueue, backref='policy_queue_files'),
3301 file = relation(ChangePendingFile, lazy='joined')))
3303 mapper(DBBinary, self.tbl_binaries,
3304 properties = dict(binary_id = self.tbl_binaries.c.id,
3305 package = self.tbl_binaries.c.package,
3306 version = self.tbl_binaries.c.version,
3307 maintainer_id = self.tbl_binaries.c.maintainer,
3308 maintainer = relation(Maintainer),
3309 source_id = self.tbl_binaries.c.source,
3310 source = relation(DBSource, backref='binaries'),
3311 arch_id = self.tbl_binaries.c.architecture,
3312 architecture = relation(Architecture),
3313 poolfile_id = self.tbl_binaries.c.file,
3314 poolfile = relation(PoolFile, backref=backref('binary', uselist = False)),
3315 binarytype = self.tbl_binaries.c.type,
3316 fingerprint_id = self.tbl_binaries.c.sig_fpr,
3317 fingerprint = relation(Fingerprint),
3318 install_date = self.tbl_binaries.c.install_date,
3319 suites = relation(Suite, secondary=self.tbl_bin_associations,
3320 backref=backref('binaries', lazy='dynamic')),
3321 extra_sources = relation(DBSource, secondary=self.tbl_extra_src_references,
3322 backref=backref('extra_binary_references', lazy='dynamic')),
3323 key = relation(BinaryMetadata, cascade='all',
3324 collection_class=attribute_mapped_collection('key'))),
3325 extension = validator)
3327 mapper(BinaryACL, self.tbl_binary_acl,
3328 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
3330 mapper(BinaryACLMap, self.tbl_binary_acl_map,
3331 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
3332 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
3333 architecture = relation(Architecture)))
3335 mapper(Component, self.tbl_component,
3336 properties = dict(component_id = self.tbl_component.c.id,
3337 component_name = self.tbl_component.c.name),
3338 extension = validator)
3340 mapper(DBConfig, self.tbl_config,
3341 properties = dict(config_id = self.tbl_config.c.id))
3343 mapper(DSCFile, self.tbl_dsc_files,
3344 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
3345 source_id = self.tbl_dsc_files.c.source,
3346 source = relation(DBSource),
3347 poolfile_id = self.tbl_dsc_files.c.file,
3348 poolfile = relation(PoolFile)))
3350 mapper(ExternalOverride, self.tbl_external_overrides,
3352 suite_id = self.tbl_external_overrides.c.suite,
3353 suite = relation(Suite),
3354 component_id = self.tbl_external_overrides.c.component,
3355 component = relation(Component)))
3357 mapper(PoolFile, self.tbl_files,
3358 properties = dict(file_id = self.tbl_files.c.id,
3359 filesize = self.tbl_files.c.size,
3360 location_id = self.tbl_files.c.location,
3361 location = relation(Location,
3362 # using lazy='dynamic' in the back
3363 # reference because we have A LOT of
3364 # files in one location
3365 backref=backref('files', lazy='dynamic'))),
3366 extension = validator)
3368 mapper(Fingerprint, self.tbl_fingerprint,
3369 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
3370 uid_id = self.tbl_fingerprint.c.uid,
3371 uid = relation(Uid),
3372 keyring_id = self.tbl_fingerprint.c.keyring,
3373 keyring = relation(Keyring),
3374 source_acl = relation(SourceACL),
3375 binary_acl = relation(BinaryACL)),
3376 extension = validator)
3378 mapper(Keyring, self.tbl_keyrings,
3379 properties = dict(keyring_name = self.tbl_keyrings.c.name,
3380 keyring_id = self.tbl_keyrings.c.id))
3382 mapper(DBChange, self.tbl_changes,
3383 properties = dict(change_id = self.tbl_changes.c.id,
3384 poolfiles = relation(PoolFile,
3385 secondary=self.tbl_changes_pool_files,
3386 backref="changeslinks"),
3387 seen = self.tbl_changes.c.seen,
3388 source = self.tbl_changes.c.source,
3389 binaries = self.tbl_changes.c.binaries,
3390 architecture = self.tbl_changes.c.architecture,
3391 distribution = self.tbl_changes.c.distribution,
3392 urgency = self.tbl_changes.c.urgency,
3393 maintainer = self.tbl_changes.c.maintainer,
3394 changedby = self.tbl_changes.c.changedby,
3395 date = self.tbl_changes.c.date,
3396 version = self.tbl_changes.c.version,
3397 files = relation(ChangePendingFile,
3398 secondary=self.tbl_changes_pending_files_map,
3399 backref="changesfile"),
3400 in_queue_id = self.tbl_changes.c.in_queue,
3401 in_queue = relation(PolicyQueue,
3402 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
3403 approved_for_id = self.tbl_changes.c.approved_for))
3405 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
3406 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
3408 mapper(ChangePendingFile, self.tbl_changes_pending_files,
3409 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id,
3410 filename = self.tbl_changes_pending_files.c.filename,
3411 size = self.tbl_changes_pending_files.c.size,
3412 md5sum = self.tbl_changes_pending_files.c.md5sum,
3413 sha1sum = self.tbl_changes_pending_files.c.sha1sum,
3414 sha256sum = self.tbl_changes_pending_files.c.sha256sum))
3416 mapper(ChangePendingSource, self.tbl_changes_pending_source,
3417 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
3418 change = relation(DBChange),
3419 maintainer = relation(Maintainer,
3420 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
3421 changedby = relation(Maintainer,
3422 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
3423 fingerprint = relation(Fingerprint),
3424 source_files = relation(ChangePendingFile,
3425 secondary=self.tbl_changes_pending_source_files,
3426 backref="pending_sources")))
3429 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
3430 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
3431 keyring = relation(Keyring, backref="keyring_acl_map"),
3432 architecture = relation(Architecture)))
3434 mapper(Location, self.tbl_location,
3435 properties = dict(location_id = self.tbl_location.c.id,
3436 component_id = self.tbl_location.c.component,
3437 component = relation(Component, backref='location'),
3438 archive_id = self.tbl_location.c.archive,
3439 archive = relation(Archive),
3440 # FIXME: the 'type' column is old cruft and
3441 # should be removed in the future.
3442 archive_type = self.tbl_location.c.type),
3443 extension = validator)
3445 mapper(Maintainer, self.tbl_maintainer,
3446 properties = dict(maintainer_id = self.tbl_maintainer.c.id,
3447 maintains_sources = relation(DBSource, backref='maintainer',
3448 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.maintainer)),
3449 changed_sources = relation(DBSource, backref='changedby',
3450 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.changedby))),
3451 extension = validator)
3453 mapper(NewComment, self.tbl_new_comments,
3454 properties = dict(comment_id = self.tbl_new_comments.c.id))
3456 mapper(Override, self.tbl_override,
3457 properties = dict(suite_id = self.tbl_override.c.suite,
3458 suite = relation(Suite, \
3459 backref=backref('overrides', lazy='dynamic')),
3460 package = self.tbl_override.c.package,
3461 component_id = self.tbl_override.c.component,
3462 component = relation(Component, \
3463 backref=backref('overrides', lazy='dynamic')),
3464 priority_id = self.tbl_override.c.priority,
3465 priority = relation(Priority, \
3466 backref=backref('overrides', lazy='dynamic')),
3467 section_id = self.tbl_override.c.section,
3468 section = relation(Section, \
3469 backref=backref('overrides', lazy='dynamic')),
3470 overridetype_id = self.tbl_override.c.type,
3471 overridetype = relation(OverrideType, \
3472 backref=backref('overrides', lazy='dynamic'))))
3474 mapper(OverrideType, self.tbl_override_type,
3475 properties = dict(overridetype = self.tbl_override_type.c.type,
3476 overridetype_id = self.tbl_override_type.c.id))
3478 mapper(PolicyQueue, self.tbl_policy_queue,
3479 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
3481 mapper(Priority, self.tbl_priority,
3482 properties = dict(priority_id = self.tbl_priority.c.id))
3484 mapper(Section, self.tbl_section,
3485 properties = dict(section_id = self.tbl_section.c.id,
3486 section=self.tbl_section.c.section))
3488 mapper(DBSource, self.tbl_source,
3489 properties = dict(source_id = self.tbl_source.c.id,
3490 version = self.tbl_source.c.version,
3491 maintainer_id = self.tbl_source.c.maintainer,
3492 poolfile_id = self.tbl_source.c.file,
3493 poolfile = relation(PoolFile, backref=backref('source', uselist = False)),
3494 fingerprint_id = self.tbl_source.c.sig_fpr,
3495 fingerprint = relation(Fingerprint),
3496 changedby_id = self.tbl_source.c.changedby,
3497 srcfiles = relation(DSCFile,
3498 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
3499 suites = relation(Suite, secondary=self.tbl_src_associations,
3500 backref=backref('sources', lazy='dynamic')),
3501 uploaders = relation(Maintainer,
3502 secondary=self.tbl_src_uploaders),
3503 key = relation(SourceMetadata, cascade='all',
3504 collection_class=attribute_mapped_collection('key'))),
3505 extension = validator)
3507 mapper(SourceACL, self.tbl_source_acl,
3508 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
3510 mapper(SrcFormat, self.tbl_src_format,
3511 properties = dict(src_format_id = self.tbl_src_format.c.id,
3512 format_name = self.tbl_src_format.c.format_name))
3514 mapper(Suite, self.tbl_suite,
3515 properties = dict(suite_id = self.tbl_suite.c.id,
3516 policy_queue = relation(PolicyQueue),
3517 copy_queues = relation(BuildQueue,
3518 secondary=self.tbl_suite_build_queue_copy)),
3519 extension = validator)
3521 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
3522 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
3523 suite = relation(Suite, backref='suitesrcformats'),
3524 src_format_id = self.tbl_suite_src_formats.c.src_format,
3525 src_format = relation(SrcFormat)))
3527 mapper(Uid, self.tbl_uid,
3528 properties = dict(uid_id = self.tbl_uid.c.id,
3529 fingerprint = relation(Fingerprint)),
3530 extension = validator)
3532 mapper(UploadBlock, self.tbl_upload_blocks,
3533 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
3534 fingerprint = relation(Fingerprint, backref="uploadblocks"),
3535 uid = relation(Uid, backref="uploadblocks")))
3537 mapper(BinContents, self.tbl_bin_contents,
3539 binary = relation(DBBinary,
3540 backref=backref('contents', lazy='dynamic', cascade='all')),
3541 file = self.tbl_bin_contents.c.file))
3543 mapper(SrcContents, self.tbl_src_contents,
3545 source = relation(DBSource,
3546 backref=backref('contents', lazy='dynamic', cascade='all')),
3547 file = self.tbl_src_contents.c.file))
3549 mapper(MetadataKey, self.tbl_metadata_keys,
3551 key_id = self.tbl_metadata_keys.c.key_id,
3552 key = self.tbl_metadata_keys.c.key))
3554 mapper(BinaryMetadata, self.tbl_binaries_metadata,
3556 binary_id = self.tbl_binaries_metadata.c.bin_id,
3557 binary = relation(DBBinary),
3558 key_id = self.tbl_binaries_metadata.c.key_id,
3559 key = relation(MetadataKey),
3560 value = self.tbl_binaries_metadata.c.value))
3562 mapper(SourceMetadata, self.tbl_source_metadata,
3564 source_id = self.tbl_source_metadata.c.src_id,
3565 source = relation(DBSource),
3566 key_id = self.tbl_source_metadata.c.key_id,
3567 key = relation(MetadataKey),
3568 value = self.tbl_source_metadata.c.value))
3570 mapper(VersionCheck, self.tbl_version_check,
3572 suite_id = self.tbl_version_check.c.suite,
3573 suite = relation(Suite, primaryjoin=self.tbl_version_check.c.suite==self.tbl_suite.c.id),
3574 reference_id = self.tbl_version_check.c.reference,
3575 reference = relation(Suite, primaryjoin=self.tbl_version_check.c.reference==self.tbl_suite.c.id, lazy='joined')))
3577 ## Connection functions
3578 def __createconn(self):
3579 from config import Config
3581 if cnf.has_key("DB::Service"):
3582 connstr = "postgresql://service=%s" % cnf["DB::Service"]
3583 elif cnf.has_key("DB::Host"):
3585 connstr = "postgresql://%s" % cnf["DB::Host"]
3586 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
3587 connstr += ":%s" % cnf["DB::Port"]
3588 connstr += "/%s" % cnf["DB::Name"]
3591 connstr = "postgresql:///%s" % cnf["DB::Name"]
3592 if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
3593 connstr += "?port=%s" % cnf["DB::Port"]
3595 engine_args = { 'echo': self.debug }
3596 if cnf.has_key('DB::PoolSize'):
3597 engine_args['pool_size'] = int(cnf['DB::PoolSize'])
3598 if cnf.has_key('DB::MaxOverflow'):
3599 engine_args['max_overflow'] = int(cnf['DB::MaxOverflow'])
3600 if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \
3601 cnf['DB::Unicode'] == 'false':
3602 engine_args['use_native_unicode'] = False
3604 # Monkey patch a new dialect in in order to support service= syntax
3605 import sqlalchemy.dialects.postgresql
3606 from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
3607 class PGDialect_psycopg2_dak(PGDialect_psycopg2):
3608 def create_connect_args(self, url):
3609 if str(url).startswith('postgresql://service='):
3611 servicename = str(url)[21:]
3612 return (['service=%s' % servicename], {})
3614 return PGDialect_psycopg2.create_connect_args(self, url)
3616 sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak
3618 self.db_pg = create_engine(connstr, **engine_args)
3619 self.db_meta = MetaData()
3620 self.db_meta.bind = self.db_pg
3621 self.db_smaker = sessionmaker(bind=self.db_pg,
3625 self.__setuptables()
3626 self.__setupmappers()
3627 self.pid = os.getpid()
3629 def session(self, work_mem = 0):
3631 Returns a new session object. If a work_mem parameter is provided a new
3632 transaction is started and the work_mem parameter is set for this
3633 transaction. The work_mem parameter is measured in MB. A default value
3634 will be used if the parameter is not set.
3636 # reinitialize DBConn in new processes
3637 if self.pid != os.getpid():
3640 session = self.db_smaker()
3642 session.execute("SET LOCAL work_mem TO '%d MB'" % work_mem)
3645 __all__.append('DBConn')