5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
47 import simplejson as json
49 from datetime import datetime, timedelta
50 from errno import ENOENT
51 from tempfile import mkstemp, mkdtemp
53 from inspect import getargspec
56 from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc
57 from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
58 backref, MapperExtension, EXT_CONTINUE
59 from sqlalchemy import types as sqltypes
61 # Don't remove this, we re-export the exceptions to scripts which import us
62 from sqlalchemy.exc import *
63 from sqlalchemy.orm.exc import NoResultFound
65 # Only import Config until Queue stuff is changed to store its config
67 from config import Config
68 from textutils import fix_maintainer
69 from dak_exceptions import DBUpdateError, NoSourceFieldError
71 # suppress some deprecation warnings in squeeze related to sqlalchemy
73 warnings.filterwarnings('ignore', \
74 "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
76 # TODO: sqlalchemy needs some extra configuration to correctly reflect
77 # the ind_deb_contents_* indexes - we ignore the warnings at the moment
78 warnings.filterwarnings("ignore", 'Predicate of partial index', SAWarning)
81 ################################################################################
83 # Patch in support for the debversion field type so that it works during
87 # that is for sqlalchemy 0.6
88 UserDefinedType = sqltypes.UserDefinedType
90 # this one for sqlalchemy 0.5
91 UserDefinedType = sqltypes.TypeEngine
93 class DebVersion(UserDefinedType):
94 def get_col_spec(self):
97 def bind_processor(self, dialect):
100 # ' = None' is needed for sqlalchemy 0.5:
101 def result_processor(self, dialect, coltype = None):
104 sa_major_version = sqlalchemy.__version__[0:3]
105 if sa_major_version in ["0.5", "0.6"]:
106 from sqlalchemy.databases import postgres
107 postgres.ischema_names['debversion'] = DebVersion
109 raise Exception("dak only ported to SQLA versions 0.5 and 0.6. See daklib/dbconn.py")
111 ################################################################################
113 __all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion']
115 ################################################################################
117 def session_wrapper(fn):
119 Wrapper around common ".., session=None):" handling. If the wrapped
120 function is called without passing 'session', we create a local one
121 and destroy it when the function ends.
123 Also attaches a commit_or_flush method to the session; if we created a
124 local session, this is a synonym for session.commit(), otherwise it is a
125 synonym for session.flush().
128 def wrapped(*args, **kwargs):
129 private_transaction = False
131 # Find the session object
132 session = kwargs.get('session')
135 if len(args) <= len(getargspec(fn)[0]) - 1:
136 # No session specified as last argument or in kwargs
137 private_transaction = True
138 session = kwargs['session'] = DBConn().session()
140 # Session is last argument in args
144 session = args[-1] = DBConn().session()
145 private_transaction = True
147 if private_transaction:
148 session.commit_or_flush = session.commit
150 session.commit_or_flush = session.flush
153 return fn(*args, **kwargs)
155 if private_transaction:
156 # We created a session; close it.
159 wrapped.__doc__ = fn.__doc__
160 wrapped.func_name = fn.func_name
164 __all__.append('session_wrapper')
166 ################################################################################
168 class ORMObject(object):
170 ORMObject is a base class for all ORM classes mapped by SQLalchemy. All
171 derived classes must implement the properties() method.
174 def properties(self):
176 This method should be implemented by all derived classes and returns a
177 list of the important properties. The properties 'created' and
178 'modified' will be added automatically. A suffix '_count' should be
179 added to properties that are lists or query objects. The most important
180 property name should be returned as the first element in the list
181 because it is used by repr().
187 Returns a JSON representation of the object based on the properties
188 returned from the properties() method.
191 # add created and modified
192 all_properties = self.properties() + ['created', 'modified']
193 for property in all_properties:
194 # check for list or query
195 if property[-6:] == '_count':
196 real_property = property[:-6]
197 if not hasattr(self, real_property):
199 value = getattr(self, real_property)
200 if hasattr(value, '__len__'):
203 elif hasattr(value, 'count'):
205 value = value.count()
207 raise KeyError('Do not understand property %s.' % property)
209 if not hasattr(self, property):
212 value = getattr(self, property)
216 elif isinstance(value, ORMObject):
217 # use repr() for ORMObject types
220 # we want a string for all other types because json cannot
223 data[property] = value
224 return json.dumps(data)
228 Returns the name of the class.
230 return type(self).__name__
234 Returns a short string representation of the object using the first
235 element from the properties() method.
237 primary_property = self.properties()[0]
238 value = getattr(self, primary_property)
239 return '<%s %s>' % (self.classname(), str(value))
243 Returns a human readable form of the object using the properties()
246 return '<%s %s>' % (self.classname(), self.json())
248 def not_null_constraints(self):
250 Returns a list of properties that must be not NULL. Derived classes
251 should override this method if needed.
255 validation_message = \
256 "Validation failed because property '%s' must not be empty in object\n%s"
260 This function validates the not NULL constraints as returned by
261 not_null_constraints(). It raises the DBUpdateError exception if
264 for property in self.not_null_constraints():
265 # TODO: It is a bit awkward that the mapper configuration allow
266 # directly setting the numeric _id columns. We should get rid of it
268 if hasattr(self, property + '_id') and \
269 getattr(self, property + '_id') is not None:
271 if not hasattr(self, property) or getattr(self, property) is None:
272 raise DBUpdateError(self.validation_message % \
273 (property, str(self)))
277 def get(cls, primary_key, session = None):
279 This is a support function that allows getting an object by its primary
282 Architecture.get(3[, session])
284 instead of the more verbose
286 session.query(Architecture).get(3)
288 return session.query(cls).get(primary_key)
290 __all__.append('ORMObject')
292 ################################################################################
294 class Validator(MapperExtension):
296 This class calls the validate() method for each instance for the
297 'before_update' and 'before_insert' events. A global object validator is
298 used for configuring the individual mappers.
301 def before_update(self, mapper, connection, instance):
305 def before_insert(self, mapper, connection, instance):
309 validator = Validator()
311 ################################################################################
313 class Architecture(ORMObject):
314 def __init__(self, arch_string = None, description = None):
315 self.arch_string = arch_string
316 self.description = description
318 def __eq__(self, val):
319 if isinstance(val, str):
320 return (self.arch_string== val)
321 # This signals to use the normal comparison operator
322 return NotImplemented
324 def __ne__(self, val):
325 if isinstance(val, str):
326 return (self.arch_string != val)
327 # This signals to use the normal comparison operator
328 return NotImplemented
330 def properties(self):
331 return ['arch_string', 'arch_id', 'suites_count']
333 def not_null_constraints(self):
334 return ['arch_string']
336 __all__.append('Architecture')
339 def get_architecture(architecture, session=None):
341 Returns database id for given C{architecture}.
343 @type architecture: string
344 @param architecture: The name of the architecture
346 @type session: Session
347 @param session: Optional SQLA session object (a temporary one will be
348 generated if not supplied)
351 @return: Architecture object for the given arch (None if not present)
354 q = session.query(Architecture).filter_by(arch_string=architecture)
358 except NoResultFound:
361 __all__.append('get_architecture')
363 # TODO: should be removed because the implementation is too trivial
365 def get_architecture_suites(architecture, session=None):
367 Returns list of Suite objects for given C{architecture} name
369 @type architecture: str
370 @param architecture: Architecture name to search for
372 @type session: Session
373 @param session: Optional SQL session object (a temporary one will be
374 generated if not supplied)
377 @return: list of Suite objects for the given name (may be empty)
380 return get_architecture(architecture, session).suites
382 __all__.append('get_architecture_suites')
384 ################################################################################
386 class Archive(object):
387 def __init__(self, *args, **kwargs):
391 return '<Archive %s>' % self.archive_name
393 __all__.append('Archive')
396 def get_archive(archive, session=None):
398 returns database id for given C{archive}.
400 @type archive: string
401 @param archive: the name of the arhive
403 @type session: Session
404 @param session: Optional SQLA session object (a temporary one will be
405 generated if not supplied)
408 @return: Archive object for the given name (None if not present)
411 archive = archive.lower()
413 q = session.query(Archive).filter_by(archive_name=archive)
417 except NoResultFound:
420 __all__.append('get_archive')
422 ################################################################################
424 class BinContents(object):
425 def __init__(self, *args, **kwargs):
429 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
431 __all__.append('BinContents')
433 ################################################################################
435 class DBBinary(ORMObject):
436 def __init__(self, package = None, source = None, version = None, \
437 maintainer = None, architecture = None, poolfile = None, \
439 self.package = package
441 self.version = version
442 self.maintainer = maintainer
443 self.architecture = architecture
444 self.poolfile = poolfile
445 self.binarytype = binarytype
447 def properties(self):
448 return ['package', 'version', 'maintainer', 'source', 'architecture', \
449 'poolfile', 'binarytype', 'fingerprint', 'install_date', \
450 'suites_count', 'binary_id']
452 def not_null_constraints(self):
453 return ['package', 'version', 'maintainer', 'source', 'poolfile', \
456 def get_component_name(self):
457 return self.poolfile.location.component.component_name
459 __all__.append('DBBinary')
462 def get_suites_binary_in(package, session=None):
464 Returns list of Suite objects which given C{package} name is in
467 @param package: DBBinary package name to search for
470 @return: list of Suite objects for the given package
473 return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all()
475 __all__.append('get_suites_binary_in')
478 def get_component_by_package_suite(package, suite_list, arch_list=[], session=None):
480 Returns the component name of the newest binary package in suite_list or
481 None if no package is found. The result can be optionally filtered by a list
482 of architecture names.
485 @param package: DBBinary package name to search for
487 @type suite_list: list of str
488 @param suite_list: list of suite_name items
490 @type arch_list: list of str
491 @param arch_list: optional list of arch_string items that defaults to []
493 @rtype: str or NoneType
494 @return: name of component or None
497 q = session.query(DBBinary).filter_by(package = package). \
498 join(DBBinary.suites).filter(Suite.suite_name.in_(suite_list))
499 if len(arch_list) > 0:
500 q = q.join(DBBinary.architecture). \
501 filter(Architecture.arch_string.in_(arch_list))
502 binary = q.order_by(desc(DBBinary.version)).first()
506 return binary.get_component_name()
508 __all__.append('get_component_by_package_suite')
510 ################################################################################
512 class BinaryACL(object):
513 def __init__(self, *args, **kwargs):
517 return '<BinaryACL %s>' % self.binary_acl_id
519 __all__.append('BinaryACL')
521 ################################################################################
523 class BinaryACLMap(object):
524 def __init__(self, *args, **kwargs):
528 return '<BinaryACLMap %s>' % self.binary_acl_map_id
530 __all__.append('BinaryACLMap')
532 ################################################################################
537 ArchiveDir "%(archivepath)s";
538 OverrideDir "%(overridedir)s";
539 CacheDir "%(cachedir)s";
544 Packages::Compress ". bzip2 gzip";
545 Sources::Compress ". bzip2 gzip";
550 bindirectory "incoming"
555 BinOverride "override.sid.all3";
556 BinCacheDB "packages-accepted.db";
558 FileList "%(filelist)s";
561 Packages::Extensions ".deb .udeb";
564 bindirectory "incoming/"
567 BinOverride "override.sid.all3";
568 SrcOverride "override.sid.all3.src";
569 FileList "%(filelist)s";
573 class BuildQueue(object):
574 def __init__(self, *args, **kwargs):
578 return '<BuildQueue %s>' % self.queue_name
580 def write_metadata(self, starttime, force=False):
581 # Do we write out metafiles?
582 if not (force or self.generate_metadata):
585 session = DBConn().session().object_session(self)
587 fl_fd = fl_name = ac_fd = ac_name = None
589 arches = " ".join([ a.arch_string for a in session.query(Architecture).all() if a.arch_string != 'source' ])
590 startdir = os.getcwd()
593 # Grab files we want to include
594 newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
595 # Write file list with newer files
596 (fl_fd, fl_name) = mkstemp()
598 os.write(fl_fd, '%s\n' % n.fullpath)
603 # Write minimal apt.conf
604 # TODO: Remove hardcoding from template
605 (ac_fd, ac_name) = mkstemp()
606 os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path,
608 'cachedir': cnf["Dir::Cache"],
609 'overridedir': cnf["Dir::Override"],
613 # Run apt-ftparchive generate
614 os.chdir(os.path.dirname(ac_name))
615 os.system('apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
617 # Run apt-ftparchive release
618 # TODO: Eww - fix this
619 bname = os.path.basename(self.path)
623 # We have to remove the Release file otherwise it'll be included in the
626 os.unlink(os.path.join(bname, 'Release'))
630 os.system("""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="%s" release %s > Release""" % (self.origin, self.label, self.releasedescription, arches, bname))
632 # Crude hack with open and append, but this whole section is and should be redone.
633 if self.notautomatic:
634 release=open("Release", "a")
635 release.write("NotAutomatic: yes")
640 keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
641 if cnf.has_key("Dinstall::SigningPubKeyring"):
642 keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
644 os.system("gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey))
646 # Move the files if we got this far
647 os.rename('Release', os.path.join(bname, 'Release'))
649 os.rename('Release.gpg', os.path.join(bname, 'Release.gpg'))
651 # Clean up any left behind files
678 def clean_and_update(self, starttime, Logger, dryrun=False):
679 """WARNING: This routine commits for you"""
680 session = DBConn().session().object_session(self)
682 if self.generate_metadata and not dryrun:
683 self.write_metadata(starttime)
685 # Grab files older than our execution time
686 older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
692 Logger.log(["I: Would have removed %s from the queue" % o.fullpath])
694 Logger.log(["I: Removing %s from the queue" % o.fullpath])
695 os.unlink(o.fullpath)
698 # If it wasn't there, don't worry
699 if e.errno == ENOENT:
702 # TODO: Replace with proper logging call
703 Logger.log(["E: Could not remove %s" % o.fullpath])
710 for f in os.listdir(self.path):
711 if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release') or f.startswith('advisory'):
715 r = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter_by(filename = f).one()
716 except NoResultFound:
717 fp = os.path.join(self.path, f)
719 Logger.log(["I: Would remove unused link %s" % fp])
721 Logger.log(["I: Removing unused link %s" % fp])
725 Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath])
727 def add_file_from_pool(self, poolfile):
728 """Copies a file into the pool. Assumes that the PoolFile object is
729 attached to the same SQLAlchemy session as the Queue object is.
731 The caller is responsible for committing after calling this function."""
732 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
734 # Check if we have a file of this name or this ID already
735 for f in self.queuefiles:
736 if f.fileid is not None and f.fileid == poolfile.file_id or \
737 f.poolfile.filename == poolfile_basename:
738 # In this case, update the BuildQueueFile entry so we
739 # don't remove it too early
740 f.lastused = datetime.now()
741 DBConn().session().object_session(poolfile).add(f)
744 # Prepare BuildQueueFile object
745 qf = BuildQueueFile()
746 qf.build_queue_id = self.queue_id
747 qf.lastused = datetime.now()
748 qf.filename = poolfile_basename
750 targetpath = poolfile.fullpath
751 queuepath = os.path.join(self.path, poolfile_basename)
755 # We need to copy instead of symlink
757 utils.copy(targetpath, queuepath)
758 # NULL in the fileid field implies a copy
761 os.symlink(targetpath, queuepath)
762 qf.fileid = poolfile.file_id
766 # Get the same session as the PoolFile is using and add the qf to it
767 DBConn().session().object_session(poolfile).add(qf)
772 __all__.append('BuildQueue')
775 def get_build_queue(queuename, session=None):
777 Returns BuildQueue object for given C{queue name}, creating it if it does not
780 @type queuename: string
781 @param queuename: The name of the queue
783 @type session: Session
784 @param session: Optional SQLA session object (a temporary one will be
785 generated if not supplied)
788 @return: BuildQueue object for the given queue
791 q = session.query(BuildQueue).filter_by(queue_name=queuename)
795 except NoResultFound:
798 __all__.append('get_build_queue')
800 ################################################################################
802 class BuildQueueFile(object):
803 def __init__(self, *args, **kwargs):
807 return '<BuildQueueFile %s (%s)>' % (self.filename, self.build_queue_id)
811 return os.path.join(self.buildqueue.path, self.filename)
814 __all__.append('BuildQueueFile')
816 ################################################################################
818 class ChangePendingBinary(object):
819 def __init__(self, *args, **kwargs):
823 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
825 __all__.append('ChangePendingBinary')
827 ################################################################################
829 class ChangePendingFile(object):
830 def __init__(self, *args, **kwargs):
834 return '<ChangePendingFile %s>' % self.change_pending_file_id
836 __all__.append('ChangePendingFile')
838 ################################################################################
840 class ChangePendingSource(object):
841 def __init__(self, *args, **kwargs):
845 return '<ChangePendingSource %s>' % self.change_pending_source_id
847 __all__.append('ChangePendingSource')
849 ################################################################################
851 class Component(ORMObject):
852 def __init__(self, component_name = None):
853 self.component_name = component_name
855 def __eq__(self, val):
856 if isinstance(val, str):
857 return (self.component_name == val)
858 # This signals to use the normal comparison operator
859 return NotImplemented
861 def __ne__(self, val):
862 if isinstance(val, str):
863 return (self.component_name != val)
864 # This signals to use the normal comparison operator
865 return NotImplemented
867 def properties(self):
868 return ['component_name', 'component_id', 'description', 'location', \
871 def not_null_constraints(self):
872 return ['component_name']
875 __all__.append('Component')
878 def get_component(component, session=None):
880 Returns database id for given C{component}.
882 @type component: string
883 @param component: The name of the override type
886 @return: the database id for the given component
889 component = component.lower()
891 q = session.query(Component).filter_by(component_name=component)
895 except NoResultFound:
898 __all__.append('get_component')
900 ################################################################################
902 class DBConfig(object):
903 def __init__(self, *args, **kwargs):
907 return '<DBConfig %s>' % self.name
909 __all__.append('DBConfig')
911 ################################################################################
914 def get_or_set_contents_file_id(filename, session=None):
916 Returns database id for given filename.
918 If no matching file is found, a row is inserted.
920 @type filename: string
921 @param filename: The filename
922 @type session: SQLAlchemy
923 @param session: Optional SQL session object (a temporary one will be
924 generated if not supplied). If not passed, a commit will be performed at
925 the end of the function, otherwise the caller is responsible for commiting.
928 @return: the database id for the given component
931 q = session.query(ContentFilename).filter_by(filename=filename)
934 ret = q.one().cafilename_id
935 except NoResultFound:
936 cf = ContentFilename()
937 cf.filename = filename
939 session.commit_or_flush()
940 ret = cf.cafilename_id
944 __all__.append('get_or_set_contents_file_id')
947 def get_contents(suite, overridetype, section=None, session=None):
949 Returns contents for a suite / overridetype combination, limiting
950 to a section if not None.
953 @param suite: Suite object
955 @type overridetype: OverrideType
956 @param overridetype: OverrideType object
958 @type section: Section
959 @param section: Optional section object to limit results to
961 @type session: SQLAlchemy
962 @param session: Optional SQL session object (a temporary one will be
963 generated if not supplied)
966 @return: ResultsProxy object set up to return tuples of (filename, section,
970 # find me all of the contents for a given suite
971 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
975 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
976 JOIN content_file_names n ON (c.filename=n.id)
977 JOIN binaries b ON (b.id=c.binary_pkg)
978 JOIN override o ON (o.package=b.package)
979 JOIN section s ON (s.id=o.section)
980 WHERE o.suite = :suiteid AND o.type = :overridetypeid
981 AND b.type=:overridetypename"""
983 vals = {'suiteid': suite.suite_id,
984 'overridetypeid': overridetype.overridetype_id,
985 'overridetypename': overridetype.overridetype}
987 if section is not None:
988 contents_q += " AND s.id = :sectionid"
989 vals['sectionid'] = section.section_id
991 contents_q += " ORDER BY fn"
993 return session.execute(contents_q, vals)
995 __all__.append('get_contents')
997 ################################################################################
999 class ContentFilepath(object):
1000 def __init__(self, *args, **kwargs):
1004 return '<ContentFilepath %s>' % self.filepath
1006 __all__.append('ContentFilepath')
1009 def get_or_set_contents_path_id(filepath, session=None):
1011 Returns database id for given path.
1013 If no matching file is found, a row is inserted.
1015 @type filepath: string
1016 @param filepath: The filepath
1018 @type session: SQLAlchemy
1019 @param session: Optional SQL session object (a temporary one will be
1020 generated if not supplied). If not passed, a commit will be performed at
1021 the end of the function, otherwise the caller is responsible for commiting.
1024 @return: the database id for the given path
1027 q = session.query(ContentFilepath).filter_by(filepath=filepath)
1030 ret = q.one().cafilepath_id
1031 except NoResultFound:
1032 cf = ContentFilepath()
1033 cf.filepath = filepath
1035 session.commit_or_flush()
1036 ret = cf.cafilepath_id
1040 __all__.append('get_or_set_contents_path_id')
1042 ################################################################################
1044 class ContentAssociation(object):
1045 def __init__(self, *args, **kwargs):
1049 return '<ContentAssociation %s>' % self.ca_id
1051 __all__.append('ContentAssociation')
1053 def insert_content_paths(binary_id, fullpaths, session=None):
1055 Make sure given path is associated with given binary id
1057 @type binary_id: int
1058 @param binary_id: the id of the binary
1059 @type fullpaths: list
1060 @param fullpaths: the list of paths of the file being associated with the binary
1061 @type session: SQLAlchemy session
1062 @param session: Optional SQLAlchemy session. If this is passed, the caller
1063 is responsible for ensuring a transaction has begun and committing the
1064 results or rolling back based on the result code. If not passed, a commit
1065 will be performed at the end of the function, otherwise the caller is
1066 responsible for commiting.
1068 @return: True upon success
1071 privatetrans = False
1073 session = DBConn().session()
1078 def generate_path_dicts():
1079 for fullpath in fullpaths:
1080 if fullpath.startswith( './' ):
1081 fullpath = fullpath[2:]
1083 yield {'filename':fullpath, 'id': binary_id }
1085 for d in generate_path_dicts():
1086 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
1095 traceback.print_exc()
1097 # Only rollback if we set up the session ourself
1104 __all__.append('insert_content_paths')
1106 ################################################################################
1108 class DSCFile(object):
1109 def __init__(self, *args, **kwargs):
1113 return '<DSCFile %s>' % self.dscfile_id
1115 __all__.append('DSCFile')
1118 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
1120 Returns a list of DSCFiles which may be empty
1122 @type dscfile_id: int (optional)
1123 @param dscfile_id: the dscfile_id of the DSCFiles to find
1125 @type source_id: int (optional)
1126 @param source_id: the source id related to the DSCFiles to find
1128 @type poolfile_id: int (optional)
1129 @param poolfile_id: the poolfile id related to the DSCFiles to find
1132 @return: Possibly empty list of DSCFiles
1135 q = session.query(DSCFile)
1137 if dscfile_id is not None:
1138 q = q.filter_by(dscfile_id=dscfile_id)
1140 if source_id is not None:
1141 q = q.filter_by(source_id=source_id)
1143 if poolfile_id is not None:
1144 q = q.filter_by(poolfile_id=poolfile_id)
1148 __all__.append('get_dscfiles')
1150 ################################################################################
1152 class PoolFile(ORMObject):
1153 def __init__(self, filename = None, location = None, filesize = -1, \
1155 self.filename = filename
1156 self.location = location
1157 self.filesize = filesize
1158 self.md5sum = md5sum
1162 return os.path.join(self.location.path, self.filename)
1164 def is_valid(self, filesize = -1, md5sum = None):\
1165 return self.filesize == long(filesize) and self.md5sum == md5sum
1167 def properties(self):
1168 return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \
1169 'sha256sum', 'location', 'source', 'binary', 'last_used']
1171 def not_null_constraints(self):
1172 return ['filename', 'md5sum', 'location']
1174 __all__.append('PoolFile')
1177 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
1180 (ValidFileFound [boolean], PoolFile object or None)
1182 @type filename: string
1183 @param filename: the filename of the file to check against the DB
1186 @param filesize: the size of the file to check against the DB
1188 @type md5sum: string
1189 @param md5sum: the md5sum of the file to check against the DB
1191 @type location_id: int
1192 @param location_id: the id of the location to look in
1195 @return: Tuple of length 2.
1196 - If valid pool file found: (C{True}, C{PoolFile object})
1197 - If valid pool file not found:
1198 - (C{False}, C{None}) if no file found
1199 - (C{False}, C{PoolFile object}) if file found with size/md5sum mismatch
1202 poolfile = session.query(Location).get(location_id). \
1203 files.filter_by(filename=filename).first()
1205 if poolfile and poolfile.is_valid(filesize = filesize, md5sum = md5sum):
1208 return (valid, poolfile)
1210 __all__.append('check_poolfile')
1212 # TODO: the implementation can trivially be inlined at the place where the
1213 # function is called
1215 def get_poolfile_by_id(file_id, session=None):
1217 Returns a PoolFile objects or None for the given id
1220 @param file_id: the id of the file to look for
1222 @rtype: PoolFile or None
1223 @return: either the PoolFile object or None
1226 return session.query(PoolFile).get(file_id)
1228 __all__.append('get_poolfile_by_id')
1231 def get_poolfile_like_name(filename, session=None):
1233 Returns an array of PoolFile objects which are like the given name
1235 @type filename: string
1236 @param filename: the filename of the file to check against the DB
1239 @return: array of PoolFile objects
1242 # TODO: There must be a way of properly using bind parameters with %FOO%
1243 q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
1247 __all__.append('get_poolfile_like_name')
1250 def add_poolfile(filename, datadict, location_id, session=None):
1252 Add a new file to the pool
1254 @type filename: string
1255 @param filename: filename
1257 @type datadict: dict
1258 @param datadict: dict with needed data
1260 @type location_id: int
1261 @param location_id: database id of the location
1264 @return: the PoolFile object created
1266 poolfile = PoolFile()
1267 poolfile.filename = filename
1268 poolfile.filesize = datadict["size"]
1269 poolfile.md5sum = datadict["md5sum"]
1270 poolfile.sha1sum = datadict["sha1sum"]
1271 poolfile.sha256sum = datadict["sha256sum"]
1272 poolfile.location_id = location_id
1274 session.add(poolfile)
1275 # Flush to get a file id (NB: This is not a commit)
1280 __all__.append('add_poolfile')
1282 ################################################################################
1284 class Fingerprint(ORMObject):
1285 def __init__(self, fingerprint = None):
1286 self.fingerprint = fingerprint
1288 def properties(self):
1289 return ['fingerprint', 'fingerprint_id', 'keyring', 'uid', \
1292 def not_null_constraints(self):
1293 return ['fingerprint']
1295 __all__.append('Fingerprint')
1298 def get_fingerprint(fpr, session=None):
1300 Returns Fingerprint object for given fpr.
1303 @param fpr: The fpr to find / add
1305 @type session: SQLAlchemy
1306 @param session: Optional SQL session object (a temporary one will be
1307 generated if not supplied).
1310 @return: the Fingerprint object for the given fpr or None
1313 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1317 except NoResultFound:
1322 __all__.append('get_fingerprint')
1325 def get_or_set_fingerprint(fpr, session=None):
1327 Returns Fingerprint object for given fpr.
1329 If no matching fpr is found, a row is inserted.
1332 @param fpr: The fpr to find / add
1334 @type session: SQLAlchemy
1335 @param session: Optional SQL session object (a temporary one will be
1336 generated if not supplied). If not passed, a commit will be performed at
1337 the end of the function, otherwise the caller is responsible for commiting.
1338 A flush will be performed either way.
1341 @return: the Fingerprint object for the given fpr
1344 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1348 except NoResultFound:
1349 fingerprint = Fingerprint()
1350 fingerprint.fingerprint = fpr
1351 session.add(fingerprint)
1352 session.commit_or_flush()
1357 __all__.append('get_or_set_fingerprint')
1359 ################################################################################
1361 # Helper routine for Keyring class
1362 def get_ldap_name(entry):
1364 for k in ["cn", "mn", "sn"]:
1366 if ret and ret[0] != "" and ret[0] != "-":
1368 return " ".join(name)
1370 ################################################################################
1372 class Keyring(object):
1373 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1374 " --with-colons --fingerprint --fingerprint"
1379 def __init__(self, *args, **kwargs):
1383 return '<Keyring %s>' % self.keyring_name
1385 def de_escape_gpg_str(self, txt):
1386 esclist = re.split(r'(\\x..)', txt)
1387 for x in range(1,len(esclist),2):
1388 esclist[x] = "%c" % (int(esclist[x][2:],16))
1389 return "".join(esclist)
1391 def parse_address(self, uid):
1392 """parses uid and returns a tuple of real name and email address"""
1394 (name, address) = email.Utils.parseaddr(uid)
1395 name = re.sub(r"\s*[(].*[)]", "", name)
1396 name = self.de_escape_gpg_str(name)
1399 return (name, address)
1401 def load_keys(self, keyring):
1402 if not self.keyring_id:
1403 raise Exception('Must be initialized with database information')
1405 k = os.popen(self.gpg_invocation % keyring, "r")
1409 for line in k.xreadlines():
1410 field = line.split(":")
1411 if field[0] == "pub":
1414 (name, addr) = self.parse_address(field[9])
1416 self.keys[key]["email"] = addr
1417 self.keys[key]["name"] = name
1418 self.keys[key]["fingerprints"] = []
1420 elif key and field[0] == "sub" and len(field) >= 12:
1421 signingkey = ("s" in field[11])
1422 elif key and field[0] == "uid":
1423 (name, addr) = self.parse_address(field[9])
1424 if "email" not in self.keys[key] and "@" in addr:
1425 self.keys[key]["email"] = addr
1426 self.keys[key]["name"] = name
1427 elif signingkey and field[0] == "fpr":
1428 self.keys[key]["fingerprints"].append(field[9])
1429 self.fpr_lookup[field[9]] = key
1431 def import_users_from_ldap(self, session):
1435 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1436 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1438 l = ldap.open(LDAPServer)
1439 l.simple_bind_s("","")
1440 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1441 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1442 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1444 ldap_fin_uid_id = {}
1451 uid = entry["uid"][0]
1452 name = get_ldap_name(entry)
1453 fingerprints = entry["keyFingerPrint"]
1455 for f in fingerprints:
1456 key = self.fpr_lookup.get(f, None)
1457 if key not in self.keys:
1459 self.keys[key]["uid"] = uid
1463 keyid = get_or_set_uid(uid, session).uid_id
1464 byuid[keyid] = (uid, name)
1465 byname[uid] = (keyid, name)
1467 return (byname, byuid)
1469 def generate_users_from_keyring(self, format, session):
1473 for x in self.keys.keys():
1474 if "email" not in self.keys[x]:
1476 self.keys[x]["uid"] = format % "invalid-uid"
1478 uid = format % self.keys[x]["email"]
1479 keyid = get_or_set_uid(uid, session).uid_id
1480 byuid[keyid] = (uid, self.keys[x]["name"])
1481 byname[uid] = (keyid, self.keys[x]["name"])
1482 self.keys[x]["uid"] = uid
1485 uid = format % "invalid-uid"
1486 keyid = get_or_set_uid(uid, session).uid_id
1487 byuid[keyid] = (uid, "ungeneratable user id")
1488 byname[uid] = (keyid, "ungeneratable user id")
1490 return (byname, byuid)
1492 __all__.append('Keyring')
1495 def get_keyring(keyring, session=None):
1497 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1498 If C{keyring} already has an entry, simply return the existing Keyring
1500 @type keyring: string
1501 @param keyring: the keyring name
1504 @return: the Keyring object for this keyring
1507 q = session.query(Keyring).filter_by(keyring_name=keyring)
1511 except NoResultFound:
1514 __all__.append('get_keyring')
1516 ################################################################################
1518 class KeyringACLMap(object):
1519 def __init__(self, *args, **kwargs):
1523 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1525 __all__.append('KeyringACLMap')
1527 ################################################################################
1529 class DBChange(object):
1530 def __init__(self, *args, **kwargs):
1534 return '<DBChange %s>' % self.changesname
1536 def clean_from_queue(self):
1537 session = DBConn().session().object_session(self)
1539 # Remove changes_pool_files entries
1542 # Remove changes_pending_files references
1545 # Clear out of queue
1546 self.in_queue = None
1547 self.approved_for_id = None
1549 __all__.append('DBChange')
1552 def get_dbchange(filename, session=None):
1554 returns DBChange object for given C{filename}.
1556 @type filename: string
1557 @param filename: the name of the file
1559 @type session: Session
1560 @param session: Optional SQLA session object (a temporary one will be
1561 generated if not supplied)
1564 @return: DBChange object for the given filename (C{None} if not present)
1567 q = session.query(DBChange).filter_by(changesname=filename)
1571 except NoResultFound:
1574 __all__.append('get_dbchange')
1576 ################################################################################
1578 # TODO: Why do we have a separate Location class? Can't it be fully integrated
1579 # into class Component?
1580 class Location(ORMObject):
1581 def __init__(self, path = None, component = None):
1583 self.component = component
1584 # the column 'type' should go away, see comment at mapper
1585 self.archive_type = 'pool'
1587 def properties(self):
1588 return ['path', 'archive_type', 'component', 'files_count']
1590 def not_null_constraints(self):
1591 return ['path', 'archive_type']
1593 __all__.append('Location')
1596 def get_location(location, component=None, archive=None, session=None):
1598 Returns Location object for the given combination of location, component
1601 @type location: string
1602 @param location: the path of the location, e.g. I{/srv/ftp-master.debian.org/ftp/pool/}
1604 @type component: string
1605 @param component: the component name (if None, no restriction applied)
1607 @type archive: string
1608 @param archive: the archive name (if None, no restriction applied)
1610 @rtype: Location / None
1611 @return: Either a Location object or None if one can't be found
1614 q = session.query(Location).filter_by(path=location)
1616 if archive is not None:
1617 q = q.join(Archive).filter_by(archive_name=archive)
1619 if component is not None:
1620 q = q.join(Component).filter_by(component_name=component)
1624 except NoResultFound:
1627 __all__.append('get_location')
1629 ################################################################################
1631 class Maintainer(ORMObject):
1632 def __init__(self, name = None):
1635 def properties(self):
1636 return ['name', 'maintainer_id']
1638 def not_null_constraints(self):
1641 def get_split_maintainer(self):
1642 if not hasattr(self, 'name') or self.name is None:
1643 return ('', '', '', '')
1645 return fix_maintainer(self.name.strip())
1647 __all__.append('Maintainer')
1650 def get_or_set_maintainer(name, session=None):
1652 Returns Maintainer object for given maintainer name.
1654 If no matching maintainer name is found, a row is inserted.
1657 @param name: The maintainer name to add
1659 @type session: SQLAlchemy
1660 @param session: Optional SQL session object (a temporary one will be
1661 generated if not supplied). If not passed, a commit will be performed at
1662 the end of the function, otherwise the caller is responsible for commiting.
1663 A flush will be performed either way.
1666 @return: the Maintainer object for the given maintainer
1669 q = session.query(Maintainer).filter_by(name=name)
1672 except NoResultFound:
1673 maintainer = Maintainer()
1674 maintainer.name = name
1675 session.add(maintainer)
1676 session.commit_or_flush()
1681 __all__.append('get_or_set_maintainer')
1684 def get_maintainer(maintainer_id, session=None):
1686 Return the name of the maintainer behind C{maintainer_id} or None if that
1687 maintainer_id is invalid.
1689 @type maintainer_id: int
1690 @param maintainer_id: the id of the maintainer
1693 @return: the Maintainer with this C{maintainer_id}
1696 return session.query(Maintainer).get(maintainer_id)
1698 __all__.append('get_maintainer')
1700 ################################################################################
1702 class NewComment(object):
1703 def __init__(self, *args, **kwargs):
1707 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1709 __all__.append('NewComment')
1712 def has_new_comment(package, version, session=None):
1714 Returns true if the given combination of C{package}, C{version} has a comment.
1716 @type package: string
1717 @param package: name of the package
1719 @type version: string
1720 @param version: package version
1722 @type session: Session
1723 @param session: Optional SQLA session object (a temporary one will be
1724 generated if not supplied)
1730 q = session.query(NewComment)
1731 q = q.filter_by(package=package)
1732 q = q.filter_by(version=version)
1734 return bool(q.count() > 0)
1736 __all__.append('has_new_comment')
1739 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1741 Returns (possibly empty) list of NewComment objects for the given
1744 @type package: string (optional)
1745 @param package: name of the package
1747 @type version: string (optional)
1748 @param version: package version
1750 @type comment_id: int (optional)
1751 @param comment_id: An id of a comment
1753 @type session: Session
1754 @param session: Optional SQLA session object (a temporary one will be
1755 generated if not supplied)
1758 @return: A (possibly empty) list of NewComment objects will be returned
1761 q = session.query(NewComment)
1762 if package is not None: q = q.filter_by(package=package)
1763 if version is not None: q = q.filter_by(version=version)
1764 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1768 __all__.append('get_new_comments')
1770 ################################################################################
1772 class Override(object):
1773 def __init__(self, *args, **kwargs):
1777 return '<Override %s (%s)>' % (self.package, self.suite_id)
1779 __all__.append('Override')
1782 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1784 Returns Override object for the given parameters
1786 @type package: string
1787 @param package: The name of the package
1789 @type suite: string, list or None
1790 @param suite: The name of the suite (or suites if a list) to limit to. If
1791 None, don't limit. Defaults to None.
1793 @type component: string, list or None
1794 @param component: The name of the component (or components if a list) to
1795 limit to. If None, don't limit. Defaults to None.
1797 @type overridetype: string, list or None
1798 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1799 limit to. If None, don't limit. Defaults to None.
1801 @type session: Session
1802 @param session: Optional SQLA session object (a temporary one will be
1803 generated if not supplied)
1806 @return: A (possibly empty) list of Override objects will be returned
1809 q = session.query(Override)
1810 q = q.filter_by(package=package)
1812 if suite is not None:
1813 if not isinstance(suite, list): suite = [suite]
1814 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1816 if component is not None:
1817 if not isinstance(component, list): component = [component]
1818 q = q.join(Component).filter(Component.component_name.in_(component))
1820 if overridetype is not None:
1821 if not isinstance(overridetype, list): overridetype = [overridetype]
1822 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1826 __all__.append('get_override')
1829 ################################################################################
1831 class OverrideType(object):
1832 def __init__(self, *args, **kwargs):
1836 return '<OverrideType %s>' % self.overridetype
1838 __all__.append('OverrideType')
1841 def get_override_type(override_type, session=None):
1843 Returns OverrideType object for given C{override type}.
1845 @type override_type: string
1846 @param override_type: The name of the override type
1848 @type session: Session
1849 @param session: Optional SQLA session object (a temporary one will be
1850 generated if not supplied)
1853 @return: the database id for the given override type
1856 q = session.query(OverrideType).filter_by(overridetype=override_type)
1860 except NoResultFound:
1863 __all__.append('get_override_type')
1865 ################################################################################
1867 class DebContents(object):
1868 def __init__(self, *args, **kwargs):
1872 return '<DebConetnts %s: %s>' % (self.package.package,self.file)
1874 __all__.append('DebContents')
1877 class UdebContents(object):
1878 def __init__(self, *args, **kwargs):
1882 return '<UdebConetnts %s: %s>' % (self.package.package,self.file)
1884 __all__.append('UdebContents')
1886 class PendingBinContents(object):
1887 def __init__(self, *args, **kwargs):
1891 return '<PendingBinContents %s>' % self.contents_id
1893 __all__.append('PendingBinContents')
1895 def insert_pending_content_paths(package,
1900 Make sure given paths are temporarily associated with given
1904 @param package: the package to associate with should have been read in from the binary control file
1905 @type fullpaths: list
1906 @param fullpaths: the list of paths of the file being associated with the binary
1907 @type session: SQLAlchemy session
1908 @param session: Optional SQLAlchemy session. If this is passed, the caller
1909 is responsible for ensuring a transaction has begun and committing the
1910 results or rolling back based on the result code. If not passed, a commit
1911 will be performed at the end of the function
1913 @return: True upon success, False if there is a problem
1916 privatetrans = False
1919 session = DBConn().session()
1923 arch = get_architecture(package['Architecture'], session)
1924 arch_id = arch.arch_id
1926 # Remove any already existing recorded files for this package
1927 q = session.query(PendingBinContents)
1928 q = q.filter_by(package=package['Package'])
1929 q = q.filter_by(version=package['Version'])
1930 q = q.filter_by(architecture=arch_id)
1933 for fullpath in fullpaths:
1935 if fullpath.startswith( "./" ):
1936 fullpath = fullpath[2:]
1938 pca = PendingBinContents()
1939 pca.package = package['Package']
1940 pca.version = package['Version']
1942 pca.architecture = arch_id
1945 pca.type = 8 # gross
1947 pca.type = 7 # also gross
1950 # Only commit if we set up the session ourself
1958 except Exception, e:
1959 traceback.print_exc()
1961 # Only rollback if we set up the session ourself
1968 __all__.append('insert_pending_content_paths')
1970 ################################################################################
1972 class PolicyQueue(object):
1973 def __init__(self, *args, **kwargs):
1977 return '<PolicyQueue %s>' % self.queue_name
1979 __all__.append('PolicyQueue')
1982 def get_policy_queue(queuename, session=None):
1984 Returns PolicyQueue object for given C{queue name}
1986 @type queuename: string
1987 @param queuename: The name of the queue
1989 @type session: Session
1990 @param session: Optional SQLA session object (a temporary one will be
1991 generated if not supplied)
1994 @return: PolicyQueue object for the given queue
1997 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
2001 except NoResultFound:
2004 __all__.append('get_policy_queue')
2007 def get_policy_queue_from_path(pathname, session=None):
2009 Returns PolicyQueue object for given C{path name}
2011 @type queuename: string
2012 @param queuename: The path
2014 @type session: Session
2015 @param session: Optional SQLA session object (a temporary one will be
2016 generated if not supplied)
2019 @return: PolicyQueue object for the given queue
2022 q = session.query(PolicyQueue).filter_by(path=pathname)
2026 except NoResultFound:
2029 __all__.append('get_policy_queue_from_path')
2031 ################################################################################
2033 class Priority(object):
2034 def __init__(self, *args, **kwargs):
2037 def __eq__(self, val):
2038 if isinstance(val, str):
2039 return (self.priority == val)
2040 # This signals to use the normal comparison operator
2041 return NotImplemented
2043 def __ne__(self, val):
2044 if isinstance(val, str):
2045 return (self.priority != val)
2046 # This signals to use the normal comparison operator
2047 return NotImplemented
2050 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
2052 __all__.append('Priority')
2055 def get_priority(priority, session=None):
2057 Returns Priority object for given C{priority name}.
2059 @type priority: string
2060 @param priority: The name of the priority
2062 @type session: Session
2063 @param session: Optional SQLA session object (a temporary one will be
2064 generated if not supplied)
2067 @return: Priority object for the given priority
2070 q = session.query(Priority).filter_by(priority=priority)
2074 except NoResultFound:
2077 __all__.append('get_priority')
2080 def get_priorities(session=None):
2082 Returns dictionary of priority names -> id mappings
2084 @type session: Session
2085 @param session: Optional SQL session object (a temporary one will be
2086 generated if not supplied)
2089 @return: dictionary of priority names -> id mappings
2093 q = session.query(Priority)
2095 ret[x.priority] = x.priority_id
2099 __all__.append('get_priorities')
2101 ################################################################################
2103 class Section(object):
2104 def __init__(self, *args, **kwargs):
2107 def __eq__(self, val):
2108 if isinstance(val, str):
2109 return (self.section == val)
2110 # This signals to use the normal comparison operator
2111 return NotImplemented
2113 def __ne__(self, val):
2114 if isinstance(val, str):
2115 return (self.section != val)
2116 # This signals to use the normal comparison operator
2117 return NotImplemented
2120 return '<Section %s>' % self.section
2122 __all__.append('Section')
2125 def get_section(section, session=None):
2127 Returns Section object for given C{section name}.
2129 @type section: string
2130 @param section: The name of the section
2132 @type session: Session
2133 @param session: Optional SQLA session object (a temporary one will be
2134 generated if not supplied)
2137 @return: Section object for the given section name
2140 q = session.query(Section).filter_by(section=section)
2144 except NoResultFound:
2147 __all__.append('get_section')
2150 def get_sections(session=None):
2152 Returns dictionary of section names -> id mappings
2154 @type session: Session
2155 @param session: Optional SQL session object (a temporary one will be
2156 generated if not supplied)
2159 @return: dictionary of section names -> id mappings
2163 q = session.query(Section)
2165 ret[x.section] = x.section_id
2169 __all__.append('get_sections')
2171 ################################################################################
2173 class DBSource(ORMObject):
2174 def __init__(self, source = None, version = None, maintainer = None, \
2175 changedby = None, poolfile = None, install_date = None):
2176 self.source = source
2177 self.version = version
2178 self.maintainer = maintainer
2179 self.changedby = changedby
2180 self.poolfile = poolfile
2181 self.install_date = install_date
2183 def properties(self):
2184 return ['source', 'source_id', 'maintainer', 'changedby', \
2185 'fingerprint', 'poolfile', 'version', 'suites_count', \
2186 'install_date', 'binaries_count']
2188 def not_null_constraints(self):
2189 return ['source', 'version', 'install_date', 'maintainer', \
2190 'changedby', 'poolfile', 'install_date']
2192 __all__.append('DBSource')
2195 def source_exists(source, source_version, suites = ["any"], session=None):
2197 Ensure that source exists somewhere in the archive for the binary
2198 upload being processed.
2199 1. exact match => 1.0-3
2200 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
2202 @type source: string
2203 @param source: source name
2205 @type source_version: string
2206 @param source_version: expected source version
2209 @param suites: list of suites to check in, default I{any}
2211 @type session: Session
2212 @param session: Optional SQLA session object (a temporary one will be
2213 generated if not supplied)
2216 @return: returns 1 if a source with expected version is found, otherwise 0
2223 from daklib.regexes import re_bin_only_nmu
2224 orig_source_version = re_bin_only_nmu.sub('', source_version)
2226 for suite in suites:
2227 q = session.query(DBSource).filter_by(source=source). \
2228 filter(DBSource.version.in_([source_version, orig_source_version]))
2230 # source must exist in suite X, or in some other suite that's
2231 # mapped to X, recursively... silent-maps are counted too,
2232 # unreleased-maps aren't.
2233 maps = cnf.ValueList("SuiteMappings")[:]
2235 maps = [ m.split() for m in maps ]
2236 maps = [ (x[1], x[2]) for x in maps
2237 if x[0] == "map" or x[0] == "silent-map" ]
2240 if x[1] in s and x[0] not in s:
2243 q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
2248 # No source found so return not ok
2253 __all__.append('source_exists')
2256 def get_suites_source_in(source, session=None):
2258 Returns list of Suite objects which given C{source} name is in
2261 @param source: DBSource package name to search for
2264 @return: list of Suite objects for the given source
2267 return session.query(Suite).filter(Suite.sources.any(source=source)).all()
2269 __all__.append('get_suites_source_in')
2272 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
2274 Returns list of DBSource objects for given C{source} name and other parameters
2277 @param source: DBSource package name to search for
2279 @type version: str or None
2280 @param version: DBSource version name to search for or None if not applicable
2282 @type dm_upload_allowed: bool
2283 @param dm_upload_allowed: If None, no effect. If True or False, only
2284 return packages with that dm_upload_allowed setting
2286 @type session: Session
2287 @param session: Optional SQL session object (a temporary one will be
2288 generated if not supplied)
2291 @return: list of DBSource objects for the given name (may be empty)
2294 q = session.query(DBSource).filter_by(source=source)
2296 if version is not None:
2297 q = q.filter_by(version=version)
2299 if dm_upload_allowed is not None:
2300 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
2304 __all__.append('get_sources_from_name')
2306 # FIXME: This function fails badly if it finds more than 1 source package and
2307 # its implementation is trivial enough to be inlined.
2309 def get_source_in_suite(source, suite, session=None):
2311 Returns a DBSource object for a combination of C{source} and C{suite}.
2313 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2314 - B{suite} - a suite name, eg. I{unstable}
2316 @type source: string
2317 @param source: source package name
2320 @param suite: the suite name
2323 @return: the version for I{source} in I{suite}
2327 q = get_suite(suite, session).get_sources(source)
2330 except NoResultFound:
2333 __all__.append('get_source_in_suite')
2335 ################################################################################
2338 def add_dsc_to_db(u, filename, session=None):
2339 entry = u.pkg.files[filename]
2343 source.source = u.pkg.dsc["source"]
2344 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2345 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2346 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2347 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2348 source.install_date = datetime.now().date()
2350 dsc_component = entry["component"]
2351 dsc_location_id = entry["location id"]
2353 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2355 # Set up a new poolfile if necessary
2356 if not entry.has_key("files id") or not entry["files id"]:
2357 filename = entry["pool name"] + filename
2358 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2360 pfs.append(poolfile)
2361 entry["files id"] = poolfile.file_id
2363 source.poolfile_id = entry["files id"]
2366 suite_names = u.pkg.changes["distribution"].keys()
2367 source.suites = session.query(Suite). \
2368 filter(Suite.suite_name.in_(suite_names)).all()
2370 # Add the source files to the DB (files and dsc_files)
2372 dscfile.source_id = source.source_id
2373 dscfile.poolfile_id = entry["files id"]
2374 session.add(dscfile)
2376 for dsc_file, dentry in u.pkg.dsc_files.items():
2378 df.source_id = source.source_id
2380 # If the .orig tarball is already in the pool, it's
2381 # files id is stored in dsc_files by check_dsc().
2382 files_id = dentry.get("files id", None)
2384 # Find the entry in the files hash
2385 # TODO: Bail out here properly
2387 for f, e in u.pkg.files.items():
2392 if files_id is None:
2393 filename = dfentry["pool name"] + dsc_file
2395 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2396 # FIXME: needs to check for -1/-2 and or handle exception
2397 if found and obj is not None:
2398 files_id = obj.file_id
2401 # If still not found, add it
2402 if files_id is None:
2403 # HACK: Force sha1sum etc into dentry
2404 dentry["sha1sum"] = dfentry["sha1sum"]
2405 dentry["sha256sum"] = dfentry["sha256sum"]
2406 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2407 pfs.append(poolfile)
2408 files_id = poolfile.file_id
2410 poolfile = get_poolfile_by_id(files_id, session)
2411 if poolfile is None:
2412 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2413 pfs.append(poolfile)
2415 df.poolfile_id = files_id
2418 # Add the src_uploaders to the DB
2419 uploader_ids = [source.maintainer_id]
2420 if u.pkg.dsc.has_key("uploaders"):
2421 for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
2423 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2426 for up_id in uploader_ids:
2427 if added_ids.has_key(up_id):
2429 utils.warn("Already saw uploader %s for source %s" % (up_id, source.source))
2435 su.maintainer_id = up_id
2436 su.source_id = source.source_id
2441 return source, dsc_component, dsc_location_id, pfs
2443 __all__.append('add_dsc_to_db')
2446 def add_deb_to_db(u, filename, session=None):
2448 Contrary to what you might expect, this routine deals with both
2449 debs and udebs. That info is in 'dbtype', whilst 'type' is
2450 'deb' for both of them
2453 entry = u.pkg.files[filename]
2456 bin.package = entry["package"]
2457 bin.version = entry["version"]
2458 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2459 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2460 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2461 bin.binarytype = entry["dbtype"]
2464 filename = entry["pool name"] + filename
2465 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2466 if not entry.get("location id", None):
2467 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2469 if entry.get("files id", None):
2470 poolfile = get_poolfile_by_id(bin.poolfile_id)
2471 bin.poolfile_id = entry["files id"]
2473 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2474 bin.poolfile_id = entry["files id"] = poolfile.file_id
2477 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2478 if len(bin_sources) != 1:
2479 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2480 (bin.package, bin.version, entry["architecture"],
2481 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2483 bin.source_id = bin_sources[0].source_id
2485 # Add and flush object so it has an ID
2488 suite_names = u.pkg.changes["distribution"].keys()
2489 bin.suites = session.query(Suite). \
2490 filter(Suite.suite_name.in_(suite_names)).all()
2494 # Deal with contents - disabled for now
2495 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2497 # print "REJECT\nCould not determine contents of package %s" % bin.package
2498 # session.rollback()
2499 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2503 __all__.append('add_deb_to_db')
2505 ################################################################################
2507 class SourceACL(object):
2508 def __init__(self, *args, **kwargs):
2512 return '<SourceACL %s>' % self.source_acl_id
2514 __all__.append('SourceACL')
2516 ################################################################################
2518 class SrcFormat(object):
2519 def __init__(self, *args, **kwargs):
2523 return '<SrcFormat %s>' % (self.format_name)
2525 __all__.append('SrcFormat')
2527 ################################################################################
2529 class SrcUploader(object):
2530 def __init__(self, *args, **kwargs):
2534 return '<SrcUploader %s>' % self.uploader_id
2536 __all__.append('SrcUploader')
2538 ################################################################################
2540 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2541 ('SuiteID', 'suite_id'),
2542 ('Version', 'version'),
2543 ('Origin', 'origin'),
2545 ('Description', 'description'),
2546 ('Untouchable', 'untouchable'),
2547 ('Announce', 'announce'),
2548 ('Codename', 'codename'),
2549 ('OverrideCodename', 'overridecodename'),
2550 ('ValidTime', 'validtime'),
2551 ('Priority', 'priority'),
2552 ('NotAutomatic', 'notautomatic'),
2553 ('CopyChanges', 'copychanges'),
2554 ('OverrideSuite', 'overridesuite')]
2556 # Why the heck don't we have any UNIQUE constraints in table suite?
2557 # TODO: Add UNIQUE constraints for appropriate columns.
2558 class Suite(ORMObject):
2559 def __init__(self, suite_name = None, version = None):
2560 self.suite_name = suite_name
2561 self.version = version
2563 def properties(self):
2564 return ['suite_name', 'version', 'sources_count', 'binaries_count']
2566 def not_null_constraints(self):
2567 return ['suite_name', 'version']
2569 def __eq__(self, val):
2570 if isinstance(val, str):
2571 return (self.suite_name == val)
2572 # This signals to use the normal comparison operator
2573 return NotImplemented
2575 def __ne__(self, val):
2576 if isinstance(val, str):
2577 return (self.suite_name != val)
2578 # This signals to use the normal comparison operator
2579 return NotImplemented
2583 for disp, field in SUITE_FIELDS:
2584 val = getattr(self, field, None)
2586 ret.append("%s: %s" % (disp, val))
2588 return "\n".join(ret)
2590 def get_architectures(self, skipsrc=False, skipall=False):
2592 Returns list of Architecture objects
2594 @type skipsrc: boolean
2595 @param skipsrc: Whether to skip returning the 'source' architecture entry
2598 @type skipall: boolean
2599 @param skipall: Whether to skip returning the 'all' architecture entry
2603 @return: list of Architecture objects for the given name (may be empty)
2606 q = object_session(self).query(Architecture).with_parent(self)
2608 q = q.filter(Architecture.arch_string != 'source')
2610 q = q.filter(Architecture.arch_string != 'all')
2611 return q.order_by(Architecture.arch_string).all()
2613 def get_sources(self, source):
2615 Returns a query object representing DBSource that is part of C{suite}.
2617 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2619 @type source: string
2620 @param source: source package name
2622 @rtype: sqlalchemy.orm.query.Query
2623 @return: a query of DBSource
2627 session = object_session(self)
2628 return session.query(DBSource).filter_by(source = source). \
2631 __all__.append('Suite')
2634 def get_suite(suite, session=None):
2636 Returns Suite object for given C{suite name}.
2639 @param suite: The name of the suite
2641 @type session: Session
2642 @param session: Optional SQLA session object (a temporary one will be
2643 generated if not supplied)
2646 @return: Suite object for the requested suite name (None if not present)
2649 q = session.query(Suite).filter_by(suite_name=suite)
2653 except NoResultFound:
2656 __all__.append('get_suite')
2658 ################################################################################
2660 # TODO: should be removed because the implementation is too trivial
2662 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2664 Returns list of Architecture objects for given C{suite} name
2667 @param suite: Suite name to search for
2669 @type skipsrc: boolean
2670 @param skipsrc: Whether to skip returning the 'source' architecture entry
2673 @type skipall: boolean
2674 @param skipall: Whether to skip returning the 'all' architecture entry
2677 @type session: Session
2678 @param session: Optional SQL session object (a temporary one will be
2679 generated if not supplied)
2682 @return: list of Architecture objects for the given name (may be empty)
2685 return get_suite(suite, session).get_architectures(skipsrc, skipall)
2687 __all__.append('get_suite_architectures')
2689 ################################################################################
2691 class SuiteSrcFormat(object):
2692 def __init__(self, *args, **kwargs):
2696 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2698 __all__.append('SuiteSrcFormat')
2701 def get_suite_src_formats(suite, session=None):
2703 Returns list of allowed SrcFormat for C{suite}.
2706 @param suite: Suite name to search for
2708 @type session: Session
2709 @param session: Optional SQL session object (a temporary one will be
2710 generated if not supplied)
2713 @return: the list of allowed source formats for I{suite}
2716 q = session.query(SrcFormat)
2717 q = q.join(SuiteSrcFormat)
2718 q = q.join(Suite).filter_by(suite_name=suite)
2719 q = q.order_by('format_name')
2723 __all__.append('get_suite_src_formats')
2725 ################################################################################
2727 class Uid(ORMObject):
2728 def __init__(self, uid = None, name = None):
2732 def __eq__(self, val):
2733 if isinstance(val, str):
2734 return (self.uid == val)
2735 # This signals to use the normal comparison operator
2736 return NotImplemented
2738 def __ne__(self, val):
2739 if isinstance(val, str):
2740 return (self.uid != val)
2741 # This signals to use the normal comparison operator
2742 return NotImplemented
2744 def properties(self):
2745 return ['uid', 'name', 'fingerprint']
2747 def not_null_constraints(self):
2750 __all__.append('Uid')
2753 def get_or_set_uid(uidname, session=None):
2755 Returns uid object for given uidname.
2757 If no matching uidname is found, a row is inserted.
2759 @type uidname: string
2760 @param uidname: The uid to add
2762 @type session: SQLAlchemy
2763 @param session: Optional SQL session object (a temporary one will be
2764 generated if not supplied). If not passed, a commit will be performed at
2765 the end of the function, otherwise the caller is responsible for commiting.
2768 @return: the uid object for the given uidname
2771 q = session.query(Uid).filter_by(uid=uidname)
2775 except NoResultFound:
2779 session.commit_or_flush()
2784 __all__.append('get_or_set_uid')
2787 def get_uid_from_fingerprint(fpr, session=None):
2788 q = session.query(Uid)
2789 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2793 except NoResultFound:
2796 __all__.append('get_uid_from_fingerprint')
2798 ################################################################################
2800 class UploadBlock(object):
2801 def __init__(self, *args, **kwargs):
2805 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2807 __all__.append('UploadBlock')
2809 ################################################################################
2811 class DBConn(object):
2813 database module init.
2817 def __init__(self, *args, **kwargs):
2818 self.__dict__ = self.__shared_state
2820 if not getattr(self, 'initialised', False):
2821 self.initialised = True
2822 self.debug = kwargs.has_key('debug')
2825 def __setuptables(self):
2826 tables_with_primary = (
2837 'changes_pending_binaries',
2838 'changes_pending_files',
2839 'changes_pending_source',
2849 'pending_bin_contents',
2861 # The following tables have primary keys but sqlalchemy
2862 # version 0.5 fails to reflect them correctly with database
2863 # versions before upgrade #41.
2865 #'build_queue_files',
2868 tables_no_primary = (
2870 'changes_pending_files_map',
2871 'changes_pending_source_files',
2872 'changes_pool_files',
2875 'suite_architectures',
2876 'suite_src_formats',
2877 'suite_build_queue_copy',
2879 # see the comment above
2881 'build_queue_files',
2885 'almost_obsolete_all_associations',
2886 'almost_obsolete_src_associations',
2887 'any_associations_source',
2888 'bin_assoc_by_arch',
2889 'bin_associations_binaries',
2890 'binaries_suite_arch',
2891 'binfiles_suite_component_arch',
2894 'newest_all_associations',
2895 'newest_any_associations',
2897 'newest_src_association',
2898 'obsolete_all_associations',
2899 'obsolete_any_associations',
2900 'obsolete_any_by_all_associations',
2901 'obsolete_src_associations',
2903 'src_associations_bin',
2904 'src_associations_src',
2905 'suite_arch_by_name',
2908 # Sqlalchemy version 0.5 fails to reflect the SERIAL type
2909 # correctly and that is why we have to use a workaround. It can
2910 # be removed as soon as we switch to version 0.6.
2911 for table_name in tables_with_primary:
2912 table = Table(table_name, self.db_meta, \
2913 Column('id', Integer, primary_key = True), \
2914 autoload=True, useexisting=True)
2915 setattr(self, 'tbl_%s' % table_name, table)
2917 for table_name in tables_no_primary:
2918 table = Table(table_name, self.db_meta, autoload=True)
2919 setattr(self, 'tbl_%s' % table_name, table)
2921 for view_name in views:
2922 view = Table(view_name, self.db_meta, autoload=True)
2923 setattr(self, 'view_%s' % view_name, view)
2925 def __setupmappers(self):
2926 mapper(Architecture, self.tbl_architecture,
2927 properties = dict(arch_id = self.tbl_architecture.c.id,
2928 suites = relation(Suite, secondary=self.tbl_suite_architectures,
2929 order_by='suite_name',
2930 backref=backref('architectures', order_by='arch_string'))),
2931 extension = validator)
2933 mapper(Archive, self.tbl_archive,
2934 properties = dict(archive_id = self.tbl_archive.c.id,
2935 archive_name = self.tbl_archive.c.name))
2937 mapper(PendingBinContents, self.tbl_pending_bin_contents,
2938 properties = dict(contents_id =self.tbl_pending_bin_contents.c.id,
2939 filename = self.tbl_pending_bin_contents.c.filename,
2940 package = self.tbl_pending_bin_contents.c.package,
2941 version = self.tbl_pending_bin_contents.c.version,
2942 arch = self.tbl_pending_bin_contents.c.arch,
2943 otype = self.tbl_pending_bin_contents.c.type))
2945 mapper(DebContents, self.tbl_deb_contents,
2946 properties = dict(binary_id=self.tbl_deb_contents.c.binary_id,
2947 package=self.tbl_deb_contents.c.package,
2948 suite=self.tbl_deb_contents.c.suite,
2949 arch=self.tbl_deb_contents.c.arch,
2950 section=self.tbl_deb_contents.c.section,
2951 filename=self.tbl_deb_contents.c.filename))
2953 mapper(UdebContents, self.tbl_udeb_contents,
2954 properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id,
2955 package=self.tbl_udeb_contents.c.package,
2956 suite=self.tbl_udeb_contents.c.suite,
2957 arch=self.tbl_udeb_contents.c.arch,
2958 section=self.tbl_udeb_contents.c.section,
2959 filename=self.tbl_udeb_contents.c.filename))
2961 mapper(BuildQueue, self.tbl_build_queue,
2962 properties = dict(queue_id = self.tbl_build_queue.c.id))
2964 mapper(BuildQueueFile, self.tbl_build_queue_files,
2965 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
2966 poolfile = relation(PoolFile, backref='buildqueueinstances')))
2968 mapper(DBBinary, self.tbl_binaries,
2969 properties = dict(binary_id = self.tbl_binaries.c.id,
2970 package = self.tbl_binaries.c.package,
2971 version = self.tbl_binaries.c.version,
2972 maintainer_id = self.tbl_binaries.c.maintainer,
2973 maintainer = relation(Maintainer),
2974 source_id = self.tbl_binaries.c.source,
2975 source = relation(DBSource, backref='binaries'),
2976 arch_id = self.tbl_binaries.c.architecture,
2977 architecture = relation(Architecture),
2978 poolfile_id = self.tbl_binaries.c.file,
2979 poolfile = relation(PoolFile, backref=backref('binary', uselist = False)),
2980 binarytype = self.tbl_binaries.c.type,
2981 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2982 fingerprint = relation(Fingerprint),
2983 install_date = self.tbl_binaries.c.install_date,
2984 suites = relation(Suite, secondary=self.tbl_bin_associations,
2985 backref=backref('binaries', lazy='dynamic'))),
2986 extension = validator)
2988 mapper(BinaryACL, self.tbl_binary_acl,
2989 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2991 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2992 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2993 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2994 architecture = relation(Architecture)))
2996 mapper(Component, self.tbl_component,
2997 properties = dict(component_id = self.tbl_component.c.id,
2998 component_name = self.tbl_component.c.name),
2999 extension = validator)
3001 mapper(DBConfig, self.tbl_config,
3002 properties = dict(config_id = self.tbl_config.c.id))
3004 mapper(DSCFile, self.tbl_dsc_files,
3005 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
3006 source_id = self.tbl_dsc_files.c.source,
3007 source = relation(DBSource),
3008 poolfile_id = self.tbl_dsc_files.c.file,
3009 poolfile = relation(PoolFile)))
3011 mapper(PoolFile, self.tbl_files,
3012 properties = dict(file_id = self.tbl_files.c.id,
3013 filesize = self.tbl_files.c.size,
3014 location_id = self.tbl_files.c.location,
3015 location = relation(Location,
3016 # using lazy='dynamic' in the back
3017 # reference because we have A LOT of
3018 # files in one location
3019 backref=backref('files', lazy='dynamic'))),
3020 extension = validator)
3022 mapper(Fingerprint, self.tbl_fingerprint,
3023 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
3024 uid_id = self.tbl_fingerprint.c.uid,
3025 uid = relation(Uid),
3026 keyring_id = self.tbl_fingerprint.c.keyring,
3027 keyring = relation(Keyring),
3028 source_acl = relation(SourceACL),
3029 binary_acl = relation(BinaryACL)),
3030 extension = validator)
3032 mapper(Keyring, self.tbl_keyrings,
3033 properties = dict(keyring_name = self.tbl_keyrings.c.name,
3034 keyring_id = self.tbl_keyrings.c.id))
3036 mapper(DBChange, self.tbl_changes,
3037 properties = dict(change_id = self.tbl_changes.c.id,
3038 poolfiles = relation(PoolFile,
3039 secondary=self.tbl_changes_pool_files,
3040 backref="changeslinks"),
3041 seen = self.tbl_changes.c.seen,
3042 source = self.tbl_changes.c.source,
3043 binaries = self.tbl_changes.c.binaries,
3044 architecture = self.tbl_changes.c.architecture,
3045 distribution = self.tbl_changes.c.distribution,
3046 urgency = self.tbl_changes.c.urgency,
3047 maintainer = self.tbl_changes.c.maintainer,
3048 changedby = self.tbl_changes.c.changedby,
3049 date = self.tbl_changes.c.date,
3050 version = self.tbl_changes.c.version,
3051 files = relation(ChangePendingFile,
3052 secondary=self.tbl_changes_pending_files_map,
3053 backref="changesfile"),
3054 in_queue_id = self.tbl_changes.c.in_queue,
3055 in_queue = relation(PolicyQueue,
3056 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
3057 approved_for_id = self.tbl_changes.c.approved_for))
3059 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
3060 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
3062 mapper(ChangePendingFile, self.tbl_changes_pending_files,
3063 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id,
3064 filename = self.tbl_changes_pending_files.c.filename,
3065 size = self.tbl_changes_pending_files.c.size,
3066 md5sum = self.tbl_changes_pending_files.c.md5sum,
3067 sha1sum = self.tbl_changes_pending_files.c.sha1sum,
3068 sha256sum = self.tbl_changes_pending_files.c.sha256sum))
3070 mapper(ChangePendingSource, self.tbl_changes_pending_source,
3071 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
3072 change = relation(DBChange),
3073 maintainer = relation(Maintainer,
3074 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
3075 changedby = relation(Maintainer,
3076 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
3077 fingerprint = relation(Fingerprint),
3078 source_files = relation(ChangePendingFile,
3079 secondary=self.tbl_changes_pending_source_files,
3080 backref="pending_sources")))
3083 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
3084 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
3085 keyring = relation(Keyring, backref="keyring_acl_map"),
3086 architecture = relation(Architecture)))
3088 mapper(Location, self.tbl_location,
3089 properties = dict(location_id = self.tbl_location.c.id,
3090 component_id = self.tbl_location.c.component,
3091 component = relation(Component, \
3092 backref=backref('location', uselist = False)),
3093 archive_id = self.tbl_location.c.archive,
3094 archive = relation(Archive),
3095 # FIXME: the 'type' column is old cruft and
3096 # should be removed in the future.
3097 archive_type = self.tbl_location.c.type),
3098 extension = validator)
3100 mapper(Maintainer, self.tbl_maintainer,
3101 properties = dict(maintainer_id = self.tbl_maintainer.c.id,
3102 maintains_sources = relation(DBSource, backref='maintainer',
3103 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.maintainer)),
3104 changed_sources = relation(DBSource, backref='changedby',
3105 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.changedby))),
3106 extension = validator)
3108 mapper(NewComment, self.tbl_new_comments,
3109 properties = dict(comment_id = self.tbl_new_comments.c.id))
3111 mapper(Override, self.tbl_override,
3112 properties = dict(suite_id = self.tbl_override.c.suite,
3113 suite = relation(Suite),
3114 package = self.tbl_override.c.package,
3115 component_id = self.tbl_override.c.component,
3116 component = relation(Component),
3117 priority_id = self.tbl_override.c.priority,
3118 priority = relation(Priority),
3119 section_id = self.tbl_override.c.section,
3120 section = relation(Section),
3121 overridetype_id = self.tbl_override.c.type,
3122 overridetype = relation(OverrideType)))
3124 mapper(OverrideType, self.tbl_override_type,
3125 properties = dict(overridetype = self.tbl_override_type.c.type,
3126 overridetype_id = self.tbl_override_type.c.id))
3128 mapper(PolicyQueue, self.tbl_policy_queue,
3129 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
3131 mapper(Priority, self.tbl_priority,
3132 properties = dict(priority_id = self.tbl_priority.c.id))
3134 mapper(Section, self.tbl_section,
3135 properties = dict(section_id = self.tbl_section.c.id,
3136 section=self.tbl_section.c.section))
3138 mapper(DBSource, self.tbl_source,
3139 properties = dict(source_id = self.tbl_source.c.id,
3140 version = self.tbl_source.c.version,
3141 maintainer_id = self.tbl_source.c.maintainer,
3142 poolfile_id = self.tbl_source.c.file,
3143 poolfile = relation(PoolFile, backref=backref('source', uselist = False)),
3144 fingerprint_id = self.tbl_source.c.sig_fpr,
3145 fingerprint = relation(Fingerprint),
3146 changedby_id = self.tbl_source.c.changedby,
3147 srcfiles = relation(DSCFile,
3148 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
3149 suites = relation(Suite, secondary=self.tbl_src_associations,
3150 backref=backref('sources', lazy='dynamic')),
3151 srcuploaders = relation(SrcUploader)),
3152 extension = validator)
3154 mapper(SourceACL, self.tbl_source_acl,
3155 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
3157 mapper(SrcFormat, self.tbl_src_format,
3158 properties = dict(src_format_id = self.tbl_src_format.c.id,
3159 format_name = self.tbl_src_format.c.format_name))
3161 mapper(SrcUploader, self.tbl_src_uploaders,
3162 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
3163 source_id = self.tbl_src_uploaders.c.source,
3164 source = relation(DBSource,
3165 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
3166 maintainer_id = self.tbl_src_uploaders.c.maintainer,
3167 maintainer = relation(Maintainer,
3168 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
3170 mapper(Suite, self.tbl_suite,
3171 properties = dict(suite_id = self.tbl_suite.c.id,
3172 policy_queue = relation(PolicyQueue),
3173 copy_queues = relation(BuildQueue,
3174 secondary=self.tbl_suite_build_queue_copy)),
3175 extension = validator)
3177 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
3178 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
3179 suite = relation(Suite, backref='suitesrcformats'),
3180 src_format_id = self.tbl_suite_src_formats.c.src_format,
3181 src_format = relation(SrcFormat)))
3183 mapper(Uid, self.tbl_uid,
3184 properties = dict(uid_id = self.tbl_uid.c.id,
3185 fingerprint = relation(Fingerprint)),
3186 extension = validator)
3188 mapper(UploadBlock, self.tbl_upload_blocks,
3189 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
3190 fingerprint = relation(Fingerprint, backref="uploadblocks"),
3191 uid = relation(Uid, backref="uploadblocks")))
3193 ## Connection functions
3194 def __createconn(self):
3195 from config import Config
3199 connstr = "postgres://%s" % cnf["DB::Host"]
3200 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
3201 connstr += ":%s" % cnf["DB::Port"]
3202 connstr += "/%s" % cnf["DB::Name"]
3205 connstr = "postgres:///%s" % cnf["DB::Name"]
3206 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
3207 connstr += "?port=%s" % cnf["DB::Port"]
3209 self.db_pg = create_engine(connstr, echo=self.debug)
3210 self.db_meta = MetaData()
3211 self.db_meta.bind = self.db_pg
3212 self.db_smaker = sessionmaker(bind=self.db_pg,
3216 self.__setuptables()
3217 self.__setupmappers()
3220 return self.db_smaker()
3222 __all__.append('DBConn')