5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
47 import simplejson as json
49 from datetime import datetime, timedelta
50 from errno import ENOENT
51 from tempfile import mkstemp, mkdtemp
53 from inspect import getargspec
56 from sqlalchemy import create_engine, Table, MetaData, Column, Integer
57 from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
58 backref, MapperExtension, EXT_CONTINUE
59 from sqlalchemy import types as sqltypes
61 # Don't remove this, we re-export the exceptions to scripts which import us
62 from sqlalchemy.exc import *
63 from sqlalchemy.orm.exc import NoResultFound
65 # Only import Config until Queue stuff is changed to store its config
67 from config import Config
68 from textutils import fix_maintainer
69 from dak_exceptions import DBUpdateError, NoSourceFieldError
71 # suppress some deprecation warnings in squeeze related to sqlalchemy
73 warnings.filterwarnings('ignore', \
74 "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
76 # TODO: sqlalchemy needs some extra configuration to correctly reflect
77 # the ind_deb_contents_* indexes - we ignore the warnings at the moment
78 warnings.filterwarnings("ignore", 'Predicate of partial index', SAWarning)
81 ################################################################################
83 # Patch in support for the debversion field type so that it works during
87 # that is for sqlalchemy 0.6
88 UserDefinedType = sqltypes.UserDefinedType
90 # this one for sqlalchemy 0.5
91 UserDefinedType = sqltypes.TypeEngine
93 class DebVersion(UserDefinedType):
94 def get_col_spec(self):
97 def bind_processor(self, dialect):
100 # ' = None' is needed for sqlalchemy 0.5:
101 def result_processor(self, dialect, coltype = None):
104 sa_major_version = sqlalchemy.__version__[0:3]
105 if sa_major_version in ["0.5", "0.6"]:
106 from sqlalchemy.databases import postgres
107 postgres.ischema_names['debversion'] = DebVersion
109 raise Exception("dak only ported to SQLA versions 0.5 and 0.6. See daklib/dbconn.py")
111 ################################################################################
113 __all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion']
115 ################################################################################
117 def session_wrapper(fn):
119 Wrapper around common ".., session=None):" handling. If the wrapped
120 function is called without passing 'session', we create a local one
121 and destroy it when the function ends.
123 Also attaches a commit_or_flush method to the session; if we created a
124 local session, this is a synonym for session.commit(), otherwise it is a
125 synonym for session.flush().
128 def wrapped(*args, **kwargs):
129 private_transaction = False
131 # Find the session object
132 session = kwargs.get('session')
135 if len(args) <= len(getargspec(fn)[0]) - 1:
136 # No session specified as last argument or in kwargs
137 private_transaction = True
138 session = kwargs['session'] = DBConn().session()
140 # Session is last argument in args
144 session = args[-1] = DBConn().session()
145 private_transaction = True
147 if private_transaction:
148 session.commit_or_flush = session.commit
150 session.commit_or_flush = session.flush
153 return fn(*args, **kwargs)
155 if private_transaction:
156 # We created a session; close it.
159 wrapped.__doc__ = fn.__doc__
160 wrapped.func_name = fn.func_name
164 __all__.append('session_wrapper')
166 ################################################################################
168 class ORMObject(object):
170 ORMObject is a base class for all ORM classes mapped by SQLalchemy. All
171 derived classes must implement the properties() method.
174 def properties(self):
176 This method should be implemented by all derived classes and returns a
177 list of the important properties. The properties 'created' and
178 'modified' will be added automatically. A suffix '_count' should be
179 added to properties that are lists or query objects. The most important
180 property name should be returned as the first element in the list
181 because it is used by repr().
187 Returns a JSON representation of the object based on the properties
188 returned from the properties() method.
191 # add created and modified
192 all_properties = self.properties() + ['created', 'modified']
193 for property in all_properties:
194 # check for list or query
195 if property[-6:] == '_count':
196 real_property = property[:-6]
197 if not hasattr(self, real_property):
199 value = getattr(self, real_property)
200 if hasattr(value, '__len__'):
203 elif hasattr(value, 'count'):
205 value = value.count()
207 raise KeyError('Do not understand property %s.' % property)
209 if not hasattr(self, property):
212 value = getattr(self, property)
216 elif isinstance(value, ORMObject):
217 # use repr() for ORMObject types
220 # we want a string for all other types because json cannot
223 data[property] = value
224 return json.dumps(data)
228 Returns the name of the class.
230 return type(self).__name__
234 Returns a short string representation of the object using the first
235 element from the properties() method.
237 primary_property = self.properties()[0]
238 value = getattr(self, primary_property)
239 return '<%s %s>' % (self.classname(), str(value))
243 Returns a human readable form of the object using the properties()
246 return '<%s %s>' % (self.classname(), self.json())
248 def not_null_constraints(self):
250 Returns a list of properties that must be not NULL. Derived classes
251 should override this method if needed.
255 validation_message = \
256 "Validation failed because property '%s' must not be empty in object\n%s"
260 This function validates the not NULL constraints as returned by
261 not_null_constraints(). It raises the DBUpdateError exception if
264 for property in self.not_null_constraints():
265 # TODO: It is a bit awkward that the mapper configuration allow
266 # directly setting the numeric _id columns. We should get rid of it
268 if hasattr(self, property + '_id') and \
269 getattr(self, property + '_id') is not None:
271 if not hasattr(self, property) or getattr(self, property) is None:
272 raise DBUpdateError(self.validation_message % \
273 (property, str(self)))
277 def get(cls, primary_key, session = None):
279 This is a support function that allows getting an object by its primary
282 Architecture.get(3[, session])
284 instead of the more verbose
286 session.query(Architecture).get(3)
288 return session.query(cls).get(primary_key)
290 __all__.append('ORMObject')
292 ################################################################################
294 class Validator(MapperExtension):
296 This class calls the validate() method for each instance for the
297 'before_update' and 'before_insert' events. A global object validator is
298 used for configuring the individual mappers.
301 def before_update(self, mapper, connection, instance):
305 def before_insert(self, mapper, connection, instance):
309 validator = Validator()
311 ################################################################################
313 class Architecture(ORMObject):
314 def __init__(self, arch_string = None, description = None):
315 self.arch_string = arch_string
316 self.description = description
318 def __eq__(self, val):
319 if isinstance(val, str):
320 return (self.arch_string== val)
321 # This signals to use the normal comparison operator
322 return NotImplemented
324 def __ne__(self, val):
325 if isinstance(val, str):
326 return (self.arch_string != val)
327 # This signals to use the normal comparison operator
328 return NotImplemented
330 def properties(self):
331 return ['arch_string', 'arch_id', 'suites_count']
333 def not_null_constraints(self):
334 return ['arch_string']
336 __all__.append('Architecture')
339 def get_architecture(architecture, session=None):
341 Returns database id for given C{architecture}.
343 @type architecture: string
344 @param architecture: The name of the architecture
346 @type session: Session
347 @param session: Optional SQLA session object (a temporary one will be
348 generated if not supplied)
351 @return: Architecture object for the given arch (None if not present)
354 q = session.query(Architecture).filter_by(arch_string=architecture)
358 except NoResultFound:
361 __all__.append('get_architecture')
363 # TODO: should be removed because the implementation is too trivial
365 def get_architecture_suites(architecture, session=None):
367 Returns list of Suite objects for given C{architecture} name
369 @type architecture: str
370 @param architecture: Architecture name to search for
372 @type session: Session
373 @param session: Optional SQL session object (a temporary one will be
374 generated if not supplied)
377 @return: list of Suite objects for the given name (may be empty)
380 return get_architecture(architecture, session).suites
382 __all__.append('get_architecture_suites')
384 ################################################################################
386 class Archive(object):
387 def __init__(self, *args, **kwargs):
391 return '<Archive %s>' % self.archive_name
393 __all__.append('Archive')
396 def get_archive(archive, session=None):
398 returns database id for given C{archive}.
400 @type archive: string
401 @param archive: the name of the arhive
403 @type session: Session
404 @param session: Optional SQLA session object (a temporary one will be
405 generated if not supplied)
408 @return: Archive object for the given name (None if not present)
411 archive = archive.lower()
413 q = session.query(Archive).filter_by(archive_name=archive)
417 except NoResultFound:
420 __all__.append('get_archive')
422 ################################################################################
424 class BinAssociation(object):
425 def __init__(self, *args, **kwargs):
429 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
431 __all__.append('BinAssociation')
433 ################################################################################
435 class BinContents(object):
436 def __init__(self, *args, **kwargs):
440 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
442 __all__.append('BinContents')
444 ################################################################################
446 class DBBinary(ORMObject):
447 def __init__(self, package = None, source = None, version = None, \
448 maintainer = None, architecture = None, poolfile = None, \
450 self.package = package
452 self.version = version
453 self.maintainer = maintainer
454 self.architecture = architecture
455 self.poolfile = poolfile
456 self.binarytype = binarytype
458 def properties(self):
459 return ['package', 'version', 'maintainer', 'source', 'architecture', \
460 'poolfile', 'binarytype', 'fingerprint', 'install_date', \
461 'suites_count', 'binary_id']
463 def not_null_constraints(self):
464 return ['package', 'version', 'maintainer', 'source', 'poolfile', \
467 __all__.append('DBBinary')
470 def get_suites_binary_in(package, session=None):
472 Returns list of Suite objects which given C{package} name is in
475 @param package: DBBinary package name to search for
478 @return: list of Suite objects for the given package
481 return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all()
483 __all__.append('get_suites_binary_in')
486 def get_binary_from_name_suite(package, suitename, session=None):
487 ### For dak examine-package
488 ### XXX: Doesn't use object API yet
490 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
491 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
492 WHERE b.package='%(package)s'
494 AND fi.location = l.id
495 AND l.component = c.id
498 AND su.suite_name %(suitename)s
499 ORDER BY b.version DESC"""
501 return session.execute(sql % {'package': package, 'suitename': suitename})
503 __all__.append('get_binary_from_name_suite')
506 def get_binary_components(package, suitename, arch, session=None):
507 # Check for packages that have moved from one component to another
508 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
509 WHERE b.package=:package AND s.suite_name=:suitename
510 AND (a.arch_string = :arch OR a.arch_string = 'all')
511 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
512 AND f.location = l.id
513 AND l.component = c.id
516 vals = {'package': package, 'suitename': suitename, 'arch': arch}
518 return session.execute(query, vals)
520 __all__.append('get_binary_components')
522 ################################################################################
524 class BinaryACL(object):
525 def __init__(self, *args, **kwargs):
529 return '<BinaryACL %s>' % self.binary_acl_id
531 __all__.append('BinaryACL')
533 ################################################################################
535 class BinaryACLMap(object):
536 def __init__(self, *args, **kwargs):
540 return '<BinaryACLMap %s>' % self.binary_acl_map_id
542 __all__.append('BinaryACLMap')
544 ################################################################################
549 ArchiveDir "%(archivepath)s";
550 OverrideDir "%(overridedir)s";
551 CacheDir "%(cachedir)s";
556 Packages::Compress ". bzip2 gzip";
557 Sources::Compress ". bzip2 gzip";
562 bindirectory "incoming"
567 BinOverride "override.sid.all3";
568 BinCacheDB "packages-accepted.db";
570 FileList "%(filelist)s";
573 Packages::Extensions ".deb .udeb";
576 bindirectory "incoming/"
579 BinOverride "override.sid.all3";
580 SrcOverride "override.sid.all3.src";
581 FileList "%(filelist)s";
585 class BuildQueue(object):
586 def __init__(self, *args, **kwargs):
590 return '<BuildQueue %s>' % self.queue_name
592 def write_metadata(self, starttime, force=False):
593 # Do we write out metafiles?
594 if not (force or self.generate_metadata):
597 session = DBConn().session().object_session(self)
599 fl_fd = fl_name = ac_fd = ac_name = None
601 arches = " ".join([ a.arch_string for a in session.query(Architecture).all() if a.arch_string != 'source' ])
602 startdir = os.getcwd()
605 # Grab files we want to include
606 newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
607 # Write file list with newer files
608 (fl_fd, fl_name) = mkstemp()
610 os.write(fl_fd, '%s\n' % n.fullpath)
615 # Write minimal apt.conf
616 # TODO: Remove hardcoding from template
617 (ac_fd, ac_name) = mkstemp()
618 os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path,
620 'cachedir': cnf["Dir::Cache"],
621 'overridedir': cnf["Dir::Override"],
625 # Run apt-ftparchive generate
626 os.chdir(os.path.dirname(ac_name))
627 os.system('apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
629 # Run apt-ftparchive release
630 # TODO: Eww - fix this
631 bname = os.path.basename(self.path)
635 # We have to remove the Release file otherwise it'll be included in the
638 os.unlink(os.path.join(bname, 'Release'))
642 os.system("""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="%s" release %s > Release""" % (self.origin, self.label, self.releasedescription, arches, bname))
644 # Crude hack with open and append, but this whole section is and should be redone.
645 if self.notautomatic:
646 release=open("Release", "a")
647 release.write("NotAutomatic: yes")
652 keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
653 if cnf.has_key("Dinstall::SigningPubKeyring"):
654 keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
656 os.system("gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey))
658 # Move the files if we got this far
659 os.rename('Release', os.path.join(bname, 'Release'))
661 os.rename('Release.gpg', os.path.join(bname, 'Release.gpg'))
663 # Clean up any left behind files
690 def clean_and_update(self, starttime, Logger, dryrun=False):
691 """WARNING: This routine commits for you"""
692 session = DBConn().session().object_session(self)
694 if self.generate_metadata and not dryrun:
695 self.write_metadata(starttime)
697 # Grab files older than our execution time
698 older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
704 Logger.log(["I: Would have removed %s from the queue" % o.fullpath])
706 Logger.log(["I: Removing %s from the queue" % o.fullpath])
707 os.unlink(o.fullpath)
710 # If it wasn't there, don't worry
711 if e.errno == ENOENT:
714 # TODO: Replace with proper logging call
715 Logger.log(["E: Could not remove %s" % o.fullpath])
722 for f in os.listdir(self.path):
723 if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release') or f.startswith('advisory'):
727 r = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter_by(filename = f).one()
728 except NoResultFound:
729 fp = os.path.join(self.path, f)
731 Logger.log(["I: Would remove unused link %s" % fp])
733 Logger.log(["I: Removing unused link %s" % fp])
737 Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath])
739 def add_file_from_pool(self, poolfile):
740 """Copies a file into the pool. Assumes that the PoolFile object is
741 attached to the same SQLAlchemy session as the Queue object is.
743 The caller is responsible for committing after calling this function."""
744 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
746 # Check if we have a file of this name or this ID already
747 for f in self.queuefiles:
748 if f.fileid is not None and f.fileid == poolfile.file_id or \
749 f.poolfile.filename == poolfile_basename:
750 # In this case, update the BuildQueueFile entry so we
751 # don't remove it too early
752 f.lastused = datetime.now()
753 DBConn().session().object_session(poolfile).add(f)
756 # Prepare BuildQueueFile object
757 qf = BuildQueueFile()
758 qf.build_queue_id = self.queue_id
759 qf.lastused = datetime.now()
760 qf.filename = poolfile_basename
762 targetpath = poolfile.fullpath
763 queuepath = os.path.join(self.path, poolfile_basename)
767 # We need to copy instead of symlink
769 utils.copy(targetpath, queuepath)
770 # NULL in the fileid field implies a copy
773 os.symlink(targetpath, queuepath)
774 qf.fileid = poolfile.file_id
778 # Get the same session as the PoolFile is using and add the qf to it
779 DBConn().session().object_session(poolfile).add(qf)
784 __all__.append('BuildQueue')
787 def get_build_queue(queuename, session=None):
789 Returns BuildQueue object for given C{queue name}, creating it if it does not
792 @type queuename: string
793 @param queuename: The name of the queue
795 @type session: Session
796 @param session: Optional SQLA session object (a temporary one will be
797 generated if not supplied)
800 @return: BuildQueue object for the given queue
803 q = session.query(BuildQueue).filter_by(queue_name=queuename)
807 except NoResultFound:
810 __all__.append('get_build_queue')
812 ################################################################################
814 class BuildQueueFile(object):
815 def __init__(self, *args, **kwargs):
819 return '<BuildQueueFile %s (%s)>' % (self.filename, self.build_queue_id)
823 return os.path.join(self.buildqueue.path, self.filename)
826 __all__.append('BuildQueueFile')
828 ################################################################################
830 class ChangePendingBinary(object):
831 def __init__(self, *args, **kwargs):
835 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
837 __all__.append('ChangePendingBinary')
839 ################################################################################
841 class ChangePendingFile(object):
842 def __init__(self, *args, **kwargs):
846 return '<ChangePendingFile %s>' % self.change_pending_file_id
848 __all__.append('ChangePendingFile')
850 ################################################################################
852 class ChangePendingSource(object):
853 def __init__(self, *args, **kwargs):
857 return '<ChangePendingSource %s>' % self.change_pending_source_id
859 __all__.append('ChangePendingSource')
861 ################################################################################
863 class Component(object):
864 def __init__(self, *args, **kwargs):
867 def __eq__(self, val):
868 if isinstance(val, str):
869 return (self.component_name == val)
870 # This signals to use the normal comparison operator
871 return NotImplemented
873 def __ne__(self, val):
874 if isinstance(val, str):
875 return (self.component_name != val)
876 # This signals to use the normal comparison operator
877 return NotImplemented
880 return '<Component %s>' % self.component_name
883 __all__.append('Component')
886 def get_component(component, session=None):
888 Returns database id for given C{component}.
890 @type component: string
891 @param component: The name of the override type
894 @return: the database id for the given component
897 component = component.lower()
899 q = session.query(Component).filter_by(component_name=component)
903 except NoResultFound:
906 __all__.append('get_component')
908 ################################################################################
910 class DBConfig(object):
911 def __init__(self, *args, **kwargs):
915 return '<DBConfig %s>' % self.name
917 __all__.append('DBConfig')
919 ################################################################################
922 def get_or_set_contents_file_id(filename, session=None):
924 Returns database id for given filename.
926 If no matching file is found, a row is inserted.
928 @type filename: string
929 @param filename: The filename
930 @type session: SQLAlchemy
931 @param session: Optional SQL session object (a temporary one will be
932 generated if not supplied). If not passed, a commit will be performed at
933 the end of the function, otherwise the caller is responsible for commiting.
936 @return: the database id for the given component
939 q = session.query(ContentFilename).filter_by(filename=filename)
942 ret = q.one().cafilename_id
943 except NoResultFound:
944 cf = ContentFilename()
945 cf.filename = filename
947 session.commit_or_flush()
948 ret = cf.cafilename_id
952 __all__.append('get_or_set_contents_file_id')
955 def get_contents(suite, overridetype, section=None, session=None):
957 Returns contents for a suite / overridetype combination, limiting
958 to a section if not None.
961 @param suite: Suite object
963 @type overridetype: OverrideType
964 @param overridetype: OverrideType object
966 @type section: Section
967 @param section: Optional section object to limit results to
969 @type session: SQLAlchemy
970 @param session: Optional SQL session object (a temporary one will be
971 generated if not supplied)
974 @return: ResultsProxy object set up to return tuples of (filename, section,
978 # find me all of the contents for a given suite
979 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
983 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
984 JOIN content_file_names n ON (c.filename=n.id)
985 JOIN binaries b ON (b.id=c.binary_pkg)
986 JOIN override o ON (o.package=b.package)
987 JOIN section s ON (s.id=o.section)
988 WHERE o.suite = :suiteid AND o.type = :overridetypeid
989 AND b.type=:overridetypename"""
991 vals = {'suiteid': suite.suite_id,
992 'overridetypeid': overridetype.overridetype_id,
993 'overridetypename': overridetype.overridetype}
995 if section is not None:
996 contents_q += " AND s.id = :sectionid"
997 vals['sectionid'] = section.section_id
999 contents_q += " ORDER BY fn"
1001 return session.execute(contents_q, vals)
1003 __all__.append('get_contents')
1005 ################################################################################
1007 class ContentFilepath(object):
1008 def __init__(self, *args, **kwargs):
1012 return '<ContentFilepath %s>' % self.filepath
1014 __all__.append('ContentFilepath')
1017 def get_or_set_contents_path_id(filepath, session=None):
1019 Returns database id for given path.
1021 If no matching file is found, a row is inserted.
1023 @type filepath: string
1024 @param filepath: The filepath
1026 @type session: SQLAlchemy
1027 @param session: Optional SQL session object (a temporary one will be
1028 generated if not supplied). If not passed, a commit will be performed at
1029 the end of the function, otherwise the caller is responsible for commiting.
1032 @return: the database id for the given path
1035 q = session.query(ContentFilepath).filter_by(filepath=filepath)
1038 ret = q.one().cafilepath_id
1039 except NoResultFound:
1040 cf = ContentFilepath()
1041 cf.filepath = filepath
1043 session.commit_or_flush()
1044 ret = cf.cafilepath_id
1048 __all__.append('get_or_set_contents_path_id')
1050 ################################################################################
1052 class ContentAssociation(object):
1053 def __init__(self, *args, **kwargs):
1057 return '<ContentAssociation %s>' % self.ca_id
1059 __all__.append('ContentAssociation')
1061 def insert_content_paths(binary_id, fullpaths, session=None):
1063 Make sure given path is associated with given binary id
1065 @type binary_id: int
1066 @param binary_id: the id of the binary
1067 @type fullpaths: list
1068 @param fullpaths: the list of paths of the file being associated with the binary
1069 @type session: SQLAlchemy session
1070 @param session: Optional SQLAlchemy session. If this is passed, the caller
1071 is responsible for ensuring a transaction has begun and committing the
1072 results or rolling back based on the result code. If not passed, a commit
1073 will be performed at the end of the function, otherwise the caller is
1074 responsible for commiting.
1076 @return: True upon success
1079 privatetrans = False
1081 session = DBConn().session()
1086 def generate_path_dicts():
1087 for fullpath in fullpaths:
1088 if fullpath.startswith( './' ):
1089 fullpath = fullpath[2:]
1091 yield {'filename':fullpath, 'id': binary_id }
1093 for d in generate_path_dicts():
1094 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
1103 traceback.print_exc()
1105 # Only rollback if we set up the session ourself
1112 __all__.append('insert_content_paths')
1114 ################################################################################
1116 class DSCFile(object):
1117 def __init__(self, *args, **kwargs):
1121 return '<DSCFile %s>' % self.dscfile_id
1123 __all__.append('DSCFile')
1126 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
1128 Returns a list of DSCFiles which may be empty
1130 @type dscfile_id: int (optional)
1131 @param dscfile_id: the dscfile_id of the DSCFiles to find
1133 @type source_id: int (optional)
1134 @param source_id: the source id related to the DSCFiles to find
1136 @type poolfile_id: int (optional)
1137 @param poolfile_id: the poolfile id related to the DSCFiles to find
1140 @return: Possibly empty list of DSCFiles
1143 q = session.query(DSCFile)
1145 if dscfile_id is not None:
1146 q = q.filter_by(dscfile_id=dscfile_id)
1148 if source_id is not None:
1149 q = q.filter_by(source_id=source_id)
1151 if poolfile_id is not None:
1152 q = q.filter_by(poolfile_id=poolfile_id)
1156 __all__.append('get_dscfiles')
1158 ################################################################################
1160 class PoolFile(ORMObject):
1161 def __init__(self, filename = None, location = None, filesize = -1, \
1163 self.filename = filename
1164 self.location = location
1165 self.filesize = filesize
1166 self.md5sum = md5sum
1170 return os.path.join(self.location.path, self.filename)
1172 def is_valid(self, filesize = -1, md5sum = None):\
1173 return self.filesize == filesize and self.md5sum == md5sum
1175 def properties(self):
1176 return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \
1177 'sha256sum', 'location', 'source', 'binary', 'last_used']
1179 def not_null_constraints(self):
1180 return ['filename', 'md5sum', 'location']
1182 __all__.append('PoolFile')
1185 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
1188 (ValidFileFound [boolean], PoolFile object or None)
1190 @type filename: string
1191 @param filename: the filename of the file to check against the DB
1194 @param filesize: the size of the file to check against the DB
1196 @type md5sum: string
1197 @param md5sum: the md5sum of the file to check against the DB
1199 @type location_id: int
1200 @param location_id: the id of the location to look in
1203 @return: Tuple of length 2.
1204 - If valid pool file found: (C{True}, C{PoolFile object})
1205 - If valid pool file not found:
1206 - (C{False}, C{None}) if no file found
1207 - (C{False}, C{PoolFile object}) if file found with size/md5sum mismatch
1210 poolfile = session.query(Location).get(location_id). \
1211 files.filter_by(filename=filename).first()
1213 if poolfile and poolfile.is_valid(filesize = filesize, md5sum = md5sum):
1216 return (valid, poolfile)
1218 __all__.append('check_poolfile')
1220 # TODO: the implementation can trivially be inlined at the place where the
1221 # function is called
1223 def get_poolfile_by_id(file_id, session=None):
1225 Returns a PoolFile objects or None for the given id
1228 @param file_id: the id of the file to look for
1230 @rtype: PoolFile or None
1231 @return: either the PoolFile object or None
1234 return session.query(PoolFile).get(file_id)
1236 __all__.append('get_poolfile_by_id')
1239 def get_poolfile_like_name(filename, session=None):
1241 Returns an array of PoolFile objects which are like the given name
1243 @type filename: string
1244 @param filename: the filename of the file to check against the DB
1247 @return: array of PoolFile objects
1250 # TODO: There must be a way of properly using bind parameters with %FOO%
1251 q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
1255 __all__.append('get_poolfile_like_name')
1258 def add_poolfile(filename, datadict, location_id, session=None):
1260 Add a new file to the pool
1262 @type filename: string
1263 @param filename: filename
1265 @type datadict: dict
1266 @param datadict: dict with needed data
1268 @type location_id: int
1269 @param location_id: database id of the location
1272 @return: the PoolFile object created
1274 poolfile = PoolFile()
1275 poolfile.filename = filename
1276 poolfile.filesize = datadict["size"]
1277 poolfile.md5sum = datadict["md5sum"]
1278 poolfile.sha1sum = datadict["sha1sum"]
1279 poolfile.sha256sum = datadict["sha256sum"]
1280 poolfile.location_id = location_id
1282 session.add(poolfile)
1283 # Flush to get a file id (NB: This is not a commit)
1288 __all__.append('add_poolfile')
1290 ################################################################################
1292 class Fingerprint(ORMObject):
1293 def __init__(self, fingerprint = None):
1294 self.fingerprint = fingerprint
1296 def properties(self):
1297 return ['fingerprint', 'fingerprint_id', 'keyring', 'uid', \
1300 def not_null_constraints(self):
1301 return ['fingerprint']
1303 __all__.append('Fingerprint')
1306 def get_fingerprint(fpr, session=None):
1308 Returns Fingerprint object for given fpr.
1311 @param fpr: The fpr to find / add
1313 @type session: SQLAlchemy
1314 @param session: Optional SQL session object (a temporary one will be
1315 generated if not supplied).
1318 @return: the Fingerprint object for the given fpr or None
1321 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1325 except NoResultFound:
1330 __all__.append('get_fingerprint')
1333 def get_or_set_fingerprint(fpr, session=None):
1335 Returns Fingerprint object for given fpr.
1337 If no matching fpr is found, a row is inserted.
1340 @param fpr: The fpr to find / add
1342 @type session: SQLAlchemy
1343 @param session: Optional SQL session object (a temporary one will be
1344 generated if not supplied). If not passed, a commit will be performed at
1345 the end of the function, otherwise the caller is responsible for commiting.
1346 A flush will be performed either way.
1349 @return: the Fingerprint object for the given fpr
1352 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1356 except NoResultFound:
1357 fingerprint = Fingerprint()
1358 fingerprint.fingerprint = fpr
1359 session.add(fingerprint)
1360 session.commit_or_flush()
1365 __all__.append('get_or_set_fingerprint')
1367 ################################################################################
1369 # Helper routine for Keyring class
1370 def get_ldap_name(entry):
1372 for k in ["cn", "mn", "sn"]:
1374 if ret and ret[0] != "" and ret[0] != "-":
1376 return " ".join(name)
1378 ################################################################################
1380 class Keyring(object):
1381 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1382 " --with-colons --fingerprint --fingerprint"
1387 def __init__(self, *args, **kwargs):
1391 return '<Keyring %s>' % self.keyring_name
1393 def de_escape_gpg_str(self, txt):
1394 esclist = re.split(r'(\\x..)', txt)
1395 for x in range(1,len(esclist),2):
1396 esclist[x] = "%c" % (int(esclist[x][2:],16))
1397 return "".join(esclist)
1399 def parse_address(self, uid):
1400 """parses uid and returns a tuple of real name and email address"""
1402 (name, address) = email.Utils.parseaddr(uid)
1403 name = re.sub(r"\s*[(].*[)]", "", name)
1404 name = self.de_escape_gpg_str(name)
1407 return (name, address)
1409 def load_keys(self, keyring):
1410 if not self.keyring_id:
1411 raise Exception('Must be initialized with database information')
1413 k = os.popen(self.gpg_invocation % keyring, "r")
1417 for line in k.xreadlines():
1418 field = line.split(":")
1419 if field[0] == "pub":
1422 (name, addr) = self.parse_address(field[9])
1424 self.keys[key]["email"] = addr
1425 self.keys[key]["name"] = name
1426 self.keys[key]["fingerprints"] = []
1428 elif key and field[0] == "sub" and len(field) >= 12:
1429 signingkey = ("s" in field[11])
1430 elif key and field[0] == "uid":
1431 (name, addr) = self.parse_address(field[9])
1432 if "email" not in self.keys[key] and "@" in addr:
1433 self.keys[key]["email"] = addr
1434 self.keys[key]["name"] = name
1435 elif signingkey and field[0] == "fpr":
1436 self.keys[key]["fingerprints"].append(field[9])
1437 self.fpr_lookup[field[9]] = key
1439 def import_users_from_ldap(self, session):
1443 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1444 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1446 l = ldap.open(LDAPServer)
1447 l.simple_bind_s("","")
1448 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1449 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1450 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1452 ldap_fin_uid_id = {}
1459 uid = entry["uid"][0]
1460 name = get_ldap_name(entry)
1461 fingerprints = entry["keyFingerPrint"]
1463 for f in fingerprints:
1464 key = self.fpr_lookup.get(f, None)
1465 if key not in self.keys:
1467 self.keys[key]["uid"] = uid
1471 keyid = get_or_set_uid(uid, session).uid_id
1472 byuid[keyid] = (uid, name)
1473 byname[uid] = (keyid, name)
1475 return (byname, byuid)
1477 def generate_users_from_keyring(self, format, session):
1481 for x in self.keys.keys():
1482 if "email" not in self.keys[x]:
1484 self.keys[x]["uid"] = format % "invalid-uid"
1486 uid = format % self.keys[x]["email"]
1487 keyid = get_or_set_uid(uid, session).uid_id
1488 byuid[keyid] = (uid, self.keys[x]["name"])
1489 byname[uid] = (keyid, self.keys[x]["name"])
1490 self.keys[x]["uid"] = uid
1493 uid = format % "invalid-uid"
1494 keyid = get_or_set_uid(uid, session).uid_id
1495 byuid[keyid] = (uid, "ungeneratable user id")
1496 byname[uid] = (keyid, "ungeneratable user id")
1498 return (byname, byuid)
1500 __all__.append('Keyring')
1503 def get_keyring(keyring, session=None):
1505 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1506 If C{keyring} already has an entry, simply return the existing Keyring
1508 @type keyring: string
1509 @param keyring: the keyring name
1512 @return: the Keyring object for this keyring
1515 q = session.query(Keyring).filter_by(keyring_name=keyring)
1519 except NoResultFound:
1522 __all__.append('get_keyring')
1524 ################################################################################
1526 class KeyringACLMap(object):
1527 def __init__(self, *args, **kwargs):
1531 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1533 __all__.append('KeyringACLMap')
1535 ################################################################################
1537 class DBChange(object):
1538 def __init__(self, *args, **kwargs):
1542 return '<DBChange %s>' % self.changesname
1544 def clean_from_queue(self):
1545 session = DBConn().session().object_session(self)
1547 # Remove changes_pool_files entries
1550 # Remove changes_pending_files references
1553 # Clear out of queue
1554 self.in_queue = None
1555 self.approved_for_id = None
1557 __all__.append('DBChange')
1560 def get_dbchange(filename, session=None):
1562 returns DBChange object for given C{filename}.
1564 @type filename: string
1565 @param filename: the name of the file
1567 @type session: Session
1568 @param session: Optional SQLA session object (a temporary one will be
1569 generated if not supplied)
1572 @return: DBChange object for the given filename (C{None} if not present)
1575 q = session.query(DBChange).filter_by(changesname=filename)
1579 except NoResultFound:
1582 __all__.append('get_dbchange')
1584 ################################################################################
1586 class Location(ORMObject):
1587 def __init__(self, path = None):
1589 # the column 'type' should go away, see comment at mapper
1590 self.archive_type = 'pool'
1592 def properties(self):
1593 return ['path', 'archive_type', 'component', 'files_count']
1595 def not_null_constraints(self):
1596 return ['path', 'archive_type']
1598 __all__.append('Location')
1601 def get_location(location, component=None, archive=None, session=None):
1603 Returns Location object for the given combination of location, component
1606 @type location: string
1607 @param location: the path of the location, e.g. I{/srv/ftp-master.debian.org/ftp/pool/}
1609 @type component: string
1610 @param component: the component name (if None, no restriction applied)
1612 @type archive: string
1613 @param archive: the archive name (if None, no restriction applied)
1615 @rtype: Location / None
1616 @return: Either a Location object or None if one can't be found
1619 q = session.query(Location).filter_by(path=location)
1621 if archive is not None:
1622 q = q.join(Archive).filter_by(archive_name=archive)
1624 if component is not None:
1625 q = q.join(Component).filter_by(component_name=component)
1629 except NoResultFound:
1632 __all__.append('get_location')
1634 ################################################################################
1636 class Maintainer(ORMObject):
1637 def __init__(self, name = None):
1640 def properties(self):
1641 return ['name', 'maintainer_id']
1643 def not_null_constraints(self):
1646 def get_split_maintainer(self):
1647 if not hasattr(self, 'name') or self.name is None:
1648 return ('', '', '', '')
1650 return fix_maintainer(self.name.strip())
1652 __all__.append('Maintainer')
1655 def get_or_set_maintainer(name, session=None):
1657 Returns Maintainer object for given maintainer name.
1659 If no matching maintainer name is found, a row is inserted.
1662 @param name: The maintainer name to add
1664 @type session: SQLAlchemy
1665 @param session: Optional SQL session object (a temporary one will be
1666 generated if not supplied). If not passed, a commit will be performed at
1667 the end of the function, otherwise the caller is responsible for commiting.
1668 A flush will be performed either way.
1671 @return: the Maintainer object for the given maintainer
1674 q = session.query(Maintainer).filter_by(name=name)
1677 except NoResultFound:
1678 maintainer = Maintainer()
1679 maintainer.name = name
1680 session.add(maintainer)
1681 session.commit_or_flush()
1686 __all__.append('get_or_set_maintainer')
1689 def get_maintainer(maintainer_id, session=None):
1691 Return the name of the maintainer behind C{maintainer_id} or None if that
1692 maintainer_id is invalid.
1694 @type maintainer_id: int
1695 @param maintainer_id: the id of the maintainer
1698 @return: the Maintainer with this C{maintainer_id}
1701 return session.query(Maintainer).get(maintainer_id)
1703 __all__.append('get_maintainer')
1705 ################################################################################
1707 class NewComment(object):
1708 def __init__(self, *args, **kwargs):
1712 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1714 __all__.append('NewComment')
1717 def has_new_comment(package, version, session=None):
1719 Returns true if the given combination of C{package}, C{version} has a comment.
1721 @type package: string
1722 @param package: name of the package
1724 @type version: string
1725 @param version: package version
1727 @type session: Session
1728 @param session: Optional SQLA session object (a temporary one will be
1729 generated if not supplied)
1735 q = session.query(NewComment)
1736 q = q.filter_by(package=package)
1737 q = q.filter_by(version=version)
1739 return bool(q.count() > 0)
1741 __all__.append('has_new_comment')
1744 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1746 Returns (possibly empty) list of NewComment objects for the given
1749 @type package: string (optional)
1750 @param package: name of the package
1752 @type version: string (optional)
1753 @param version: package version
1755 @type comment_id: int (optional)
1756 @param comment_id: An id of a comment
1758 @type session: Session
1759 @param session: Optional SQLA session object (a temporary one will be
1760 generated if not supplied)
1763 @return: A (possibly empty) list of NewComment objects will be returned
1766 q = session.query(NewComment)
1767 if package is not None: q = q.filter_by(package=package)
1768 if version is not None: q = q.filter_by(version=version)
1769 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1773 __all__.append('get_new_comments')
1775 ################################################################################
1777 class Override(object):
1778 def __init__(self, *args, **kwargs):
1782 return '<Override %s (%s)>' % (self.package, self.suite_id)
1784 __all__.append('Override')
1787 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1789 Returns Override object for the given parameters
1791 @type package: string
1792 @param package: The name of the package
1794 @type suite: string, list or None
1795 @param suite: The name of the suite (or suites if a list) to limit to. If
1796 None, don't limit. Defaults to None.
1798 @type component: string, list or None
1799 @param component: The name of the component (or components if a list) to
1800 limit to. If None, don't limit. Defaults to None.
1802 @type overridetype: string, list or None
1803 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1804 limit to. If None, don't limit. Defaults to None.
1806 @type session: Session
1807 @param session: Optional SQLA session object (a temporary one will be
1808 generated if not supplied)
1811 @return: A (possibly empty) list of Override objects will be returned
1814 q = session.query(Override)
1815 q = q.filter_by(package=package)
1817 if suite is not None:
1818 if not isinstance(suite, list): suite = [suite]
1819 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1821 if component is not None:
1822 if not isinstance(component, list): component = [component]
1823 q = q.join(Component).filter(Component.component_name.in_(component))
1825 if overridetype is not None:
1826 if not isinstance(overridetype, list): overridetype = [overridetype]
1827 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1831 __all__.append('get_override')
1834 ################################################################################
1836 class OverrideType(object):
1837 def __init__(self, *args, **kwargs):
1841 return '<OverrideType %s>' % self.overridetype
1843 __all__.append('OverrideType')
1846 def get_override_type(override_type, session=None):
1848 Returns OverrideType object for given C{override type}.
1850 @type override_type: string
1851 @param override_type: The name of the override type
1853 @type session: Session
1854 @param session: Optional SQLA session object (a temporary one will be
1855 generated if not supplied)
1858 @return: the database id for the given override type
1861 q = session.query(OverrideType).filter_by(overridetype=override_type)
1865 except NoResultFound:
1868 __all__.append('get_override_type')
1870 ################################################################################
1872 class DebContents(object):
1873 def __init__(self, *args, **kwargs):
1877 return '<DebConetnts %s: %s>' % (self.package.package,self.file)
1879 __all__.append('DebContents')
1882 class UdebContents(object):
1883 def __init__(self, *args, **kwargs):
1887 return '<UdebConetnts %s: %s>' % (self.package.package,self.file)
1889 __all__.append('UdebContents')
1891 class PendingBinContents(object):
1892 def __init__(self, *args, **kwargs):
1896 return '<PendingBinContents %s>' % self.contents_id
1898 __all__.append('PendingBinContents')
1900 def insert_pending_content_paths(package,
1905 Make sure given paths are temporarily associated with given
1909 @param package: the package to associate with should have been read in from the binary control file
1910 @type fullpaths: list
1911 @param fullpaths: the list of paths of the file being associated with the binary
1912 @type session: SQLAlchemy session
1913 @param session: Optional SQLAlchemy session. If this is passed, the caller
1914 is responsible for ensuring a transaction has begun and committing the
1915 results or rolling back based on the result code. If not passed, a commit
1916 will be performed at the end of the function
1918 @return: True upon success, False if there is a problem
1921 privatetrans = False
1924 session = DBConn().session()
1928 arch = get_architecture(package['Architecture'], session)
1929 arch_id = arch.arch_id
1931 # Remove any already existing recorded files for this package
1932 q = session.query(PendingBinContents)
1933 q = q.filter_by(package=package['Package'])
1934 q = q.filter_by(version=package['Version'])
1935 q = q.filter_by(architecture=arch_id)
1938 for fullpath in fullpaths:
1940 if fullpath.startswith( "./" ):
1941 fullpath = fullpath[2:]
1943 pca = PendingBinContents()
1944 pca.package = package['Package']
1945 pca.version = package['Version']
1947 pca.architecture = arch_id
1950 pca.type = 8 # gross
1952 pca.type = 7 # also gross
1955 # Only commit if we set up the session ourself
1963 except Exception, e:
1964 traceback.print_exc()
1966 # Only rollback if we set up the session ourself
1973 __all__.append('insert_pending_content_paths')
1975 ################################################################################
1977 class PolicyQueue(object):
1978 def __init__(self, *args, **kwargs):
1982 return '<PolicyQueue %s>' % self.queue_name
1984 __all__.append('PolicyQueue')
1987 def get_policy_queue(queuename, session=None):
1989 Returns PolicyQueue object for given C{queue name}
1991 @type queuename: string
1992 @param queuename: The name of the queue
1994 @type session: Session
1995 @param session: Optional SQLA session object (a temporary one will be
1996 generated if not supplied)
1999 @return: PolicyQueue object for the given queue
2002 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
2006 except NoResultFound:
2009 __all__.append('get_policy_queue')
2012 def get_policy_queue_from_path(pathname, session=None):
2014 Returns PolicyQueue object for given C{path name}
2016 @type queuename: string
2017 @param queuename: The path
2019 @type session: Session
2020 @param session: Optional SQLA session object (a temporary one will be
2021 generated if not supplied)
2024 @return: PolicyQueue object for the given queue
2027 q = session.query(PolicyQueue).filter_by(path=pathname)
2031 except NoResultFound:
2034 __all__.append('get_policy_queue_from_path')
2036 ################################################################################
2038 class Priority(object):
2039 def __init__(self, *args, **kwargs):
2042 def __eq__(self, val):
2043 if isinstance(val, str):
2044 return (self.priority == val)
2045 # This signals to use the normal comparison operator
2046 return NotImplemented
2048 def __ne__(self, val):
2049 if isinstance(val, str):
2050 return (self.priority != val)
2051 # This signals to use the normal comparison operator
2052 return NotImplemented
2055 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
2057 __all__.append('Priority')
2060 def get_priority(priority, session=None):
2062 Returns Priority object for given C{priority name}.
2064 @type priority: string
2065 @param priority: The name of the priority
2067 @type session: Session
2068 @param session: Optional SQLA session object (a temporary one will be
2069 generated if not supplied)
2072 @return: Priority object for the given priority
2075 q = session.query(Priority).filter_by(priority=priority)
2079 except NoResultFound:
2082 __all__.append('get_priority')
2085 def get_priorities(session=None):
2087 Returns dictionary of priority names -> id mappings
2089 @type session: Session
2090 @param session: Optional SQL session object (a temporary one will be
2091 generated if not supplied)
2094 @return: dictionary of priority names -> id mappings
2098 q = session.query(Priority)
2100 ret[x.priority] = x.priority_id
2104 __all__.append('get_priorities')
2106 ################################################################################
2108 class Section(object):
2109 def __init__(self, *args, **kwargs):
2112 def __eq__(self, val):
2113 if isinstance(val, str):
2114 return (self.section == val)
2115 # This signals to use the normal comparison operator
2116 return NotImplemented
2118 def __ne__(self, val):
2119 if isinstance(val, str):
2120 return (self.section != val)
2121 # This signals to use the normal comparison operator
2122 return NotImplemented
2125 return '<Section %s>' % self.section
2127 __all__.append('Section')
2130 def get_section(section, session=None):
2132 Returns Section object for given C{section name}.
2134 @type section: string
2135 @param section: The name of the section
2137 @type session: Session
2138 @param session: Optional SQLA session object (a temporary one will be
2139 generated if not supplied)
2142 @return: Section object for the given section name
2145 q = session.query(Section).filter_by(section=section)
2149 except NoResultFound:
2152 __all__.append('get_section')
2155 def get_sections(session=None):
2157 Returns dictionary of section names -> id mappings
2159 @type session: Session
2160 @param session: Optional SQL session object (a temporary one will be
2161 generated if not supplied)
2164 @return: dictionary of section names -> id mappings
2168 q = session.query(Section)
2170 ret[x.section] = x.section_id
2174 __all__.append('get_sections')
2176 ################################################################################
2178 class DBSource(ORMObject):
2179 def __init__(self, source = None, version = None, maintainer = None, \
2180 changedby = None, poolfile = None, install_date = None):
2181 self.source = source
2182 self.version = version
2183 self.maintainer = maintainer
2184 self.changedby = changedby
2185 self.poolfile = poolfile
2186 self.install_date = install_date
2188 def properties(self):
2189 return ['source', 'source_id', 'maintainer', 'changedby', \
2190 'fingerprint', 'poolfile', 'version', 'suites_count', \
2191 'install_date', 'binaries_count']
2193 def not_null_constraints(self):
2194 return ['source', 'version', 'install_date', 'maintainer', \
2195 'changedby', 'poolfile', 'install_date']
2197 __all__.append('DBSource')
2200 def source_exists(source, source_version, suites = ["any"], session=None):
2202 Ensure that source exists somewhere in the archive for the binary
2203 upload being processed.
2204 1. exact match => 1.0-3
2205 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
2207 @type source: string
2208 @param source: source name
2210 @type source_version: string
2211 @param source_version: expected source version
2214 @param suites: list of suites to check in, default I{any}
2216 @type session: Session
2217 @param session: Optional SQLA session object (a temporary one will be
2218 generated if not supplied)
2221 @return: returns 1 if a source with expected version is found, otherwise 0
2228 from daklib.regexes import re_bin_only_nmu
2229 orig_source_version = re_bin_only_nmu.sub('', source_version)
2231 for suite in suites:
2232 q = session.query(DBSource).filter_by(source=source). \
2233 filter(DBSource.version.in_([source_version, orig_source_version]))
2235 # source must exist in suite X, or in some other suite that's
2236 # mapped to X, recursively... silent-maps are counted too,
2237 # unreleased-maps aren't.
2238 maps = cnf.ValueList("SuiteMappings")[:]
2240 maps = [ m.split() for m in maps ]
2241 maps = [ (x[1], x[2]) for x in maps
2242 if x[0] == "map" or x[0] == "silent-map" ]
2245 if x[1] in s and x[0] not in s:
2248 q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
2253 # No source found so return not ok
2258 __all__.append('source_exists')
2261 def get_suites_source_in(source, session=None):
2263 Returns list of Suite objects which given C{source} name is in
2266 @param source: DBSource package name to search for
2269 @return: list of Suite objects for the given source
2272 return session.query(Suite).filter(Suite.sources.any(source=source)).all()
2274 __all__.append('get_suites_source_in')
2277 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
2279 Returns list of DBSource objects for given C{source} name and other parameters
2282 @param source: DBSource package name to search for
2284 @type version: str or None
2285 @param version: DBSource version name to search for or None if not applicable
2287 @type dm_upload_allowed: bool
2288 @param dm_upload_allowed: If None, no effect. If True or False, only
2289 return packages with that dm_upload_allowed setting
2291 @type session: Session
2292 @param session: Optional SQL session object (a temporary one will be
2293 generated if not supplied)
2296 @return: list of DBSource objects for the given name (may be empty)
2299 q = session.query(DBSource).filter_by(source=source)
2301 if version is not None:
2302 q = q.filter_by(version=version)
2304 if dm_upload_allowed is not None:
2305 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
2309 __all__.append('get_sources_from_name')
2311 # FIXME: This function fails badly if it finds more than 1 source package and
2312 # its implementation is trivial enough to be inlined.
2314 def get_source_in_suite(source, suite, session=None):
2316 Returns a DBSource object for a combination of C{source} and C{suite}.
2318 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2319 - B{suite} - a suite name, eg. I{unstable}
2321 @type source: string
2322 @param source: source package name
2325 @param suite: the suite name
2328 @return: the version for I{source} in I{suite}
2332 q = get_suite(suite, session).get_sources(source)
2335 except NoResultFound:
2338 __all__.append('get_source_in_suite')
2340 ################################################################################
2343 def add_dsc_to_db(u, filename, session=None):
2344 entry = u.pkg.files[filename]
2348 source.source = u.pkg.dsc["source"]
2349 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2350 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2351 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2352 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2353 source.install_date = datetime.now().date()
2355 dsc_component = entry["component"]
2356 dsc_location_id = entry["location id"]
2358 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2360 # Set up a new poolfile if necessary
2361 if not entry.has_key("files id") or not entry["files id"]:
2362 filename = entry["pool name"] + filename
2363 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2365 pfs.append(poolfile)
2366 entry["files id"] = poolfile.file_id
2368 source.poolfile_id = entry["files id"]
2371 suite_names = u.pkg.changes["distribution"].keys()
2372 source.suites = session.query(Suite). \
2373 filter(Suite.suite_name.in_(suite_names)).all()
2375 # Add the source files to the DB (files and dsc_files)
2377 dscfile.source_id = source.source_id
2378 dscfile.poolfile_id = entry["files id"]
2379 session.add(dscfile)
2381 for dsc_file, dentry in u.pkg.dsc_files.items():
2383 df.source_id = source.source_id
2385 # If the .orig tarball is already in the pool, it's
2386 # files id is stored in dsc_files by check_dsc().
2387 files_id = dentry.get("files id", None)
2389 # Find the entry in the files hash
2390 # TODO: Bail out here properly
2392 for f, e in u.pkg.files.items():
2397 if files_id is None:
2398 filename = dfentry["pool name"] + dsc_file
2400 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2401 # FIXME: needs to check for -1/-2 and or handle exception
2402 if found and obj is not None:
2403 files_id = obj.file_id
2406 # If still not found, add it
2407 if files_id is None:
2408 # HACK: Force sha1sum etc into dentry
2409 dentry["sha1sum"] = dfentry["sha1sum"]
2410 dentry["sha256sum"] = dfentry["sha256sum"]
2411 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2412 pfs.append(poolfile)
2413 files_id = poolfile.file_id
2415 poolfile = get_poolfile_by_id(files_id, session)
2416 if poolfile is None:
2417 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2418 pfs.append(poolfile)
2420 df.poolfile_id = files_id
2423 # Add the src_uploaders to the DB
2424 uploader_ids = [source.maintainer_id]
2425 if u.pkg.dsc.has_key("uploaders"):
2426 for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
2428 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2431 for up_id in uploader_ids:
2432 if added_ids.has_key(up_id):
2434 utils.warn("Already saw uploader %s for source %s" % (up_id, source.source))
2440 su.maintainer_id = up_id
2441 su.source_id = source.source_id
2446 return source, dsc_component, dsc_location_id, pfs
2448 __all__.append('add_dsc_to_db')
2451 def add_deb_to_db(u, filename, session=None):
2453 Contrary to what you might expect, this routine deals with both
2454 debs and udebs. That info is in 'dbtype', whilst 'type' is
2455 'deb' for both of them
2458 entry = u.pkg.files[filename]
2461 bin.package = entry["package"]
2462 bin.version = entry["version"]
2463 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2464 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2465 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2466 bin.binarytype = entry["dbtype"]
2469 filename = entry["pool name"] + filename
2470 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2471 if not entry.get("location id", None):
2472 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2474 if entry.get("files id", None):
2475 poolfile = get_poolfile_by_id(bin.poolfile_id)
2476 bin.poolfile_id = entry["files id"]
2478 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2479 bin.poolfile_id = entry["files id"] = poolfile.file_id
2482 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2483 if len(bin_sources) != 1:
2484 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2485 (bin.package, bin.version, entry["architecture"],
2486 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2488 bin.source_id = bin_sources[0].source_id
2490 # Add and flush object so it has an ID
2493 suite_names = u.pkg.changes["distribution"].keys()
2494 bin.suites = session.query(Suite). \
2495 filter(Suite.suite_name.in_(suite_names)).all()
2499 # Deal with contents - disabled for now
2500 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2502 # print "REJECT\nCould not determine contents of package %s" % bin.package
2503 # session.rollback()
2504 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2508 __all__.append('add_deb_to_db')
2510 ################################################################################
2512 class SourceACL(object):
2513 def __init__(self, *args, **kwargs):
2517 return '<SourceACL %s>' % self.source_acl_id
2519 __all__.append('SourceACL')
2521 ################################################################################
2523 class SrcFormat(object):
2524 def __init__(self, *args, **kwargs):
2528 return '<SrcFormat %s>' % (self.format_name)
2530 __all__.append('SrcFormat')
2532 ################################################################################
2534 class SrcUploader(object):
2535 def __init__(self, *args, **kwargs):
2539 return '<SrcUploader %s>' % self.uploader_id
2541 __all__.append('SrcUploader')
2543 ################################################################################
2545 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2546 ('SuiteID', 'suite_id'),
2547 ('Version', 'version'),
2548 ('Origin', 'origin'),
2550 ('Description', 'description'),
2551 ('Untouchable', 'untouchable'),
2552 ('Announce', 'announce'),
2553 ('Codename', 'codename'),
2554 ('OverrideCodename', 'overridecodename'),
2555 ('ValidTime', 'validtime'),
2556 ('Priority', 'priority'),
2557 ('NotAutomatic', 'notautomatic'),
2558 ('CopyChanges', 'copychanges'),
2559 ('OverrideSuite', 'overridesuite')]
2561 # Why the heck don't we have any UNIQUE constraints in table suite?
2562 # TODO: Add UNIQUE constraints for appropriate columns.
2563 class Suite(ORMObject):
2564 def __init__(self, suite_name = None, version = None):
2565 self.suite_name = suite_name
2566 self.version = version
2568 def properties(self):
2569 return ['suite_name', 'version', 'sources_count', 'binaries_count']
2571 def not_null_constraints(self):
2572 return ['suite_name', 'version']
2574 def __eq__(self, val):
2575 if isinstance(val, str):
2576 return (self.suite_name == val)
2577 # This signals to use the normal comparison operator
2578 return NotImplemented
2580 def __ne__(self, val):
2581 if isinstance(val, str):
2582 return (self.suite_name != val)
2583 # This signals to use the normal comparison operator
2584 return NotImplemented
2588 for disp, field in SUITE_FIELDS:
2589 val = getattr(self, field, None)
2591 ret.append("%s: %s" % (disp, val))
2593 return "\n".join(ret)
2595 def get_architectures(self, skipsrc=False, skipall=False):
2597 Returns list of Architecture objects
2599 @type skipsrc: boolean
2600 @param skipsrc: Whether to skip returning the 'source' architecture entry
2603 @type skipall: boolean
2604 @param skipall: Whether to skip returning the 'all' architecture entry
2608 @return: list of Architecture objects for the given name (may be empty)
2611 q = object_session(self).query(Architecture).with_parent(self)
2613 q = q.filter(Architecture.arch_string != 'source')
2615 q = q.filter(Architecture.arch_string != 'all')
2616 return q.order_by(Architecture.arch_string).all()
2618 def get_sources(self, source):
2620 Returns a query object representing DBSource that is part of C{suite}.
2622 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2624 @type source: string
2625 @param source: source package name
2627 @rtype: sqlalchemy.orm.query.Query
2628 @return: a query of DBSource
2632 session = object_session(self)
2633 return session.query(DBSource).filter_by(source = source). \
2636 __all__.append('Suite')
2639 def get_suite(suite, session=None):
2641 Returns Suite object for given C{suite name}.
2644 @param suite: The name of the suite
2646 @type session: Session
2647 @param session: Optional SQLA session object (a temporary one will be
2648 generated if not supplied)
2651 @return: Suite object for the requested suite name (None if not present)
2654 q = session.query(Suite).filter_by(suite_name=suite)
2658 except NoResultFound:
2661 __all__.append('get_suite')
2663 ################################################################################
2665 # TODO: should be removed because the implementation is too trivial
2667 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2669 Returns list of Architecture objects for given C{suite} name
2672 @param suite: Suite name to search for
2674 @type skipsrc: boolean
2675 @param skipsrc: Whether to skip returning the 'source' architecture entry
2678 @type skipall: boolean
2679 @param skipall: Whether to skip returning the 'all' architecture entry
2682 @type session: Session
2683 @param session: Optional SQL session object (a temporary one will be
2684 generated if not supplied)
2687 @return: list of Architecture objects for the given name (may be empty)
2690 return get_suite(suite, session).get_architectures(skipsrc, skipall)
2692 __all__.append('get_suite_architectures')
2694 ################################################################################
2696 class SuiteSrcFormat(object):
2697 def __init__(self, *args, **kwargs):
2701 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2703 __all__.append('SuiteSrcFormat')
2706 def get_suite_src_formats(suite, session=None):
2708 Returns list of allowed SrcFormat for C{suite}.
2711 @param suite: Suite name to search for
2713 @type session: Session
2714 @param session: Optional SQL session object (a temporary one will be
2715 generated if not supplied)
2718 @return: the list of allowed source formats for I{suite}
2721 q = session.query(SrcFormat)
2722 q = q.join(SuiteSrcFormat)
2723 q = q.join(Suite).filter_by(suite_name=suite)
2724 q = q.order_by('format_name')
2728 __all__.append('get_suite_src_formats')
2730 ################################################################################
2732 class Uid(ORMObject):
2733 def __init__(self, uid = None, name = None):
2737 def __eq__(self, val):
2738 if isinstance(val, str):
2739 return (self.uid == val)
2740 # This signals to use the normal comparison operator
2741 return NotImplemented
2743 def __ne__(self, val):
2744 if isinstance(val, str):
2745 return (self.uid != val)
2746 # This signals to use the normal comparison operator
2747 return NotImplemented
2749 def properties(self):
2750 return ['uid', 'name', 'fingerprint']
2752 def not_null_constraints(self):
2755 __all__.append('Uid')
2758 def get_or_set_uid(uidname, session=None):
2760 Returns uid object for given uidname.
2762 If no matching uidname is found, a row is inserted.
2764 @type uidname: string
2765 @param uidname: The uid to add
2767 @type session: SQLAlchemy
2768 @param session: Optional SQL session object (a temporary one will be
2769 generated if not supplied). If not passed, a commit will be performed at
2770 the end of the function, otherwise the caller is responsible for commiting.
2773 @return: the uid object for the given uidname
2776 q = session.query(Uid).filter_by(uid=uidname)
2780 except NoResultFound:
2784 session.commit_or_flush()
2789 __all__.append('get_or_set_uid')
2792 def get_uid_from_fingerprint(fpr, session=None):
2793 q = session.query(Uid)
2794 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2798 except NoResultFound:
2801 __all__.append('get_uid_from_fingerprint')
2803 ################################################################################
2805 class UploadBlock(object):
2806 def __init__(self, *args, **kwargs):
2810 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2812 __all__.append('UploadBlock')
2814 ################################################################################
2816 class DBConn(object):
2818 database module init.
2822 def __init__(self, *args, **kwargs):
2823 self.__dict__ = self.__shared_state
2825 if not getattr(self, 'initialised', False):
2826 self.initialised = True
2827 self.debug = kwargs.has_key('debug')
2830 def __setuptables(self):
2831 tables_with_primary = (
2842 'changes_pending_binaries',
2843 'changes_pending_files',
2844 'changes_pending_source',
2854 'pending_bin_contents',
2866 # The following tables have primary keys but sqlalchemy
2867 # version 0.5 fails to reflect them correctly with database
2868 # versions before upgrade #41.
2870 #'build_queue_files',
2873 tables_no_primary = (
2875 'changes_pending_files_map',
2876 'changes_pending_source_files',
2877 'changes_pool_files',
2880 'suite_architectures',
2881 'suite_src_formats',
2882 'suite_build_queue_copy',
2884 # see the comment above
2886 'build_queue_files',
2890 'almost_obsolete_all_associations',
2891 'almost_obsolete_src_associations',
2892 'any_associations_source',
2893 'bin_assoc_by_arch',
2894 'bin_associations_binaries',
2895 'binaries_suite_arch',
2896 'binfiles_suite_component_arch',
2899 'newest_all_associations',
2900 'newest_any_associations',
2902 'newest_src_association',
2903 'obsolete_all_associations',
2904 'obsolete_any_associations',
2905 'obsolete_any_by_all_associations',
2906 'obsolete_src_associations',
2908 'src_associations_bin',
2909 'src_associations_src',
2910 'suite_arch_by_name',
2913 # Sqlalchemy version 0.5 fails to reflect the SERIAL type
2914 # correctly and that is why we have to use a workaround. It can
2915 # be removed as soon as we switch to version 0.6.
2916 for table_name in tables_with_primary:
2917 table = Table(table_name, self.db_meta, \
2918 Column('id', Integer, primary_key = True), \
2919 autoload=True, useexisting=True)
2920 setattr(self, 'tbl_%s' % table_name, table)
2922 for table_name in tables_no_primary:
2923 table = Table(table_name, self.db_meta, autoload=True)
2924 setattr(self, 'tbl_%s' % table_name, table)
2926 for view_name in views:
2927 view = Table(view_name, self.db_meta, autoload=True)
2928 setattr(self, 'view_%s' % view_name, view)
2930 def __setupmappers(self):
2931 mapper(Architecture, self.tbl_architecture,
2932 properties = dict(arch_id = self.tbl_architecture.c.id,
2933 suites = relation(Suite, secondary=self.tbl_suite_architectures,
2934 order_by='suite_name',
2935 backref=backref('architectures', order_by='arch_string'))),
2936 extension = validator)
2938 mapper(Archive, self.tbl_archive,
2939 properties = dict(archive_id = self.tbl_archive.c.id,
2940 archive_name = self.tbl_archive.c.name))
2942 mapper(BinAssociation, self.tbl_bin_associations,
2943 properties = dict(ba_id = self.tbl_bin_associations.c.id,
2944 suite_id = self.tbl_bin_associations.c.suite,
2945 suite = relation(Suite),
2946 binary_id = self.tbl_bin_associations.c.bin,
2947 binary = relation(DBBinary)))
2949 mapper(PendingBinContents, self.tbl_pending_bin_contents,
2950 properties = dict(contents_id =self.tbl_pending_bin_contents.c.id,
2951 filename = self.tbl_pending_bin_contents.c.filename,
2952 package = self.tbl_pending_bin_contents.c.package,
2953 version = self.tbl_pending_bin_contents.c.version,
2954 arch = self.tbl_pending_bin_contents.c.arch,
2955 otype = self.tbl_pending_bin_contents.c.type))
2957 mapper(DebContents, self.tbl_deb_contents,
2958 properties = dict(binary_id=self.tbl_deb_contents.c.binary_id,
2959 package=self.tbl_deb_contents.c.package,
2960 suite=self.tbl_deb_contents.c.suite,
2961 arch=self.tbl_deb_contents.c.arch,
2962 section=self.tbl_deb_contents.c.section,
2963 filename=self.tbl_deb_contents.c.filename))
2965 mapper(UdebContents, self.tbl_udeb_contents,
2966 properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id,
2967 package=self.tbl_udeb_contents.c.package,
2968 suite=self.tbl_udeb_contents.c.suite,
2969 arch=self.tbl_udeb_contents.c.arch,
2970 section=self.tbl_udeb_contents.c.section,
2971 filename=self.tbl_udeb_contents.c.filename))
2973 mapper(BuildQueue, self.tbl_build_queue,
2974 properties = dict(queue_id = self.tbl_build_queue.c.id))
2976 mapper(BuildQueueFile, self.tbl_build_queue_files,
2977 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
2978 poolfile = relation(PoolFile, backref='buildqueueinstances')))
2980 mapper(DBBinary, self.tbl_binaries,
2981 properties = dict(binary_id = self.tbl_binaries.c.id,
2982 package = self.tbl_binaries.c.package,
2983 version = self.tbl_binaries.c.version,
2984 maintainer_id = self.tbl_binaries.c.maintainer,
2985 maintainer = relation(Maintainer),
2986 source_id = self.tbl_binaries.c.source,
2987 source = relation(DBSource, backref='binaries'),
2988 arch_id = self.tbl_binaries.c.architecture,
2989 architecture = relation(Architecture),
2990 poolfile_id = self.tbl_binaries.c.file,
2991 poolfile = relation(PoolFile, backref=backref('binary', uselist = False)),
2992 binarytype = self.tbl_binaries.c.type,
2993 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2994 fingerprint = relation(Fingerprint),
2995 install_date = self.tbl_binaries.c.install_date,
2996 suites = relation(Suite, secondary=self.tbl_bin_associations,
2997 backref=backref('binaries', lazy='dynamic')),
2998 binassociations = relation(BinAssociation,
2999 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))),
3000 extension = validator)
3002 mapper(BinaryACL, self.tbl_binary_acl,
3003 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
3005 mapper(BinaryACLMap, self.tbl_binary_acl_map,
3006 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
3007 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
3008 architecture = relation(Architecture)))
3010 mapper(Component, self.tbl_component,
3011 properties = dict(component_id = self.tbl_component.c.id,
3012 component_name = self.tbl_component.c.name))
3014 mapper(DBConfig, self.tbl_config,
3015 properties = dict(config_id = self.tbl_config.c.id))
3017 mapper(DSCFile, self.tbl_dsc_files,
3018 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
3019 source_id = self.tbl_dsc_files.c.source,
3020 source = relation(DBSource),
3021 poolfile_id = self.tbl_dsc_files.c.file,
3022 poolfile = relation(PoolFile)))
3024 mapper(PoolFile, self.tbl_files,
3025 properties = dict(file_id = self.tbl_files.c.id,
3026 filesize = self.tbl_files.c.size,
3027 location_id = self.tbl_files.c.location,
3028 location = relation(Location,
3029 # using lazy='dynamic' in the back
3030 # reference because we have A LOT of
3031 # files in one location
3032 backref=backref('files', lazy='dynamic'))),
3033 extension = validator)
3035 mapper(Fingerprint, self.tbl_fingerprint,
3036 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
3037 uid_id = self.tbl_fingerprint.c.uid,
3038 uid = relation(Uid),
3039 keyring_id = self.tbl_fingerprint.c.keyring,
3040 keyring = relation(Keyring),
3041 source_acl = relation(SourceACL),
3042 binary_acl = relation(BinaryACL)),
3043 extension = validator)
3045 mapper(Keyring, self.tbl_keyrings,
3046 properties = dict(keyring_name = self.tbl_keyrings.c.name,
3047 keyring_id = self.tbl_keyrings.c.id))
3049 mapper(DBChange, self.tbl_changes,
3050 properties = dict(change_id = self.tbl_changes.c.id,
3051 poolfiles = relation(PoolFile,
3052 secondary=self.tbl_changes_pool_files,
3053 backref="changeslinks"),
3054 seen = self.tbl_changes.c.seen,
3055 source = self.tbl_changes.c.source,
3056 binaries = self.tbl_changes.c.binaries,
3057 architecture = self.tbl_changes.c.architecture,
3058 distribution = self.tbl_changes.c.distribution,
3059 urgency = self.tbl_changes.c.urgency,
3060 maintainer = self.tbl_changes.c.maintainer,
3061 changedby = self.tbl_changes.c.changedby,
3062 date = self.tbl_changes.c.date,
3063 version = self.tbl_changes.c.version,
3064 files = relation(ChangePendingFile,
3065 secondary=self.tbl_changes_pending_files_map,
3066 backref="changesfile"),
3067 in_queue_id = self.tbl_changes.c.in_queue,
3068 in_queue = relation(PolicyQueue,
3069 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
3070 approved_for_id = self.tbl_changes.c.approved_for))
3072 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
3073 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
3075 mapper(ChangePendingFile, self.tbl_changes_pending_files,
3076 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id,
3077 filename = self.tbl_changes_pending_files.c.filename,
3078 size = self.tbl_changes_pending_files.c.size,
3079 md5sum = self.tbl_changes_pending_files.c.md5sum,
3080 sha1sum = self.tbl_changes_pending_files.c.sha1sum,
3081 sha256sum = self.tbl_changes_pending_files.c.sha256sum))
3083 mapper(ChangePendingSource, self.tbl_changes_pending_source,
3084 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
3085 change = relation(DBChange),
3086 maintainer = relation(Maintainer,
3087 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
3088 changedby = relation(Maintainer,
3089 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
3090 fingerprint = relation(Fingerprint),
3091 source_files = relation(ChangePendingFile,
3092 secondary=self.tbl_changes_pending_source_files,
3093 backref="pending_sources")))
3096 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
3097 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
3098 keyring = relation(Keyring, backref="keyring_acl_map"),
3099 architecture = relation(Architecture)))
3101 mapper(Location, self.tbl_location,
3102 properties = dict(location_id = self.tbl_location.c.id,
3103 component_id = self.tbl_location.c.component,
3104 component = relation(Component),
3105 archive_id = self.tbl_location.c.archive,
3106 archive = relation(Archive),
3107 # FIXME: the 'type' column is old cruft and
3108 # should be removed in the future.
3109 archive_type = self.tbl_location.c.type),
3110 extension = validator)
3112 mapper(Maintainer, self.tbl_maintainer,
3113 properties = dict(maintainer_id = self.tbl_maintainer.c.id,
3114 maintains_sources = relation(DBSource, backref='maintainer',
3115 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.maintainer)),
3116 changed_sources = relation(DBSource, backref='changedby',
3117 primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.changedby))),
3118 extension = validator)
3120 mapper(NewComment, self.tbl_new_comments,
3121 properties = dict(comment_id = self.tbl_new_comments.c.id))
3123 mapper(Override, self.tbl_override,
3124 properties = dict(suite_id = self.tbl_override.c.suite,
3125 suite = relation(Suite),
3126 package = self.tbl_override.c.package,
3127 component_id = self.tbl_override.c.component,
3128 component = relation(Component),
3129 priority_id = self.tbl_override.c.priority,
3130 priority = relation(Priority),
3131 section_id = self.tbl_override.c.section,
3132 section = relation(Section),
3133 overridetype_id = self.tbl_override.c.type,
3134 overridetype = relation(OverrideType)))
3136 mapper(OverrideType, self.tbl_override_type,
3137 properties = dict(overridetype = self.tbl_override_type.c.type,
3138 overridetype_id = self.tbl_override_type.c.id))
3140 mapper(PolicyQueue, self.tbl_policy_queue,
3141 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
3143 mapper(Priority, self.tbl_priority,
3144 properties = dict(priority_id = self.tbl_priority.c.id))
3146 mapper(Section, self.tbl_section,
3147 properties = dict(section_id = self.tbl_section.c.id,
3148 section=self.tbl_section.c.section))
3150 mapper(DBSource, self.tbl_source,
3151 properties = dict(source_id = self.tbl_source.c.id,
3152 version = self.tbl_source.c.version,
3153 maintainer_id = self.tbl_source.c.maintainer,
3154 poolfile_id = self.tbl_source.c.file,
3155 poolfile = relation(PoolFile, backref=backref('source', uselist = False)),
3156 fingerprint_id = self.tbl_source.c.sig_fpr,
3157 fingerprint = relation(Fingerprint),
3158 changedby_id = self.tbl_source.c.changedby,
3159 srcfiles = relation(DSCFile,
3160 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
3161 suites = relation(Suite, secondary=self.tbl_src_associations,
3162 backref=backref('sources', lazy='dynamic')),
3163 srcuploaders = relation(SrcUploader)),
3164 extension = validator)
3166 mapper(SourceACL, self.tbl_source_acl,
3167 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
3169 mapper(SrcFormat, self.tbl_src_format,
3170 properties = dict(src_format_id = self.tbl_src_format.c.id,
3171 format_name = self.tbl_src_format.c.format_name))
3173 mapper(SrcUploader, self.tbl_src_uploaders,
3174 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
3175 source_id = self.tbl_src_uploaders.c.source,
3176 source = relation(DBSource,
3177 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
3178 maintainer_id = self.tbl_src_uploaders.c.maintainer,
3179 maintainer = relation(Maintainer,
3180 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
3182 mapper(Suite, self.tbl_suite,
3183 properties = dict(suite_id = self.tbl_suite.c.id,
3184 policy_queue = relation(PolicyQueue),
3185 copy_queues = relation(BuildQueue,
3186 secondary=self.tbl_suite_build_queue_copy)),
3187 extension = validator)
3189 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
3190 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
3191 suite = relation(Suite, backref='suitesrcformats'),
3192 src_format_id = self.tbl_suite_src_formats.c.src_format,
3193 src_format = relation(SrcFormat)))
3195 mapper(Uid, self.tbl_uid,
3196 properties = dict(uid_id = self.tbl_uid.c.id,
3197 fingerprint = relation(Fingerprint)),
3198 extension = validator)
3200 mapper(UploadBlock, self.tbl_upload_blocks,
3201 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
3202 fingerprint = relation(Fingerprint, backref="uploadblocks"),
3203 uid = relation(Uid, backref="uploadblocks")))
3205 ## Connection functions
3206 def __createconn(self):
3207 from config import Config
3211 connstr = "postgres://%s" % cnf["DB::Host"]
3212 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
3213 connstr += ":%s" % cnf["DB::Port"]
3214 connstr += "/%s" % cnf["DB::Name"]
3217 connstr = "postgres:///%s" % cnf["DB::Name"]
3218 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
3219 connstr += "?port=%s" % cnf["DB::Port"]
3221 self.db_pg = create_engine(connstr, echo=self.debug)
3222 self.db_meta = MetaData()
3223 self.db_meta.bind = self.db_pg
3224 self.db_smaker = sessionmaker(bind=self.db_pg,
3228 self.__setuptables()
3229 self.__setupmappers()
3232 return self.db_smaker()
3234 __all__.append('DBConn')