5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
40 from sqlalchemy import create_engine, Table, MetaData, select
41 from sqlalchemy.orm import sessionmaker, mapper, relation
43 # Don't remove this, we re-export the exceptions to scripts which import us
44 from sqlalchemy.exc import *
46 # Only import Config until Queue stuff is changed to store its config
48 from config import Config
49 from singleton import Singleton
50 from textutils import fix_maintainer
52 ################################################################################
54 __all__ = ['IntegrityError', 'SQLAlchemyError']
56 ################################################################################
58 class Architecture(object):
59 def __init__(self, *args, **kwargs):
63 return '<Architecture %s>' % self.arch_string
65 __all__.append('Architecture')
67 def get_architecture(architecture, session=None):
69 Returns database id for given C{architecture}.
71 @type architecture: string
72 @param architecture: The name of the architecture
74 @type session: Session
75 @param session: Optional SQLA session object (a temporary one will be
76 generated if not supplied)
79 @return: Architecture object for the given arch (None if not present)
83 session = DBConn().session()
84 q = session.query(Architecture).filter_by(arch_string=architecture)
89 __all__.append('get_architecture')
91 def get_architecture_suites(architecture, session=None):
93 Returns list of Suite objects for given C{architecture} name
96 @param source: Architecture name to search for
98 @type session: Session
99 @param session: Optional SQL session object (a temporary one will be
100 generated if not supplied)
103 @return: list of Suite objects for the given name (may be empty)
107 session = DBConn().session()
109 q = session.query(Suite)
110 q = q.join(SuiteArchitecture)
111 q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
114 __all__.append('get_architecture_suites')
116 ################################################################################
118 class Archive(object):
119 def __init__(self, *args, **kwargs):
123 return '<Archive %s>' % self.name
125 __all__.append('Archive')
127 def get_archive(archive, session=None):
129 returns database id for given c{archive}.
131 @type archive: string
132 @param archive: the name of the arhive
134 @type session: Session
135 @param session: Optional SQLA session object (a temporary one will be
136 generated if not supplied)
139 @return: Archive object for the given name (None if not present)
142 archive = archive.lower()
144 session = DBConn().session()
145 q = session.query(Archive).filter_by(archive_name=archive)
150 __all__.append('get_archive')
152 ################################################################################
154 class BinAssociation(object):
155 def __init__(self, *args, **kwargs):
159 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
161 __all__.append('BinAssociation')
163 ################################################################################
165 class DBBinary(object):
166 def __init__(self, *args, **kwargs):
170 return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
172 __all__.append('DBBinary')
174 def get_suites_binary_in(package, session=None):
176 Returns list of Suite objects which given C{package} name is in
179 @param source: DBBinary package name to search for
182 @return: list of Suite objects for the given package
186 session = DBConn().session()
188 return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
190 __all__.append('get_suites_binary_in')
192 def get_binary_from_id(id, session=None):
194 Returns DBBinary object for given C{id}
197 @param id: Id of the required binary
199 @type session: Session
200 @param session: Optional SQLA session object (a temporary one will be
201 generated if not supplied)
204 @return: DBBinary object for the given binary (None if not present)
207 session = DBConn().session()
208 q = session.query(DBBinary).filter_by(binary_id=id)
213 __all__.append('get_binary_from_id')
215 def get_binaries_from_name(package, version=None, architecture=None, session=None):
217 Returns list of DBBinary objects for given C{package} name
220 @param package: DBBinary package name to search for
222 @type version: str or None
223 @param version: Version to search for (or None)
225 @type package: str, list or None
226 @param package: Architectures to limit to (or None if no limit)
228 @type session: Session
229 @param session: Optional SQL session object (a temporary one will be
230 generated if not supplied)
233 @return: list of DBBinary objects for the given name (may be empty)
236 session = DBConn().session()
238 q = session.query(DBBinary).filter_by(package=package)
240 if version is not None:
241 q = q.filter_by(version=version)
243 if architecture is not None:
244 if not isinstance(architecture, list):
245 architecture = [architecture]
246 q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
250 __all__.append('get_binaries_from_name')
252 def get_binaries_from_source_id(source_id, session=None):
254 Returns list of DBBinary objects for given C{source_id}
257 @param source_id: source_id to search for
259 @type session: Session
260 @param session: Optional SQL session object (a temporary one will be
261 generated if not supplied)
264 @return: list of DBBinary objects for the given name (may be empty)
267 session = DBConn().session()
268 return session.query(DBBinary).filter_by(source_id=source_id).all()
270 __all__.append('get_binaries_from_source_id')
273 def get_binary_from_name_suite(package, suitename, session=None):
274 ### For dak examine-package
275 ### XXX: Doesn't use object API yet
277 session = DBConn().session()
279 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
280 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
281 WHERE b.package=:package
283 AND fi.location = l.id
284 AND l.component = c.id
287 AND su.suite_name=:suitename
288 ORDER BY b.version DESC"""
290 return session.execute(sql, {'package': package, 'suitename': suitename})
292 __all__.append('get_binary_from_name_suite')
294 def get_binary_components(package, suitename, arch, session=None):
295 # Check for packages that have moved from one component to another
296 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
297 WHERE b.package=:package AND s.suite_name=:suitename
298 AND (a.arch_string = :arch OR a.arch_string = 'all')
299 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
300 AND f.location = l.id
301 AND l.component = c.id
304 vals = {'package': package, 'suitename': suitename, 'arch': arch}
307 session = DBConn().session()
308 return session.execute(query, vals)
310 __all__.append('get_binary_components')
312 ################################################################################
314 class Component(object):
315 def __init__(self, *args, **kwargs):
319 return '<Component %s>' % self.component_name
322 __all__.append('Component')
324 def get_component(component, session=None):
326 Returns database id for given C{component}.
328 @type component: string
329 @param component: The name of the override type
332 @return: the database id for the given component
335 component = component.lower()
337 session = DBConn().session()
338 q = session.query(Component).filter_by(component_name=component)
343 __all__.append('get_component')
345 ################################################################################
347 class DBConfig(object):
348 def __init__(self, *args, **kwargs):
352 return '<DBConfig %s>' % self.name
354 __all__.append('DBConfig')
356 ################################################################################
358 class ContentFilename(object):
359 def __init__(self, *args, **kwargs):
363 return '<ContentFilename %s>' % self.filename
365 __all__.append('ContentFilename')
367 def get_or_set_contents_file_id(filename, session=None):
369 Returns database id for given filename.
371 If no matching file is found, a row is inserted.
373 @type filename: string
374 @param filename: The filename
375 @type session: SQLAlchemy
376 @param session: Optional SQL session object (a temporary one will be
377 generated if not supplied). If not passed, a commit will be performed at
378 the end of the function, otherwise the caller is responsible for commiting.
381 @return: the database id for the given component
385 session = DBConn().session()
389 q = session.query(ContentFilename).filter_by(filename=filename)
391 cf = ContentFilename()
392 cf.filename = filename
396 return cf.cafilename_id
398 return q.one().cafilename_id
401 traceback.print_exc()
404 __all__.append('get_or_set_contents_file_id')
406 def get_contents(suite, overridetype, section=None, session=None):
408 Returns contents for a suite / overridetype combination, limiting
409 to a section if not None.
412 @param suite: Suite object
414 @type overridetype: OverrideType
415 @param overridetype: OverrideType object
417 @type section: Section
418 @param section: Optional section object to limit results to
420 @type session: SQLAlchemy
421 @param session: Optional SQL session object (a temporary one will be
422 generated if not supplied)
425 @return: ResultsProxy object set up to return tuples of (filename, section,
430 session = DBConn().session()
432 # find me all of the contents for a given suite
433 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
437 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
438 JOIN content_file_names n ON (c.filename=n.id)
439 JOIN binaries b ON (b.id=c.binary_pkg)
440 JOIN override o ON (o.package=b.package)
441 JOIN section s ON (s.id=o.section)
442 WHERE o.suite = :suiteid AND o.type = :overridetypeid
443 AND b.type=:overridetypename"""
445 vals = {'suiteid': suite.suite_id,
446 'overridetypeid': overridetype.overridetype_id,
447 'overridetypename': overridetype.overridetype}
449 if section is not None:
450 contents_q += " AND s.id = :sectionid"
451 vals['sectionid'] = section.section_id
453 contents_q += " ORDER BY fn"
455 return session.execute(contents_q, vals)
457 __all__.append('get_contents')
459 ################################################################################
461 class ContentFilepath(object):
462 def __init__(self, *args, **kwargs):
466 return '<ContentFilepath %s>' % self.filepath
468 __all__.append('ContentFilepath')
470 def get_or_set_contents_path_id(filepath, session):
472 Returns database id for given path.
474 If no matching file is found, a row is inserted.
476 @type filename: string
477 @param filename: The filepath
478 @type session: SQLAlchemy
479 @param session: Optional SQL session object (a temporary one will be
480 generated if not supplied). If not passed, a commit will be performed at
481 the end of the function, otherwise the caller is responsible for commiting.
484 @return: the database id for the given path
488 session = DBConn().session()
492 q = session.query(ContentFilepath).filter_by(filepath=filepath)
494 cf = ContentFilepath()
495 cf.filepath = filepath
499 return cf.cafilepath_id
501 return q.one().cafilepath_id
504 traceback.print_exc()
507 __all__.append('get_or_set_contents_path_id')
509 ################################################################################
511 class ContentAssociation(object):
512 def __init__(self, *args, **kwargs):
516 return '<ContentAssociation %s>' % self.ca_id
518 __all__.append('ContentAssociation')
520 def insert_content_paths(binary_id, fullpaths, session=None):
522 Make sure given path is associated with given binary id
525 @param binary_id: the id of the binary
526 @type fullpaths: list
527 @param fullpaths: the list of paths of the file being associated with the binary
528 @type session: SQLAlchemy session
529 @param session: Optional SQLAlchemy session. If this is passed, the caller
530 is responsible for ensuring a transaction has begun and committing the
531 results or rolling back based on the result code. If not passed, a commit
532 will be performed at the end of the function, otherwise the caller is
533 responsible for commiting.
535 @return: True upon success
541 session = DBConn().session()
545 for fullpath in fullpaths:
546 (path, file) = os.path.split(fullpath)
548 # Get the necessary IDs ...
549 ca = ContentAssociation()
550 ca.binary_id = binary_id
551 ca.filename_id = get_or_set_contents_file_id(file)
552 ca.filepath_id = get_or_set_contents_path_id(path)
555 # Only commit if we set up the session ourself
561 traceback.print_exc()
563 # Only rollback if we set up the session ourself
569 __all__.append('insert_content_paths')
571 ################################################################################
573 class DSCFile(object):
574 def __init__(self, *args, **kwargs):
578 return '<DSCFile %s>' % self.dscfile_id
580 __all__.append('DSCFile')
582 ################################################################################
584 class PoolFile(object):
585 def __init__(self, *args, **kwargs):
589 return '<PoolFile %s>' % self.filename
591 __all__.append('PoolFile')
593 def get_poolfile_by_name(filename, location_id=None, session=None):
595 Returns an array of PoolFile objects for the given filename and
596 (optionally) location_id
598 @type filename: string
599 @param filename: the filename of the file to check against the DB
601 @type location_id: int
602 @param location_id: the id of the location to look in (optional)
605 @return: array of PoolFile objects
608 if session is not None:
609 session = DBConn().session()
611 q = session.query(PoolFile).filter_by(filename=filename)
613 if location_id is not None:
614 q = q.join(Location).filter_by(location_id=location_id)
618 __all__.append('get_poolfile_by_name')
620 def get_poolfile_like_name(filename, session=None):
622 Returns an array of PoolFile objects which are like the given name
624 @type filename: string
625 @param filename: the filename of the file to check against the DB
628 @return: array of PoolFile objects
631 if session is not None:
632 session = DBConn().session()
634 # TODO: There must be a way of properly using bind parameters with %FOO%
635 q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
639 __all__.append('get_poolfile_like_name')
641 ################################################################################
643 class Fingerprint(object):
644 def __init__(self, *args, **kwargs):
648 return '<Fingerprint %s>' % self.fingerprint
650 __all__.append('Fingerprint')
652 ################################################################################
654 class Keyring(object):
655 def __init__(self, *args, **kwargs):
659 return '<Keyring %s>' % self.keyring_name
661 __all__.append('Keyring')
663 ################################################################################
665 class Location(object):
666 def __init__(self, *args, **kwargs):
670 return '<Location %s (%s)>' % (self.path, self.location_id)
672 __all__.append('Location')
674 def get_location(location, component=None, archive=None, session=None):
676 Returns Location object for the given combination of location, component
679 @type location: string
680 @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
682 @type component: string
683 @param component: the component name (if None, no restriction applied)
685 @type archive: string
686 @param archive_id: the archive name (if None, no restriction applied)
688 @rtype: Location / None
689 @return: Either a Location object or None if one can't be found
693 session = DBConn().session()
695 q = session.query(Location).filter_by(path=location)
697 if archive is not None:
698 q = q.join(Archive).filter_by(archive_name=archive)
700 if component is not None:
701 q = q.join(Component).filter_by(component_name=component)
708 __all__.append('get_location')
710 ################################################################################
712 class Maintainer(object):
713 def __init__(self, *args, **kwargs):
717 return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
719 def get_split_maintainer(self):
720 if not hasattr(self, 'name') or self.name is None:
721 return ('', '', '', '')
723 return fix_maintainer(self.name.strip())
725 __all__.append('Maintainer')
727 ################################################################################
729 class NewComment(object):
730 def __init__(self, *args, **kwargs):
734 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
736 __all__.append('NewComment')
738 def has_new_comment(package, version, session=None):
740 Returns true if the given combination of C{package}, C{version} has a comment.
742 @type package: string
743 @param package: name of the package
745 @type version: string
746 @param version: package version
748 @type session: Session
749 @param session: Optional SQLA session object (a temporary one will be
750 generated if not supplied)
757 session = DBConn().session()
759 q = session.query(NewComment)
760 q = q.filter_by(package=package)
761 q = q.filter_by(version=version)
764 __all__.append('has_new_comment')
766 def get_new_comments(package=None, version=None, comment_id=None, session=None):
768 Returns (possibly empty) list of NewComment objects for the given
771 @type package: string (optional)
772 @param package: name of the package
774 @type version: string (optional)
775 @param version: package version
777 @type comment_id: int (optional)
778 @param comment_id: An id of a comment
780 @type session: Session
781 @param session: Optional SQLA session object (a temporary one will be
782 generated if not supplied)
785 @return: A (possibly empty) list of NewComment objects will be returned
790 session = DBConn().session()
792 q = session.query(NewComment)
793 if package is not None: q = q.filter_by(package=package)
794 if version is not None: q = q.filter_by(version=version)
795 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
799 __all__.append('get_new_comments')
801 ################################################################################
803 class Override(object):
804 def __init__(self, *args, **kwargs):
808 return '<Override %s (%s)>' % (self.package, self.suite_id)
810 __all__.append('Override')
812 def get_override(package, suite=None, component=None, overridetype=None, session=None):
814 Returns Override object for the given parameters
816 @type package: string
817 @param package: The name of the package
819 @type suite: string, list or None
820 @param suite: The name of the suite (or suites if a list) to limit to. If
821 None, don't limit. Defaults to None.
823 @type component: string, list or None
824 @param component: The name of the component (or components if a list) to
825 limit to. If None, don't limit. Defaults to None.
827 @type overridetype: string, list or None
828 @param overridetype: The name of the overridetype (or overridetypes if a list) to
829 limit to. If None, don't limit. Defaults to None.
831 @type session: Session
832 @param session: Optional SQLA session object (a temporary one will be
833 generated if not supplied)
836 @return: A (possibly empty) list of Override objects will be returned
840 session = DBConn().session()
842 q = session.query(Override)
843 q = q.filter_by(package=package)
845 if suite is not None:
846 if not isinstance(suite, list): suite = [suite]
847 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
849 if component is not None:
850 if not isinstance(component, list): component = [component]
851 q = q.join(Component).filter(Component.component_name.in_(component))
853 if overridetype is not None:
854 if not isinstance(overridetype, list): overridetype = [overridetype]
855 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
859 __all__.append('get_override')
862 ################################################################################
864 class OverrideType(object):
865 def __init__(self, *args, **kwargs):
869 return '<OverrideType %s>' % self.overridetype
871 __all__.append('OverrideType')
873 def get_override_type(override_type, session=None):
875 Returns OverrideType object for given C{override type}.
877 @type override_type: string
878 @param override_type: The name of the override type
880 @type session: Session
881 @param session: Optional SQLA session object (a temporary one will be
882 generated if not supplied)
885 @return: the database id for the given override type
889 session = DBConn().session()
890 q = session.query(OverrideType).filter_by(overridetype=override_type)
895 __all__.append('get_override_type')
897 ################################################################################
899 class PendingContentAssociation(object):
900 def __init__(self, *args, **kwargs):
904 return '<PendingContentAssociation %s>' % self.pca_id
906 __all__.append('PendingContentAssociation')
908 def insert_pending_content_paths(package, fullpaths, session=None):
910 Make sure given paths are temporarily associated with given
914 @param package: the package to associate with should have been read in from the binary control file
915 @type fullpaths: list
916 @param fullpaths: the list of paths of the file being associated with the binary
917 @type session: SQLAlchemy session
918 @param session: Optional SQLAlchemy session. If this is passed, the caller
919 is responsible for ensuring a transaction has begun and committing the
920 results or rolling back based on the result code. If not passed, a commit
921 will be performed at the end of the function
923 @return: True upon success, False if there is a problem
929 session = DBConn().session()
933 arch = get_architecture(package['Architecture'], session)
934 arch_id = arch.arch_id
936 # Remove any already existing recorded files for this package
937 q = session.query(PendingContentAssociation)
938 q = q.filter_by(package=package['Package'])
939 q = q.filter_by(version=package['Version'])
940 q = q.filter_by(architecture=arch_id)
944 for fullpath in fullpaths:
945 (path, file) = os.path.split(fullpath)
947 if path.startswith( "./" ):
950 pca = PendingContentAssociation()
951 pca.package = package['Package']
952 pca.version = package['Version']
953 pca.filename_id = get_or_set_contents_file_id(file, session)
954 pca.filepath_id = get_or_set_contents_path_id(path, session)
955 pca.architecture = arch_id
958 # Only commit if we set up the session ourself
964 traceback.print_exc()
966 # Only rollback if we set up the session ourself
972 __all__.append('insert_pending_content_paths')
974 ################################################################################
976 class Priority(object):
977 def __init__(self, *args, **kwargs):
981 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
983 __all__.append('Priority')
985 def get_priority(priority, session=None):
987 Returns Priority object for given C{priority name}.
989 @type priority: string
990 @param priority: The name of the priority
992 @type session: Session
993 @param session: Optional SQLA session object (a temporary one will be
994 generated if not supplied)
997 @return: Priority object for the given priority
1001 session = DBConn().session()
1002 q = session.query(Priority).filter_by(priority=priority)
1007 __all__.append('get_priority')
1009 ################################################################################
1011 class Queue(object):
1012 def __init__(self, *args, **kwargs):
1016 return '<Queue %s>' % self.queue_name
1018 def autobuild_upload(self, changes, srcpath, session=None):
1020 Update queue_build database table used for incoming autobuild support.
1022 @type changes: Changes
1023 @param changes: changes object for the upload to process
1025 @type srcpath: string
1026 @param srcpath: path for the queue file entries/link destinations
1028 @type session: SQLAlchemy session
1029 @param session: Optional SQLAlchemy session. If this is passed, the
1030 caller is responsible for ensuring a transaction has begun and
1031 committing the results or rolling back based on the result code. If
1032 not passed, a commit will be performed at the end of the function,
1033 otherwise the caller is responsible for commiting.
1035 @rtype: NoneType or string
1036 @return: None if the operation failed, a string describing the error if not
1041 session = DBConn().session()
1044 # TODO: Remove by moving queue config into the database
1047 for suitename in changes.changes["distribution"].keys():
1048 # TODO: Move into database as:
1049 # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build)
1050 # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink)
1051 # This also gets rid of the SecurityQueueBuild hack below
1052 if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"):
1056 s = get_suite(suitename, session)
1058 return "INTERNAL ERROR: Could not find suite %s" % suitename
1060 # TODO: Get from database as above
1061 dest_dir = conf["Dir::QueueBuild"]
1063 # TODO: Move into database as above
1064 if conf.FindB("Dinstall::SecurityQueueBuild"):
1065 dest_dir = os.path.join(dest_dir, suitename)
1067 for file_entry in changes.files.keys():
1068 src = os.path.join(srcpath, file_entry)
1069 dest = os.path.join(dest_dir, file_entry)
1071 # TODO: Move into database as above
1072 if Cnf.FindB("Dinstall::SecurityQueueBuild"):
1073 # Copy it since the original won't be readable by www-data
1074 utils.copy(src, dest)
1076 # Create a symlink to it
1077 os.symlink(src, dest)
1080 qb.suite_id = s.suite_id
1081 qb.queue_id = self.queue_id
1087 # If the .orig.tar.gz is in the pool, create a symlink to
1088 # it (if one doesn't already exist)
1089 if changes.orig_tar_id:
1090 # Determine the .orig.tar.gz file name
1091 for dsc_file in changes.dsc_files.keys():
1092 if dsc_file.endswith(".orig.tar.gz"):
1095 dest = os.path.join(dest_dir, filename)
1097 # If it doesn't exist, create a symlink
1098 if not os.path.exists(dest):
1099 q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id",
1100 {'id': changes.orig_tar_id})
1103 return "[INTERNAL ERROR] Couldn't find id %s in files table." % (changes.orig_tar_id)
1105 src = os.path.join(res[0], res[1])
1106 os.symlink(src, dest)
1108 # Add it to the list of packages for later processing by apt-ftparchive
1110 qb.suite_id = s.suite_id
1111 qb.queue_id = self.queue_id
1116 # If it does, update things to ensure it's not removed prematurely
1118 qb = get_queue_build(dest, suite_id, session)
1129 __all__.append('Queue')
1131 def get_queue(queuename, session=None):
1133 Returns Queue object for given C{queue name}.
1135 @type queuename: string
1136 @param queuename: The name of the queue
1138 @type session: Session
1139 @param session: Optional SQLA session object (a temporary one will be
1140 generated if not supplied)
1143 @return: Queue object for the given queue
1147 session = DBConn().session()
1148 q = session.query(Queue).filter_by(queue_name=queuename)
1153 __all__.append('get_queue')
1155 ################################################################################
1157 class QueueBuild(object):
1158 def __init__(self, *args, **kwargs):
1162 return '<QueueBuild %s (%s)>' % (self.filename, self.queue_id)
1164 __all__.append('QueueBuild')
1166 def get_queue_build(filename, suite_id, session=None):
1168 Returns QueueBuild object for given C{filename} and C{suite id}.
1170 @type filename: string
1171 @param filename: The name of the file
1174 @param suiteid: Suite ID
1176 @type session: Session
1177 @param session: Optional SQLA session object (a temporary one will be
1178 generated if not supplied)
1181 @return: Queue object for the given queue
1185 session = DBConn().session()
1186 q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite_id)
1191 __all__.append('get_queue_build')
1193 ################################################################################
1195 class Section(object):
1196 def __init__(self, *args, **kwargs):
1200 return '<Section %s>' % self.section
1202 __all__.append('Section')
1204 def get_section(section, session=None):
1206 Returns Section object for given C{section name}.
1208 @type section: string
1209 @param section: The name of the section
1211 @type session: Session
1212 @param session: Optional SQLA session object (a temporary one will be
1213 generated if not supplied)
1216 @return: Section object for the given section name
1220 session = DBConn().session()
1221 q = session.query(Section).filter_by(section=section)
1226 __all__.append('get_section')
1228 ################################################################################
1230 class DBSource(object):
1231 def __init__(self, *args, **kwargs):
1235 return '<DBSource %s (%s)>' % (self.source, self.version)
1237 __all__.append('DBSource')
1239 def source_exists(source, source_version, suites = ["any"], session=None):
1241 Ensure that source exists somewhere in the archive for the binary
1242 upload being processed.
1243 1. exact match => 1.0-3
1244 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1246 @type package: string
1247 @param package: package source name
1249 @type source_version: string
1250 @param source_version: expected source version
1253 @param suites: list of suites to check in, default I{any}
1255 @type session: Session
1256 @param session: Optional SQLA session object (a temporary one will be
1257 generated if not supplied)
1260 @return: returns 1 if a source with expected version is found, otherwise 0
1265 session = DBConn().session()
1269 for suite in suites:
1270 q = session.query(DBSource).filter_by(source=source)
1272 # source must exist in suite X, or in some other suite that's
1273 # mapped to X, recursively... silent-maps are counted too,
1274 # unreleased-maps aren't.
1275 maps = cnf.ValueList("SuiteMappings")[:]
1277 maps = [ m.split() for m in maps ]
1278 maps = [ (x[1], x[2]) for x in maps
1279 if x[0] == "map" or x[0] == "silent-map" ]
1282 if x[1] in s and x[0] not in s:
1285 q = q.join(SrcAssociation).join(Suite)
1286 q = q.filter(Suite.suite_name.in_(s))
1288 # Reduce the query results to a list of version numbers
1289 ql = [ j.version for j in q.all() ]
1292 if source_version in ql:
1296 from daklib.regexes import re_bin_only_nmu
1297 orig_source_version = re_bin_only_nmu.sub('', source_version)
1298 if orig_source_version in ql:
1301 # No source found so return not ok
1307 __all__.append('source_exists')
1309 def get_suites_source_in(source, session=None):
1311 Returns list of Suite objects which given C{source} name is in
1314 @param source: DBSource package name to search for
1317 @return: list of Suite objects for the given source
1321 session = DBConn().session()
1323 return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
1325 __all__.append('get_suites_source_in')
1327 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
1329 Returns list of DBSource objects for given C{source} name and other parameters
1332 @param source: DBSource package name to search for
1334 @type source: str or None
1335 @param source: DBSource version name to search for or None if not applicable
1337 @type dm_upload_allowed: bool
1338 @param dm_upload_allowed: If None, no effect. If True or False, only
1339 return packages with that dm_upload_allowed setting
1341 @type session: Session
1342 @param session: Optional SQL session object (a temporary one will be
1343 generated if not supplied)
1346 @return: list of DBSource objects for the given name (may be empty)
1349 session = DBConn().session()
1351 q = session.query(DBSource).filter_by(source=source)
1353 if version is not None:
1354 q = q.filter_by(version=version)
1356 if dm_upload_allowed is not None:
1357 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
1361 __all__.append('get_sources_from_name')
1363 def get_source_in_suite(source, suite, session=None):
1365 Returns list of DBSource objects for a combination of C{source} and C{suite}.
1367 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
1368 - B{suite} - a suite name, eg. I{unstable}
1370 @type source: string
1371 @param source: source package name
1374 @param suite: the suite name
1377 @return: the version for I{source} in I{suite}
1381 session = DBConn().session()
1382 q = session.query(SrcAssociation)
1383 q = q.join('source').filter_by(source=source)
1384 q = q.join('suite').filter_by(suite_name=suite)
1387 # ???: Maybe we should just return the SrcAssociation object instead
1388 return q.one().source
1390 __all__.append('get_source_in_suite')
1392 ################################################################################
1394 class SrcAssociation(object):
1395 def __init__(self, *args, **kwargs):
1399 return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
1401 __all__.append('SrcAssociation')
1403 ################################################################################
1405 class SrcUploader(object):
1406 def __init__(self, *args, **kwargs):
1410 return '<SrcUploader %s>' % self.uploader_id
1412 __all__.append('SrcUploader')
1414 ################################################################################
1416 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
1417 ('SuiteID', 'suite_id'),
1418 ('Version', 'version'),
1419 ('Origin', 'origin'),
1421 ('Description', 'description'),
1422 ('Untouchable', 'untouchable'),
1423 ('Announce', 'announce'),
1424 ('Codename', 'codename'),
1425 ('OverrideCodename', 'overridecodename'),
1426 ('ValidTime', 'validtime'),
1427 ('Priority', 'priority'),
1428 ('NotAutomatic', 'notautomatic'),
1429 ('CopyChanges', 'copychanges'),
1430 ('CopyDotDak', 'copydotdak'),
1431 ('CommentsDir', 'commentsdir'),
1432 ('OverrideSuite', 'overridesuite'),
1433 ('ChangelogBase', 'changelogbase')]
1436 class Suite(object):
1437 def __init__(self, *args, **kwargs):
1441 return '<Suite %s>' % self.suite_name
1445 for disp, field in SUITE_FIELDS:
1446 val = getattr(self, field, None)
1448 ret.append("%s: %s" % (disp, val))
1450 return "\n".join(ret)
1452 __all__.append('Suite')
1454 def get_suite_architecture(suite, architecture, session=None):
1456 Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
1460 @param suite: Suite name to search for
1462 @type architecture: str
1463 @param architecture: Architecture name to search for
1465 @type session: Session
1466 @param session: Optional SQL session object (a temporary one will be
1467 generated if not supplied)
1469 @rtype: SuiteArchitecture
1470 @return: the SuiteArchitecture object or None
1474 session = DBConn().session()
1476 q = session.query(SuiteArchitecture)
1477 q = q.join(Architecture).filter_by(arch_string=architecture)
1478 q = q.join(Suite).filter_by(suite_name=suite)
1483 __all__.append('get_suite_architecture')
1485 def get_suite(suite, session=None):
1487 Returns Suite object for given C{suite name}.
1490 @param suite: The name of the suite
1492 @type session: Session
1493 @param session: Optional SQLA session object (a temporary one will be
1494 generated if not supplied)
1497 @return: Suite object for the requested suite name (None if not presenT)
1501 session = DBConn().session()
1502 q = session.query(Suite).filter_by(suite_name=suite)
1507 __all__.append('get_suite')
1509 ################################################################################
1511 class SuiteArchitecture(object):
1512 def __init__(self, *args, **kwargs):
1516 return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
1518 __all__.append('SuiteArchitecture')
1520 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
1522 Returns list of Architecture objects for given C{suite} name
1525 @param source: Suite name to search for
1527 @type skipsrc: boolean
1528 @param skipsrc: Whether to skip returning the 'source' architecture entry
1531 @type skipall: boolean
1532 @param skipall: Whether to skip returning the 'all' architecture entry
1535 @type session: Session
1536 @param session: Optional SQL session object (a temporary one will be
1537 generated if not supplied)
1540 @return: list of Architecture objects for the given name (may be empty)
1544 session = DBConn().session()
1546 q = session.query(Architecture)
1547 q = q.join(SuiteArchitecture)
1548 q = q.join(Suite).filter_by(suite_name=suite)
1550 q = q.filter(Architecture.arch_string != 'source')
1552 q = q.filter(Architecture.arch_string != 'all')
1553 q = q.order_by('arch_string')
1556 __all__.append('get_suite_architectures')
1558 ################################################################################
1561 def __init__(self, *args, **kwargs):
1565 return '<Uid %s (%s)>' % (self.uid, self.name)
1567 __all__.append('Uid')
1569 def add_database_user(uidname, session=None):
1571 Adds a database user
1573 @type uidname: string
1574 @param uidname: The uid of the user to add
1576 @type session: SQLAlchemy
1577 @param session: Optional SQL session object (a temporary one will be
1578 generated if not supplied). If not passed, a commit will be performed at
1579 the end of the function, otherwise the caller is responsible for commiting.
1582 @return: the uid object for the given uidname
1584 privatetrans = False
1586 session = DBConn().session()
1590 session.execute("CREATE USER :uid", {'uid': uidname})
1594 traceback.print_exc()
1597 __all__.append('add_database_user')
1599 def get_or_set_uid(uidname, session=None):
1601 Returns uid object for given uidname.
1603 If no matching uidname is found, a row is inserted.
1605 @type uidname: string
1606 @param uidname: The uid to add
1608 @type session: SQLAlchemy
1609 @param session: Optional SQL session object (a temporary one will be
1610 generated if not supplied). If not passed, a commit will be performed at
1611 the end of the function, otherwise the caller is responsible for commiting.
1614 @return: the uid object for the given uidname
1616 privatetrans = False
1618 session = DBConn().session()
1622 q = session.query(Uid).filter_by(uid=uidname)
1634 traceback.print_exc()
1637 __all__.append('get_or_set_uid')
1640 def get_uid_from_fingerprint(fpr, session=None):
1642 session = DBConn().session()
1644 q = session.query(Uid)
1645 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
1652 __all__.append('get_uid_from_fingerprint')
1654 ################################################################################
1656 class DBConn(Singleton):
1658 database module init.
1660 def __init__(self, *args, **kwargs):
1661 super(DBConn, self).__init__(*args, **kwargs)
1663 def _startup(self, *args, **kwargs):
1665 if kwargs.has_key('debug'):
1669 def __setuptables(self):
1670 self.tbl_architecture = Table('architecture', self.db_meta, autoload=True)
1671 self.tbl_archive = Table('archive', self.db_meta, autoload=True)
1672 self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True)
1673 self.tbl_binaries = Table('binaries', self.db_meta, autoload=True)
1674 self.tbl_component = Table('component', self.db_meta, autoload=True)
1675 self.tbl_config = Table('config', self.db_meta, autoload=True)
1676 self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True)
1677 self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True)
1678 self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True)
1679 self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True)
1680 self.tbl_files = Table('files', self.db_meta, autoload=True)
1681 self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True)
1682 self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True)
1683 self.tbl_location = Table('location', self.db_meta, autoload=True)
1684 self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True)
1685 self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True)
1686 self.tbl_override = Table('override', self.db_meta, autoload=True)
1687 self.tbl_override_type = Table('override_type', self.db_meta, autoload=True)
1688 self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True)
1689 self.tbl_priority = Table('priority', self.db_meta, autoload=True)
1690 self.tbl_queue = Table('queue', self.db_meta, autoload=True)
1691 self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True)
1692 self.tbl_section = Table('section', self.db_meta, autoload=True)
1693 self.tbl_source = Table('source', self.db_meta, autoload=True)
1694 self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True)
1695 self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True)
1696 self.tbl_suite = Table('suite', self.db_meta, autoload=True)
1697 self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True)
1698 self.tbl_uid = Table('uid', self.db_meta, autoload=True)
1700 def __setupmappers(self):
1701 mapper(Architecture, self.tbl_architecture,
1702 properties = dict(arch_id = self.tbl_architecture.c.id))
1704 mapper(Archive, self.tbl_archive,
1705 properties = dict(archive_id = self.tbl_archive.c.id,
1706 archive_name = self.tbl_archive.c.name))
1708 mapper(BinAssociation, self.tbl_bin_associations,
1709 properties = dict(ba_id = self.tbl_bin_associations.c.id,
1710 suite_id = self.tbl_bin_associations.c.suite,
1711 suite = relation(Suite),
1712 binary_id = self.tbl_bin_associations.c.bin,
1713 binary = relation(DBBinary)))
1715 mapper(DBBinary, self.tbl_binaries,
1716 properties = dict(binary_id = self.tbl_binaries.c.id,
1717 package = self.tbl_binaries.c.package,
1718 version = self.tbl_binaries.c.version,
1719 maintainer_id = self.tbl_binaries.c.maintainer,
1720 maintainer = relation(Maintainer),
1721 source_id = self.tbl_binaries.c.source,
1722 source = relation(DBSource),
1723 arch_id = self.tbl_binaries.c.architecture,
1724 architecture = relation(Architecture),
1725 poolfile_id = self.tbl_binaries.c.file,
1726 poolfile = relation(PoolFile),
1727 binarytype = self.tbl_binaries.c.type,
1728 fingerprint_id = self.tbl_binaries.c.sig_fpr,
1729 fingerprint = relation(Fingerprint),
1730 install_date = self.tbl_binaries.c.install_date,
1731 binassociations = relation(BinAssociation,
1732 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
1734 mapper(Component, self.tbl_component,
1735 properties = dict(component_id = self.tbl_component.c.id,
1736 component_name = self.tbl_component.c.name))
1738 mapper(DBConfig, self.tbl_config,
1739 properties = dict(config_id = self.tbl_config.c.id))
1741 mapper(ContentAssociation, self.tbl_content_associations,
1742 properties = dict(ca_id = self.tbl_content_associations.c.id,
1743 filename_id = self.tbl_content_associations.c.filename,
1744 filename = relation(ContentFilename),
1745 filepath_id = self.tbl_content_associations.c.filepath,
1746 filepath = relation(ContentFilepath),
1747 binary_id = self.tbl_content_associations.c.binary_pkg,
1748 binary = relation(DBBinary)))
1751 mapper(ContentFilename, self.tbl_content_file_names,
1752 properties = dict(cafilename_id = self.tbl_content_file_names.c.id,
1753 filename = self.tbl_content_file_names.c.file))
1755 mapper(ContentFilepath, self.tbl_content_file_paths,
1756 properties = dict(cafilepath_id = self.tbl_content_file_paths.c.id,
1757 filepath = self.tbl_content_file_paths.c.path))
1759 mapper(DSCFile, self.tbl_dsc_files,
1760 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
1761 source_id = self.tbl_dsc_files.c.source,
1762 source = relation(DBSource),
1763 poolfile_id = self.tbl_dsc_files.c.file,
1764 poolfile = relation(PoolFile)))
1766 mapper(PoolFile, self.tbl_files,
1767 properties = dict(file_id = self.tbl_files.c.id,
1768 filesize = self.tbl_files.c.size,
1769 location_id = self.tbl_files.c.location,
1770 location = relation(Location)))
1772 mapper(Fingerprint, self.tbl_fingerprint,
1773 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
1774 uid_id = self.tbl_fingerprint.c.uid,
1775 uid = relation(Uid),
1776 keyring_id = self.tbl_fingerprint.c.keyring,
1777 keyring = relation(Keyring)))
1779 mapper(Keyring, self.tbl_keyrings,
1780 properties = dict(keyring_name = self.tbl_keyrings.c.name,
1781 keyring_id = self.tbl_keyrings.c.id))
1783 mapper(Location, self.tbl_location,
1784 properties = dict(location_id = self.tbl_location.c.id,
1785 component_id = self.tbl_location.c.component,
1786 component = relation(Component),
1787 archive_id = self.tbl_location.c.archive,
1788 archive = relation(Archive),
1789 archive_type = self.tbl_location.c.type))
1791 mapper(Maintainer, self.tbl_maintainer,
1792 properties = dict(maintainer_id = self.tbl_maintainer.c.id))
1794 mapper(NewComment, self.tbl_new_comments,
1795 properties = dict(comment_id = self.tbl_new_comments.c.id))
1797 mapper(Override, self.tbl_override,
1798 properties = dict(suite_id = self.tbl_override.c.suite,
1799 suite = relation(Suite),
1800 component_id = self.tbl_override.c.component,
1801 component = relation(Component),
1802 priority_id = self.tbl_override.c.priority,
1803 priority = relation(Priority),
1804 section_id = self.tbl_override.c.section,
1805 section = relation(Section),
1806 overridetype_id = self.tbl_override.c.type,
1807 overridetype = relation(OverrideType)))
1809 mapper(OverrideType, self.tbl_override_type,
1810 properties = dict(overridetype = self.tbl_override_type.c.type,
1811 overridetype_id = self.tbl_override_type.c.id))
1813 mapper(PendingContentAssociation, self.tbl_pending_content_associations,
1814 properties = dict(pca_id = self.tbl_pending_content_associations.c.id,
1815 filepath_id = self.tbl_pending_content_associations.c.filepath,
1816 filepath = relation(ContentFilepath),
1817 filename_id = self.tbl_pending_content_associations.c.filename,
1818 filename = relation(ContentFilename)))
1820 mapper(Priority, self.tbl_priority,
1821 properties = dict(priority_id = self.tbl_priority.c.id))
1823 mapper(Queue, self.tbl_queue,
1824 properties = dict(queue_id = self.tbl_queue.c.id))
1826 mapper(QueueBuild, self.tbl_queue_build,
1827 properties = dict(suite_id = self.tbl_queue_build.c.suite,
1828 queue_id = self.tbl_queue_build.c.queue,
1829 queue = relation(Queue, backref='queuebuild')))
1831 mapper(Section, self.tbl_section,
1832 properties = dict(section_id = self.tbl_section.c.id))
1834 mapper(DBSource, self.tbl_source,
1835 properties = dict(source_id = self.tbl_source.c.id,
1836 version = self.tbl_source.c.version,
1837 maintainer_id = self.tbl_source.c.maintainer,
1838 maintainer = relation(Maintainer,
1839 primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
1840 poolfile_id = self.tbl_source.c.file,
1841 poolfile = relation(PoolFile),
1842 fingerprint_id = self.tbl_source.c.sig_fpr,
1843 fingerprint = relation(Fingerprint),
1844 changedby_id = self.tbl_source.c.changedby,
1845 changedby = relation(Maintainer,
1846 primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
1847 srcfiles = relation(DSCFile,
1848 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
1849 srcassociations = relation(SrcAssociation,
1850 primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source))))
1852 mapper(SrcAssociation, self.tbl_src_associations,
1853 properties = dict(sa_id = self.tbl_src_associations.c.id,
1854 suite_id = self.tbl_src_associations.c.suite,
1855 suite = relation(Suite),
1856 source_id = self.tbl_src_associations.c.source,
1857 source = relation(DBSource)))
1859 mapper(SrcUploader, self.tbl_src_uploaders,
1860 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
1861 source_id = self.tbl_src_uploaders.c.source,
1862 source = relation(DBSource,
1863 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
1864 maintainer_id = self.tbl_src_uploaders.c.maintainer,
1865 maintainer = relation(Maintainer,
1866 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
1868 mapper(Suite, self.tbl_suite,
1869 properties = dict(suite_id = self.tbl_suite.c.id))
1871 mapper(SuiteArchitecture, self.tbl_suite_architectures,
1872 properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
1873 suite = relation(Suite, backref='suitearchitectures'),
1874 arch_id = self.tbl_suite_architectures.c.architecture,
1875 architecture = relation(Architecture)))
1877 mapper(Uid, self.tbl_uid,
1878 properties = dict(uid_id = self.tbl_uid.c.id,
1879 fingerprint = relation(Fingerprint)))
1881 ## Connection functions
1882 def __createconn(self):
1883 from config import Config
1887 connstr = "postgres://%s" % cnf["DB::Host"]
1888 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
1889 connstr += ":%s" % cnf["DB::Port"]
1890 connstr += "/%s" % cnf["DB::Name"]
1893 connstr = "postgres:///%s" % cnf["DB::Name"]
1894 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
1895 connstr += "?port=%s" % cnf["DB::Port"]
1897 self.db_pg = create_engine(connstr, echo=self.debug)
1898 self.db_meta = MetaData()
1899 self.db_meta.bind = self.db_pg
1900 self.db_smaker = sessionmaker(bind=self.db_pg,
1904 self.__setuptables()
1905 self.__setupmappers()
1908 return self.db_smaker()
1910 __all__.append('DBConn')