5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
40 from datetime import datetime
42 from inspect import getargspec
45 from sqlalchemy import create_engine, Table, MetaData
46 from sqlalchemy.orm import sessionmaker, mapper, relation
47 from sqlalchemy import types as sqltypes
49 # Don't remove this, we re-export the exceptions to scripts which import us
50 from sqlalchemy.exc import *
51 from sqlalchemy.orm.exc import NoResultFound
53 from config import Config
54 from textutils import fix_maintainer
56 ################################################################################
58 # Patch in support for the debversion field type so that it works during
61 class DebVersion(sqltypes.Text):
62 def get_col_spec(self):
65 sa_major_version = sqlalchemy.__version__[0:3]
66 if sa_major_version == "0.5":
67 from sqlalchemy.databases import postgres
68 postgres.ischema_names['debversion'] = DebVersion
70 raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
72 ################################################################################
74 __all__ = ['IntegrityError', 'SQLAlchemyError']
76 ################################################################################
78 def session_wrapper(fn):
80 Wrapper around common ".., session=None):" handling. If the wrapped
81 function is called without passing 'session', we create a local one
82 and destroy it when the function ends.
84 Also attaches a commit_or_flush method to the session; if we created a
85 local session, this is a synonym for session.commit(), otherwise it is a
86 synonym for session.flush().
89 def wrapped(*args, **kwargs):
90 private_transaction = False
92 # Find the session object
93 session = kwargs.get('session')
96 if len(args) <= len(getargspec(fn)[0]) - 1:
97 # No session specified as last argument or in kwargs
98 private_transaction = True
99 session = kwargs['session'] = DBConn().session()
101 # Session is last argument in args
105 session = args[-1] = DBConn().session()
106 private_transaction = True
108 if private_transaction:
109 session.commit_or_flush = session.commit
111 session.commit_or_flush = session.flush
114 return fn(*args, **kwargs)
116 if private_transaction:
117 # We created a session; close it.
120 wrapped.__doc__ = fn.__doc__
121 wrapped.func_name = fn.func_name
125 __all__.append('session_wrapper')
127 ################################################################################
129 class Architecture(object):
130 def __init__(self, *args, **kwargs):
133 def __eq__(self, val):
134 if isinstance(val, str):
135 return (self.arch_string== val)
136 # This signals to use the normal comparison operator
137 return NotImplemented
139 def __ne__(self, val):
140 if isinstance(val, str):
141 return (self.arch_string != val)
142 # This signals to use the normal comparison operator
143 return NotImplemented
146 return '<Architecture %s>' % self.arch_string
148 __all__.append('Architecture')
151 def get_architecture(architecture, session=None):
153 Returns database id for given C{architecture}.
155 @type architecture: string
156 @param architecture: The name of the architecture
158 @type session: Session
159 @param session: Optional SQLA session object (a temporary one will be
160 generated if not supplied)
163 @return: Architecture object for the given arch (None if not present)
166 q = session.query(Architecture).filter_by(arch_string=architecture)
170 except NoResultFound:
173 __all__.append('get_architecture')
176 def get_architecture_suites(architecture, session=None):
178 Returns list of Suite objects for given C{architecture} name
181 @param source: Architecture name to search for
183 @type session: Session
184 @param session: Optional SQL session object (a temporary one will be
185 generated if not supplied)
188 @return: list of Suite objects for the given name (may be empty)
191 q = session.query(Suite)
192 q = q.join(SuiteArchitecture)
193 q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
199 __all__.append('get_architecture_suites')
201 ################################################################################
203 class Archive(object):
204 def __init__(self, *args, **kwargs):
208 return '<Archive %s>' % self.archive_name
210 __all__.append('Archive')
213 def get_archive(archive, session=None):
215 returns database id for given C{archive}.
217 @type archive: string
218 @param archive: the name of the arhive
220 @type session: Session
221 @param session: Optional SQLA session object (a temporary one will be
222 generated if not supplied)
225 @return: Archive object for the given name (None if not present)
228 archive = archive.lower()
230 q = session.query(Archive).filter_by(archive_name=archive)
234 except NoResultFound:
237 __all__.append('get_archive')
239 ################################################################################
241 class BinAssociation(object):
242 def __init__(self, *args, **kwargs):
246 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
248 __all__.append('BinAssociation')
250 ################################################################################
252 class BinContents(object):
253 def __init__(self, *args, **kwargs):
257 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
259 __all__.append('BinContents')
261 ################################################################################
263 class DBBinary(object):
264 def __init__(self, *args, **kwargs):
268 return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
270 __all__.append('DBBinary')
273 def get_suites_binary_in(package, session=None):
275 Returns list of Suite objects which given C{package} name is in
278 @param source: DBBinary package name to search for
281 @return: list of Suite objects for the given package
284 return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
286 __all__.append('get_suites_binary_in')
289 def get_binary_from_id(binary_id, session=None):
291 Returns DBBinary object for given C{id}
294 @param binary_id: Id of the required binary
296 @type session: Session
297 @param session: Optional SQLA session object (a temporary one will be
298 generated if not supplied)
301 @return: DBBinary object for the given binary (None if not present)
304 q = session.query(DBBinary).filter_by(binary_id=binary_id)
308 except NoResultFound:
311 __all__.append('get_binary_from_id')
314 def get_binaries_from_name(package, version=None, architecture=None, session=None):
316 Returns list of DBBinary objects for given C{package} name
319 @param package: DBBinary package name to search for
321 @type version: str or None
322 @param version: Version to search for (or None)
324 @type package: str, list or None
325 @param package: Architectures to limit to (or None if no limit)
327 @type session: Session
328 @param session: Optional SQL session object (a temporary one will be
329 generated if not supplied)
332 @return: list of DBBinary objects for the given name (may be empty)
335 q = session.query(DBBinary).filter_by(package=package)
337 if version is not None:
338 q = q.filter_by(version=version)
340 if architecture is not None:
341 if not isinstance(architecture, list):
342 architecture = [architecture]
343 q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
349 __all__.append('get_binaries_from_name')
352 def get_binaries_from_source_id(source_id, session=None):
354 Returns list of DBBinary objects for given C{source_id}
357 @param source_id: source_id to search for
359 @type session: Session
360 @param session: Optional SQL session object (a temporary one will be
361 generated if not supplied)
364 @return: list of DBBinary objects for the given name (may be empty)
367 return session.query(DBBinary).filter_by(source_id=source_id).all()
369 __all__.append('get_binaries_from_source_id')
372 def get_binary_from_name_suite(package, suitename, session=None):
373 ### For dak examine-package
374 ### XXX: Doesn't use object API yet
376 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
377 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
378 WHERE b.package=:package
380 AND fi.location = l.id
381 AND l.component = c.id
384 AND su.suite_name=:suitename
385 ORDER BY b.version DESC"""
387 return session.execute(sql, {'package': package, 'suitename': suitename})
389 __all__.append('get_binary_from_name_suite')
392 def get_binary_components(package, suitename, arch, session=None):
393 # Check for packages that have moved from one component to another
394 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
395 WHERE b.package=:package AND s.suite_name=:suitename
396 AND (a.arch_string = :arch OR a.arch_string = 'all')
397 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
398 AND f.location = l.id
399 AND l.component = c.id
402 vals = {'package': package, 'suitename': suitename, 'arch': arch}
404 return session.execute(query, vals)
406 __all__.append('get_binary_components')
408 ################################################################################
410 class BinaryACL(object):
411 def __init__(self, *args, **kwargs):
415 return '<BinaryACL %s>' % self.binary_acl_id
417 __all__.append('BinaryACL')
419 ################################################################################
421 class BinaryACLMap(object):
422 def __init__(self, *args, **kwargs):
426 return '<BinaryACLMap %s>' % self.binary_acl_map_id
428 __all__.append('BinaryACLMap')
430 ################################################################################
432 class BuildQueue(object):
433 def __init__(self, *args, **kwargs):
437 return '<BuildQueue %s>' % self.queue_name
439 def add_file_from_pool(self, poolfile):
440 """Copies a file into the pool. Assumes that the PoolFile object is
441 attached to the same SQLAlchemy session as the Queue object is.
443 The caller is responsible for committing after calling this function."""
444 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
446 # Check if we have a file of this name or this ID already
447 for f in self.queuefiles:
448 if f.fileid is not None and f.fileid == poolfile.file_id or \
449 f.poolfile.filename == poolfile_basename:
450 # In this case, update the BuildQueueFile entry so we
451 # don't remove it too early
452 f.lastused = datetime.now()
453 DBConn().session().object_session(pf).add(f)
456 # Prepare BuildQueueFile object
457 qf = BuildQueueFile()
458 qf.build_queue_id = self.queue_id
459 qf.lastused = datetime.now()
460 qf.filename = poolfile_basename
462 targetpath = poolfile.fullpath
463 queuepath = os.path.join(self.path, poolfile_basename)
467 # We need to copy instead of symlink
469 utils.copy(targetpath, queuepath)
470 # NULL in the fileid field implies a copy
473 os.symlink(targetpath, queuepath)
474 qf.fileid = poolfile.file_id
478 # Get the same session as the PoolFile is using and add the qf to it
479 DBConn().session().object_session(poolfile).add(qf)
484 __all__.append('BuildQueue')
487 def get_build_queue(queuename, session=None):
489 Returns BuildQueue object for given C{queue name}, creating it if it does not
492 @type queuename: string
493 @param queuename: The name of the queue
495 @type session: Session
496 @param session: Optional SQLA session object (a temporary one will be
497 generated if not supplied)
500 @return: BuildQueue object for the given queue
503 q = session.query(BuildQueue).filter_by(queue_name=queuename)
507 except NoResultFound:
510 __all__.append('get_build_queue')
512 ################################################################################
514 class BuildQueueFile(object):
515 def __init__(self, *args, **kwargs):
519 return '<BuildQueueFile %s (%s)>' % (self.filename, self.queue_id)
521 __all__.append('BuildQueueFile')
523 ################################################################################
525 class ChangePendingBinary(object):
526 def __init__(self, *args, **kwargs):
530 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
532 __all__.append('ChangePendingBinary')
534 ################################################################################
536 class ChangePendingFile(object):
537 def __init__(self, *args, **kwargs):
541 return '<ChangePendingFile %s>' % self.change_pending_file_id
543 __all__.append('ChangePendingFile')
545 ################################################################################
547 class ChangePendingSource(object):
548 def __init__(self, *args, **kwargs):
552 return '<ChangePendingSource %s>' % self.change_pending_source_id
554 __all__.append('ChangePendingSource')
556 ################################################################################
558 class Component(object):
559 def __init__(self, *args, **kwargs):
562 def __eq__(self, val):
563 if isinstance(val, str):
564 return (self.component_name == val)
565 # This signals to use the normal comparison operator
566 return NotImplemented
568 def __ne__(self, val):
569 if isinstance(val, str):
570 return (self.component_name != val)
571 # This signals to use the normal comparison operator
572 return NotImplemented
575 return '<Component %s>' % self.component_name
578 __all__.append('Component')
581 def get_component(component, session=None):
583 Returns database id for given C{component}.
585 @type component: string
586 @param component: The name of the override type
589 @return: the database id for the given component
592 component = component.lower()
594 q = session.query(Component).filter_by(component_name=component)
598 except NoResultFound:
601 __all__.append('get_component')
603 ################################################################################
605 class DBConfig(object):
606 def __init__(self, *args, **kwargs):
610 return '<DBConfig %s>' % self.name
612 __all__.append('DBConfig')
614 ################################################################################
617 def get_or_set_contents_file_id(filename, session=None):
619 Returns database id for given filename.
621 If no matching file is found, a row is inserted.
623 @type filename: string
624 @param filename: The filename
625 @type session: SQLAlchemy
626 @param session: Optional SQL session object (a temporary one will be
627 generated if not supplied). If not passed, a commit will be performed at
628 the end of the function, otherwise the caller is responsible for commiting.
631 @return: the database id for the given component
634 q = session.query(ContentFilename).filter_by(filename=filename)
637 ret = q.one().cafilename_id
638 except NoResultFound:
639 cf = ContentFilename()
640 cf.filename = filename
642 session.commit_or_flush()
643 ret = cf.cafilename_id
647 __all__.append('get_or_set_contents_file_id')
650 def get_contents(suite, overridetype, section=None, session=None):
652 Returns contents for a suite / overridetype combination, limiting
653 to a section if not None.
656 @param suite: Suite object
658 @type overridetype: OverrideType
659 @param overridetype: OverrideType object
661 @type section: Section
662 @param section: Optional section object to limit results to
664 @type session: SQLAlchemy
665 @param session: Optional SQL session object (a temporary one will be
666 generated if not supplied)
669 @return: ResultsProxy object set up to return tuples of (filename, section,
673 # find me all of the contents for a given suite
674 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
678 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
679 JOIN content_file_names n ON (c.filename=n.id)
680 JOIN binaries b ON (b.id=c.binary_pkg)
681 JOIN override o ON (o.package=b.package)
682 JOIN section s ON (s.id=o.section)
683 WHERE o.suite = :suiteid AND o.type = :overridetypeid
684 AND b.type=:overridetypename"""
686 vals = {'suiteid': suite.suite_id,
687 'overridetypeid': overridetype.overridetype_id,
688 'overridetypename': overridetype.overridetype}
690 if section is not None:
691 contents_q += " AND s.id = :sectionid"
692 vals['sectionid'] = section.section_id
694 contents_q += " ORDER BY fn"
696 return session.execute(contents_q, vals)
698 __all__.append('get_contents')
700 ################################################################################
702 class ContentFilepath(object):
703 def __init__(self, *args, **kwargs):
707 return '<ContentFilepath %s>' % self.filepath
709 __all__.append('ContentFilepath')
712 def get_or_set_contents_path_id(filepath, session=None):
714 Returns database id for given path.
716 If no matching file is found, a row is inserted.
718 @type filename: string
719 @param filename: The filepath
720 @type session: SQLAlchemy
721 @param session: Optional SQL session object (a temporary one will be
722 generated if not supplied). If not passed, a commit will be performed at
723 the end of the function, otherwise the caller is responsible for commiting.
726 @return: the database id for the given path
729 q = session.query(ContentFilepath).filter_by(filepath=filepath)
732 ret = q.one().cafilepath_id
733 except NoResultFound:
734 cf = ContentFilepath()
735 cf.filepath = filepath
737 session.commit_or_flush()
738 ret = cf.cafilepath_id
742 __all__.append('get_or_set_contents_path_id')
744 ################################################################################
746 class ContentAssociation(object):
747 def __init__(self, *args, **kwargs):
751 return '<ContentAssociation %s>' % self.ca_id
753 __all__.append('ContentAssociation')
755 def insert_content_paths(binary_id, fullpaths, session=None):
757 Make sure given path is associated with given binary id
760 @param binary_id: the id of the binary
761 @type fullpaths: list
762 @param fullpaths: the list of paths of the file being associated with the binary
763 @type session: SQLAlchemy session
764 @param session: Optional SQLAlchemy session. If this is passed, the caller
765 is responsible for ensuring a transaction has begun and committing the
766 results or rolling back based on the result code. If not passed, a commit
767 will be performed at the end of the function, otherwise the caller is
768 responsible for commiting.
770 @return: True upon success
775 session = DBConn().session()
781 for fullpath in fullpaths:
782 if fullpath.startswith( './' ):
783 fullpath = fullpath[2:]
785 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} )
793 traceback.print_exc()
795 # Only rollback if we set up the session ourself
802 __all__.append('insert_content_paths')
804 ################################################################################
806 class DSCFile(object):
807 def __init__(self, *args, **kwargs):
811 return '<DSCFile %s>' % self.dscfile_id
813 __all__.append('DSCFile')
816 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
818 Returns a list of DSCFiles which may be empty
820 @type dscfile_id: int (optional)
821 @param dscfile_id: the dscfile_id of the DSCFiles to find
823 @type source_id: int (optional)
824 @param source_id: the source id related to the DSCFiles to find
826 @type poolfile_id: int (optional)
827 @param poolfile_id: the poolfile id related to the DSCFiles to find
830 @return: Possibly empty list of DSCFiles
833 q = session.query(DSCFile)
835 if dscfile_id is not None:
836 q = q.filter_by(dscfile_id=dscfile_id)
838 if source_id is not None:
839 q = q.filter_by(source_id=source_id)
841 if poolfile_id is not None:
842 q = q.filter_by(poolfile_id=poolfile_id)
846 __all__.append('get_dscfiles')
848 ################################################################################
850 class PoolFile(object):
851 def __init__(self, *args, **kwargs):
855 return '<PoolFile %s>' % self.filename
859 return os.path.join(self.location.path, self.filename)
861 __all__.append('PoolFile')
864 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
867 (ValidFileFound [boolean or None], PoolFile object or None)
869 @type filename: string
870 @param filename: the filename of the file to check against the DB
873 @param filesize: the size of the file to check against the DB
876 @param md5sum: the md5sum of the file to check against the DB
878 @type location_id: int
879 @param location_id: the id of the location to look in
882 @return: Tuple of length 2.
883 If more than one file found with that name:
885 If valid pool file found: (True, PoolFile object)
886 If valid pool file not found:
887 (False, None) if no file found
888 (False, PoolFile object) if file found with size/md5sum mismatch
891 q = session.query(PoolFile).filter_by(filename=filename)
892 q = q.join(Location).filter_by(location_id=location_id)
902 if obj.md5sum != md5sum or obj.filesize != int(filesize):
910 __all__.append('check_poolfile')
913 def get_poolfile_by_id(file_id, session=None):
915 Returns a PoolFile objects or None for the given id
918 @param file_id: the id of the file to look for
920 @rtype: PoolFile or None
921 @return: either the PoolFile object or None
924 q = session.query(PoolFile).filter_by(file_id=file_id)
928 except NoResultFound:
931 __all__.append('get_poolfile_by_id')
935 def get_poolfile_by_name(filename, location_id=None, session=None):
937 Returns an array of PoolFile objects for the given filename and
938 (optionally) location_id
940 @type filename: string
941 @param filename: the filename of the file to check against the DB
943 @type location_id: int
944 @param location_id: the id of the location to look in (optional)
947 @return: array of PoolFile objects
950 q = session.query(PoolFile).filter_by(filename=filename)
952 if location_id is not None:
953 q = q.join(Location).filter_by(location_id=location_id)
957 __all__.append('get_poolfile_by_name')
960 def get_poolfile_like_name(filename, session=None):
962 Returns an array of PoolFile objects which are like the given name
964 @type filename: string
965 @param filename: the filename of the file to check against the DB
968 @return: array of PoolFile objects
971 # TODO: There must be a way of properly using bind parameters with %FOO%
972 q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
976 __all__.append('get_poolfile_like_name')
979 def add_poolfile(filename, datadict, location_id, session=None):
981 Add a new file to the pool
983 @type filename: string
984 @param filename: filename
987 @param datadict: dict with needed data
989 @type location_id: int
990 @param location_id: database id of the location
993 @return: the PoolFile object created
995 poolfile = PoolFile()
996 poolfile.filename = filename
997 poolfile.filesize = datadict["size"]
998 poolfile.md5sum = datadict["md5sum"]
999 poolfile.sha1sum = datadict["sha1sum"]
1000 poolfile.sha256sum = datadict["sha256sum"]
1001 poolfile.location_id = location_id
1003 session.add(poolfile)
1004 # Flush to get a file id (NB: This is not a commit)
1009 __all__.append('add_poolfile')
1011 ################################################################################
1013 class Fingerprint(object):
1014 def __init__(self, *args, **kwargs):
1018 return '<Fingerprint %s>' % self.fingerprint
1020 __all__.append('Fingerprint')
1023 def get_fingerprint(fpr, session=None):
1025 Returns Fingerprint object for given fpr.
1028 @param fpr: The fpr to find / add
1030 @type session: SQLAlchemy
1031 @param session: Optional SQL session object (a temporary one will be
1032 generated if not supplied).
1035 @return: the Fingerprint object for the given fpr or None
1038 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1042 except NoResultFound:
1047 __all__.append('get_fingerprint')
1050 def get_or_set_fingerprint(fpr, session=None):
1052 Returns Fingerprint object for given fpr.
1054 If no matching fpr is found, a row is inserted.
1057 @param fpr: The fpr to find / add
1059 @type session: SQLAlchemy
1060 @param session: Optional SQL session object (a temporary one will be
1061 generated if not supplied). If not passed, a commit will be performed at
1062 the end of the function, otherwise the caller is responsible for commiting.
1063 A flush will be performed either way.
1066 @return: the Fingerprint object for the given fpr
1069 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1073 except NoResultFound:
1074 fingerprint = Fingerprint()
1075 fingerprint.fingerprint = fpr
1076 session.add(fingerprint)
1077 session.commit_or_flush()
1082 __all__.append('get_or_set_fingerprint')
1084 ################################################################################
1086 # Helper routine for Keyring class
1087 def get_ldap_name(entry):
1089 for k in ["cn", "mn", "sn"]:
1091 if ret and ret[0] != "" and ret[0] != "-":
1093 return " ".join(name)
1095 ################################################################################
1097 class Keyring(object):
1098 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1099 " --with-colons --fingerprint --fingerprint"
1104 def __init__(self, *args, **kwargs):
1108 return '<Keyring %s>' % self.keyring_name
1110 def de_escape_gpg_str(self, txt):
1111 esclist = re.split(r'(\\x..)', txt)
1112 for x in range(1,len(esclist),2):
1113 esclist[x] = "%c" % (int(esclist[x][2:],16))
1114 return "".join(esclist)
1116 def load_keys(self, keyring):
1119 if not self.keyring_id:
1120 raise Exception('Must be initialized with database information')
1122 k = os.popen(self.gpg_invocation % keyring, "r")
1126 for line in k.xreadlines():
1127 field = line.split(":")
1128 if field[0] == "pub":
1130 (name, addr) = email.Utils.parseaddr(field[9])
1131 name = re.sub(r"\s*[(].*[)]", "", name)
1132 if name == "" or addr == "" or "@" not in addr:
1134 addr = "invalid-uid"
1135 name = self.de_escape_gpg_str(name)
1136 self.keys[key] = {"email": addr}
1138 self.keys[key]["name"] = name
1139 self.keys[key]["aliases"] = [name]
1140 self.keys[key]["fingerprints"] = []
1142 elif key and field[0] == "sub" and len(field) >= 12:
1143 signingkey = ("s" in field[11])
1144 elif key and field[0] == "uid":
1145 (name, addr) = email.Utils.parseaddr(field[9])
1146 if name and name not in self.keys[key]["aliases"]:
1147 self.keys[key]["aliases"].append(name)
1148 elif signingkey and field[0] == "fpr":
1149 self.keys[key]["fingerprints"].append(field[9])
1150 self.fpr_lookup[field[9]] = key
1152 def import_users_from_ldap(self, session):
1156 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1157 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1159 l = ldap.open(LDAPServer)
1160 l.simple_bind_s("","")
1161 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1162 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1163 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1165 ldap_fin_uid_id = {}
1172 uid = entry["uid"][0]
1173 name = get_ldap_name(entry)
1174 fingerprints = entry["keyFingerPrint"]
1176 for f in fingerprints:
1177 key = self.fpr_lookup.get(f, None)
1178 if key not in self.keys:
1180 self.keys[key]["uid"] = uid
1184 keyid = get_or_set_uid(uid, session).uid_id
1185 byuid[keyid] = (uid, name)
1186 byname[uid] = (keyid, name)
1188 return (byname, byuid)
1190 def generate_users_from_keyring(self, format, session):
1194 for x in self.keys.keys():
1195 if self.keys[x]["email"] == "invalid-uid":
1197 self.keys[x]["uid"] = format % "invalid-uid"
1199 uid = format % self.keys[x]["email"]
1200 keyid = get_or_set_uid(uid, session).uid_id
1201 byuid[keyid] = (uid, self.keys[x]["name"])
1202 byname[uid] = (keyid, self.keys[x]["name"])
1203 self.keys[x]["uid"] = uid
1206 uid = format % "invalid-uid"
1207 keyid = get_or_set_uid(uid, session).uid_id
1208 byuid[keyid] = (uid, "ungeneratable user id")
1209 byname[uid] = (keyid, "ungeneratable user id")
1211 return (byname, byuid)
1213 __all__.append('Keyring')
1216 def get_keyring(keyring, session=None):
1218 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1219 If C{keyring} already has an entry, simply return the existing Keyring
1221 @type keyring: string
1222 @param keyring: the keyring name
1225 @return: the Keyring object for this keyring
1228 q = session.query(Keyring).filter_by(keyring_name=keyring)
1232 except NoResultFound:
1235 __all__.append('get_keyring')
1237 ################################################################################
1239 class KeyringACLMap(object):
1240 def __init__(self, *args, **kwargs):
1244 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1246 __all__.append('KeyringACLMap')
1248 ################################################################################
1250 class DBChange(object):
1251 def __init__(self, *args, **kwargs):
1255 return '<DBChange %s>' % self.changesname
1257 __all__.append('DBChange')
1260 def get_dbchange(filename, session=None):
1262 returns DBChange object for given C{filename}.
1264 @type archive: string
1265 @param archive: the name of the arhive
1267 @type session: Session
1268 @param session: Optional SQLA session object (a temporary one will be
1269 generated if not supplied)
1272 @return: Archive object for the given name (None if not present)
1275 q = session.query(DBChange).filter_by(changesname=filename)
1279 except NoResultFound:
1282 __all__.append('get_dbchange')
1284 ################################################################################
1286 class Location(object):
1287 def __init__(self, *args, **kwargs):
1291 return '<Location %s (%s)>' % (self.path, self.location_id)
1293 __all__.append('Location')
1296 def get_location(location, component=None, archive=None, session=None):
1298 Returns Location object for the given combination of location, component
1301 @type location: string
1302 @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
1304 @type component: string
1305 @param component: the component name (if None, no restriction applied)
1307 @type archive: string
1308 @param archive_id: the archive name (if None, no restriction applied)
1310 @rtype: Location / None
1311 @return: Either a Location object or None if one can't be found
1314 q = session.query(Location).filter_by(path=location)
1316 if archive is not None:
1317 q = q.join(Archive).filter_by(archive_name=archive)
1319 if component is not None:
1320 q = q.join(Component).filter_by(component_name=component)
1324 except NoResultFound:
1327 __all__.append('get_location')
1329 ################################################################################
1331 class Maintainer(object):
1332 def __init__(self, *args, **kwargs):
1336 return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
1338 def get_split_maintainer(self):
1339 if not hasattr(self, 'name') or self.name is None:
1340 return ('', '', '', '')
1342 return fix_maintainer(self.name.strip())
1344 __all__.append('Maintainer')
1347 def get_or_set_maintainer(name, session=None):
1349 Returns Maintainer object for given maintainer name.
1351 If no matching maintainer name is found, a row is inserted.
1354 @param name: The maintainer name to add
1356 @type session: SQLAlchemy
1357 @param session: Optional SQL session object (a temporary one will be
1358 generated if not supplied). If not passed, a commit will be performed at
1359 the end of the function, otherwise the caller is responsible for commiting.
1360 A flush will be performed either way.
1363 @return: the Maintainer object for the given maintainer
1366 q = session.query(Maintainer).filter_by(name=name)
1369 except NoResultFound:
1370 maintainer = Maintainer()
1371 maintainer.name = name
1372 session.add(maintainer)
1373 session.commit_or_flush()
1378 __all__.append('get_or_set_maintainer')
1381 def get_maintainer(maintainer_id, session=None):
1383 Return the name of the maintainer behind C{maintainer_id} or None if that
1384 maintainer_id is invalid.
1386 @type maintainer_id: int
1387 @param maintainer_id: the id of the maintainer
1390 @return: the Maintainer with this C{maintainer_id}
1393 return session.query(Maintainer).get(maintainer_id)
1395 __all__.append('get_maintainer')
1397 ################################################################################
1399 class NewComment(object):
1400 def __init__(self, *args, **kwargs):
1404 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1406 __all__.append('NewComment')
1409 def has_new_comment(package, version, session=None):
1411 Returns true if the given combination of C{package}, C{version} has a comment.
1413 @type package: string
1414 @param package: name of the package
1416 @type version: string
1417 @param version: package version
1419 @type session: Session
1420 @param session: Optional SQLA session object (a temporary one will be
1421 generated if not supplied)
1427 q = session.query(NewComment)
1428 q = q.filter_by(package=package)
1429 q = q.filter_by(version=version)
1431 return bool(q.count() > 0)
1433 __all__.append('has_new_comment')
1436 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1438 Returns (possibly empty) list of NewComment objects for the given
1441 @type package: string (optional)
1442 @param package: name of the package
1444 @type version: string (optional)
1445 @param version: package version
1447 @type comment_id: int (optional)
1448 @param comment_id: An id of a comment
1450 @type session: Session
1451 @param session: Optional SQLA session object (a temporary one will be
1452 generated if not supplied)
1455 @return: A (possibly empty) list of NewComment objects will be returned
1458 q = session.query(NewComment)
1459 if package is not None: q = q.filter_by(package=package)
1460 if version is not None: q = q.filter_by(version=version)
1461 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1465 __all__.append('get_new_comments')
1467 ################################################################################
1469 class Override(object):
1470 def __init__(self, *args, **kwargs):
1474 return '<Override %s (%s)>' % (self.package, self.suite_id)
1476 __all__.append('Override')
1479 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1481 Returns Override object for the given parameters
1483 @type package: string
1484 @param package: The name of the package
1486 @type suite: string, list or None
1487 @param suite: The name of the suite (or suites if a list) to limit to. If
1488 None, don't limit. Defaults to None.
1490 @type component: string, list or None
1491 @param component: The name of the component (or components if a list) to
1492 limit to. If None, don't limit. Defaults to None.
1494 @type overridetype: string, list or None
1495 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1496 limit to. If None, don't limit. Defaults to None.
1498 @type session: Session
1499 @param session: Optional SQLA session object (a temporary one will be
1500 generated if not supplied)
1503 @return: A (possibly empty) list of Override objects will be returned
1506 q = session.query(Override)
1507 q = q.filter_by(package=package)
1509 if suite is not None:
1510 if not isinstance(suite, list): suite = [suite]
1511 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1513 if component is not None:
1514 if not isinstance(component, list): component = [component]
1515 q = q.join(Component).filter(Component.component_name.in_(component))
1517 if overridetype is not None:
1518 if not isinstance(overridetype, list): overridetype = [overridetype]
1519 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1523 __all__.append('get_override')
1526 ################################################################################
1528 class OverrideType(object):
1529 def __init__(self, *args, **kwargs):
1533 return '<OverrideType %s>' % self.overridetype
1535 __all__.append('OverrideType')
1538 def get_override_type(override_type, session=None):
1540 Returns OverrideType object for given C{override type}.
1542 @type override_type: string
1543 @param override_type: The name of the override type
1545 @type session: Session
1546 @param session: Optional SQLA session object (a temporary one will be
1547 generated if not supplied)
1550 @return: the database id for the given override type
1553 q = session.query(OverrideType).filter_by(overridetype=override_type)
1557 except NoResultFound:
1560 __all__.append('get_override_type')
1562 ################################################################################
1564 class PendingContentAssociation(object):
1565 def __init__(self, *args, **kwargs):
1569 return '<PendingContentAssociation %s>' % self.pca_id
1571 __all__.append('PendingContentAssociation')
1573 def insert_pending_content_paths(package, fullpaths, session=None):
1575 Make sure given paths are temporarily associated with given
1579 @param package: the package to associate with should have been read in from the binary control file
1580 @type fullpaths: list
1581 @param fullpaths: the list of paths of the file being associated with the binary
1582 @type session: SQLAlchemy session
1583 @param session: Optional SQLAlchemy session. If this is passed, the caller
1584 is responsible for ensuring a transaction has begun and committing the
1585 results or rolling back based on the result code. If not passed, a commit
1586 will be performed at the end of the function
1588 @return: True upon success, False if there is a problem
1591 privatetrans = False
1594 session = DBConn().session()
1598 arch = get_architecture(package['Architecture'], session)
1599 arch_id = arch.arch_id
1601 # Remove any already existing recorded files for this package
1602 q = session.query(PendingContentAssociation)
1603 q = q.filter_by(package=package['Package'])
1604 q = q.filter_by(version=package['Version'])
1605 q = q.filter_by(architecture=arch_id)
1610 for fullpath in fullpaths:
1611 (path, filename) = os.path.split(fullpath)
1613 if path.startswith( "./" ):
1616 filepath_id = get_or_set_contents_path_id(path, session)
1617 filename_id = get_or_set_contents_file_id(filename, session)
1619 pathcache[fullpath] = (filepath_id, filename_id)
1621 for fullpath, dat in pathcache.items():
1622 pca = PendingContentAssociation()
1623 pca.package = package['Package']
1624 pca.version = package['Version']
1625 pca.filepath_id = dat[0]
1626 pca.filename_id = dat[1]
1627 pca.architecture = arch_id
1630 # Only commit if we set up the session ourself
1638 except Exception, e:
1639 traceback.print_exc()
1641 # Only rollback if we set up the session ourself
1648 __all__.append('insert_pending_content_paths')
1650 ################################################################################
1652 class PolicyQueue(object):
1653 def __init__(self, *args, **kwargs):
1657 return '<PolicyQueue %s>' % self.queue_name
1659 __all__.append('PolicyQueue')
1662 def get_policy_queue(queuename, session=None):
1664 Returns PolicyQueue object for given C{queue name}
1666 @type queuename: string
1667 @param queuename: The name of the queue
1669 @type session: Session
1670 @param session: Optional SQLA session object (a temporary one will be
1671 generated if not supplied)
1674 @return: PolicyQueue object for the given queue
1677 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
1681 except NoResultFound:
1684 __all__.append('get_policy_queue')
1686 ################################################################################
1688 class Priority(object):
1689 def __init__(self, *args, **kwargs):
1692 def __eq__(self, val):
1693 if isinstance(val, str):
1694 return (self.priority == val)
1695 # This signals to use the normal comparison operator
1696 return NotImplemented
1698 def __ne__(self, val):
1699 if isinstance(val, str):
1700 return (self.priority != val)
1701 # This signals to use the normal comparison operator
1702 return NotImplemented
1705 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
1707 __all__.append('Priority')
1710 def get_priority(priority, session=None):
1712 Returns Priority object for given C{priority name}.
1714 @type priority: string
1715 @param priority: The name of the priority
1717 @type session: Session
1718 @param session: Optional SQLA session object (a temporary one will be
1719 generated if not supplied)
1722 @return: Priority object for the given priority
1725 q = session.query(Priority).filter_by(priority=priority)
1729 except NoResultFound:
1732 __all__.append('get_priority')
1735 def get_priorities(session=None):
1737 Returns dictionary of priority names -> id mappings
1739 @type session: Session
1740 @param session: Optional SQL session object (a temporary one will be
1741 generated if not supplied)
1744 @return: dictionary of priority names -> id mappings
1748 q = session.query(Priority)
1750 ret[x.priority] = x.priority_id
1754 __all__.append('get_priorities')
1756 ################################################################################
1758 class Section(object):
1759 def __init__(self, *args, **kwargs):
1762 def __eq__(self, val):
1763 if isinstance(val, str):
1764 return (self.section == val)
1765 # This signals to use the normal comparison operator
1766 return NotImplemented
1768 def __ne__(self, val):
1769 if isinstance(val, str):
1770 return (self.section != val)
1771 # This signals to use the normal comparison operator
1772 return NotImplemented
1775 return '<Section %s>' % self.section
1777 __all__.append('Section')
1780 def get_section(section, session=None):
1782 Returns Section object for given C{section name}.
1784 @type section: string
1785 @param section: The name of the section
1787 @type session: Session
1788 @param session: Optional SQLA session object (a temporary one will be
1789 generated if not supplied)
1792 @return: Section object for the given section name
1795 q = session.query(Section).filter_by(section=section)
1799 except NoResultFound:
1802 __all__.append('get_section')
1805 def get_sections(session=None):
1807 Returns dictionary of section names -> id mappings
1809 @type session: Session
1810 @param session: Optional SQL session object (a temporary one will be
1811 generated if not supplied)
1814 @return: dictionary of section names -> id mappings
1818 q = session.query(Section)
1820 ret[x.section] = x.section_id
1824 __all__.append('get_sections')
1826 ################################################################################
1828 class DBSource(object):
1829 def __init__(self, *args, **kwargs):
1833 return '<DBSource %s (%s)>' % (self.source, self.version)
1835 __all__.append('DBSource')
1838 def source_exists(source, source_version, suites = ["any"], session=None):
1840 Ensure that source exists somewhere in the archive for the binary
1841 upload being processed.
1842 1. exact match => 1.0-3
1843 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1845 @type package: string
1846 @param package: package source name
1848 @type source_version: string
1849 @param source_version: expected source version
1852 @param suites: list of suites to check in, default I{any}
1854 @type session: Session
1855 @param session: Optional SQLA session object (a temporary one will be
1856 generated if not supplied)
1859 @return: returns 1 if a source with expected version is found, otherwise 0
1866 for suite in suites:
1867 q = session.query(DBSource).filter_by(source=source)
1869 # source must exist in suite X, or in some other suite that's
1870 # mapped to X, recursively... silent-maps are counted too,
1871 # unreleased-maps aren't.
1872 maps = cnf.ValueList("SuiteMappings")[:]
1874 maps = [ m.split() for m in maps ]
1875 maps = [ (x[1], x[2]) for x in maps
1876 if x[0] == "map" or x[0] == "silent-map" ]
1879 if x[1] in s and x[0] not in s:
1882 q = q.join(SrcAssociation).join(Suite)
1883 q = q.filter(Suite.suite_name.in_(s))
1885 # Reduce the query results to a list of version numbers
1886 ql = [ j.version for j in q.all() ]
1889 if source_version in ql:
1893 from daklib.regexes import re_bin_only_nmu
1894 orig_source_version = re_bin_only_nmu.sub('', source_version)
1895 if orig_source_version in ql:
1898 # No source found so return not ok
1903 __all__.append('source_exists')
1906 def get_suites_source_in(source, session=None):
1908 Returns list of Suite objects which given C{source} name is in
1911 @param source: DBSource package name to search for
1914 @return: list of Suite objects for the given source
1917 return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
1919 __all__.append('get_suites_source_in')
1922 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
1924 Returns list of DBSource objects for given C{source} name and other parameters
1927 @param source: DBSource package name to search for
1929 @type source: str or None
1930 @param source: DBSource version name to search for or None if not applicable
1932 @type dm_upload_allowed: bool
1933 @param dm_upload_allowed: If None, no effect. If True or False, only
1934 return packages with that dm_upload_allowed setting
1936 @type session: Session
1937 @param session: Optional SQL session object (a temporary one will be
1938 generated if not supplied)
1941 @return: list of DBSource objects for the given name (may be empty)
1944 q = session.query(DBSource).filter_by(source=source)
1946 if version is not None:
1947 q = q.filter_by(version=version)
1949 if dm_upload_allowed is not None:
1950 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
1954 __all__.append('get_sources_from_name')
1957 def get_source_in_suite(source, suite, session=None):
1959 Returns list of DBSource objects for a combination of C{source} and C{suite}.
1961 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
1962 - B{suite} - a suite name, eg. I{unstable}
1964 @type source: string
1965 @param source: source package name
1968 @param suite: the suite name
1971 @return: the version for I{source} in I{suite}
1975 q = session.query(SrcAssociation)
1976 q = q.join('source').filter_by(source=source)
1977 q = q.join('suite').filter_by(suite_name=suite)
1980 return q.one().source
1981 except NoResultFound:
1984 __all__.append('get_source_in_suite')
1986 ################################################################################
1989 def add_dsc_to_db(u, filename, session=None):
1990 entry = u.pkg.files[filename]
1994 source.source = u.pkg.dsc["source"]
1995 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
1996 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
1997 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
1998 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
1999 source.install_date = datetime.now().date()
2001 dsc_component = entry["component"]
2002 dsc_location_id = entry["location id"]
2004 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2006 # Set up a new poolfile if necessary
2007 if not entry.has_key("files id") or not entry["files id"]:
2008 filename = entry["pool name"] + filename
2009 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2011 pfs.append(poolfile)
2012 entry["files id"] = poolfile.file_id
2014 source.poolfile_id = entry["files id"]
2018 for suite_name in u.pkg.changes["distribution"].keys():
2019 sa = SrcAssociation()
2020 sa.source_id = source.source_id
2021 sa.suite_id = get_suite(suite_name).suite_id
2026 # Add the source files to the DB (files and dsc_files)
2028 dscfile.source_id = source.source_id
2029 dscfile.poolfile_id = entry["files id"]
2030 session.add(dscfile)
2032 for dsc_file, dentry in u.pkg.dsc_files.items():
2034 df.source_id = source.source_id
2036 # If the .orig tarball is already in the pool, it's
2037 # files id is stored in dsc_files by check_dsc().
2038 files_id = dentry.get("files id", None)
2040 # Find the entry in the files hash
2041 # TODO: Bail out here properly
2043 for f, e in u.pkg.files.items():
2048 if files_id is None:
2049 filename = dfentry["pool name"] + dsc_file
2051 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2052 # FIXME: needs to check for -1/-2 and or handle exception
2053 if found and obj is not None:
2054 files_id = obj.file_id
2057 # If still not found, add it
2058 if files_id is None:
2059 # HACK: Force sha1sum etc into dentry
2060 dentry["sha1sum"] = dfentry["sha1sum"]
2061 dentry["sha256sum"] = dfentry["sha256sum"]
2062 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2063 pfs.append(poolfile)
2064 files_id = poolfile.file_id
2066 df.poolfile_id = files_id
2071 # Add the src_uploaders to the DB
2072 uploader_ids = [source.maintainer_id]
2073 if u.pkg.dsc.has_key("uploaders"):
2074 for up in u.pkg.dsc["uploaders"].split(","):
2076 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2079 for up in uploader_ids:
2080 if added_ids.has_key(up):
2081 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
2087 su.maintainer_id = up
2088 su.source_id = source.source_id
2093 return dsc_component, dsc_location_id, pfs
2095 __all__.append('add_dsc_to_db')
2098 def add_deb_to_db(u, filename, session=None):
2100 Contrary to what you might expect, this routine deals with both
2101 debs and udebs. That info is in 'dbtype', whilst 'type' is
2102 'deb' for both of them
2105 entry = u.pkg.files[filename]
2108 bin.package = entry["package"]
2109 bin.version = entry["version"]
2110 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2111 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2112 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2113 bin.binarytype = entry["dbtype"]
2116 filename = entry["pool name"] + filename
2117 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2118 if not entry.get("location id", None):
2119 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2121 if entry.get("files id", None):
2122 poolfile = get_poolfile_by_id(bin.poolfile_id)
2123 bin.poolfile_id = entry["files id"]
2125 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2126 bin.poolfile_id = entry["files id"] = poolfile.file_id
2129 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2130 if len(bin_sources) != 1:
2131 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2132 (bin.package, bin.version, bin.architecture.arch_string,
2133 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2135 bin.source_id = bin_sources[0].source_id
2137 # Add and flush object so it has an ID
2141 # Add BinAssociations
2142 for suite_name in u.pkg.changes["distribution"].keys():
2143 ba = BinAssociation()
2144 ba.binary_id = bin.binary_id
2145 ba.suite_id = get_suite(suite_name).suite_id
2150 # Deal with contents - disabled for now
2151 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2153 # print "REJECT\nCould not determine contents of package %s" % bin.package
2154 # session.rollback()
2155 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2159 __all__.append('add_deb_to_db')
2161 ################################################################################
2163 class SourceACL(object):
2164 def __init__(self, *args, **kwargs):
2168 return '<SourceACL %s>' % self.source_acl_id
2170 __all__.append('SourceACL')
2172 ################################################################################
2174 class SrcAssociation(object):
2175 def __init__(self, *args, **kwargs):
2179 return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
2181 __all__.append('SrcAssociation')
2183 ################################################################################
2185 class SrcFormat(object):
2186 def __init__(self, *args, **kwargs):
2190 return '<SrcFormat %s>' % (self.format_name)
2192 __all__.append('SrcFormat')
2194 ################################################################################
2196 class SrcUploader(object):
2197 def __init__(self, *args, **kwargs):
2201 return '<SrcUploader %s>' % self.uploader_id
2203 __all__.append('SrcUploader')
2205 ################################################################################
2207 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2208 ('SuiteID', 'suite_id'),
2209 ('Version', 'version'),
2210 ('Origin', 'origin'),
2212 ('Description', 'description'),
2213 ('Untouchable', 'untouchable'),
2214 ('Announce', 'announce'),
2215 ('Codename', 'codename'),
2216 ('OverrideCodename', 'overridecodename'),
2217 ('ValidTime', 'validtime'),
2218 ('Priority', 'priority'),
2219 ('NotAutomatic', 'notautomatic'),
2220 ('CopyChanges', 'copychanges'),
2221 ('CopyDotDak', 'copydotdak'),
2222 ('CommentsDir', 'commentsdir'),
2223 ('OverrideSuite', 'overridesuite'),
2224 ('ChangelogBase', 'changelogbase')]
2227 class Suite(object):
2228 def __init__(self, *args, **kwargs):
2232 return '<Suite %s>' % self.suite_name
2234 def __eq__(self, val):
2235 if isinstance(val, str):
2236 return (self.suite_name == val)
2237 # This signals to use the normal comparison operator
2238 return NotImplemented
2240 def __ne__(self, val):
2241 if isinstance(val, str):
2242 return (self.suite_name != val)
2243 # This signals to use the normal comparison operator
2244 return NotImplemented
2248 for disp, field in SUITE_FIELDS:
2249 val = getattr(self, field, None)
2251 ret.append("%s: %s" % (disp, val))
2253 return "\n".join(ret)
2255 __all__.append('Suite')
2258 def get_suite_architecture(suite, architecture, session=None):
2260 Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
2264 @param suite: Suite name to search for
2266 @type architecture: str
2267 @param architecture: Architecture name to search for
2269 @type session: Session
2270 @param session: Optional SQL session object (a temporary one will be
2271 generated if not supplied)
2273 @rtype: SuiteArchitecture
2274 @return: the SuiteArchitecture object or None
2277 q = session.query(SuiteArchitecture)
2278 q = q.join(Architecture).filter_by(arch_string=architecture)
2279 q = q.join(Suite).filter_by(suite_name=suite)
2283 except NoResultFound:
2286 __all__.append('get_suite_architecture')
2289 def get_suite(suite, session=None):
2291 Returns Suite object for given C{suite name}.
2294 @param suite: The name of the suite
2296 @type session: Session
2297 @param session: Optional SQLA session object (a temporary one will be
2298 generated if not supplied)
2301 @return: Suite object for the requested suite name (None if not present)
2304 q = session.query(Suite).filter_by(suite_name=suite)
2308 except NoResultFound:
2311 __all__.append('get_suite')
2313 ################################################################################
2315 class SuiteArchitecture(object):
2316 def __init__(self, *args, **kwargs):
2320 return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
2322 __all__.append('SuiteArchitecture')
2325 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2327 Returns list of Architecture objects for given C{suite} name
2330 @param source: Suite name to search for
2332 @type skipsrc: boolean
2333 @param skipsrc: Whether to skip returning the 'source' architecture entry
2336 @type skipall: boolean
2337 @param skipall: Whether to skip returning the 'all' architecture entry
2340 @type session: Session
2341 @param session: Optional SQL session object (a temporary one will be
2342 generated if not supplied)
2345 @return: list of Architecture objects for the given name (may be empty)
2348 q = session.query(Architecture)
2349 q = q.join(SuiteArchitecture)
2350 q = q.join(Suite).filter_by(suite_name=suite)
2353 q = q.filter(Architecture.arch_string != 'source')
2356 q = q.filter(Architecture.arch_string != 'all')
2358 q = q.order_by('arch_string')
2362 __all__.append('get_suite_architectures')
2364 ################################################################################
2366 class SuiteSrcFormat(object):
2367 def __init__(self, *args, **kwargs):
2371 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2373 __all__.append('SuiteSrcFormat')
2376 def get_suite_src_formats(suite, session=None):
2378 Returns list of allowed SrcFormat for C{suite}.
2381 @param suite: Suite name to search for
2383 @type session: Session
2384 @param session: Optional SQL session object (a temporary one will be
2385 generated if not supplied)
2388 @return: the list of allowed source formats for I{suite}
2391 q = session.query(SrcFormat)
2392 q = q.join(SuiteSrcFormat)
2393 q = q.join(Suite).filter_by(suite_name=suite)
2394 q = q.order_by('format_name')
2398 __all__.append('get_suite_src_formats')
2400 ################################################################################
2403 def __init__(self, *args, **kwargs):
2406 def __eq__(self, val):
2407 if isinstance(val, str):
2408 return (self.uid == val)
2409 # This signals to use the normal comparison operator
2410 return NotImplemented
2412 def __ne__(self, val):
2413 if isinstance(val, str):
2414 return (self.uid != val)
2415 # This signals to use the normal comparison operator
2416 return NotImplemented
2419 return '<Uid %s (%s)>' % (self.uid, self.name)
2421 __all__.append('Uid')
2424 def add_database_user(uidname, session=None):
2426 Adds a database user
2428 @type uidname: string
2429 @param uidname: The uid of the user to add
2431 @type session: SQLAlchemy
2432 @param session: Optional SQL session object (a temporary one will be
2433 generated if not supplied). If not passed, a commit will be performed at
2434 the end of the function, otherwise the caller is responsible for commiting.
2437 @return: the uid object for the given uidname
2440 session.execute("CREATE USER :uid", {'uid': uidname})
2441 session.commit_or_flush()
2443 __all__.append('add_database_user')
2446 def get_or_set_uid(uidname, session=None):
2448 Returns uid object for given uidname.
2450 If no matching uidname is found, a row is inserted.
2452 @type uidname: string
2453 @param uidname: The uid to add
2455 @type session: SQLAlchemy
2456 @param session: Optional SQL session object (a temporary one will be
2457 generated if not supplied). If not passed, a commit will be performed at
2458 the end of the function, otherwise the caller is responsible for commiting.
2461 @return: the uid object for the given uidname
2464 q = session.query(Uid).filter_by(uid=uidname)
2468 except NoResultFound:
2472 session.commit_or_flush()
2477 __all__.append('get_or_set_uid')
2480 def get_uid_from_fingerprint(fpr, session=None):
2481 q = session.query(Uid)
2482 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2486 except NoResultFound:
2489 __all__.append('get_uid_from_fingerprint')
2491 ################################################################################
2493 class UploadBlock(object):
2494 def __init__(self, *args, **kwargs):
2498 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2500 __all__.append('UploadBlock')
2502 ################################################################################
2504 class DBConn(object):
2506 database module init.
2510 def __init__(self, *args, **kwargs):
2511 self.__dict__ = self.__shared_state
2513 if not getattr(self, 'initialised', False):
2514 self.initialised = True
2515 self.debug = kwargs.has_key('debug')
2518 def __setuptables(self):
2527 'build_queue_files',
2530 'content_associations',
2531 'content_file_names',
2532 'content_file_paths',
2533 'changes_pending_binaries',
2534 'changes_pending_files',
2535 'changes_pending_files_map',
2536 'changes_pending_source',
2537 'changes_pending_source_files',
2538 'changes_pool_files',
2550 'pending_content_associations',
2560 'suite_architectures',
2561 'suite_src_formats',
2562 'suite_build_queue_copy',
2567 for table_name in tables:
2568 table = Table(table_name, self.db_meta, autoload=True)
2569 setattr(self, 'tbl_%s' % table_name, table)
2571 def __setupmappers(self):
2572 mapper(Architecture, self.tbl_architecture,
2573 properties = dict(arch_id = self.tbl_architecture.c.id))
2575 mapper(Archive, self.tbl_archive,
2576 properties = dict(archive_id = self.tbl_archive.c.id,
2577 archive_name = self.tbl_archive.c.name))
2579 mapper(BinAssociation, self.tbl_bin_associations,
2580 properties = dict(ba_id = self.tbl_bin_associations.c.id,
2581 suite_id = self.tbl_bin_associations.c.suite,
2582 suite = relation(Suite),
2583 binary_id = self.tbl_bin_associations.c.bin,
2584 binary = relation(DBBinary)))
2586 mapper(BuildQueue, self.tbl_build_queue,
2587 properties = dict(queue_id = self.tbl_build_queue.c.id))
2589 mapper(BuildQueueFile, self.tbl_build_queue_files,
2590 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
2591 poolfile = relation(PoolFile, backref='buildqueueinstances')))
2593 mapper(DBBinary, self.tbl_binaries,
2594 properties = dict(binary_id = self.tbl_binaries.c.id,
2595 package = self.tbl_binaries.c.package,
2596 version = self.tbl_binaries.c.version,
2597 maintainer_id = self.tbl_binaries.c.maintainer,
2598 maintainer = relation(Maintainer),
2599 source_id = self.tbl_binaries.c.source,
2600 source = relation(DBSource),
2601 arch_id = self.tbl_binaries.c.architecture,
2602 architecture = relation(Architecture),
2603 poolfile_id = self.tbl_binaries.c.file,
2604 poolfile = relation(PoolFile),
2605 binarytype = self.tbl_binaries.c.type,
2606 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2607 fingerprint = relation(Fingerprint),
2608 install_date = self.tbl_binaries.c.install_date,
2609 binassociations = relation(BinAssociation,
2610 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
2612 mapper(BinaryACL, self.tbl_binary_acl,
2613 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2615 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2616 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2617 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2618 architecture = relation(Architecture)))
2620 mapper(Component, self.tbl_component,
2621 properties = dict(component_id = self.tbl_component.c.id,
2622 component_name = self.tbl_component.c.name))
2624 mapper(DBConfig, self.tbl_config,
2625 properties = dict(config_id = self.tbl_config.c.id))
2627 mapper(DSCFile, self.tbl_dsc_files,
2628 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2629 source_id = self.tbl_dsc_files.c.source,
2630 source = relation(DBSource),
2631 poolfile_id = self.tbl_dsc_files.c.file,
2632 poolfile = relation(PoolFile)))
2634 mapper(PoolFile, self.tbl_files,
2635 properties = dict(file_id = self.tbl_files.c.id,
2636 filesize = self.tbl_files.c.size,
2637 location_id = self.tbl_files.c.location,
2638 location = relation(Location)))
2640 mapper(Fingerprint, self.tbl_fingerprint,
2641 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2642 uid_id = self.tbl_fingerprint.c.uid,
2643 uid = relation(Uid),
2644 keyring_id = self.tbl_fingerprint.c.keyring,
2645 keyring = relation(Keyring),
2646 source_acl = relation(SourceACL),
2647 binary_acl = relation(BinaryACL)))
2649 mapper(Keyring, self.tbl_keyrings,
2650 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2651 keyring_id = self.tbl_keyrings.c.id))
2653 mapper(DBChange, self.tbl_changes,
2654 properties = dict(change_id = self.tbl_changes.c.id,
2655 poolfiles = relation(PoolFile,
2656 secondary=self.tbl_changes_pool_files,
2657 backref="changeslinks"),
2658 files = relation(ChangePendingFile,
2659 secondary=self.tbl_changes_pending_files_map,
2660 backref="changesfile"),
2661 in_queue_id = self.tbl_changes.c.in_queue,
2662 in_queue = relation(PolicyQueue,
2663 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
2664 approved_for_id = self.tbl_changes.c.approved_for))
2666 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
2667 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
2669 mapper(ChangePendingFile, self.tbl_changes_pending_files,
2670 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id))
2672 mapper(ChangePendingSource, self.tbl_changes_pending_source,
2673 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
2674 change = relation(DBChange),
2675 maintainer = relation(Maintainer,
2676 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
2677 changedby = relation(Maintainer,
2678 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
2679 fingerprint = relation(Fingerprint),
2680 source_files = relation(ChangePendingFile,
2681 secondary=self.tbl_changes_pending_source_files,
2682 backref="pending_sources")))
2683 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
2684 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
2685 keyring = relation(Keyring, backref="keyring_acl_map"),
2686 architecture = relation(Architecture)))
2688 mapper(Location, self.tbl_location,
2689 properties = dict(location_id = self.tbl_location.c.id,
2690 component_id = self.tbl_location.c.component,
2691 component = relation(Component),
2692 archive_id = self.tbl_location.c.archive,
2693 archive = relation(Archive),
2694 archive_type = self.tbl_location.c.type))
2696 mapper(Maintainer, self.tbl_maintainer,
2697 properties = dict(maintainer_id = self.tbl_maintainer.c.id))
2699 mapper(NewComment, self.tbl_new_comments,
2700 properties = dict(comment_id = self.tbl_new_comments.c.id))
2702 mapper(Override, self.tbl_override,
2703 properties = dict(suite_id = self.tbl_override.c.suite,
2704 suite = relation(Suite),
2705 component_id = self.tbl_override.c.component,
2706 component = relation(Component),
2707 priority_id = self.tbl_override.c.priority,
2708 priority = relation(Priority),
2709 section_id = self.tbl_override.c.section,
2710 section = relation(Section),
2711 overridetype_id = self.tbl_override.c.type,
2712 overridetype = relation(OverrideType)))
2714 mapper(OverrideType, self.tbl_override_type,
2715 properties = dict(overridetype = self.tbl_override_type.c.type,
2716 overridetype_id = self.tbl_override_type.c.id))
2718 mapper(PolicyQueue, self.tbl_policy_queue,
2719 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
2721 mapper(Priority, self.tbl_priority,
2722 properties = dict(priority_id = self.tbl_priority.c.id))
2724 mapper(Section, self.tbl_section,
2725 properties = dict(section_id = self.tbl_section.c.id))
2727 mapper(DBSource, self.tbl_source,
2728 properties = dict(source_id = self.tbl_source.c.id,
2729 version = self.tbl_source.c.version,
2730 maintainer_id = self.tbl_source.c.maintainer,
2731 maintainer = relation(Maintainer,
2732 primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
2733 poolfile_id = self.tbl_source.c.file,
2734 poolfile = relation(PoolFile),
2735 fingerprint_id = self.tbl_source.c.sig_fpr,
2736 fingerprint = relation(Fingerprint),
2737 changedby_id = self.tbl_source.c.changedby,
2738 changedby = relation(Maintainer,
2739 primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
2740 srcfiles = relation(DSCFile,
2741 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
2742 srcassociations = relation(SrcAssociation,
2743 primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)),
2744 srcuploaders = relation(SrcUploader)))
2746 mapper(SourceACL, self.tbl_source_acl,
2747 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
2749 mapper(SrcAssociation, self.tbl_src_associations,
2750 properties = dict(sa_id = self.tbl_src_associations.c.id,
2751 suite_id = self.tbl_src_associations.c.suite,
2752 suite = relation(Suite),
2753 source_id = self.tbl_src_associations.c.source,
2754 source = relation(DBSource)))
2756 mapper(SrcFormat, self.tbl_src_format,
2757 properties = dict(src_format_id = self.tbl_src_format.c.id,
2758 format_name = self.tbl_src_format.c.format_name))
2760 mapper(SrcUploader, self.tbl_src_uploaders,
2761 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
2762 source_id = self.tbl_src_uploaders.c.source,
2763 source = relation(DBSource,
2764 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
2765 maintainer_id = self.tbl_src_uploaders.c.maintainer,
2766 maintainer = relation(Maintainer,
2767 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
2769 mapper(Suite, self.tbl_suite,
2770 properties = dict(suite_id = self.tbl_suite.c.id,
2771 policy_queue = relation(PolicyQueue),
2772 copy_queues = relation(BuildQueue, secondary=self.tbl_suite_build_queue_copy)))
2774 mapper(SuiteArchitecture, self.tbl_suite_architectures,
2775 properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
2776 suite = relation(Suite, backref='suitearchitectures'),
2777 arch_id = self.tbl_suite_architectures.c.architecture,
2778 architecture = relation(Architecture)))
2780 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
2781 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
2782 suite = relation(Suite, backref='suitesrcformats'),
2783 src_format_id = self.tbl_suite_src_formats.c.src_format,
2784 src_format = relation(SrcFormat)))
2786 mapper(Uid, self.tbl_uid,
2787 properties = dict(uid_id = self.tbl_uid.c.id,
2788 fingerprint = relation(Fingerprint)))
2790 mapper(UploadBlock, self.tbl_upload_blocks,
2791 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
2792 fingerprint = relation(Fingerprint, backref="uploadblocks"),
2793 uid = relation(Uid, backref="uploadblocks")))
2795 ## Connection functions
2796 def __createconn(self):
2797 from config import Config
2801 connstr = "postgres://%s" % cnf["DB::Host"]
2802 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2803 connstr += ":%s" % cnf["DB::Port"]
2804 connstr += "/%s" % cnf["DB::Name"]
2807 connstr = "postgres:///%s" % cnf["DB::Name"]
2808 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2809 connstr += "?port=%s" % cnf["DB::Port"]
2811 self.db_pg = create_engine(connstr, echo=self.debug)
2812 self.db_meta = MetaData()
2813 self.db_meta.bind = self.db_pg
2814 self.db_smaker = sessionmaker(bind=self.db_pg,
2818 self.__setuptables()
2819 self.__setupmappers()
2822 return self.db_smaker()
2824 __all__.append('DBConn')