5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
40 from datetime import datetime
42 from inspect import getargspec
45 from sqlalchemy import create_engine, Table, MetaData
46 from sqlalchemy.orm import sessionmaker, mapper, relation
47 from sqlalchemy import types as sqltypes
49 # Don't remove this, we re-export the exceptions to scripts which import us
50 from sqlalchemy.exc import *
51 from sqlalchemy.orm.exc import NoResultFound
53 from config import Config
54 from textutils import fix_maintainer
56 ################################################################################
58 # Patch in support for the debversion field type so that it works during
61 class DebVersion(sqltypes.Text):
62 def get_col_spec(self):
65 sa_major_version = sqlalchemy.__version__[0:3]
66 if sa_major_version == "0.5":
67 from sqlalchemy.databases import postgres
68 postgres.ischema_names['debversion'] = DebVersion
70 raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
72 ################################################################################
74 __all__ = ['IntegrityError', 'SQLAlchemyError']
76 ################################################################################
78 def session_wrapper(fn):
80 Wrapper around common ".., session=None):" handling. If the wrapped
81 function is called without passing 'session', we create a local one
82 and destroy it when the function ends.
84 Also attaches a commit_or_flush method to the session; if we created a
85 local session, this is a synonym for session.commit(), otherwise it is a
86 synonym for session.flush().
89 def wrapped(*args, **kwargs):
90 private_transaction = False
92 # Find the session object
93 session = kwargs.get('session')
96 if len(args) <= len(getargspec(fn)[0]) - 1:
97 # No session specified as last argument or in kwargs
98 private_transaction = True
99 session = kwargs['session'] = DBConn().session()
101 # Session is last argument in args
105 session = args[-1] = DBConn().session()
106 private_transaction = True
108 if private_transaction:
109 session.commit_or_flush = session.commit
111 session.commit_or_flush = session.flush
114 return fn(*args, **kwargs)
116 if private_transaction:
117 # We created a session; close it.
120 wrapped.__doc__ = fn.__doc__
121 wrapped.func_name = fn.func_name
125 __all__.append('session_wrapper')
127 ################################################################################
129 class Architecture(object):
130 def __init__(self, *args, **kwargs):
133 def __eq__(self, val):
134 if isinstance(val, str):
135 return (self.arch_string== val)
136 # This signals to use the normal comparison operator
137 return NotImplemented
139 def __ne__(self, val):
140 if isinstance(val, str):
141 return (self.arch_string != val)
142 # This signals to use the normal comparison operator
143 return NotImplemented
146 return '<Architecture %s>' % self.arch_string
148 __all__.append('Architecture')
151 def get_architecture(architecture, session=None):
153 Returns database id for given C{architecture}.
155 @type architecture: string
156 @param architecture: The name of the architecture
158 @type session: Session
159 @param session: Optional SQLA session object (a temporary one will be
160 generated if not supplied)
163 @return: Architecture object for the given arch (None if not present)
166 q = session.query(Architecture).filter_by(arch_string=architecture)
170 except NoResultFound:
173 __all__.append('get_architecture')
176 def get_architecture_suites(architecture, session=None):
178 Returns list of Suite objects for given C{architecture} name
181 @param source: Architecture name to search for
183 @type session: Session
184 @param session: Optional SQL session object (a temporary one will be
185 generated if not supplied)
188 @return: list of Suite objects for the given name (may be empty)
191 q = session.query(Suite)
192 q = q.join(SuiteArchitecture)
193 q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
199 __all__.append('get_architecture_suites')
201 ################################################################################
203 class Archive(object):
204 def __init__(self, *args, **kwargs):
208 return '<Archive %s>' % self.archive_name
210 __all__.append('Archive')
213 def get_archive(archive, session=None):
215 returns database id for given C{archive}.
217 @type archive: string
218 @param archive: the name of the arhive
220 @type session: Session
221 @param session: Optional SQLA session object (a temporary one will be
222 generated if not supplied)
225 @return: Archive object for the given name (None if not present)
228 archive = archive.lower()
230 q = session.query(Archive).filter_by(archive_name=archive)
234 except NoResultFound:
237 __all__.append('get_archive')
239 ################################################################################
241 class BinAssociation(object):
242 def __init__(self, *args, **kwargs):
246 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
248 __all__.append('BinAssociation')
250 ################################################################################
252 class BinContents(object):
253 def __init__(self, *args, **kwargs):
257 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
259 __all__.append('BinContents')
261 ################################################################################
263 class DBBinary(object):
264 def __init__(self, *args, **kwargs):
268 return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
270 __all__.append('DBBinary')
273 def get_suites_binary_in(package, session=None):
275 Returns list of Suite objects which given C{package} name is in
278 @param source: DBBinary package name to search for
281 @return: list of Suite objects for the given package
284 return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
286 __all__.append('get_suites_binary_in')
289 def get_binary_from_id(binary_id, session=None):
291 Returns DBBinary object for given C{id}
294 @param binary_id: Id of the required binary
296 @type session: Session
297 @param session: Optional SQLA session object (a temporary one will be
298 generated if not supplied)
301 @return: DBBinary object for the given binary (None if not present)
304 q = session.query(DBBinary).filter_by(binary_id=binary_id)
308 except NoResultFound:
311 __all__.append('get_binary_from_id')
314 def get_binaries_from_name(package, version=None, architecture=None, session=None):
316 Returns list of DBBinary objects for given C{package} name
319 @param package: DBBinary package name to search for
321 @type version: str or None
322 @param version: Version to search for (or None)
324 @type package: str, list or None
325 @param package: Architectures to limit to (or None if no limit)
327 @type session: Session
328 @param session: Optional SQL session object (a temporary one will be
329 generated if not supplied)
332 @return: list of DBBinary objects for the given name (may be empty)
335 q = session.query(DBBinary).filter_by(package=package)
337 if version is not None:
338 q = q.filter_by(version=version)
340 if architecture is not None:
341 if not isinstance(architecture, list):
342 architecture = [architecture]
343 q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
349 __all__.append('get_binaries_from_name')
352 def get_binaries_from_source_id(source_id, session=None):
354 Returns list of DBBinary objects for given C{source_id}
357 @param source_id: source_id to search for
359 @type session: Session
360 @param session: Optional SQL session object (a temporary one will be
361 generated if not supplied)
364 @return: list of DBBinary objects for the given name (may be empty)
367 return session.query(DBBinary).filter_by(source_id=source_id).all()
369 __all__.append('get_binaries_from_source_id')
372 def get_binary_from_name_suite(package, suitename, session=None):
373 ### For dak examine-package
374 ### XXX: Doesn't use object API yet
376 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
377 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
378 WHERE b.package=:package
380 AND fi.location = l.id
381 AND l.component = c.id
384 AND su.suite_name=:suitename
385 ORDER BY b.version DESC"""
387 return session.execute(sql, {'package': package, 'suitename': suitename})
389 __all__.append('get_binary_from_name_suite')
392 def get_binary_components(package, suitename, arch, session=None):
393 # Check for packages that have moved from one component to another
394 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
395 WHERE b.package=:package AND s.suite_name=:suitename
396 AND (a.arch_string = :arch OR a.arch_string = 'all')
397 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
398 AND f.location = l.id
399 AND l.component = c.id
402 vals = {'package': package, 'suitename': suitename, 'arch': arch}
404 return session.execute(query, vals)
406 __all__.append('get_binary_components')
408 ################################################################################
410 class BinaryACL(object):
411 def __init__(self, *args, **kwargs):
415 return '<BinaryACL %s>' % self.binary_acl_id
417 __all__.append('BinaryACL')
419 ################################################################################
421 class BinaryACLMap(object):
422 def __init__(self, *args, **kwargs):
426 return '<BinaryACLMap %s>' % self.binary_acl_map_id
428 __all__.append('BinaryACLMap')
430 ################################################################################
432 class BuildQueue(object):
433 def __init__(self, *args, **kwargs):
437 return '<BuildQueue %s>' % self.queue_name
439 def add_file_from_pool(self, poolfile):
440 """Copies a file into the pool. Assumes that the PoolFile object is
441 attached to the same SQLAlchemy session as the Queue object is.
443 The caller is responsible for committing after calling this function."""
444 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
446 # Check if we have a file of this name or this ID already
447 for f in self.queuefiles:
448 if f.fileid is not None and f.fileid == poolfile.file_id or \
449 f.poolfile.filename == poolfile_basename:
450 # In this case, update the BuildQueueFile entry so we
451 # don't remove it too early
452 f.lastused = datetime.now()
453 DBConn().session().object_session(pf).add(f)
456 # Prepare BuildQueueFile object
457 qf = BuildQueueFile()
458 qf.build_queue_id = self.queue_id
459 qf.lastused = datetime.now()
460 qf.filename = poolfile_basename
462 targetpath = poolfile.fullpath
463 queuepath = os.path.join(self.path, poolfile_basename)
467 # We need to copy instead of symlink
469 utils.copy(targetpath, queuepath)
470 # NULL in the fileid field implies a copy
473 os.symlink(targetpath, queuepath)
474 qf.fileid = poolfile.file_id
478 # Get the same session as the PoolFile is using and add the qf to it
479 DBConn().session().object_session(poolfile).add(qf)
484 __all__.append('BuildQueue')
487 def get_build_queue(queuename, session=None):
489 Returns BuildQueue object for given C{queue name}, creating it if it does not
492 @type queuename: string
493 @param queuename: The name of the queue
495 @type session: Session
496 @param session: Optional SQLA session object (a temporary one will be
497 generated if not supplied)
500 @return: BuildQueue object for the given queue
503 q = session.query(BuildQueue).filter_by(queue_name=queuename)
507 except NoResultFound:
510 __all__.append('get_build_queue')
512 ################################################################################
514 class BuildQueueFile(object):
515 def __init__(self, *args, **kwargs):
519 return '<BuildQueueFile %s (%s)>' % (self.filename, self.queue_id)
521 __all__.append('BuildQueueFile')
523 ################################################################################
525 class ChangePendingBinary(object):
526 def __init__(self, *args, **kwargs):
530 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
532 __all__.append('ChangePendingBinary')
534 ################################################################################
536 class ChangePendingFile(object):
537 def __init__(self, *args, **kwargs):
541 return '<ChangePendingFile %s>' % self.change_pending_file_id
543 __all__.append('ChangePendingFile')
545 ################################################################################
547 class ChangePendingSource(object):
548 def __init__(self, *args, **kwargs):
552 return '<ChangePendingSource %s>' % self.change_pending_source_id
554 __all__.append('ChangePendingSource')
556 ################################################################################
558 class Component(object):
559 def __init__(self, *args, **kwargs):
562 def __eq__(self, val):
563 if isinstance(val, str):
564 return (self.component_name == val)
565 # This signals to use the normal comparison operator
566 return NotImplemented
568 def __ne__(self, val):
569 if isinstance(val, str):
570 return (self.component_name != val)
571 # This signals to use the normal comparison operator
572 return NotImplemented
575 return '<Component %s>' % self.component_name
578 __all__.append('Component')
581 def get_component(component, session=None):
583 Returns database id for given C{component}.
585 @type component: string
586 @param component: The name of the override type
589 @return: the database id for the given component
592 component = component.lower()
594 q = session.query(Component).filter_by(component_name=component)
598 except NoResultFound:
601 __all__.append('get_component')
603 ################################################################################
605 class DBConfig(object):
606 def __init__(self, *args, **kwargs):
610 return '<DBConfig %s>' % self.name
612 __all__.append('DBConfig')
614 ################################################################################
617 def get_or_set_contents_file_id(filename, session=None):
619 Returns database id for given filename.
621 If no matching file is found, a row is inserted.
623 @type filename: string
624 @param filename: The filename
625 @type session: SQLAlchemy
626 @param session: Optional SQL session object (a temporary one will be
627 generated if not supplied). If not passed, a commit will be performed at
628 the end of the function, otherwise the caller is responsible for commiting.
631 @return: the database id for the given component
634 q = session.query(ContentFilename).filter_by(filename=filename)
637 ret = q.one().cafilename_id
638 except NoResultFound:
639 cf = ContentFilename()
640 cf.filename = filename
642 session.commit_or_flush()
643 ret = cf.cafilename_id
647 __all__.append('get_or_set_contents_file_id')
650 def get_contents(suite, overridetype, section=None, session=None):
652 Returns contents for a suite / overridetype combination, limiting
653 to a section if not None.
656 @param suite: Suite object
658 @type overridetype: OverrideType
659 @param overridetype: OverrideType object
661 @type section: Section
662 @param section: Optional section object to limit results to
664 @type session: SQLAlchemy
665 @param session: Optional SQL session object (a temporary one will be
666 generated if not supplied)
669 @return: ResultsProxy object set up to return tuples of (filename, section,
673 # find me all of the contents for a given suite
674 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
678 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
679 JOIN content_file_names n ON (c.filename=n.id)
680 JOIN binaries b ON (b.id=c.binary_pkg)
681 JOIN override o ON (o.package=b.package)
682 JOIN section s ON (s.id=o.section)
683 WHERE o.suite = :suiteid AND o.type = :overridetypeid
684 AND b.type=:overridetypename"""
686 vals = {'suiteid': suite.suite_id,
687 'overridetypeid': overridetype.overridetype_id,
688 'overridetypename': overridetype.overridetype}
690 if section is not None:
691 contents_q += " AND s.id = :sectionid"
692 vals['sectionid'] = section.section_id
694 contents_q += " ORDER BY fn"
696 return session.execute(contents_q, vals)
698 __all__.append('get_contents')
700 ################################################################################
702 class ContentFilepath(object):
703 def __init__(self, *args, **kwargs):
707 return '<ContentFilepath %s>' % self.filepath
709 __all__.append('ContentFilepath')
712 def get_or_set_contents_path_id(filepath, session=None):
714 Returns database id for given path.
716 If no matching file is found, a row is inserted.
718 @type filename: string
719 @param filename: The filepath
720 @type session: SQLAlchemy
721 @param session: Optional SQL session object (a temporary one will be
722 generated if not supplied). If not passed, a commit will be performed at
723 the end of the function, otherwise the caller is responsible for commiting.
726 @return: the database id for the given path
729 q = session.query(ContentFilepath).filter_by(filepath=filepath)
732 ret = q.one().cafilepath_id
733 except NoResultFound:
734 cf = ContentFilepath()
735 cf.filepath = filepath
737 session.commit_or_flush()
738 ret = cf.cafilepath_id
742 __all__.append('get_or_set_contents_path_id')
744 ################################################################################
746 class ContentAssociation(object):
747 def __init__(self, *args, **kwargs):
751 return '<ContentAssociation %s>' % self.ca_id
753 __all__.append('ContentAssociation')
755 def insert_content_paths(binary_id, fullpaths, session=None):
757 Make sure given path is associated with given binary id
760 @param binary_id: the id of the binary
761 @type fullpaths: list
762 @param fullpaths: the list of paths of the file being associated with the binary
763 @type session: SQLAlchemy session
764 @param session: Optional SQLAlchemy session. If this is passed, the caller
765 is responsible for ensuring a transaction has begun and committing the
766 results or rolling back based on the result code. If not passed, a commit
767 will be performed at the end of the function, otherwise the caller is
768 responsible for commiting.
770 @return: True upon success
775 session = DBConn().session()
781 for fullpath in fullpaths:
782 if fullpath.startswith( './' ):
783 fullpath = fullpath[2:]
785 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} )
793 traceback.print_exc()
795 # Only rollback if we set up the session ourself
802 __all__.append('insert_content_paths')
804 ################################################################################
806 class DSCFile(object):
807 def __init__(self, *args, **kwargs):
811 return '<DSCFile %s>' % self.dscfile_id
813 __all__.append('DSCFile')
816 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
818 Returns a list of DSCFiles which may be empty
820 @type dscfile_id: int (optional)
821 @param dscfile_id: the dscfile_id of the DSCFiles to find
823 @type source_id: int (optional)
824 @param source_id: the source id related to the DSCFiles to find
826 @type poolfile_id: int (optional)
827 @param poolfile_id: the poolfile id related to the DSCFiles to find
830 @return: Possibly empty list of DSCFiles
833 q = session.query(DSCFile)
835 if dscfile_id is not None:
836 q = q.filter_by(dscfile_id=dscfile_id)
838 if source_id is not None:
839 q = q.filter_by(source_id=source_id)
841 if poolfile_id is not None:
842 q = q.filter_by(poolfile_id=poolfile_id)
846 __all__.append('get_dscfiles')
848 ################################################################################
850 class PoolFile(object):
851 def __init__(self, *args, **kwargs):
855 return '<PoolFile %s>' % self.filename
859 return os.path.join(self.location.path, self.filename)
861 __all__.append('PoolFile')
864 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
867 (ValidFileFound [boolean or None], PoolFile object or None)
869 @type filename: string
870 @param filename: the filename of the file to check against the DB
873 @param filesize: the size of the file to check against the DB
876 @param md5sum: the md5sum of the file to check against the DB
878 @type location_id: int
879 @param location_id: the id of the location to look in
882 @return: Tuple of length 2.
883 If more than one file found with that name:
885 If valid pool file found: (True, PoolFile object)
886 If valid pool file not found:
887 (False, None) if no file found
888 (False, PoolFile object) if file found with size/md5sum mismatch
891 q = session.query(PoolFile).filter_by(filename=filename)
892 q = q.join(Location).filter_by(location_id=location_id)
902 if obj.md5sum != md5sum or obj.filesize != int(filesize):
910 __all__.append('check_poolfile')
913 def get_poolfile_by_id(file_id, session=None):
915 Returns a PoolFile objects or None for the given id
918 @param file_id: the id of the file to look for
920 @rtype: PoolFile or None
921 @return: either the PoolFile object or None
924 q = session.query(PoolFile).filter_by(file_id=file_id)
928 except NoResultFound:
931 __all__.append('get_poolfile_by_id')
935 def get_poolfile_by_name(filename, location_id=None, session=None):
937 Returns an array of PoolFile objects for the given filename and
938 (optionally) location_id
940 @type filename: string
941 @param filename: the filename of the file to check against the DB
943 @type location_id: int
944 @param location_id: the id of the location to look in (optional)
947 @return: array of PoolFile objects
950 q = session.query(PoolFile).filter_by(filename=filename)
952 if location_id is not None:
953 q = q.join(Location).filter_by(location_id=location_id)
957 __all__.append('get_poolfile_by_name')
960 def get_poolfile_like_name(filename, session=None):
962 Returns an array of PoolFile objects which are like the given name
964 @type filename: string
965 @param filename: the filename of the file to check against the DB
968 @return: array of PoolFile objects
971 # TODO: There must be a way of properly using bind parameters with %FOO%
972 q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
976 __all__.append('get_poolfile_like_name')
979 def add_poolfile(filename, datadict, location_id, session=None):
981 Add a new file to the pool
983 @type filename: string
984 @param filename: filename
987 @param datadict: dict with needed data
989 @type location_id: int
990 @param location_id: database id of the location
993 @return: the PoolFile object created
995 poolfile = PoolFile()
996 poolfile.filename = filename
997 poolfile.filesize = datadict["size"]
998 poolfile.md5sum = datadict["md5sum"]
999 poolfile.sha1sum = datadict["sha1sum"]
1000 poolfile.sha256sum = datadict["sha256sum"]
1001 poolfile.location_id = location_id
1003 session.add(poolfile)
1004 # Flush to get a file id (NB: This is not a commit)
1009 __all__.append('add_poolfile')
1011 ################################################################################
1013 class Fingerprint(object):
1014 def __init__(self, *args, **kwargs):
1018 return '<Fingerprint %s>' % self.fingerprint
1020 __all__.append('Fingerprint')
1023 def get_fingerprint(fpr, session=None):
1025 Returns Fingerprint object for given fpr.
1028 @param fpr: The fpr to find / add
1030 @type session: SQLAlchemy
1031 @param session: Optional SQL session object (a temporary one will be
1032 generated if not supplied).
1035 @return: the Fingerprint object for the given fpr or None
1038 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1042 except NoResultFound:
1047 __all__.append('get_fingerprint')
1050 def get_or_set_fingerprint(fpr, session=None):
1052 Returns Fingerprint object for given fpr.
1054 If no matching fpr is found, a row is inserted.
1057 @param fpr: The fpr to find / add
1059 @type session: SQLAlchemy
1060 @param session: Optional SQL session object (a temporary one will be
1061 generated if not supplied). If not passed, a commit will be performed at
1062 the end of the function, otherwise the caller is responsible for commiting.
1063 A flush will be performed either way.
1066 @return: the Fingerprint object for the given fpr
1069 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1073 except NoResultFound:
1074 fingerprint = Fingerprint()
1075 fingerprint.fingerprint = fpr
1076 session.add(fingerprint)
1077 session.commit_or_flush()
1082 __all__.append('get_or_set_fingerprint')
1084 ################################################################################
1086 # Helper routine for Keyring class
1087 def get_ldap_name(entry):
1089 for k in ["cn", "mn", "sn"]:
1091 if ret and ret[0] != "" and ret[0] != "-":
1093 return " ".join(name)
1095 ################################################################################
1097 class Keyring(object):
1098 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1099 " --with-colons --fingerprint --fingerprint"
1104 def __init__(self, *args, **kwargs):
1108 return '<Keyring %s>' % self.keyring_name
1110 def de_escape_gpg_str(self, txt):
1111 esclist = re.split(r'(\\x..)', txt)
1112 for x in range(1,len(esclist),2):
1113 esclist[x] = "%c" % (int(esclist[x][2:],16))
1114 return "".join(esclist)
1116 def load_keys(self, keyring):
1119 if not self.keyring_id:
1120 raise Exception('Must be initialized with database information')
1122 k = os.popen(self.gpg_invocation % keyring, "r")
1126 for line in k.xreadlines():
1127 field = line.split(":")
1128 if field[0] == "pub":
1130 (name, addr) = email.Utils.parseaddr(field[9])
1131 name = re.sub(r"\s*[(].*[)]", "", name)
1132 if name == "" or addr == "" or "@" not in addr:
1134 addr = "invalid-uid"
1135 name = self.de_escape_gpg_str(name)
1136 self.keys[key] = {"email": addr}
1138 self.keys[key]["name"] = name
1139 self.keys[key]["aliases"] = [name]
1140 self.keys[key]["fingerprints"] = []
1142 elif key and field[0] == "sub" and len(field) >= 12:
1143 signingkey = ("s" in field[11])
1144 elif key and field[0] == "uid":
1145 (name, addr) = email.Utils.parseaddr(field[9])
1146 if name and name not in self.keys[key]["aliases"]:
1147 self.keys[key]["aliases"].append(name)
1148 elif signingkey and field[0] == "fpr":
1149 self.keys[key]["fingerprints"].append(field[9])
1150 self.fpr_lookup[field[9]] = key
1152 def import_users_from_ldap(self, session):
1156 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1157 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1159 l = ldap.open(LDAPServer)
1160 l.simple_bind_s("","")
1161 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1162 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1163 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1165 ldap_fin_uid_id = {}
1172 uid = entry["uid"][0]
1173 name = get_ldap_name(entry)
1174 fingerprints = entry["keyFingerPrint"]
1176 for f in fingerprints:
1177 key = self.fpr_lookup.get(f, None)
1178 if key not in self.keys:
1180 self.keys[key]["uid"] = uid
1184 keyid = get_or_set_uid(uid, session).uid_id
1185 byuid[keyid] = (uid, name)
1186 byname[uid] = (keyid, name)
1188 return (byname, byuid)
1190 def generate_users_from_keyring(self, format, session):
1194 for x in self.keys.keys():
1195 if self.keys[x]["email"] == "invalid-uid":
1197 self.keys[x]["uid"] = format % "invalid-uid"
1199 uid = format % self.keys[x]["email"]
1200 keyid = get_or_set_uid(uid, session).uid_id
1201 byuid[keyid] = (uid, self.keys[x]["name"])
1202 byname[uid] = (keyid, self.keys[x]["name"])
1203 self.keys[x]["uid"] = uid
1206 uid = format % "invalid-uid"
1207 keyid = get_or_set_uid(uid, session).uid_id
1208 byuid[keyid] = (uid, "ungeneratable user id")
1209 byname[uid] = (keyid, "ungeneratable user id")
1211 return (byname, byuid)
1213 __all__.append('Keyring')
1216 def get_keyring(keyring, session=None):
1218 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1219 If C{keyring} already has an entry, simply return the existing Keyring
1221 @type keyring: string
1222 @param keyring: the keyring name
1225 @return: the Keyring object for this keyring
1228 q = session.query(Keyring).filter_by(keyring_name=keyring)
1232 except NoResultFound:
1235 __all__.append('get_keyring')
1237 ################################################################################
1239 class KeyringACLMap(object):
1240 def __init__(self, *args, **kwargs):
1244 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1246 __all__.append('KeyringACLMap')
1248 ################################################################################
1250 class DBChange(object):
1251 def __init__(self, *args, **kwargs):
1255 return '<DBChange %s>' % self.changesname
1257 __all__.append('DBChange')
1260 def get_dbchange(filename, session=None):
1262 returns DBChange object for given C{filename}.
1264 @type archive: string
1265 @param archive: the name of the arhive
1267 @type session: Session
1268 @param session: Optional SQLA session object (a temporary one will be
1269 generated if not supplied)
1272 @return: Archive object for the given name (None if not present)
1275 q = session.query(DBChange).filter_by(changesname=filename)
1279 except NoResultFound:
1282 __all__.append('get_dbchange')
1284 ################################################################################
1286 class Location(object):
1287 def __init__(self, *args, **kwargs):
1291 return '<Location %s (%s)>' % (self.path, self.location_id)
1293 __all__.append('Location')
1296 def get_location(location, component=None, archive=None, session=None):
1298 Returns Location object for the given combination of location, component
1301 @type location: string
1302 @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
1304 @type component: string
1305 @param component: the component name (if None, no restriction applied)
1307 @type archive: string
1308 @param archive_id: the archive name (if None, no restriction applied)
1310 @rtype: Location / None
1311 @return: Either a Location object or None if one can't be found
1314 q = session.query(Location).filter_by(path=location)
1316 if archive is not None:
1317 q = q.join(Archive).filter_by(archive_name=archive)
1319 if component is not None:
1320 q = q.join(Component).filter_by(component_name=component)
1324 except NoResultFound:
1327 __all__.append('get_location')
1329 ################################################################################
1331 class Maintainer(object):
1332 def __init__(self, *args, **kwargs):
1336 return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
1338 def get_split_maintainer(self):
1339 if not hasattr(self, 'name') or self.name is None:
1340 return ('', '', '', '')
1342 return fix_maintainer(self.name.strip())
1344 __all__.append('Maintainer')
1347 def get_or_set_maintainer(name, session=None):
1349 Returns Maintainer object for given maintainer name.
1351 If no matching maintainer name is found, a row is inserted.
1354 @param name: The maintainer name to add
1356 @type session: SQLAlchemy
1357 @param session: Optional SQL session object (a temporary one will be
1358 generated if not supplied). If not passed, a commit will be performed at
1359 the end of the function, otherwise the caller is responsible for commiting.
1360 A flush will be performed either way.
1363 @return: the Maintainer object for the given maintainer
1366 q = session.query(Maintainer).filter_by(name=name)
1369 except NoResultFound:
1370 maintainer = Maintainer()
1371 maintainer.name = name
1372 session.add(maintainer)
1373 session.commit_or_flush()
1378 __all__.append('get_or_set_maintainer')
1381 def get_maintainer(maintainer_id, session=None):
1383 Return the name of the maintainer behind C{maintainer_id} or None if that
1384 maintainer_id is invalid.
1386 @type maintainer_id: int
1387 @param maintainer_id: the id of the maintainer
1390 @return: the Maintainer with this C{maintainer_id}
1393 return session.query(Maintainer).get(maintainer_id)
1395 __all__.append('get_maintainer')
1397 ################################################################################
1399 class NewComment(object):
1400 def __init__(self, *args, **kwargs):
1404 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1406 __all__.append('NewComment')
1409 def has_new_comment(package, version, session=None):
1411 Returns true if the given combination of C{package}, C{version} has a comment.
1413 @type package: string
1414 @param package: name of the package
1416 @type version: string
1417 @param version: package version
1419 @type session: Session
1420 @param session: Optional SQLA session object (a temporary one will be
1421 generated if not supplied)
1427 q = session.query(NewComment)
1428 q = q.filter_by(package=package)
1429 q = q.filter_by(version=version)
1431 return bool(q.count() > 0)
1433 __all__.append('has_new_comment')
1436 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1438 Returns (possibly empty) list of NewComment objects for the given
1441 @type package: string (optional)
1442 @param package: name of the package
1444 @type version: string (optional)
1445 @param version: package version
1447 @type comment_id: int (optional)
1448 @param comment_id: An id of a comment
1450 @type session: Session
1451 @param session: Optional SQLA session object (a temporary one will be
1452 generated if not supplied)
1455 @return: A (possibly empty) list of NewComment objects will be returned
1458 q = session.query(NewComment)
1459 if package is not None: q = q.filter_by(package=package)
1460 if version is not None: q = q.filter_by(version=version)
1461 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1465 __all__.append('get_new_comments')
1467 ################################################################################
1469 class Override(object):
1470 def __init__(self, *args, **kwargs):
1474 return '<Override %s (%s)>' % (self.package, self.suite_id)
1476 __all__.append('Override')
1479 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1481 Returns Override object for the given parameters
1483 @type package: string
1484 @param package: The name of the package
1486 @type suite: string, list or None
1487 @param suite: The name of the suite (or suites if a list) to limit to. If
1488 None, don't limit. Defaults to None.
1490 @type component: string, list or None
1491 @param component: The name of the component (or components if a list) to
1492 limit to. If None, don't limit. Defaults to None.
1494 @type overridetype: string, list or None
1495 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1496 limit to. If None, don't limit. Defaults to None.
1498 @type session: Session
1499 @param session: Optional SQLA session object (a temporary one will be
1500 generated if not supplied)
1503 @return: A (possibly empty) list of Override objects will be returned
1506 q = session.query(Override)
1507 q = q.filter_by(package=package)
1509 if suite is not None:
1510 if not isinstance(suite, list): suite = [suite]
1511 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1513 if component is not None:
1514 if not isinstance(component, list): component = [component]
1515 q = q.join(Component).filter(Component.component_name.in_(component))
1517 if overridetype is not None:
1518 if not isinstance(overridetype, list): overridetype = [overridetype]
1519 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1523 __all__.append('get_override')
1526 ################################################################################
1528 class OverrideType(object):
1529 def __init__(self, *args, **kwargs):
1533 return '<OverrideType %s>' % self.overridetype
1535 __all__.append('OverrideType')
1538 def get_override_type(override_type, session=None):
1540 Returns OverrideType object for given C{override type}.
1542 @type override_type: string
1543 @param override_type: The name of the override type
1545 @type session: Session
1546 @param session: Optional SQLA session object (a temporary one will be
1547 generated if not supplied)
1550 @return: the database id for the given override type
1553 q = session.query(OverrideType).filter_by(overridetype=override_type)
1557 except NoResultFound:
1560 __all__.append('get_override_type')
1562 ################################################################################
1564 class PendingContentAssociation(object):
1565 def __init__(self, *args, **kwargs):
1569 return '<PendingContentAssociation %s>' % self.pca_id
1571 __all__.append('PendingContentAssociation')
1573 def insert_pending_content_paths(package, fullpaths, session=None):
1575 Make sure given paths are temporarily associated with given
1579 @param package: the package to associate with should have been read in from the binary control file
1580 @type fullpaths: list
1581 @param fullpaths: the list of paths of the file being associated with the binary
1582 @type session: SQLAlchemy session
1583 @param session: Optional SQLAlchemy session. If this is passed, the caller
1584 is responsible for ensuring a transaction has begun and committing the
1585 results or rolling back based on the result code. If not passed, a commit
1586 will be performed at the end of the function
1588 @return: True upon success, False if there is a problem
1591 privatetrans = False
1594 session = DBConn().session()
1598 arch = get_architecture(package['Architecture'], session)
1599 arch_id = arch.arch_id
1601 # Remove any already existing recorded files for this package
1602 q = session.query(PendingContentAssociation)
1603 q = q.filter_by(package=package['Package'])
1604 q = q.filter_by(version=package['Version'])
1605 q = q.filter_by(architecture=arch_id)
1610 for fullpath in fullpaths:
1611 (path, filename) = os.path.split(fullpath)
1613 if path.startswith( "./" ):
1616 filepath_id = get_or_set_contents_path_id(path, session)
1617 filename_id = get_or_set_contents_file_id(filename, session)
1619 pathcache[fullpath] = (filepath_id, filename_id)
1621 for fullpath, dat in pathcache.items():
1622 pca = PendingContentAssociation()
1623 pca.package = package['Package']
1624 pca.version = package['Version']
1625 pca.filepath_id = dat[0]
1626 pca.filename_id = dat[1]
1627 pca.architecture = arch_id
1630 # Only commit if we set up the session ourself
1638 except Exception, e:
1639 traceback.print_exc()
1641 # Only rollback if we set up the session ourself
1648 __all__.append('insert_pending_content_paths')
1650 ################################################################################
1652 class PolicyQueue(object):
1653 def __init__(self, *args, **kwargs):
1657 return '<PolicyQueue %s>' % self.queue_name
1659 __all__.append('PolicyQueue')
1662 def get_policy_queue(queuename, session=None):
1664 Returns PolicyQueue object for given C{queue name}
1666 @type queuename: string
1667 @param queuename: The name of the queue
1669 @type session: Session
1670 @param session: Optional SQLA session object (a temporary one will be
1671 generated if not supplied)
1674 @return: PolicyQueue object for the given queue
1677 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
1681 except NoResultFound:
1684 __all__.append('get_policy_queue')
1686 ################################################################################
1688 class Priority(object):
1689 def __init__(self, *args, **kwargs):
1692 def __eq__(self, val):
1693 if isinstance(val, str):
1694 return (self.priority == val)
1695 # This signals to use the normal comparison operator
1696 return NotImplemented
1698 def __ne__(self, val):
1699 if isinstance(val, str):
1700 return (self.priority != val)
1701 # This signals to use the normal comparison operator
1702 return NotImplemented
1705 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
1707 __all__.append('Priority')
1710 def get_priority(priority, session=None):
1712 Returns Priority object for given C{priority name}.
1714 @type priority: string
1715 @param priority: The name of the priority
1717 @type session: Session
1718 @param session: Optional SQLA session object (a temporary one will be
1719 generated if not supplied)
1722 @return: Priority object for the given priority
1725 q = session.query(Priority).filter_by(priority=priority)
1729 except NoResultFound:
1732 __all__.append('get_priority')
1735 def get_priorities(session=None):
1737 Returns dictionary of priority names -> id mappings
1739 @type session: Session
1740 @param session: Optional SQL session object (a temporary one will be
1741 generated if not supplied)
1744 @return: dictionary of priority names -> id mappings
1748 q = session.query(Priority)
1750 ret[x.priority] = x.priority_id
1754 __all__.append('get_priorities')
1756 ################################################################################
1758 class Section(object):
1759 def __init__(self, *args, **kwargs):
1762 def __eq__(self, val):
1763 if isinstance(val, str):
1764 return (self.section == val)
1765 # This signals to use the normal comparison operator
1766 return NotImplemented
1768 def __ne__(self, val):
1769 if isinstance(val, str):
1770 return (self.section != val)
1771 # This signals to use the normal comparison operator
1772 return NotImplemented
1775 return '<Section %s>' % self.section
1777 __all__.append('Section')
1780 def get_section(section, session=None):
1782 Returns Section object for given C{section name}.
1784 @type section: string
1785 @param section: The name of the section
1787 @type session: Session
1788 @param session: Optional SQLA session object (a temporary one will be
1789 generated if not supplied)
1792 @return: Section object for the given section name
1795 q = session.query(Section).filter_by(section=section)
1799 except NoResultFound:
1802 __all__.append('get_section')
1805 def get_sections(session=None):
1807 Returns dictionary of section names -> id mappings
1809 @type session: Session
1810 @param session: Optional SQL session object (a temporary one will be
1811 generated if not supplied)
1814 @return: dictionary of section names -> id mappings
1818 q = session.query(Section)
1820 ret[x.section] = x.section_id
1824 __all__.append('get_sections')
1826 ################################################################################
1828 class DBSource(object):
1829 def __init__(self, *args, **kwargs):
1833 return '<DBSource %s (%s)>' % (self.source, self.version)
1835 __all__.append('DBSource')
1838 def source_exists(source, source_version, suites = ["any"], session=None):
1840 Ensure that source exists somewhere in the archive for the binary
1841 upload being processed.
1842 1. exact match => 1.0-3
1843 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1845 @type package: string
1846 @param package: package source name
1848 @type source_version: string
1849 @param source_version: expected source version
1852 @param suites: list of suites to check in, default I{any}
1854 @type session: Session
1855 @param session: Optional SQLA session object (a temporary one will be
1856 generated if not supplied)
1859 @return: returns 1 if a source with expected version is found, otherwise 0
1866 for suite in suites:
1867 q = session.query(DBSource).filter_by(source=source)
1869 # source must exist in suite X, or in some other suite that's
1870 # mapped to X, recursively... silent-maps are counted too,
1871 # unreleased-maps aren't.
1872 maps = cnf.ValueList("SuiteMappings")[:]
1874 maps = [ m.split() for m in maps ]
1875 maps = [ (x[1], x[2]) for x in maps
1876 if x[0] == "map" or x[0] == "silent-map" ]
1879 if x[1] in s and x[0] not in s:
1882 q = q.join(SrcAssociation).join(Suite)
1883 q = q.filter(Suite.suite_name.in_(s))
1885 # Reduce the query results to a list of version numbers
1886 ql = [ j.version for j in q.all() ]
1889 if source_version in ql:
1893 from daklib.regexes import re_bin_only_nmu
1894 orig_source_version = re_bin_only_nmu.sub('', source_version)
1895 if orig_source_version in ql:
1898 # No source found so return not ok
1903 __all__.append('source_exists')
1906 def get_suites_source_in(source, session=None):
1908 Returns list of Suite objects which given C{source} name is in
1911 @param source: DBSource package name to search for
1914 @return: list of Suite objects for the given source
1917 return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
1919 __all__.append('get_suites_source_in')
1922 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
1924 Returns list of DBSource objects for given C{source} name and other parameters
1927 @param source: DBSource package name to search for
1929 @type source: str or None
1930 @param source: DBSource version name to search for or None if not applicable
1932 @type dm_upload_allowed: bool
1933 @param dm_upload_allowed: If None, no effect. If True or False, only
1934 return packages with that dm_upload_allowed setting
1936 @type session: Session
1937 @param session: Optional SQL session object (a temporary one will be
1938 generated if not supplied)
1941 @return: list of DBSource objects for the given name (may be empty)
1944 q = session.query(DBSource).filter_by(source=source)
1946 if version is not None:
1947 q = q.filter_by(version=version)
1949 if dm_upload_allowed is not None:
1950 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
1954 __all__.append('get_sources_from_name')
1957 def get_source_in_suite(source, suite, session=None):
1959 Returns list of DBSource objects for a combination of C{source} and C{suite}.
1961 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
1962 - B{suite} - a suite name, eg. I{unstable}
1964 @type source: string
1965 @param source: source package name
1968 @param suite: the suite name
1971 @return: the version for I{source} in I{suite}
1975 q = session.query(SrcAssociation)
1976 q = q.join('source').filter_by(source=source)
1977 q = q.join('suite').filter_by(suite_name=suite)
1980 return q.one().source
1981 except NoResultFound:
1984 __all__.append('get_source_in_suite')
1986 ################################################################################
1989 def add_dsc_to_db(u, filename, session=None):
1990 entry = u.pkg.files[filename]
1994 source.source = u.pkg.dsc["source"]
1995 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
1996 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
1997 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
1998 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
1999 source.install_date = datetime.now().date()
2001 dsc_component = entry["component"]
2002 dsc_location_id = entry["location id"]
2004 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2006 # Set up a new poolfile if necessary
2007 if not entry.has_key("files id") or not entry["files id"]:
2008 filename = entry["pool name"] + filename
2009 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2011 pfs.append(poolfile)
2012 entry["files id"] = poolfile.file_id
2014 source.poolfile_id = entry["files id"]
2018 for suite_name in u.pkg.changes["distribution"].keys():
2019 sa = SrcAssociation()
2020 sa.source_id = source.source_id
2021 sa.suite_id = get_suite(suite_name).suite_id
2026 # Add the source files to the DB (files and dsc_files)
2028 dscfile.source_id = source.source_id
2029 dscfile.poolfile_id = entry["files id"]
2030 session.add(dscfile)
2032 for dsc_file, dentry in u.pkg.dsc_files.items():
2034 df.source_id = source.source_id
2036 # If the .orig tarball is already in the pool, it's
2037 # files id is stored in dsc_files by check_dsc().
2038 files_id = dentry.get("files id", None)
2040 # Find the entry in the files hash
2041 # TODO: Bail out here properly
2043 for f, e in u.pkg.files.items():
2048 if files_id is None:
2049 filename = dfentry["pool name"] + dsc_file
2051 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2052 # FIXME: needs to check for -1/-2 and or handle exception
2053 if found and obj is not None:
2054 files_id = obj.file_id
2057 # If still not found, add it
2058 if files_id is None:
2059 # HACK: Force sha1sum etc into dentry
2060 dentry["sha1sum"] = dfentry["sha1sum"]
2061 dentry["sha256sum"] = dfentry["sha256sum"]
2062 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2063 pfs.append(poolfile)
2064 files_id = poolfile.file_id
2066 poolfile = get_poolfile_by_id(files_id, session)
2067 if poolfile is None:
2068 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2069 pfs.append(poolfile)
2071 df.poolfile_id = files_id
2076 # Add the src_uploaders to the DB
2077 uploader_ids = [source.maintainer_id]
2078 if u.pkg.dsc.has_key("uploaders"):
2079 for up in u.pkg.dsc["uploaders"].split(","):
2081 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2084 for up in uploader_ids:
2085 if added_ids.has_key(up):
2086 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
2092 su.maintainer_id = up
2093 su.source_id = source.source_id
2098 return dsc_component, dsc_location_id, pfs
2100 __all__.append('add_dsc_to_db')
2103 def add_deb_to_db(u, filename, session=None):
2105 Contrary to what you might expect, this routine deals with both
2106 debs and udebs. That info is in 'dbtype', whilst 'type' is
2107 'deb' for both of them
2110 entry = u.pkg.files[filename]
2113 bin.package = entry["package"]
2114 bin.version = entry["version"]
2115 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2116 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2117 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2118 bin.binarytype = entry["dbtype"]
2121 filename = entry["pool name"] + filename
2122 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2123 if not entry.get("location id", None):
2124 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2126 if entry.get("files id", None):
2127 poolfile = get_poolfile_by_id(bin.poolfile_id)
2128 bin.poolfile_id = entry["files id"]
2130 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2131 bin.poolfile_id = entry["files id"] = poolfile.file_id
2134 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2135 if len(bin_sources) != 1:
2136 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2137 (bin.package, bin.version, bin.architecture.arch_string,
2138 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2140 bin.source_id = bin_sources[0].source_id
2142 # Add and flush object so it has an ID
2146 # Add BinAssociations
2147 for suite_name in u.pkg.changes["distribution"].keys():
2148 ba = BinAssociation()
2149 ba.binary_id = bin.binary_id
2150 ba.suite_id = get_suite(suite_name).suite_id
2155 # Deal with contents - disabled for now
2156 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2158 # print "REJECT\nCould not determine contents of package %s" % bin.package
2159 # session.rollback()
2160 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2164 __all__.append('add_deb_to_db')
2166 ################################################################################
2168 class SourceACL(object):
2169 def __init__(self, *args, **kwargs):
2173 return '<SourceACL %s>' % self.source_acl_id
2175 __all__.append('SourceACL')
2177 ################################################################################
2179 class SrcAssociation(object):
2180 def __init__(self, *args, **kwargs):
2184 return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
2186 __all__.append('SrcAssociation')
2188 ################################################################################
2190 class SrcFormat(object):
2191 def __init__(self, *args, **kwargs):
2195 return '<SrcFormat %s>' % (self.format_name)
2197 __all__.append('SrcFormat')
2199 ################################################################################
2201 class SrcUploader(object):
2202 def __init__(self, *args, **kwargs):
2206 return '<SrcUploader %s>' % self.uploader_id
2208 __all__.append('SrcUploader')
2210 ################################################################################
2212 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2213 ('SuiteID', 'suite_id'),
2214 ('Version', 'version'),
2215 ('Origin', 'origin'),
2217 ('Description', 'description'),
2218 ('Untouchable', 'untouchable'),
2219 ('Announce', 'announce'),
2220 ('Codename', 'codename'),
2221 ('OverrideCodename', 'overridecodename'),
2222 ('ValidTime', 'validtime'),
2223 ('Priority', 'priority'),
2224 ('NotAutomatic', 'notautomatic'),
2225 ('CopyChanges', 'copychanges'),
2226 ('CopyDotDak', 'copydotdak'),
2227 ('CommentsDir', 'commentsdir'),
2228 ('OverrideSuite', 'overridesuite'),
2229 ('ChangelogBase', 'changelogbase')]
2232 class Suite(object):
2233 def __init__(self, *args, **kwargs):
2237 return '<Suite %s>' % self.suite_name
2239 def __eq__(self, val):
2240 if isinstance(val, str):
2241 return (self.suite_name == val)
2242 # This signals to use the normal comparison operator
2243 return NotImplemented
2245 def __ne__(self, val):
2246 if isinstance(val, str):
2247 return (self.suite_name != val)
2248 # This signals to use the normal comparison operator
2249 return NotImplemented
2253 for disp, field in SUITE_FIELDS:
2254 val = getattr(self, field, None)
2256 ret.append("%s: %s" % (disp, val))
2258 return "\n".join(ret)
2260 __all__.append('Suite')
2263 def get_suite_architecture(suite, architecture, session=None):
2265 Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
2269 @param suite: Suite name to search for
2271 @type architecture: str
2272 @param architecture: Architecture name to search for
2274 @type session: Session
2275 @param session: Optional SQL session object (a temporary one will be
2276 generated if not supplied)
2278 @rtype: SuiteArchitecture
2279 @return: the SuiteArchitecture object or None
2282 q = session.query(SuiteArchitecture)
2283 q = q.join(Architecture).filter_by(arch_string=architecture)
2284 q = q.join(Suite).filter_by(suite_name=suite)
2288 except NoResultFound:
2291 __all__.append('get_suite_architecture')
2294 def get_suite(suite, session=None):
2296 Returns Suite object for given C{suite name}.
2299 @param suite: The name of the suite
2301 @type session: Session
2302 @param session: Optional SQLA session object (a temporary one will be
2303 generated if not supplied)
2306 @return: Suite object for the requested suite name (None if not present)
2309 q = session.query(Suite).filter_by(suite_name=suite)
2313 except NoResultFound:
2316 __all__.append('get_suite')
2318 ################################################################################
2320 class SuiteArchitecture(object):
2321 def __init__(self, *args, **kwargs):
2325 return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
2327 __all__.append('SuiteArchitecture')
2330 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2332 Returns list of Architecture objects for given C{suite} name
2335 @param source: Suite name to search for
2337 @type skipsrc: boolean
2338 @param skipsrc: Whether to skip returning the 'source' architecture entry
2341 @type skipall: boolean
2342 @param skipall: Whether to skip returning the 'all' architecture entry
2345 @type session: Session
2346 @param session: Optional SQL session object (a temporary one will be
2347 generated if not supplied)
2350 @return: list of Architecture objects for the given name (may be empty)
2353 q = session.query(Architecture)
2354 q = q.join(SuiteArchitecture)
2355 q = q.join(Suite).filter_by(suite_name=suite)
2358 q = q.filter(Architecture.arch_string != 'source')
2361 q = q.filter(Architecture.arch_string != 'all')
2363 q = q.order_by('arch_string')
2367 __all__.append('get_suite_architectures')
2369 ################################################################################
2371 class SuiteSrcFormat(object):
2372 def __init__(self, *args, **kwargs):
2376 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2378 __all__.append('SuiteSrcFormat')
2381 def get_suite_src_formats(suite, session=None):
2383 Returns list of allowed SrcFormat for C{suite}.
2386 @param suite: Suite name to search for
2388 @type session: Session
2389 @param session: Optional SQL session object (a temporary one will be
2390 generated if not supplied)
2393 @return: the list of allowed source formats for I{suite}
2396 q = session.query(SrcFormat)
2397 q = q.join(SuiteSrcFormat)
2398 q = q.join(Suite).filter_by(suite_name=suite)
2399 q = q.order_by('format_name')
2403 __all__.append('get_suite_src_formats')
2405 ################################################################################
2408 def __init__(self, *args, **kwargs):
2411 def __eq__(self, val):
2412 if isinstance(val, str):
2413 return (self.uid == val)
2414 # This signals to use the normal comparison operator
2415 return NotImplemented
2417 def __ne__(self, val):
2418 if isinstance(val, str):
2419 return (self.uid != val)
2420 # This signals to use the normal comparison operator
2421 return NotImplemented
2424 return '<Uid %s (%s)>' % (self.uid, self.name)
2426 __all__.append('Uid')
2429 def add_database_user(uidname, session=None):
2431 Adds a database user
2433 @type uidname: string
2434 @param uidname: The uid of the user to add
2436 @type session: SQLAlchemy
2437 @param session: Optional SQL session object (a temporary one will be
2438 generated if not supplied). If not passed, a commit will be performed at
2439 the end of the function, otherwise the caller is responsible for commiting.
2442 @return: the uid object for the given uidname
2445 session.execute("CREATE USER :uid", {'uid': uidname})
2446 session.commit_or_flush()
2448 __all__.append('add_database_user')
2451 def get_or_set_uid(uidname, session=None):
2453 Returns uid object for given uidname.
2455 If no matching uidname is found, a row is inserted.
2457 @type uidname: string
2458 @param uidname: The uid to add
2460 @type session: SQLAlchemy
2461 @param session: Optional SQL session object (a temporary one will be
2462 generated if not supplied). If not passed, a commit will be performed at
2463 the end of the function, otherwise the caller is responsible for commiting.
2466 @return: the uid object for the given uidname
2469 q = session.query(Uid).filter_by(uid=uidname)
2473 except NoResultFound:
2477 session.commit_or_flush()
2482 __all__.append('get_or_set_uid')
2485 def get_uid_from_fingerprint(fpr, session=None):
2486 q = session.query(Uid)
2487 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2491 except NoResultFound:
2494 __all__.append('get_uid_from_fingerprint')
2496 ################################################################################
2498 class UploadBlock(object):
2499 def __init__(self, *args, **kwargs):
2503 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2505 __all__.append('UploadBlock')
2507 ################################################################################
2509 class DBConn(object):
2511 database module init.
2515 def __init__(self, *args, **kwargs):
2516 self.__dict__ = self.__shared_state
2518 if not getattr(self, 'initialised', False):
2519 self.initialised = True
2520 self.debug = kwargs.has_key('debug')
2523 def __setuptables(self):
2532 'build_queue_files',
2535 'content_associations',
2536 'content_file_names',
2537 'content_file_paths',
2538 'changes_pending_binaries',
2539 'changes_pending_files',
2540 'changes_pending_files_map',
2541 'changes_pending_source',
2542 'changes_pending_source_files',
2543 'changes_pool_files',
2555 'pending_content_associations',
2565 'suite_architectures',
2566 'suite_src_formats',
2567 'suite_build_queue_copy',
2572 for table_name in tables:
2573 table = Table(table_name, self.db_meta, autoload=True)
2574 setattr(self, 'tbl_%s' % table_name, table)
2576 def __setupmappers(self):
2577 mapper(Architecture, self.tbl_architecture,
2578 properties = dict(arch_id = self.tbl_architecture.c.id))
2580 mapper(Archive, self.tbl_archive,
2581 properties = dict(archive_id = self.tbl_archive.c.id,
2582 archive_name = self.tbl_archive.c.name))
2584 mapper(BinAssociation, self.tbl_bin_associations,
2585 properties = dict(ba_id = self.tbl_bin_associations.c.id,
2586 suite_id = self.tbl_bin_associations.c.suite,
2587 suite = relation(Suite),
2588 binary_id = self.tbl_bin_associations.c.bin,
2589 binary = relation(DBBinary)))
2591 mapper(BuildQueue, self.tbl_build_queue,
2592 properties = dict(queue_id = self.tbl_build_queue.c.id))
2594 mapper(BuildQueueFile, self.tbl_build_queue_files,
2595 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
2596 poolfile = relation(PoolFile, backref='buildqueueinstances')))
2598 mapper(DBBinary, self.tbl_binaries,
2599 properties = dict(binary_id = self.tbl_binaries.c.id,
2600 package = self.tbl_binaries.c.package,
2601 version = self.tbl_binaries.c.version,
2602 maintainer_id = self.tbl_binaries.c.maintainer,
2603 maintainer = relation(Maintainer),
2604 source_id = self.tbl_binaries.c.source,
2605 source = relation(DBSource),
2606 arch_id = self.tbl_binaries.c.architecture,
2607 architecture = relation(Architecture),
2608 poolfile_id = self.tbl_binaries.c.file,
2609 poolfile = relation(PoolFile),
2610 binarytype = self.tbl_binaries.c.type,
2611 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2612 fingerprint = relation(Fingerprint),
2613 install_date = self.tbl_binaries.c.install_date,
2614 binassociations = relation(BinAssociation,
2615 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
2617 mapper(BinaryACL, self.tbl_binary_acl,
2618 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2620 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2621 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2622 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2623 architecture = relation(Architecture)))
2625 mapper(Component, self.tbl_component,
2626 properties = dict(component_id = self.tbl_component.c.id,
2627 component_name = self.tbl_component.c.name))
2629 mapper(DBConfig, self.tbl_config,
2630 properties = dict(config_id = self.tbl_config.c.id))
2632 mapper(DSCFile, self.tbl_dsc_files,
2633 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2634 source_id = self.tbl_dsc_files.c.source,
2635 source = relation(DBSource),
2636 poolfile_id = self.tbl_dsc_files.c.file,
2637 poolfile = relation(PoolFile)))
2639 mapper(PoolFile, self.tbl_files,
2640 properties = dict(file_id = self.tbl_files.c.id,
2641 filesize = self.tbl_files.c.size,
2642 location_id = self.tbl_files.c.location,
2643 location = relation(Location)))
2645 mapper(Fingerprint, self.tbl_fingerprint,
2646 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2647 uid_id = self.tbl_fingerprint.c.uid,
2648 uid = relation(Uid),
2649 keyring_id = self.tbl_fingerprint.c.keyring,
2650 keyring = relation(Keyring),
2651 source_acl = relation(SourceACL),
2652 binary_acl = relation(BinaryACL)))
2654 mapper(Keyring, self.tbl_keyrings,
2655 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2656 keyring_id = self.tbl_keyrings.c.id))
2658 mapper(DBChange, self.tbl_changes,
2659 properties = dict(change_id = self.tbl_changes.c.id,
2660 poolfiles = relation(PoolFile,
2661 secondary=self.tbl_changes_pool_files,
2662 backref="changeslinks"),
2663 files = relation(ChangePendingFile,
2664 secondary=self.tbl_changes_pending_files_map,
2665 backref="changesfile"),
2666 in_queue_id = self.tbl_changes.c.in_queue,
2667 in_queue = relation(PolicyQueue,
2668 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
2669 approved_for_id = self.tbl_changes.c.approved_for))
2671 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
2672 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
2674 mapper(ChangePendingFile, self.tbl_changes_pending_files,
2675 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id))
2677 mapper(ChangePendingSource, self.tbl_changes_pending_source,
2678 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
2679 change = relation(DBChange),
2680 maintainer = relation(Maintainer,
2681 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
2682 changedby = relation(Maintainer,
2683 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
2684 fingerprint = relation(Fingerprint),
2685 source_files = relation(ChangePendingFile,
2686 secondary=self.tbl_changes_pending_source_files,
2687 backref="pending_sources")))
2688 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
2689 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
2690 keyring = relation(Keyring, backref="keyring_acl_map"),
2691 architecture = relation(Architecture)))
2693 mapper(Location, self.tbl_location,
2694 properties = dict(location_id = self.tbl_location.c.id,
2695 component_id = self.tbl_location.c.component,
2696 component = relation(Component),
2697 archive_id = self.tbl_location.c.archive,
2698 archive = relation(Archive),
2699 archive_type = self.tbl_location.c.type))
2701 mapper(Maintainer, self.tbl_maintainer,
2702 properties = dict(maintainer_id = self.tbl_maintainer.c.id))
2704 mapper(NewComment, self.tbl_new_comments,
2705 properties = dict(comment_id = self.tbl_new_comments.c.id))
2707 mapper(Override, self.tbl_override,
2708 properties = dict(suite_id = self.tbl_override.c.suite,
2709 suite = relation(Suite),
2710 component_id = self.tbl_override.c.component,
2711 component = relation(Component),
2712 priority_id = self.tbl_override.c.priority,
2713 priority = relation(Priority),
2714 section_id = self.tbl_override.c.section,
2715 section = relation(Section),
2716 overridetype_id = self.tbl_override.c.type,
2717 overridetype = relation(OverrideType)))
2719 mapper(OverrideType, self.tbl_override_type,
2720 properties = dict(overridetype = self.tbl_override_type.c.type,
2721 overridetype_id = self.tbl_override_type.c.id))
2723 mapper(PolicyQueue, self.tbl_policy_queue,
2724 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
2726 mapper(Priority, self.tbl_priority,
2727 properties = dict(priority_id = self.tbl_priority.c.id))
2729 mapper(Section, self.tbl_section,
2730 properties = dict(section_id = self.tbl_section.c.id))
2732 mapper(DBSource, self.tbl_source,
2733 properties = dict(source_id = self.tbl_source.c.id,
2734 version = self.tbl_source.c.version,
2735 maintainer_id = self.tbl_source.c.maintainer,
2736 maintainer = relation(Maintainer,
2737 primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
2738 poolfile_id = self.tbl_source.c.file,
2739 poolfile = relation(PoolFile),
2740 fingerprint_id = self.tbl_source.c.sig_fpr,
2741 fingerprint = relation(Fingerprint),
2742 changedby_id = self.tbl_source.c.changedby,
2743 changedby = relation(Maintainer,
2744 primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
2745 srcfiles = relation(DSCFile,
2746 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
2747 srcassociations = relation(SrcAssociation,
2748 primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)),
2749 srcuploaders = relation(SrcUploader)))
2751 mapper(SourceACL, self.tbl_source_acl,
2752 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
2754 mapper(SrcAssociation, self.tbl_src_associations,
2755 properties = dict(sa_id = self.tbl_src_associations.c.id,
2756 suite_id = self.tbl_src_associations.c.suite,
2757 suite = relation(Suite),
2758 source_id = self.tbl_src_associations.c.source,
2759 source = relation(DBSource)))
2761 mapper(SrcFormat, self.tbl_src_format,
2762 properties = dict(src_format_id = self.tbl_src_format.c.id,
2763 format_name = self.tbl_src_format.c.format_name))
2765 mapper(SrcUploader, self.tbl_src_uploaders,
2766 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
2767 source_id = self.tbl_src_uploaders.c.source,
2768 source = relation(DBSource,
2769 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
2770 maintainer_id = self.tbl_src_uploaders.c.maintainer,
2771 maintainer = relation(Maintainer,
2772 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
2774 mapper(Suite, self.tbl_suite,
2775 properties = dict(suite_id = self.tbl_suite.c.id,
2776 policy_queue = relation(PolicyQueue),
2777 copy_queues = relation(BuildQueue, secondary=self.tbl_suite_build_queue_copy)))
2779 mapper(SuiteArchitecture, self.tbl_suite_architectures,
2780 properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
2781 suite = relation(Suite, backref='suitearchitectures'),
2782 arch_id = self.tbl_suite_architectures.c.architecture,
2783 architecture = relation(Architecture)))
2785 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
2786 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
2787 suite = relation(Suite, backref='suitesrcformats'),
2788 src_format_id = self.tbl_suite_src_formats.c.src_format,
2789 src_format = relation(SrcFormat)))
2791 mapper(Uid, self.tbl_uid,
2792 properties = dict(uid_id = self.tbl_uid.c.id,
2793 fingerprint = relation(Fingerprint)))
2795 mapper(UploadBlock, self.tbl_upload_blocks,
2796 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
2797 fingerprint = relation(Fingerprint, backref="uploadblocks"),
2798 uid = relation(Uid, backref="uploadblocks")))
2800 ## Connection functions
2801 def __createconn(self):
2802 from config import Config
2806 connstr = "postgres://%s" % cnf["DB::Host"]
2807 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2808 connstr += ":%s" % cnf["DB::Port"]
2809 connstr += "/%s" % cnf["DB::Name"]
2812 connstr = "postgres:///%s" % cnf["DB::Name"]
2813 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2814 connstr += "?port=%s" % cnf["DB::Port"]
2816 self.db_pg = create_engine(connstr, echo=self.debug)
2817 self.db_meta = MetaData()
2818 self.db_meta.bind = self.db_pg
2819 self.db_smaker = sessionmaker(bind=self.db_pg,
2823 self.__setuptables()
2824 self.__setupmappers()
2827 return self.db_smaker()
2829 __all__.append('DBConn')