5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
41 from inspect import getargspec
44 from sqlalchemy import create_engine, Table, MetaData
45 from sqlalchemy.orm import sessionmaker, mapper, relation
46 from sqlalchemy import types as sqltypes
48 # Don't remove this, we re-export the exceptions to scripts which import us
49 from sqlalchemy.exc import *
50 from sqlalchemy.orm.exc import NoResultFound
52 # Only import Config until Queue stuff is changed to store its config
54 from config import Config
55 from singleton import Singleton
56 from textutils import fix_maintainer
58 ################################################################################
60 # Patch in support for the debversion field type so that it works during
63 class DebVersion(sqltypes.Text):
64 def get_col_spec(self):
67 sa_major_version = sqlalchemy.__version__[0:3]
68 if sa_major_version == "0.5":
69 from sqlalchemy.databases import postgres
70 postgres.ischema_names['debversion'] = DebVersion
72 raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
74 ################################################################################
76 __all__ = ['IntegrityError', 'SQLAlchemyError']
78 ################################################################################
80 def session_wrapper(fn):
82 Wrapper around common ".., session=None):" handling. If the wrapped
83 function is called without passing 'session', we create a local one
84 and destroy it when the function ends.
86 Also attaches a commit_or_flush method to the session; if we created a
87 local session, this is a synonym for session.commit(), otherwise it is a
88 synonym for session.flush().
91 def wrapped(*args, **kwargs):
92 private_transaction = False
94 # Find the session object
95 session = kwargs.get('session')
98 if len(args) <= len(getargspec(fn)[0]) - 1:
99 # No session specified as last argument or in kwargs
100 private_transaction = True
101 session = kwargs['session'] = DBConn().session()
103 # Session is last argument in args
107 session = args[-1] = DBConn().session()
108 private_transaction = True
110 if private_transaction:
111 session.commit_or_flush = session.commit
113 session.commit_or_flush = session.flush
116 return fn(*args, **kwargs)
118 if private_transaction:
119 # We created a session; close it.
122 wrapped.__doc__ = fn.__doc__
123 wrapped.func_name = fn.func_name
127 ################################################################################
129 class Architecture(object):
130 def __init__(self, *args, **kwargs):
133 def __eq__(self, val):
134 if isinstance(val, str):
135 return (self.arch_string== val)
136 # This signals to use the normal comparison operator
137 return NotImplemented
139 def __ne__(self, val):
140 if isinstance(val, str):
141 return (self.arch_string != val)
142 # This signals to use the normal comparison operator
143 return NotImplemented
146 return '<Architecture %s>' % self.arch_string
148 __all__.append('Architecture')
151 def get_architecture(architecture, session=None):
153 Returns database id for given C{architecture}.
155 @type architecture: string
156 @param architecture: The name of the architecture
158 @type session: Session
159 @param session: Optional SQLA session object (a temporary one will be
160 generated if not supplied)
163 @return: Architecture object for the given arch (None if not present)
166 q = session.query(Architecture).filter_by(arch_string=architecture)
170 except NoResultFound:
173 __all__.append('get_architecture')
176 def get_architecture_suites(architecture, session=None):
178 Returns list of Suite objects for given C{architecture} name
181 @param source: Architecture name to search for
183 @type session: Session
184 @param session: Optional SQL session object (a temporary one will be
185 generated if not supplied)
188 @return: list of Suite objects for the given name (may be empty)
191 q = session.query(Suite)
192 q = q.join(SuiteArchitecture)
193 q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
199 __all__.append('get_architecture_suites')
201 ################################################################################
203 class Archive(object):
204 def __init__(self, *args, **kwargs):
208 return '<Archive %s>' % self.archive_name
210 __all__.append('Archive')
213 def get_archive(archive, session=None):
215 returns database id for given C{archive}.
217 @type archive: string
218 @param archive: the name of the arhive
220 @type session: Session
221 @param session: Optional SQLA session object (a temporary one will be
222 generated if not supplied)
225 @return: Archive object for the given name (None if not present)
228 archive = archive.lower()
230 q = session.query(Archive).filter_by(archive_name=archive)
234 except NoResultFound:
237 __all__.append('get_archive')
239 ################################################################################
241 class BinAssociation(object):
242 def __init__(self, *args, **kwargs):
246 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
248 __all__.append('BinAssociation')
250 ################################################################################
252 class BinContents(object):
253 def __init__(self, *args, **kwargs):
257 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
259 __all__.append('BinContents')
261 ################################################################################
263 class DBBinary(object):
264 def __init__(self, *args, **kwargs):
268 return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
270 __all__.append('DBBinary')
273 def get_suites_binary_in(package, session=None):
275 Returns list of Suite objects which given C{package} name is in
278 @param source: DBBinary package name to search for
281 @return: list of Suite objects for the given package
284 return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
286 __all__.append('get_suites_binary_in')
289 def get_binary_from_id(binary_id, session=None):
291 Returns DBBinary object for given C{id}
294 @param binary_id: Id of the required binary
296 @type session: Session
297 @param session: Optional SQLA session object (a temporary one will be
298 generated if not supplied)
301 @return: DBBinary object for the given binary (None if not present)
304 q = session.query(DBBinary).filter_by(binary_id=binary_id)
308 except NoResultFound:
311 __all__.append('get_binary_from_id')
314 def get_binaries_from_name(package, version=None, architecture=None, session=None):
316 Returns list of DBBinary objects for given C{package} name
319 @param package: DBBinary package name to search for
321 @type version: str or None
322 @param version: Version to search for (or None)
324 @type package: str, list or None
325 @param package: Architectures to limit to (or None if no limit)
327 @type session: Session
328 @param session: Optional SQL session object (a temporary one will be
329 generated if not supplied)
332 @return: list of DBBinary objects for the given name (may be empty)
335 q = session.query(DBBinary).filter_by(package=package)
337 if version is not None:
338 q = q.filter_by(version=version)
340 if architecture is not None:
341 if not isinstance(architecture, list):
342 architecture = [architecture]
343 q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
349 __all__.append('get_binaries_from_name')
352 def get_binaries_from_source_id(source_id, session=None):
354 Returns list of DBBinary objects for given C{source_id}
357 @param source_id: source_id to search for
359 @type session: Session
360 @param session: Optional SQL session object (a temporary one will be
361 generated if not supplied)
364 @return: list of DBBinary objects for the given name (may be empty)
367 return session.query(DBBinary).filter_by(source_id=source_id).all()
369 __all__.append('get_binaries_from_source_id')
372 def get_binary_from_name_suite(package, suitename, session=None):
373 ### For dak examine-package
374 ### XXX: Doesn't use object API yet
376 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
377 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
378 WHERE b.package=:package
380 AND fi.location = l.id
381 AND l.component = c.id
384 AND su.suite_name=:suitename
385 ORDER BY b.version DESC"""
387 return session.execute(sql, {'package': package, 'suitename': suitename})
389 __all__.append('get_binary_from_name_suite')
392 def get_binary_components(package, suitename, arch, session=None):
393 # Check for packages that have moved from one component to another
394 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
395 WHERE b.package=:package AND s.suite_name=:suitename
396 AND (a.arch_string = :arch OR a.arch_string = 'all')
397 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
398 AND f.location = l.id
399 AND l.component = c.id
402 vals = {'package': package, 'suitename': suitename, 'arch': arch}
404 return session.execute(query, vals)
406 __all__.append('get_binary_components')
408 ################################################################################
410 class BinaryACL(object):
411 def __init__(self, *args, **kwargs):
415 return '<BinaryACL %s>' % self.binary_acl_id
417 __all__.append('BinaryACL')
419 ################################################################################
421 class BinaryACLMap(object):
422 def __init__(self, *args, **kwargs):
426 return '<BinaryACLMap %s>' % self.binary_acl_map_id
428 __all__.append('BinaryACLMap')
430 ################################################################################
432 class Component(object):
433 def __init__(self, *args, **kwargs):
436 def __eq__(self, val):
437 if isinstance(val, str):
438 return (self.component_name == val)
439 # This signals to use the normal comparison operator
440 return NotImplemented
442 def __ne__(self, val):
443 if isinstance(val, str):
444 return (self.component_name != val)
445 # This signals to use the normal comparison operator
446 return NotImplemented
449 return '<Component %s>' % self.component_name
452 __all__.append('Component')
455 def get_component(component, session=None):
457 Returns database id for given C{component}.
459 @type component: string
460 @param component: The name of the override type
463 @return: the database id for the given component
466 component = component.lower()
468 q = session.query(Component).filter_by(component_name=component)
472 except NoResultFound:
475 __all__.append('get_component')
477 ################################################################################
479 class DBConfig(object):
480 def __init__(self, *args, **kwargs):
484 return '<DBConfig %s>' % self.name
486 __all__.append('DBConfig')
488 ################################################################################
491 def get_or_set_contents_file_id(filename, session=None):
493 Returns database id for given filename.
495 If no matching file is found, a row is inserted.
497 @type filename: string
498 @param filename: The filename
499 @type session: SQLAlchemy
500 @param session: Optional SQL session object (a temporary one will be
501 generated if not supplied). If not passed, a commit will be performed at
502 the end of the function, otherwise the caller is responsible for commiting.
505 @return: the database id for the given component
508 q = session.query(ContentFilename).filter_by(filename=filename)
511 ret = q.one().cafilename_id
512 except NoResultFound:
513 cf = ContentFilename()
514 cf.filename = filename
516 session.commit_or_flush()
517 ret = cf.cafilename_id
521 __all__.append('get_or_set_contents_file_id')
524 def get_contents(suite, overridetype, section=None, session=None):
526 Returns contents for a suite / overridetype combination, limiting
527 to a section if not None.
530 @param suite: Suite object
532 @type overridetype: OverrideType
533 @param overridetype: OverrideType object
535 @type section: Section
536 @param section: Optional section object to limit results to
538 @type session: SQLAlchemy
539 @param session: Optional SQL session object (a temporary one will be
540 generated if not supplied)
543 @return: ResultsProxy object set up to return tuples of (filename, section,
547 # find me all of the contents for a given suite
548 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
552 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
553 JOIN content_file_names n ON (c.filename=n.id)
554 JOIN binaries b ON (b.id=c.binary_pkg)
555 JOIN override o ON (o.package=b.package)
556 JOIN section s ON (s.id=o.section)
557 WHERE o.suite = :suiteid AND o.type = :overridetypeid
558 AND b.type=:overridetypename"""
560 vals = {'suiteid': suite.suite_id,
561 'overridetypeid': overridetype.overridetype_id,
562 'overridetypename': overridetype.overridetype}
564 if section is not None:
565 contents_q += " AND s.id = :sectionid"
566 vals['sectionid'] = section.section_id
568 contents_q += " ORDER BY fn"
570 return session.execute(contents_q, vals)
572 __all__.append('get_contents')
574 ################################################################################
576 class ContentFilepath(object):
577 def __init__(self, *args, **kwargs):
581 return '<ContentFilepath %s>' % self.filepath
583 __all__.append('ContentFilepath')
586 def get_or_set_contents_path_id(filepath, session=None):
588 Returns database id for given path.
590 If no matching file is found, a row is inserted.
592 @type filename: string
593 @param filename: The filepath
594 @type session: SQLAlchemy
595 @param session: Optional SQL session object (a temporary one will be
596 generated if not supplied). If not passed, a commit will be performed at
597 the end of the function, otherwise the caller is responsible for commiting.
600 @return: the database id for the given path
603 q = session.query(ContentFilepath).filter_by(filepath=filepath)
606 ret = q.one().cafilepath_id
607 except NoResultFound:
608 cf = ContentFilepath()
609 cf.filepath = filepath
611 session.commit_or_flush()
612 ret = cf.cafilepath_id
616 __all__.append('get_or_set_contents_path_id')
618 ################################################################################
620 class ContentAssociation(object):
621 def __init__(self, *args, **kwargs):
625 return '<ContentAssociation %s>' % self.ca_id
627 __all__.append('ContentAssociation')
629 def insert_content_paths(binary_id, fullpaths, session=None):
631 Make sure given path is associated with given binary id
634 @param binary_id: the id of the binary
635 @type fullpaths: list
636 @param fullpaths: the list of paths of the file being associated with the binary
637 @type session: SQLAlchemy session
638 @param session: Optional SQLAlchemy session. If this is passed, the caller
639 is responsible for ensuring a transaction has begun and committing the
640 results or rolling back based on the result code. If not passed, a commit
641 will be performed at the end of the function, otherwise the caller is
642 responsible for commiting.
644 @return: True upon success
649 session = DBConn().session()
655 for fullpath in fullpaths:
656 if fullpath.startswith( './' ):
657 fullpath = fullpath[2:]
659 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} )
667 traceback.print_exc()
669 # Only rollback if we set up the session ourself
676 __all__.append('insert_content_paths')
678 ################################################################################
680 class DSCFile(object):
681 def __init__(self, *args, **kwargs):
685 return '<DSCFile %s>' % self.dscfile_id
687 __all__.append('DSCFile')
690 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
692 Returns a list of DSCFiles which may be empty
694 @type dscfile_id: int (optional)
695 @param dscfile_id: the dscfile_id of the DSCFiles to find
697 @type source_id: int (optional)
698 @param source_id: the source id related to the DSCFiles to find
700 @type poolfile_id: int (optional)
701 @param poolfile_id: the poolfile id related to the DSCFiles to find
704 @return: Possibly empty list of DSCFiles
707 q = session.query(DSCFile)
709 if dscfile_id is not None:
710 q = q.filter_by(dscfile_id=dscfile_id)
712 if source_id is not None:
713 q = q.filter_by(source_id=source_id)
715 if poolfile_id is not None:
716 q = q.filter_by(poolfile_id=poolfile_id)
720 __all__.append('get_dscfiles')
722 ################################################################################
724 class PoolFile(object):
725 def __init__(self, *args, **kwargs):
729 return '<PoolFile %s>' % self.filename
731 __all__.append('PoolFile')
734 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
737 (ValidFileFound [boolean or None], PoolFile object or None)
739 @type filename: string
740 @param filename: the filename of the file to check against the DB
743 @param filesize: the size of the file to check against the DB
746 @param md5sum: the md5sum of the file to check against the DB
748 @type location_id: int
749 @param location_id: the id of the location to look in
752 @return: Tuple of length 2.
753 If more than one file found with that name:
755 If valid pool file found: (True, PoolFile object)
756 If valid pool file not found:
757 (False, None) if no file found
758 (False, PoolFile object) if file found with size/md5sum mismatch
761 q = session.query(PoolFile).filter_by(filename=filename)
762 q = q.join(Location).filter_by(location_id=location_id)
772 if obj.md5sum != md5sum or obj.filesize != int(filesize):
780 __all__.append('check_poolfile')
783 def get_poolfile_by_id(file_id, session=None):
785 Returns a PoolFile objects or None for the given id
788 @param file_id: the id of the file to look for
790 @rtype: PoolFile or None
791 @return: either the PoolFile object or None
794 q = session.query(PoolFile).filter_by(file_id=file_id)
798 except NoResultFound:
801 __all__.append('get_poolfile_by_id')
805 def get_poolfile_by_name(filename, location_id=None, session=None):
807 Returns an array of PoolFile objects for the given filename and
808 (optionally) location_id
810 @type filename: string
811 @param filename: the filename of the file to check against the DB
813 @type location_id: int
814 @param location_id: the id of the location to look in (optional)
817 @return: array of PoolFile objects
820 q = session.query(PoolFile).filter_by(filename=filename)
822 if location_id is not None:
823 q = q.join(Location).filter_by(location_id=location_id)
827 __all__.append('get_poolfile_by_name')
830 def get_poolfile_like_name(filename, session=None):
832 Returns an array of PoolFile objects which are like the given name
834 @type filename: string
835 @param filename: the filename of the file to check against the DB
838 @return: array of PoolFile objects
841 # TODO: There must be a way of properly using bind parameters with %FOO%
842 q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
846 __all__.append('get_poolfile_like_name')
849 def add_poolfile(filename, datadict, location_id, session=None):
851 Add a new file to the pool
853 @type filename: string
854 @param filename: filename
857 @param datadict: dict with needed data
859 @type location_id: int
860 @param location_id: database id of the location
863 @return: the PoolFile object created
865 poolfile = PoolFile()
866 poolfile.filename = filename
867 poolfile.filesize = datadict["size"]
868 poolfile.md5sum = datadict["md5sum"]
869 poolfile.sha1sum = datadict["sha1sum"]
870 poolfile.sha256sum = datadict["sha256sum"]
871 poolfile.location_id = location_id
873 session.add(poolfile)
874 # Flush to get a file id (NB: This is not a commit)
879 __all__.append('add_poolfile')
881 ################################################################################
883 class Fingerprint(object):
884 def __init__(self, *args, **kwargs):
888 return '<Fingerprint %s>' % self.fingerprint
890 __all__.append('Fingerprint')
893 def get_fingerprint(fpr, session=None):
895 Returns Fingerprint object for given fpr.
898 @param fpr: The fpr to find / add
900 @type session: SQLAlchemy
901 @param session: Optional SQL session object (a temporary one will be
902 generated if not supplied).
905 @return: the Fingerprint object for the given fpr or None
908 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
912 except NoResultFound:
917 __all__.append('get_fingerprint')
920 def get_or_set_fingerprint(fpr, session=None):
922 Returns Fingerprint object for given fpr.
924 If no matching fpr is found, a row is inserted.
927 @param fpr: The fpr to find / add
929 @type session: SQLAlchemy
930 @param session: Optional SQL session object (a temporary one will be
931 generated if not supplied). If not passed, a commit will be performed at
932 the end of the function, otherwise the caller is responsible for commiting.
933 A flush will be performed either way.
936 @return: the Fingerprint object for the given fpr
939 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
943 except NoResultFound:
944 fingerprint = Fingerprint()
945 fingerprint.fingerprint = fpr
946 session.add(fingerprint)
947 session.commit_or_flush()
952 __all__.append('get_or_set_fingerprint')
954 ################################################################################
956 # Helper routine for Keyring class
957 def get_ldap_name(entry):
959 for k in ["cn", "mn", "sn"]:
961 if ret and ret[0] != "" and ret[0] != "-":
963 return " ".join(name)
965 ################################################################################
967 class Keyring(object):
968 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
969 " --with-colons --fingerprint --fingerprint"
974 def __init__(self, *args, **kwargs):
978 return '<Keyring %s>' % self.keyring_name
980 def de_escape_gpg_str(self, txt):
981 esclist = re.split(r'(\\x..)', txt)
982 for x in range(1,len(esclist),2):
983 esclist[x] = "%c" % (int(esclist[x][2:],16))
984 return "".join(esclist)
986 def load_keys(self, keyring):
989 if not self.keyring_id:
990 raise Exception('Must be initialized with database information')
992 k = os.popen(self.gpg_invocation % keyring, "r")
996 for line in k.xreadlines():
997 field = line.split(":")
998 if field[0] == "pub":
1000 (name, addr) = email.Utils.parseaddr(field[9])
1001 name = re.sub(r"\s*[(].*[)]", "", name)
1002 if name == "" or addr == "" or "@" not in addr:
1004 addr = "invalid-uid"
1005 name = self.de_escape_gpg_str(name)
1006 self.keys[key] = {"email": addr}
1008 self.keys[key]["name"] = name
1009 self.keys[key]["aliases"] = [name]
1010 self.keys[key]["fingerprints"] = []
1012 elif key and field[0] == "sub" and len(field) >= 12:
1013 signingkey = ("s" in field[11])
1014 elif key and field[0] == "uid":
1015 (name, addr) = email.Utils.parseaddr(field[9])
1016 if name and name not in self.keys[key]["aliases"]:
1017 self.keys[key]["aliases"].append(name)
1018 elif signingkey and field[0] == "fpr":
1019 self.keys[key]["fingerprints"].append(field[9])
1020 self.fpr_lookup[field[9]] = key
1022 def import_users_from_ldap(self, session):
1026 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1027 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1029 l = ldap.open(LDAPServer)
1030 l.simple_bind_s("","")
1031 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1032 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1033 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1035 ldap_fin_uid_id = {}
1042 uid = entry["uid"][0]
1043 name = get_ldap_name(entry)
1044 fingerprints = entry["keyFingerPrint"]
1046 for f in fingerprints:
1047 key = self.fpr_lookup.get(f, None)
1048 if key not in self.keys:
1050 self.keys[key]["uid"] = uid
1054 keyid = get_or_set_uid(uid, session).uid_id
1055 byuid[keyid] = (uid, name)
1056 byname[uid] = (keyid, name)
1058 return (byname, byuid)
1060 def generate_users_from_keyring(self, format, session):
1064 for x in self.keys.keys():
1065 if self.keys[x]["email"] == "invalid-uid":
1067 self.keys[x]["uid"] = format % "invalid-uid"
1069 uid = format % self.keys[x]["email"]
1070 keyid = get_or_set_uid(uid, session).uid_id
1071 byuid[keyid] = (uid, self.keys[x]["name"])
1072 byname[uid] = (keyid, self.keys[x]["name"])
1073 self.keys[x]["uid"] = uid
1076 uid = format % "invalid-uid"
1077 keyid = get_or_set_uid(uid, session).uid_id
1078 byuid[keyid] = (uid, "ungeneratable user id")
1079 byname[uid] = (keyid, "ungeneratable user id")
1081 return (byname, byuid)
1083 __all__.append('Keyring')
1086 def get_keyring(keyring, session=None):
1088 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1089 If C{keyring} already has an entry, simply return the existing Keyring
1091 @type keyring: string
1092 @param keyring: the keyring name
1095 @return: the Keyring object for this keyring
1098 q = session.query(Keyring).filter_by(keyring_name=keyring)
1102 except NoResultFound:
1105 __all__.append('get_keyring')
1107 ################################################################################
1109 class KeyringACLMap(object):
1110 def __init__(self, *args, **kwargs):
1114 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1116 __all__.append('KeyringACLMap')
1118 ################################################################################
1120 class KnownChange(object):
1121 def __init__(self, *args, **kwargs):
1125 return '<KnownChange %s>' % self.changesname
1127 __all__.append('KnownChange')
1130 def get_knownchange(filename, session=None):
1132 returns knownchange object for given C{filename}.
1134 @type archive: string
1135 @param archive: the name of the arhive
1137 @type session: Session
1138 @param session: Optional SQLA session object (a temporary one will be
1139 generated if not supplied)
1142 @return: Archive object for the given name (None if not present)
1145 q = session.query(KnownChange).filter_by(changesname=filename)
1149 except NoResultFound:
1152 __all__.append('get_knownchange')
1154 ################################################################################
1155 class Location(object):
1156 def __init__(self, *args, **kwargs):
1160 return '<Location %s (%s)>' % (self.path, self.location_id)
1162 __all__.append('Location')
1165 def get_location(location, component=None, archive=None, session=None):
1167 Returns Location object for the given combination of location, component
1170 @type location: string
1171 @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
1173 @type component: string
1174 @param component: the component name (if None, no restriction applied)
1176 @type archive: string
1177 @param archive_id: the archive name (if None, no restriction applied)
1179 @rtype: Location / None
1180 @return: Either a Location object or None if one can't be found
1183 q = session.query(Location).filter_by(path=location)
1185 if archive is not None:
1186 q = q.join(Archive).filter_by(archive_name=archive)
1188 if component is not None:
1189 q = q.join(Component).filter_by(component_name=component)
1193 except NoResultFound:
1196 __all__.append('get_location')
1198 ################################################################################
1200 class Maintainer(object):
1201 def __init__(self, *args, **kwargs):
1205 return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
1207 def get_split_maintainer(self):
1208 if not hasattr(self, 'name') or self.name is None:
1209 return ('', '', '', '')
1211 return fix_maintainer(self.name.strip())
1213 __all__.append('Maintainer')
1216 def get_or_set_maintainer(name, session=None):
1218 Returns Maintainer object for given maintainer name.
1220 If no matching maintainer name is found, a row is inserted.
1223 @param name: The maintainer name to add
1225 @type session: SQLAlchemy
1226 @param session: Optional SQL session object (a temporary one will be
1227 generated if not supplied). If not passed, a commit will be performed at
1228 the end of the function, otherwise the caller is responsible for commiting.
1229 A flush will be performed either way.
1232 @return: the Maintainer object for the given maintainer
1235 q = session.query(Maintainer).filter_by(name=name)
1238 except NoResultFound:
1239 maintainer = Maintainer()
1240 maintainer.name = name
1241 session.add(maintainer)
1242 session.commit_or_flush()
1247 __all__.append('get_or_set_maintainer')
1250 def get_maintainer(maintainer_id, session=None):
1252 Return the name of the maintainer behind C{maintainer_id} or None if that
1253 maintainer_id is invalid.
1255 @type maintainer_id: int
1256 @param maintainer_id: the id of the maintainer
1259 @return: the Maintainer with this C{maintainer_id}
1262 return session.query(Maintainer).get(maintainer_id)
1264 __all__.append('get_maintainer')
1266 ################################################################################
1268 class NewComment(object):
1269 def __init__(self, *args, **kwargs):
1273 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1275 __all__.append('NewComment')
1278 def has_new_comment(package, version, session=None):
1280 Returns true if the given combination of C{package}, C{version} has a comment.
1282 @type package: string
1283 @param package: name of the package
1285 @type version: string
1286 @param version: package version
1288 @type session: Session
1289 @param session: Optional SQLA session object (a temporary one will be
1290 generated if not supplied)
1296 q = session.query(NewComment)
1297 q = q.filter_by(package=package)
1298 q = q.filter_by(version=version)
1300 return bool(q.count() > 0)
1302 __all__.append('has_new_comment')
1305 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1307 Returns (possibly empty) list of NewComment objects for the given
1310 @type package: string (optional)
1311 @param package: name of the package
1313 @type version: string (optional)
1314 @param version: package version
1316 @type comment_id: int (optional)
1317 @param comment_id: An id of a comment
1319 @type session: Session
1320 @param session: Optional SQLA session object (a temporary one will be
1321 generated if not supplied)
1324 @return: A (possibly empty) list of NewComment objects will be returned
1327 q = session.query(NewComment)
1328 if package is not None: q = q.filter_by(package=package)
1329 if version is not None: q = q.filter_by(version=version)
1330 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1334 __all__.append('get_new_comments')
1336 ################################################################################
1338 class Override(object):
1339 def __init__(self, *args, **kwargs):
1343 return '<Override %s (%s)>' % (self.package, self.suite_id)
1345 __all__.append('Override')
1348 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1350 Returns Override object for the given parameters
1352 @type package: string
1353 @param package: The name of the package
1355 @type suite: string, list or None
1356 @param suite: The name of the suite (or suites if a list) to limit to. If
1357 None, don't limit. Defaults to None.
1359 @type component: string, list or None
1360 @param component: The name of the component (or components if a list) to
1361 limit to. If None, don't limit. Defaults to None.
1363 @type overridetype: string, list or None
1364 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1365 limit to. If None, don't limit. Defaults to None.
1367 @type session: Session
1368 @param session: Optional SQLA session object (a temporary one will be
1369 generated if not supplied)
1372 @return: A (possibly empty) list of Override objects will be returned
1375 q = session.query(Override)
1376 q = q.filter_by(package=package)
1378 if suite is not None:
1379 if not isinstance(suite, list): suite = [suite]
1380 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1382 if component is not None:
1383 if not isinstance(component, list): component = [component]
1384 q = q.join(Component).filter(Component.component_name.in_(component))
1386 if overridetype is not None:
1387 if not isinstance(overridetype, list): overridetype = [overridetype]
1388 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1392 __all__.append('get_override')
1395 ################################################################################
1397 class OverrideType(object):
1398 def __init__(self, *args, **kwargs):
1402 return '<OverrideType %s>' % self.overridetype
1404 __all__.append('OverrideType')
1407 def get_override_type(override_type, session=None):
1409 Returns OverrideType object for given C{override type}.
1411 @type override_type: string
1412 @param override_type: The name of the override type
1414 @type session: Session
1415 @param session: Optional SQLA session object (a temporary one will be
1416 generated if not supplied)
1419 @return: the database id for the given override type
1422 q = session.query(OverrideType).filter_by(overridetype=override_type)
1426 except NoResultFound:
1429 __all__.append('get_override_type')
1431 ################################################################################
1433 class PendingContentAssociation(object):
1434 def __init__(self, *args, **kwargs):
1438 return '<PendingContentAssociation %s>' % self.pca_id
1440 __all__.append('PendingContentAssociation')
1442 def insert_pending_content_paths(package, fullpaths, session=None):
1444 Make sure given paths are temporarily associated with given
1448 @param package: the package to associate with should have been read in from the binary control file
1449 @type fullpaths: list
1450 @param fullpaths: the list of paths of the file being associated with the binary
1451 @type session: SQLAlchemy session
1452 @param session: Optional SQLAlchemy session. If this is passed, the caller
1453 is responsible for ensuring a transaction has begun and committing the
1454 results or rolling back based on the result code. If not passed, a commit
1455 will be performed at the end of the function
1457 @return: True upon success, False if there is a problem
1460 privatetrans = False
1463 session = DBConn().session()
1467 arch = get_architecture(package['Architecture'], session)
1468 arch_id = arch.arch_id
1470 # Remove any already existing recorded files for this package
1471 q = session.query(PendingContentAssociation)
1472 q = q.filter_by(package=package['Package'])
1473 q = q.filter_by(version=package['Version'])
1474 q = q.filter_by(architecture=arch_id)
1479 for fullpath in fullpaths:
1480 (path, filename) = os.path.split(fullpath)
1482 if path.startswith( "./" ):
1485 filepath_id = get_or_set_contents_path_id(path, session)
1486 filename_id = get_or_set_contents_file_id(filename, session)
1488 pathcache[fullpath] = (filepath_id, filename_id)
1490 for fullpath, dat in pathcache.items():
1491 pca = PendingContentAssociation()
1492 pca.package = package['Package']
1493 pca.version = package['Version']
1494 pca.filepath_id = dat[0]
1495 pca.filename_id = dat[1]
1496 pca.architecture = arch_id
1499 # Only commit if we set up the session ourself
1507 except Exception, e:
1508 traceback.print_exc()
1510 # Only rollback if we set up the session ourself
1517 __all__.append('insert_pending_content_paths')
1519 ################################################################################
1521 class Priority(object):
1522 def __init__(self, *args, **kwargs):
1525 def __eq__(self, val):
1526 if isinstance(val, str):
1527 return (self.priority == val)
1528 # This signals to use the normal comparison operator
1529 return NotImplemented
1531 def __ne__(self, val):
1532 if isinstance(val, str):
1533 return (self.priority != val)
1534 # This signals to use the normal comparison operator
1535 return NotImplemented
1538 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
1540 __all__.append('Priority')
1543 def get_priority(priority, session=None):
1545 Returns Priority object for given C{priority name}.
1547 @type priority: string
1548 @param priority: The name of the priority
1550 @type session: Session
1551 @param session: Optional SQLA session object (a temporary one will be
1552 generated if not supplied)
1555 @return: Priority object for the given priority
1558 q = session.query(Priority).filter_by(priority=priority)
1562 except NoResultFound:
1565 __all__.append('get_priority')
1568 def get_priorities(session=None):
1570 Returns dictionary of priority names -> id mappings
1572 @type session: Session
1573 @param session: Optional SQL session object (a temporary one will be
1574 generated if not supplied)
1577 @return: dictionary of priority names -> id mappings
1581 q = session.query(Priority)
1583 ret[x.priority] = x.priority_id
1587 __all__.append('get_priorities')
1589 ################################################################################
1591 class Queue(object):
1592 def __init__(self, *args, **kwargs):
1596 return '<Queue %s>' % self.queue_name
1598 def autobuild_upload(self, changes, srcpath, session=None):
1600 Update queue_build database table used for incoming autobuild support.
1602 @type changes: Changes
1603 @param changes: changes object for the upload to process
1605 @type srcpath: string
1606 @param srcpath: path for the queue file entries/link destinations
1608 @type session: SQLAlchemy session
1609 @param session: Optional SQLAlchemy session. If this is passed, the
1610 caller is responsible for ensuring a transaction has begun and
1611 committing the results or rolling back based on the result code. If
1612 not passed, a commit will be performed at the end of the function,
1613 otherwise the caller is responsible for commiting.
1615 @rtype: NoneType or string
1616 @return: None if the operation failed, a string describing the error if not
1619 privatetrans = False
1621 session = DBConn().session()
1624 # TODO: Remove by moving queue config into the database
1627 for suitename in changes.changes["distribution"].keys():
1628 # TODO: Move into database as:
1629 # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build)
1630 # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink)
1631 # This also gets rid of the SecurityQueueBuild hack below
1632 if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"):
1636 s = get_suite(suitename, session)
1638 return "INTERNAL ERROR: Could not find suite %s" % suitename
1640 # TODO: Get from database as above
1641 dest_dir = conf["Dir::QueueBuild"]
1643 # TODO: Move into database as above
1644 if conf.FindB("Dinstall::SecurityQueueBuild"):
1645 dest_dir = os.path.join(dest_dir, suitename)
1647 for file_entry in changes.files.keys():
1648 src = os.path.join(srcpath, file_entry)
1649 dest = os.path.join(dest_dir, file_entry)
1651 # TODO: Move into database as above
1652 if conf.FindB("Dinstall::SecurityQueueBuild"):
1653 # Copy it since the original won't be readable by www-data
1655 utils.copy(src, dest)
1657 # Create a symlink to it
1658 os.symlink(src, dest)
1661 qb.suite_id = s.suite_id
1662 qb.queue_id = self.queue_id
1668 # If the .orig tarballs are in the pool, create a symlink to
1669 # them (if one doesn't already exist)
1670 for dsc_file in changes.dsc_files.keys():
1671 # Skip all files except orig tarballs
1672 from daklib.regexes import re_is_orig_source
1673 if not re_is_orig_source.match(dsc_file):
1675 # Skip orig files not identified in the pool
1676 if not (changes.orig_files.has_key(dsc_file) and
1677 changes.orig_files[dsc_file].has_key("id")):
1679 orig_file_id = changes.orig_files[dsc_file]["id"]
1680 dest = os.path.join(dest_dir, dsc_file)
1682 # If it doesn't exist, create a symlink
1683 if not os.path.exists(dest):
1684 q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id",
1685 {'id': orig_file_id})
1688 return "[INTERNAL ERROR] Couldn't find id %s in files table." % (orig_file_id)
1690 src = os.path.join(res[0], res[1])
1691 os.symlink(src, dest)
1693 # Add it to the list of packages for later processing by apt-ftparchive
1695 qb.suite_id = s.suite_id
1696 qb.queue_id = self.queue_id
1701 # If it does, update things to ensure it's not removed prematurely
1703 qb = get_queue_build(dest, s.suite_id, session)
1715 __all__.append('Queue')
1718 def get_or_set_queue(queuename, session=None):
1720 Returns Queue object for given C{queue name}, creating it if it does not
1723 @type queuename: string
1724 @param queuename: The name of the queue
1726 @type session: Session
1727 @param session: Optional SQLA session object (a temporary one will be
1728 generated if not supplied)
1731 @return: Queue object for the given queue
1734 q = session.query(Queue).filter_by(queue_name=queuename)
1738 except NoResultFound:
1740 queue.queue_name = queuename
1742 session.commit_or_flush()
1747 __all__.append('get_or_set_queue')
1749 ################################################################################
1751 class QueueBuild(object):
1752 def __init__(self, *args, **kwargs):
1756 return '<QueueBuild %s (%s)>' % (self.filename, self.queue_id)
1758 __all__.append('QueueBuild')
1761 def get_queue_build(filename, suite, session=None):
1763 Returns QueueBuild object for given C{filename} and C{suite}.
1765 @type filename: string
1766 @param filename: The name of the file
1768 @type suiteid: int or str
1769 @param suiteid: Suite name or ID
1771 @type session: Session
1772 @param session: Optional SQLA session object (a temporary one will be
1773 generated if not supplied)
1776 @return: Queue object for the given queue
1779 if isinstance(suite, int):
1780 q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite)
1782 q = session.query(QueueBuild).filter_by(filename=filename)
1783 q = q.join(Suite).filter_by(suite_name=suite)
1787 except NoResultFound:
1790 __all__.append('get_queue_build')
1792 ################################################################################
1794 class Section(object):
1795 def __init__(self, *args, **kwargs):
1798 def __eq__(self, val):
1799 if isinstance(val, str):
1800 return (self.section == val)
1801 # This signals to use the normal comparison operator
1802 return NotImplemented
1804 def __ne__(self, val):
1805 if isinstance(val, str):
1806 return (self.section != val)
1807 # This signals to use the normal comparison operator
1808 return NotImplemented
1811 return '<Section %s>' % self.section
1813 __all__.append('Section')
1816 def get_section(section, session=None):
1818 Returns Section object for given C{section name}.
1820 @type section: string
1821 @param section: The name of the section
1823 @type session: Session
1824 @param session: Optional SQLA session object (a temporary one will be
1825 generated if not supplied)
1828 @return: Section object for the given section name
1831 q = session.query(Section).filter_by(section=section)
1835 except NoResultFound:
1838 __all__.append('get_section')
1841 def get_sections(session=None):
1843 Returns dictionary of section names -> id mappings
1845 @type session: Session
1846 @param session: Optional SQL session object (a temporary one will be
1847 generated if not supplied)
1850 @return: dictionary of section names -> id mappings
1854 q = session.query(Section)
1856 ret[x.section] = x.section_id
1860 __all__.append('get_sections')
1862 ################################################################################
1864 class DBSource(object):
1865 def __init__(self, *args, **kwargs):
1869 return '<DBSource %s (%s)>' % (self.source, self.version)
1871 __all__.append('DBSource')
1874 def source_exists(source, source_version, suites = ["any"], session=None):
1876 Ensure that source exists somewhere in the archive for the binary
1877 upload being processed.
1878 1. exact match => 1.0-3
1879 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1881 @type package: string
1882 @param package: package source name
1884 @type source_version: string
1885 @param source_version: expected source version
1888 @param suites: list of suites to check in, default I{any}
1890 @type session: Session
1891 @param session: Optional SQLA session object (a temporary one will be
1892 generated if not supplied)
1895 @return: returns 1 if a source with expected version is found, otherwise 0
1902 for suite in suites:
1903 q = session.query(DBSource).filter_by(source=source)
1905 # source must exist in suite X, or in some other suite that's
1906 # mapped to X, recursively... silent-maps are counted too,
1907 # unreleased-maps aren't.
1908 maps = cnf.ValueList("SuiteMappings")[:]
1910 maps = [ m.split() for m in maps ]
1911 maps = [ (x[1], x[2]) for x in maps
1912 if x[0] == "map" or x[0] == "silent-map" ]
1915 if x[1] in s and x[0] not in s:
1918 q = q.join(SrcAssociation).join(Suite)
1919 q = q.filter(Suite.suite_name.in_(s))
1921 # Reduce the query results to a list of version numbers
1922 ql = [ j.version for j in q.all() ]
1925 if source_version in ql:
1929 from daklib.regexes import re_bin_only_nmu
1930 orig_source_version = re_bin_only_nmu.sub('', source_version)
1931 if orig_source_version in ql:
1934 # No source found so return not ok
1939 __all__.append('source_exists')
1942 def get_suites_source_in(source, session=None):
1944 Returns list of Suite objects which given C{source} name is in
1947 @param source: DBSource package name to search for
1950 @return: list of Suite objects for the given source
1953 return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
1955 __all__.append('get_suites_source_in')
1958 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
1960 Returns list of DBSource objects for given C{source} name and other parameters
1963 @param source: DBSource package name to search for
1965 @type source: str or None
1966 @param source: DBSource version name to search for or None if not applicable
1968 @type dm_upload_allowed: bool
1969 @param dm_upload_allowed: If None, no effect. If True or False, only
1970 return packages with that dm_upload_allowed setting
1972 @type session: Session
1973 @param session: Optional SQL session object (a temporary one will be
1974 generated if not supplied)
1977 @return: list of DBSource objects for the given name (may be empty)
1980 q = session.query(DBSource).filter_by(source=source)
1982 if version is not None:
1983 q = q.filter_by(version=version)
1985 if dm_upload_allowed is not None:
1986 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
1990 __all__.append('get_sources_from_name')
1993 def get_source_in_suite(source, suite, session=None):
1995 Returns list of DBSource objects for a combination of C{source} and C{suite}.
1997 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
1998 - B{suite} - a suite name, eg. I{unstable}
2000 @type source: string
2001 @param source: source package name
2004 @param suite: the suite name
2007 @return: the version for I{source} in I{suite}
2011 q = session.query(SrcAssociation)
2012 q = q.join('source').filter_by(source=source)
2013 q = q.join('suite').filter_by(suite_name=suite)
2016 return q.one().source
2017 except NoResultFound:
2020 __all__.append('get_source_in_suite')
2022 ################################################################################
2025 def add_dsc_to_db(u, filename, session=None):
2026 entry = u.pkg.files[filename]
2029 source.source = u.pkg.dsc["source"]
2030 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2031 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2032 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2033 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2034 source.install_date = datetime.now().date()
2036 dsc_component = entry["component"]
2037 dsc_location_id = entry["location id"]
2039 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2041 # Set up a new poolfile if necessary
2042 if not entry.has_key("files id") or not entry["files id"]:
2043 filename = entry["pool name"] + filename
2044 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2045 entry["files id"] = poolfile.file_id
2047 source.poolfile_id = entry["files id"]
2051 for suite_name in u.pkg.changes["distribution"].keys():
2052 sa = SrcAssociation()
2053 sa.source_id = source.source_id
2054 sa.suite_id = get_suite(suite_name).suite_id
2059 # Add the source files to the DB (files and dsc_files)
2061 dscfile.source_id = source.source_id
2062 dscfile.poolfile_id = entry["files id"]
2063 session.add(dscfile)
2065 for dsc_file, dentry in u.pkg.dsc_files.items():
2067 df.source_id = source.source_id
2069 # If the .orig tarball is already in the pool, it's
2070 # files id is stored in dsc_files by check_dsc().
2071 files_id = dentry.get("files id", None)
2073 # Find the entry in the files hash
2074 # TODO: Bail out here properly
2076 for f, e in u.pkg.files.items():
2081 if files_id is None:
2082 filename = dfentry["pool name"] + dsc_file
2084 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2085 # FIXME: needs to check for -1/-2 and or handle exception
2086 if found and obj is not None:
2087 files_id = obj.file_id
2089 # If still not found, add it
2090 if files_id is None:
2091 # HACK: Force sha1sum etc into dentry
2092 dentry["sha1sum"] = dfentry["sha1sum"]
2093 dentry["sha256sum"] = dfentry["sha256sum"]
2094 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2095 files_id = poolfile.file_id
2097 df.poolfile_id = files_id
2102 # Add the src_uploaders to the DB
2103 uploader_ids = [source.maintainer_id]
2104 if u.pkg.dsc.has_key("uploaders"):
2105 for up in u.pkg.dsc["uploaders"].split(","):
2107 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2110 for up in uploader_ids:
2111 if added_ids.has_key(up):
2112 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
2118 su.maintainer_id = up
2119 su.source_id = source.source_id
2124 return dsc_component, dsc_location_id
2126 __all__.append('add_dsc_to_db')
2129 def add_deb_to_db(u, filename, session=None):
2131 Contrary to what you might expect, this routine deals with both
2132 debs and udebs. That info is in 'dbtype', whilst 'type' is
2133 'deb' for both of them
2136 entry = u.pkg.files[filename]
2139 bin.package = entry["package"]
2140 bin.version = entry["version"]
2141 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2142 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2143 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2144 bin.binarytype = entry["dbtype"]
2147 filename = entry["pool name"] + filename
2148 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2149 if not entry.get("location id", None):
2150 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], utils.where_am_i(), session).location_id
2152 if not entry.get("files id", None):
2153 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2154 entry["files id"] = poolfile.file_id
2156 bin.poolfile_id = entry["files id"]
2159 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2160 if len(bin_sources) != 1:
2161 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2162 (bin.package, bin.version, bin.architecture.arch_string,
2163 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2165 bin.source_id = bin_sources[0].source_id
2167 # Add and flush object so it has an ID
2171 # Add BinAssociations
2172 for suite_name in u.pkg.changes["distribution"].keys():
2173 ba = BinAssociation()
2174 ba.binary_id = bin.binary_id
2175 ba.suite_id = get_suite(suite_name).suite_id
2180 # Deal with contents - disabled for now
2181 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2183 # print "REJECT\nCould not determine contents of package %s" % bin.package
2184 # session.rollback()
2185 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2187 __all__.append('add_deb_to_db')
2189 ################################################################################
2191 class SourceACL(object):
2192 def __init__(self, *args, **kwargs):
2196 return '<SourceACL %s>' % self.source_acl_id
2198 __all__.append('SourceACL')
2200 ################################################################################
2202 class SrcAssociation(object):
2203 def __init__(self, *args, **kwargs):
2207 return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
2209 __all__.append('SrcAssociation')
2211 ################################################################################
2213 class SrcFormat(object):
2214 def __init__(self, *args, **kwargs):
2218 return '<SrcFormat %s>' % (self.format_name)
2220 __all__.append('SrcFormat')
2222 ################################################################################
2224 class SrcUploader(object):
2225 def __init__(self, *args, **kwargs):
2229 return '<SrcUploader %s>' % self.uploader_id
2231 __all__.append('SrcUploader')
2233 ################################################################################
2235 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2236 ('SuiteID', 'suite_id'),
2237 ('Version', 'version'),
2238 ('Origin', 'origin'),
2240 ('Description', 'description'),
2241 ('Untouchable', 'untouchable'),
2242 ('Announce', 'announce'),
2243 ('Codename', 'codename'),
2244 ('OverrideCodename', 'overridecodename'),
2245 ('ValidTime', 'validtime'),
2246 ('Priority', 'priority'),
2247 ('NotAutomatic', 'notautomatic'),
2248 ('CopyChanges', 'copychanges'),
2249 ('CopyDotDak', 'copydotdak'),
2250 ('CommentsDir', 'commentsdir'),
2251 ('OverrideSuite', 'overridesuite'),
2252 ('ChangelogBase', 'changelogbase')]
2255 class Suite(object):
2256 def __init__(self, *args, **kwargs):
2260 return '<Suite %s>' % self.suite_name
2262 def __eq__(self, val):
2263 if isinstance(val, str):
2264 return (self.suite_name == val)
2265 # This signals to use the normal comparison operator
2266 return NotImplemented
2268 def __ne__(self, val):
2269 if isinstance(val, str):
2270 return (self.suite_name != val)
2271 # This signals to use the normal comparison operator
2272 return NotImplemented
2276 for disp, field in SUITE_FIELDS:
2277 val = getattr(self, field, None)
2279 ret.append("%s: %s" % (disp, val))
2281 return "\n".join(ret)
2283 __all__.append('Suite')
2286 def get_suite_architecture(suite, architecture, session=None):
2288 Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
2292 @param suite: Suite name to search for
2294 @type architecture: str
2295 @param architecture: Architecture name to search for
2297 @type session: Session
2298 @param session: Optional SQL session object (a temporary one will be
2299 generated if not supplied)
2301 @rtype: SuiteArchitecture
2302 @return: the SuiteArchitecture object or None
2305 q = session.query(SuiteArchitecture)
2306 q = q.join(Architecture).filter_by(arch_string=architecture)
2307 q = q.join(Suite).filter_by(suite_name=suite)
2311 except NoResultFound:
2314 __all__.append('get_suite_architecture')
2317 def get_suite(suite, session=None):
2319 Returns Suite object for given C{suite name}.
2322 @param suite: The name of the suite
2324 @type session: Session
2325 @param session: Optional SQLA session object (a temporary one will be
2326 generated if not supplied)
2329 @return: Suite object for the requested suite name (None if not present)
2332 q = session.query(Suite).filter_by(suite_name=suite)
2336 except NoResultFound:
2339 __all__.append('get_suite')
2341 ################################################################################
2343 class SuiteArchitecture(object):
2344 def __init__(self, *args, **kwargs):
2348 return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
2350 __all__.append('SuiteArchitecture')
2353 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2355 Returns list of Architecture objects for given C{suite} name
2358 @param source: Suite name to search for
2360 @type skipsrc: boolean
2361 @param skipsrc: Whether to skip returning the 'source' architecture entry
2364 @type skipall: boolean
2365 @param skipall: Whether to skip returning the 'all' architecture entry
2368 @type session: Session
2369 @param session: Optional SQL session object (a temporary one will be
2370 generated if not supplied)
2373 @return: list of Architecture objects for the given name (may be empty)
2376 q = session.query(Architecture)
2377 q = q.join(SuiteArchitecture)
2378 q = q.join(Suite).filter_by(suite_name=suite)
2381 q = q.filter(Architecture.arch_string != 'source')
2384 q = q.filter(Architecture.arch_string != 'all')
2386 q = q.order_by('arch_string')
2390 __all__.append('get_suite_architectures')
2392 ################################################################################
2394 class SuiteSrcFormat(object):
2395 def __init__(self, *args, **kwargs):
2399 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2401 __all__.append('SuiteSrcFormat')
2404 def get_suite_src_formats(suite, session=None):
2406 Returns list of allowed SrcFormat for C{suite}.
2409 @param suite: Suite name to search for
2411 @type session: Session
2412 @param session: Optional SQL session object (a temporary one will be
2413 generated if not supplied)
2416 @return: the list of allowed source formats for I{suite}
2419 q = session.query(SrcFormat)
2420 q = q.join(SuiteSrcFormat)
2421 q = q.join(Suite).filter_by(suite_name=suite)
2422 q = q.order_by('format_name')
2426 __all__.append('get_suite_src_formats')
2428 ################################################################################
2431 def __init__(self, *args, **kwargs):
2434 def __eq__(self, val):
2435 if isinstance(val, str):
2436 return (self.uid == val)
2437 # This signals to use the normal comparison operator
2438 return NotImplemented
2440 def __ne__(self, val):
2441 if isinstance(val, str):
2442 return (self.uid != val)
2443 # This signals to use the normal comparison operator
2444 return NotImplemented
2447 return '<Uid %s (%s)>' % (self.uid, self.name)
2449 __all__.append('Uid')
2452 def add_database_user(uidname, session=None):
2454 Adds a database user
2456 @type uidname: string
2457 @param uidname: The uid of the user to add
2459 @type session: SQLAlchemy
2460 @param session: Optional SQL session object (a temporary one will be
2461 generated if not supplied). If not passed, a commit will be performed at
2462 the end of the function, otherwise the caller is responsible for commiting.
2465 @return: the uid object for the given uidname
2468 session.execute("CREATE USER :uid", {'uid': uidname})
2469 session.commit_or_flush()
2471 __all__.append('add_database_user')
2474 def get_or_set_uid(uidname, session=None):
2476 Returns uid object for given uidname.
2478 If no matching uidname is found, a row is inserted.
2480 @type uidname: string
2481 @param uidname: The uid to add
2483 @type session: SQLAlchemy
2484 @param session: Optional SQL session object (a temporary one will be
2485 generated if not supplied). If not passed, a commit will be performed at
2486 the end of the function, otherwise the caller is responsible for commiting.
2489 @return: the uid object for the given uidname
2492 q = session.query(Uid).filter_by(uid=uidname)
2496 except NoResultFound:
2500 session.commit_or_flush()
2505 __all__.append('get_or_set_uid')
2508 def get_uid_from_fingerprint(fpr, session=None):
2509 q = session.query(Uid)
2510 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2514 except NoResultFound:
2517 __all__.append('get_uid_from_fingerprint')
2519 ################################################################################
2521 class UploadBlock(object):
2522 def __init__(self, *args, **kwargs):
2526 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2528 __all__.append('UploadBlock')
2530 ################################################################################
2532 class DBConn(Singleton):
2534 database module init.
2536 def __init__(self, *args, **kwargs):
2537 super(DBConn, self).__init__(*args, **kwargs)
2539 def _startup(self, *args, **kwargs):
2541 if kwargs.has_key('debug'):
2545 def __setuptables(self):
2546 self.tbl_architecture = Table('architecture', self.db_meta, autoload=True)
2547 self.tbl_archive = Table('archive', self.db_meta, autoload=True)
2548 self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True)
2549 self.tbl_binaries = Table('binaries', self.db_meta, autoload=True)
2550 self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True)
2551 self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True)
2552 self.tbl_component = Table('component', self.db_meta, autoload=True)
2553 self.tbl_config = Table('config', self.db_meta, autoload=True)
2554 self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True)
2555 self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True)
2556 self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True)
2557 self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True)
2558 self.tbl_files = Table('files', self.db_meta, autoload=True)
2559 self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True)
2560 self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True)
2561 self.tbl_known_changes = Table('known_changes', self.db_meta, autoload=True)
2562 self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True)
2563 self.tbl_location = Table('location', self.db_meta, autoload=True)
2564 self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True)
2565 self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True)
2566 self.tbl_override = Table('override', self.db_meta, autoload=True)
2567 self.tbl_override_type = Table('override_type', self.db_meta, autoload=True)
2568 self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True)
2569 self.tbl_priority = Table('priority', self.db_meta, autoload=True)
2570 self.tbl_queue = Table('queue', self.db_meta, autoload=True)
2571 self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True)
2572 self.tbl_section = Table('section', self.db_meta, autoload=True)
2573 self.tbl_source = Table('source', self.db_meta, autoload=True)
2574 self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True)
2575 self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True)
2576 self.tbl_src_format = Table('src_format', self.db_meta, autoload=True)
2577 self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True)
2578 self.tbl_suite = Table('suite', self.db_meta, autoload=True)
2579 self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True)
2580 self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True)
2581 self.tbl_uid = Table('uid', self.db_meta, autoload=True)
2582 self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True)
2584 def __setupmappers(self):
2585 mapper(Architecture, self.tbl_architecture,
2586 properties = dict(arch_id = self.tbl_architecture.c.id))
2588 mapper(Archive, self.tbl_archive,
2589 properties = dict(archive_id = self.tbl_archive.c.id,
2590 archive_name = self.tbl_archive.c.name))
2592 mapper(BinAssociation, self.tbl_bin_associations,
2593 properties = dict(ba_id = self.tbl_bin_associations.c.id,
2594 suite_id = self.tbl_bin_associations.c.suite,
2595 suite = relation(Suite),
2596 binary_id = self.tbl_bin_associations.c.bin,
2597 binary = relation(DBBinary)))
2600 mapper(DBBinary, self.tbl_binaries,
2601 properties = dict(binary_id = self.tbl_binaries.c.id,
2602 package = self.tbl_binaries.c.package,
2603 version = self.tbl_binaries.c.version,
2604 maintainer_id = self.tbl_binaries.c.maintainer,
2605 maintainer = relation(Maintainer),
2606 source_id = self.tbl_binaries.c.source,
2607 source = relation(DBSource),
2608 arch_id = self.tbl_binaries.c.architecture,
2609 architecture = relation(Architecture),
2610 poolfile_id = self.tbl_binaries.c.file,
2611 poolfile = relation(PoolFile),
2612 binarytype = self.tbl_binaries.c.type,
2613 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2614 fingerprint = relation(Fingerprint),
2615 install_date = self.tbl_binaries.c.install_date,
2616 binassociations = relation(BinAssociation,
2617 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
2619 mapper(BinaryACL, self.tbl_binary_acl,
2620 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2622 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2623 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2624 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2625 architecture = relation(Architecture)))
2627 mapper(Component, self.tbl_component,
2628 properties = dict(component_id = self.tbl_component.c.id,
2629 component_name = self.tbl_component.c.name))
2631 mapper(DBConfig, self.tbl_config,
2632 properties = dict(config_id = self.tbl_config.c.id))
2634 mapper(DSCFile, self.tbl_dsc_files,
2635 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2636 source_id = self.tbl_dsc_files.c.source,
2637 source = relation(DBSource),
2638 poolfile_id = self.tbl_dsc_files.c.file,
2639 poolfile = relation(PoolFile)))
2641 mapper(PoolFile, self.tbl_files,
2642 properties = dict(file_id = self.tbl_files.c.id,
2643 filesize = self.tbl_files.c.size,
2644 location_id = self.tbl_files.c.location,
2645 location = relation(Location)))
2647 mapper(Fingerprint, self.tbl_fingerprint,
2648 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2649 uid_id = self.tbl_fingerprint.c.uid,
2650 uid = relation(Uid),
2651 keyring_id = self.tbl_fingerprint.c.keyring,
2652 keyring = relation(Keyring),
2653 source_acl = relation(SourceACL),
2654 binary_acl = relation(BinaryACL)))
2656 mapper(Keyring, self.tbl_keyrings,
2657 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2658 keyring_id = self.tbl_keyrings.c.id))
2660 mapper(KnownChange, self.tbl_known_changes,
2661 properties = dict(known_change_id = self.tbl_known_changes.c.id))
2663 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
2664 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
2665 keyring = relation(Keyring, backref="keyring_acl_map"),
2666 architecture = relation(Architecture)))
2668 mapper(Location, self.tbl_location,
2669 properties = dict(location_id = self.tbl_location.c.id,
2670 component_id = self.tbl_location.c.component,
2671 component = relation(Component),
2672 archive_id = self.tbl_location.c.archive,
2673 archive = relation(Archive),
2674 archive_type = self.tbl_location.c.type))
2676 mapper(Maintainer, self.tbl_maintainer,
2677 properties = dict(maintainer_id = self.tbl_maintainer.c.id))
2679 mapper(NewComment, self.tbl_new_comments,
2680 properties = dict(comment_id = self.tbl_new_comments.c.id))
2682 mapper(Override, self.tbl_override,
2683 properties = dict(suite_id = self.tbl_override.c.suite,
2684 suite = relation(Suite),
2685 component_id = self.tbl_override.c.component,
2686 component = relation(Component),
2687 priority_id = self.tbl_override.c.priority,
2688 priority = relation(Priority),
2689 section_id = self.tbl_override.c.section,
2690 section = relation(Section),
2691 overridetype_id = self.tbl_override.c.type,
2692 overridetype = relation(OverrideType)))
2694 mapper(OverrideType, self.tbl_override_type,
2695 properties = dict(overridetype = self.tbl_override_type.c.type,
2696 overridetype_id = self.tbl_override_type.c.id))
2698 mapper(Priority, self.tbl_priority,
2699 properties = dict(priority_id = self.tbl_priority.c.id))
2701 mapper(Queue, self.tbl_queue,
2702 properties = dict(queue_id = self.tbl_queue.c.id))
2704 mapper(QueueBuild, self.tbl_queue_build,
2705 properties = dict(suite_id = self.tbl_queue_build.c.suite,
2706 queue_id = self.tbl_queue_build.c.queue,
2707 queue = relation(Queue, backref='queuebuild')))
2709 mapper(Section, self.tbl_section,
2710 properties = dict(section_id = self.tbl_section.c.id))
2712 mapper(DBSource, self.tbl_source,
2713 properties = dict(source_id = self.tbl_source.c.id,
2714 version = self.tbl_source.c.version,
2715 maintainer_id = self.tbl_source.c.maintainer,
2716 maintainer = relation(Maintainer,
2717 primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
2718 poolfile_id = self.tbl_source.c.file,
2719 poolfile = relation(PoolFile),
2720 fingerprint_id = self.tbl_source.c.sig_fpr,
2721 fingerprint = relation(Fingerprint),
2722 changedby_id = self.tbl_source.c.changedby,
2723 changedby = relation(Maintainer,
2724 primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
2725 srcfiles = relation(DSCFile,
2726 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
2727 srcassociations = relation(SrcAssociation,
2728 primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)),
2729 srcuploaders = relation(SrcUploader)))
2731 mapper(SourceACL, self.tbl_source_acl,
2732 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
2734 mapper(SrcAssociation, self.tbl_src_associations,
2735 properties = dict(sa_id = self.tbl_src_associations.c.id,
2736 suite_id = self.tbl_src_associations.c.suite,
2737 suite = relation(Suite),
2738 source_id = self.tbl_src_associations.c.source,
2739 source = relation(DBSource)))
2741 mapper(SrcFormat, self.tbl_src_format,
2742 properties = dict(src_format_id = self.tbl_src_format.c.id,
2743 format_name = self.tbl_src_format.c.format_name))
2745 mapper(SrcUploader, self.tbl_src_uploaders,
2746 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
2747 source_id = self.tbl_src_uploaders.c.source,
2748 source = relation(DBSource,
2749 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
2750 maintainer_id = self.tbl_src_uploaders.c.maintainer,
2751 maintainer = relation(Maintainer,
2752 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
2754 mapper(Suite, self.tbl_suite,
2755 properties = dict(suite_id = self.tbl_suite.c.id,
2756 policy_queue = relation(Queue)))
2758 mapper(SuiteArchitecture, self.tbl_suite_architectures,
2759 properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
2760 suite = relation(Suite, backref='suitearchitectures'),
2761 arch_id = self.tbl_suite_architectures.c.architecture,
2762 architecture = relation(Architecture)))
2764 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
2765 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
2766 suite = relation(Suite, backref='suitesrcformats'),
2767 src_format_id = self.tbl_suite_src_formats.c.src_format,
2768 src_format = relation(SrcFormat)))
2770 mapper(Uid, self.tbl_uid,
2771 properties = dict(uid_id = self.tbl_uid.c.id,
2772 fingerprint = relation(Fingerprint)))
2774 mapper(UploadBlock, self.tbl_upload_blocks,
2775 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
2776 fingerprint = relation(Fingerprint, backref="uploadblocks"),
2777 uid = relation(Uid, backref="uploadblocks")))
2779 ## Connection functions
2780 def __createconn(self):
2781 from config import Config
2785 connstr = "postgres://%s" % cnf["DB::Host"]
2786 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2787 connstr += ":%s" % cnf["DB::Port"]
2788 connstr += "/%s" % cnf["DB::Name"]
2791 connstr = "postgres:///%s" % cnf["DB::Name"]
2792 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2793 connstr += "?port=%s" % cnf["DB::Port"]
2795 self.db_pg = create_engine(connstr, echo=self.debug)
2796 self.db_meta = MetaData()
2797 self.db_meta.bind = self.db_pg
2798 self.db_smaker = sessionmaker(bind=self.db_pg,
2802 self.__setuptables()
2803 self.__setupmappers()
2806 return self.db_smaker()
2808 __all__.append('DBConn')