5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
40 from datetime import datetime
42 from inspect import getargspec
45 from sqlalchemy import create_engine, Table, MetaData
46 from sqlalchemy.orm import sessionmaker, mapper, relation
47 from sqlalchemy import types as sqltypes
49 # Don't remove this, we re-export the exceptions to scripts which import us
50 from sqlalchemy.exc import *
51 from sqlalchemy.orm.exc import NoResultFound
53 # Only import Config until Queue stuff is changed to store its config
55 from config import Config
56 from singleton import Singleton
57 from textutils import fix_maintainer
59 ################################################################################
61 # Patch in support for the debversion field type so that it works during
64 class DebVersion(sqltypes.Text):
65 def get_col_spec(self):
68 sa_major_version = sqlalchemy.__version__[0:3]
69 if sa_major_version == "0.5":
70 from sqlalchemy.databases import postgres
71 postgres.ischema_names['debversion'] = DebVersion
73 raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
75 ################################################################################
77 __all__ = ['IntegrityError', 'SQLAlchemyError']
79 ################################################################################
81 def session_wrapper(fn):
83 Wrapper around common ".., session=None):" handling. If the wrapped
84 function is called without passing 'session', we create a local one
85 and destroy it when the function ends.
87 Also attaches a commit_or_flush method to the session; if we created a
88 local session, this is a synonym for session.commit(), otherwise it is a
89 synonym for session.flush().
92 def wrapped(*args, **kwargs):
93 private_transaction = False
95 # Find the session object
96 session = kwargs.get('session')
99 if len(args) <= len(getargspec(fn)[0]) - 1:
100 # No session specified as last argument or in kwargs
101 private_transaction = True
102 session = kwargs['session'] = DBConn().session()
104 # Session is last argument in args
108 session = args[-1] = DBConn().session()
109 private_transaction = True
111 if private_transaction:
112 session.commit_or_flush = session.commit
114 session.commit_or_flush = session.flush
117 return fn(*args, **kwargs)
119 if private_transaction:
120 # We created a session; close it.
123 wrapped.__doc__ = fn.__doc__
124 wrapped.func_name = fn.func_name
128 __all__.append('session_wrapper')
130 ################################################################################
132 class Architecture(object):
133 def __init__(self, *args, **kwargs):
136 def __eq__(self, val):
137 if isinstance(val, str):
138 return (self.arch_string== val)
139 # This signals to use the normal comparison operator
140 return NotImplemented
142 def __ne__(self, val):
143 if isinstance(val, str):
144 return (self.arch_string != val)
145 # This signals to use the normal comparison operator
146 return NotImplemented
149 return '<Architecture %s>' % self.arch_string
151 __all__.append('Architecture')
154 def get_architecture(architecture, session=None):
156 Returns database id for given C{architecture}.
158 @type architecture: string
159 @param architecture: The name of the architecture
161 @type session: Session
162 @param session: Optional SQLA session object (a temporary one will be
163 generated if not supplied)
166 @return: Architecture object for the given arch (None if not present)
169 q = session.query(Architecture).filter_by(arch_string=architecture)
173 except NoResultFound:
176 __all__.append('get_architecture')
179 def get_architecture_suites(architecture, session=None):
181 Returns list of Suite objects for given C{architecture} name
184 @param source: Architecture name to search for
186 @type session: Session
187 @param session: Optional SQL session object (a temporary one will be
188 generated if not supplied)
191 @return: list of Suite objects for the given name (may be empty)
194 q = session.query(Suite)
195 q = q.join(SuiteArchitecture)
196 q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
202 __all__.append('get_architecture_suites')
204 ################################################################################
206 class Archive(object):
207 def __init__(self, *args, **kwargs):
211 return '<Archive %s>' % self.archive_name
213 __all__.append('Archive')
216 def get_archive(archive, session=None):
218 returns database id for given C{archive}.
220 @type archive: string
221 @param archive: the name of the arhive
223 @type session: Session
224 @param session: Optional SQLA session object (a temporary one will be
225 generated if not supplied)
228 @return: Archive object for the given name (None if not present)
231 archive = archive.lower()
233 q = session.query(Archive).filter_by(archive_name=archive)
237 except NoResultFound:
240 __all__.append('get_archive')
242 ################################################################################
244 class BinAssociation(object):
245 def __init__(self, *args, **kwargs):
249 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
251 __all__.append('BinAssociation')
253 ################################################################################
255 class BinContents(object):
256 def __init__(self, *args, **kwargs):
260 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
262 __all__.append('BinContents')
264 ################################################################################
266 class DBBinary(object):
267 def __init__(self, *args, **kwargs):
271 return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
273 __all__.append('DBBinary')
276 def get_suites_binary_in(package, session=None):
278 Returns list of Suite objects which given C{package} name is in
281 @param source: DBBinary package name to search for
284 @return: list of Suite objects for the given package
287 return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
289 __all__.append('get_suites_binary_in')
292 def get_binary_from_id(binary_id, session=None):
294 Returns DBBinary object for given C{id}
297 @param binary_id: Id of the required binary
299 @type session: Session
300 @param session: Optional SQLA session object (a temporary one will be
301 generated if not supplied)
304 @return: DBBinary object for the given binary (None if not present)
307 q = session.query(DBBinary).filter_by(binary_id=binary_id)
311 except NoResultFound:
314 __all__.append('get_binary_from_id')
317 def get_binaries_from_name(package, version=None, architecture=None, session=None):
319 Returns list of DBBinary objects for given C{package} name
322 @param package: DBBinary package name to search for
324 @type version: str or None
325 @param version: Version to search for (or None)
327 @type package: str, list or None
328 @param package: Architectures to limit to (or None if no limit)
330 @type session: Session
331 @param session: Optional SQL session object (a temporary one will be
332 generated if not supplied)
335 @return: list of DBBinary objects for the given name (may be empty)
338 q = session.query(DBBinary).filter_by(package=package)
340 if version is not None:
341 q = q.filter_by(version=version)
343 if architecture is not None:
344 if not isinstance(architecture, list):
345 architecture = [architecture]
346 q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
352 __all__.append('get_binaries_from_name')
355 def get_binaries_from_source_id(source_id, session=None):
357 Returns list of DBBinary objects for given C{source_id}
360 @param source_id: source_id to search for
362 @type session: Session
363 @param session: Optional SQL session object (a temporary one will be
364 generated if not supplied)
367 @return: list of DBBinary objects for the given name (may be empty)
370 return session.query(DBBinary).filter_by(source_id=source_id).all()
372 __all__.append('get_binaries_from_source_id')
375 def get_binary_from_name_suite(package, suitename, session=None):
376 ### For dak examine-package
377 ### XXX: Doesn't use object API yet
379 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
380 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
381 WHERE b.package=:package
383 AND fi.location = l.id
384 AND l.component = c.id
387 AND su.suite_name=:suitename
388 ORDER BY b.version DESC"""
390 return session.execute(sql, {'package': package, 'suitename': suitename})
392 __all__.append('get_binary_from_name_suite')
395 def get_binary_components(package, suitename, arch, session=None):
396 # Check for packages that have moved from one component to another
397 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
398 WHERE b.package=:package AND s.suite_name=:suitename
399 AND (a.arch_string = :arch OR a.arch_string = 'all')
400 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
401 AND f.location = l.id
402 AND l.component = c.id
405 vals = {'package': package, 'suitename': suitename, 'arch': arch}
407 return session.execute(query, vals)
409 __all__.append('get_binary_components')
411 ################################################################################
413 class BinaryACL(object):
414 def __init__(self, *args, **kwargs):
418 return '<BinaryACL %s>' % self.binary_acl_id
420 __all__.append('BinaryACL')
422 ################################################################################
424 class BinaryACLMap(object):
425 def __init__(self, *args, **kwargs):
429 return '<BinaryACLMap %s>' % self.binary_acl_map_id
431 __all__.append('BinaryACLMap')
433 ################################################################################
435 class Component(object):
436 def __init__(self, *args, **kwargs):
439 def __eq__(self, val):
440 if isinstance(val, str):
441 return (self.component_name == val)
442 # This signals to use the normal comparison operator
443 return NotImplemented
445 def __ne__(self, val):
446 if isinstance(val, str):
447 return (self.component_name != val)
448 # This signals to use the normal comparison operator
449 return NotImplemented
452 return '<Component %s>' % self.component_name
455 __all__.append('Component')
458 def get_component(component, session=None):
460 Returns database id for given C{component}.
462 @type component: string
463 @param component: The name of the override type
466 @return: the database id for the given component
469 component = component.lower()
471 q = session.query(Component).filter_by(component_name=component)
475 except NoResultFound:
478 __all__.append('get_component')
480 ################################################################################
482 class DBConfig(object):
483 def __init__(self, *args, **kwargs):
487 return '<DBConfig %s>' % self.name
489 __all__.append('DBConfig')
491 ################################################################################
494 def get_or_set_contents_file_id(filename, session=None):
496 Returns database id for given filename.
498 If no matching file is found, a row is inserted.
500 @type filename: string
501 @param filename: The filename
502 @type session: SQLAlchemy
503 @param session: Optional SQL session object (a temporary one will be
504 generated if not supplied). If not passed, a commit will be performed at
505 the end of the function, otherwise the caller is responsible for commiting.
508 @return: the database id for the given component
511 q = session.query(ContentFilename).filter_by(filename=filename)
514 ret = q.one().cafilename_id
515 except NoResultFound:
516 cf = ContentFilename()
517 cf.filename = filename
519 session.commit_or_flush()
520 ret = cf.cafilename_id
524 __all__.append('get_or_set_contents_file_id')
527 def get_contents(suite, overridetype, section=None, session=None):
529 Returns contents for a suite / overridetype combination, limiting
530 to a section if not None.
533 @param suite: Suite object
535 @type overridetype: OverrideType
536 @param overridetype: OverrideType object
538 @type section: Section
539 @param section: Optional section object to limit results to
541 @type session: SQLAlchemy
542 @param session: Optional SQL session object (a temporary one will be
543 generated if not supplied)
546 @return: ResultsProxy object set up to return tuples of (filename, section,
550 # find me all of the contents for a given suite
551 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
555 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
556 JOIN content_file_names n ON (c.filename=n.id)
557 JOIN binaries b ON (b.id=c.binary_pkg)
558 JOIN override o ON (o.package=b.package)
559 JOIN section s ON (s.id=o.section)
560 WHERE o.suite = :suiteid AND o.type = :overridetypeid
561 AND b.type=:overridetypename"""
563 vals = {'suiteid': suite.suite_id,
564 'overridetypeid': overridetype.overridetype_id,
565 'overridetypename': overridetype.overridetype}
567 if section is not None:
568 contents_q += " AND s.id = :sectionid"
569 vals['sectionid'] = section.section_id
571 contents_q += " ORDER BY fn"
573 return session.execute(contents_q, vals)
575 __all__.append('get_contents')
577 ################################################################################
579 class ContentFilepath(object):
580 def __init__(self, *args, **kwargs):
584 return '<ContentFilepath %s>' % self.filepath
586 __all__.append('ContentFilepath')
589 def get_or_set_contents_path_id(filepath, session=None):
591 Returns database id for given path.
593 If no matching file is found, a row is inserted.
595 @type filename: string
596 @param filename: The filepath
597 @type session: SQLAlchemy
598 @param session: Optional SQL session object (a temporary one will be
599 generated if not supplied). If not passed, a commit will be performed at
600 the end of the function, otherwise the caller is responsible for commiting.
603 @return: the database id for the given path
606 q = session.query(ContentFilepath).filter_by(filepath=filepath)
609 ret = q.one().cafilepath_id
610 except NoResultFound:
611 cf = ContentFilepath()
612 cf.filepath = filepath
614 session.commit_or_flush()
615 ret = cf.cafilepath_id
619 __all__.append('get_or_set_contents_path_id')
621 ################################################################################
623 class ContentAssociation(object):
624 def __init__(self, *args, **kwargs):
628 return '<ContentAssociation %s>' % self.ca_id
630 __all__.append('ContentAssociation')
632 def insert_content_paths(binary_id, fullpaths, session=None):
634 Make sure given path is associated with given binary id
637 @param binary_id: the id of the binary
638 @type fullpaths: list
639 @param fullpaths: the list of paths of the file being associated with the binary
640 @type session: SQLAlchemy session
641 @param session: Optional SQLAlchemy session. If this is passed, the caller
642 is responsible for ensuring a transaction has begun and committing the
643 results or rolling back based on the result code. If not passed, a commit
644 will be performed at the end of the function, otherwise the caller is
645 responsible for commiting.
647 @return: True upon success
652 session = DBConn().session()
658 for fullpath in fullpaths:
659 if fullpath.startswith( './' ):
660 fullpath = fullpath[2:]
662 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} )
670 traceback.print_exc()
672 # Only rollback if we set up the session ourself
679 __all__.append('insert_content_paths')
681 ################################################################################
683 class DSCFile(object):
684 def __init__(self, *args, **kwargs):
688 return '<DSCFile %s>' % self.dscfile_id
690 __all__.append('DSCFile')
693 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
695 Returns a list of DSCFiles which may be empty
697 @type dscfile_id: int (optional)
698 @param dscfile_id: the dscfile_id of the DSCFiles to find
700 @type source_id: int (optional)
701 @param source_id: the source id related to the DSCFiles to find
703 @type poolfile_id: int (optional)
704 @param poolfile_id: the poolfile id related to the DSCFiles to find
707 @return: Possibly empty list of DSCFiles
710 q = session.query(DSCFile)
712 if dscfile_id is not None:
713 q = q.filter_by(dscfile_id=dscfile_id)
715 if source_id is not None:
716 q = q.filter_by(source_id=source_id)
718 if poolfile_id is not None:
719 q = q.filter_by(poolfile_id=poolfile_id)
723 __all__.append('get_dscfiles')
725 ################################################################################
727 class PoolFile(object):
728 def __init__(self, *args, **kwargs):
732 return '<PoolFile %s>' % self.filename
734 __all__.append('PoolFile')
737 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
740 (ValidFileFound [boolean or None], PoolFile object or None)
742 @type filename: string
743 @param filename: the filename of the file to check against the DB
746 @param filesize: the size of the file to check against the DB
749 @param md5sum: the md5sum of the file to check against the DB
751 @type location_id: int
752 @param location_id: the id of the location to look in
755 @return: Tuple of length 2.
756 If more than one file found with that name:
758 If valid pool file found: (True, PoolFile object)
759 If valid pool file not found:
760 (False, None) if no file found
761 (False, PoolFile object) if file found with size/md5sum mismatch
764 q = session.query(PoolFile).filter_by(filename=filename)
765 q = q.join(Location).filter_by(location_id=location_id)
775 if obj.md5sum != md5sum or obj.filesize != int(filesize):
783 __all__.append('check_poolfile')
786 def get_poolfile_by_id(file_id, session=None):
788 Returns a PoolFile objects or None for the given id
791 @param file_id: the id of the file to look for
793 @rtype: PoolFile or None
794 @return: either the PoolFile object or None
797 q = session.query(PoolFile).filter_by(file_id=file_id)
801 except NoResultFound:
804 __all__.append('get_poolfile_by_id')
808 def get_poolfile_by_name(filename, location_id=None, session=None):
810 Returns an array of PoolFile objects for the given filename and
811 (optionally) location_id
813 @type filename: string
814 @param filename: the filename of the file to check against the DB
816 @type location_id: int
817 @param location_id: the id of the location to look in (optional)
820 @return: array of PoolFile objects
823 q = session.query(PoolFile).filter_by(filename=filename)
825 if location_id is not None:
826 q = q.join(Location).filter_by(location_id=location_id)
830 __all__.append('get_poolfile_by_name')
833 def get_poolfile_like_name(filename, session=None):
835 Returns an array of PoolFile objects which are like the given name
837 @type filename: string
838 @param filename: the filename of the file to check against the DB
841 @return: array of PoolFile objects
844 # TODO: There must be a way of properly using bind parameters with %FOO%
845 q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
849 __all__.append('get_poolfile_like_name')
852 def add_poolfile(filename, datadict, location_id, session=None):
854 Add a new file to the pool
856 @type filename: string
857 @param filename: filename
860 @param datadict: dict with needed data
862 @type location_id: int
863 @param location_id: database id of the location
866 @return: the PoolFile object created
868 poolfile = PoolFile()
869 poolfile.filename = filename
870 poolfile.filesize = datadict["size"]
871 poolfile.md5sum = datadict["md5sum"]
872 poolfile.sha1sum = datadict["sha1sum"]
873 poolfile.sha256sum = datadict["sha256sum"]
874 poolfile.location_id = location_id
876 session.add(poolfile)
877 # Flush to get a file id (NB: This is not a commit)
882 __all__.append('add_poolfile')
884 ################################################################################
886 class Fingerprint(object):
887 def __init__(self, *args, **kwargs):
891 return '<Fingerprint %s>' % self.fingerprint
893 __all__.append('Fingerprint')
896 def get_fingerprint(fpr, session=None):
898 Returns Fingerprint object for given fpr.
901 @param fpr: The fpr to find / add
903 @type session: SQLAlchemy
904 @param session: Optional SQL session object (a temporary one will be
905 generated if not supplied).
908 @return: the Fingerprint object for the given fpr or None
911 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
915 except NoResultFound:
920 __all__.append('get_fingerprint')
923 def get_or_set_fingerprint(fpr, session=None):
925 Returns Fingerprint object for given fpr.
927 If no matching fpr is found, a row is inserted.
930 @param fpr: The fpr to find / add
932 @type session: SQLAlchemy
933 @param session: Optional SQL session object (a temporary one will be
934 generated if not supplied). If not passed, a commit will be performed at
935 the end of the function, otherwise the caller is responsible for commiting.
936 A flush will be performed either way.
939 @return: the Fingerprint object for the given fpr
942 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
946 except NoResultFound:
947 fingerprint = Fingerprint()
948 fingerprint.fingerprint = fpr
949 session.add(fingerprint)
950 session.commit_or_flush()
955 __all__.append('get_or_set_fingerprint')
957 ################################################################################
959 # Helper routine for Keyring class
960 def get_ldap_name(entry):
962 for k in ["cn", "mn", "sn"]:
964 if ret and ret[0] != "" and ret[0] != "-":
966 return " ".join(name)
968 ################################################################################
970 class Keyring(object):
971 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
972 " --with-colons --fingerprint --fingerprint"
977 def __init__(self, *args, **kwargs):
981 return '<Keyring %s>' % self.keyring_name
983 def de_escape_gpg_str(self, txt):
984 esclist = re.split(r'(\\x..)', txt)
985 for x in range(1,len(esclist),2):
986 esclist[x] = "%c" % (int(esclist[x][2:],16))
987 return "".join(esclist)
989 def load_keys(self, keyring):
992 if not self.keyring_id:
993 raise Exception('Must be initialized with database information')
995 k = os.popen(self.gpg_invocation % keyring, "r")
999 for line in k.xreadlines():
1000 field = line.split(":")
1001 if field[0] == "pub":
1003 (name, addr) = email.Utils.parseaddr(field[9])
1004 name = re.sub(r"\s*[(].*[)]", "", name)
1005 if name == "" or addr == "" or "@" not in addr:
1007 addr = "invalid-uid"
1008 name = self.de_escape_gpg_str(name)
1009 self.keys[key] = {"email": addr}
1011 self.keys[key]["name"] = name
1012 self.keys[key]["aliases"] = [name]
1013 self.keys[key]["fingerprints"] = []
1015 elif key and field[0] == "sub" and len(field) >= 12:
1016 signingkey = ("s" in field[11])
1017 elif key and field[0] == "uid":
1018 (name, addr) = email.Utils.parseaddr(field[9])
1019 if name and name not in self.keys[key]["aliases"]:
1020 self.keys[key]["aliases"].append(name)
1021 elif signingkey and field[0] == "fpr":
1022 self.keys[key]["fingerprints"].append(field[9])
1023 self.fpr_lookup[field[9]] = key
1025 def import_users_from_ldap(self, session):
1029 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1030 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1032 l = ldap.open(LDAPServer)
1033 l.simple_bind_s("","")
1034 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1035 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1036 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1038 ldap_fin_uid_id = {}
1045 uid = entry["uid"][0]
1046 name = get_ldap_name(entry)
1047 fingerprints = entry["keyFingerPrint"]
1049 for f in fingerprints:
1050 key = self.fpr_lookup.get(f, None)
1051 if key not in self.keys:
1053 self.keys[key]["uid"] = uid
1057 keyid = get_or_set_uid(uid, session).uid_id
1058 byuid[keyid] = (uid, name)
1059 byname[uid] = (keyid, name)
1061 return (byname, byuid)
1063 def generate_users_from_keyring(self, format, session):
1067 for x in self.keys.keys():
1068 if self.keys[x]["email"] == "invalid-uid":
1070 self.keys[x]["uid"] = format % "invalid-uid"
1072 uid = format % self.keys[x]["email"]
1073 keyid = get_or_set_uid(uid, session).uid_id
1074 byuid[keyid] = (uid, self.keys[x]["name"])
1075 byname[uid] = (keyid, self.keys[x]["name"])
1076 self.keys[x]["uid"] = uid
1079 uid = format % "invalid-uid"
1080 keyid = get_or_set_uid(uid, session).uid_id
1081 byuid[keyid] = (uid, "ungeneratable user id")
1082 byname[uid] = (keyid, "ungeneratable user id")
1084 return (byname, byuid)
1086 __all__.append('Keyring')
1089 def get_keyring(keyring, session=None):
1091 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1092 If C{keyring} already has an entry, simply return the existing Keyring
1094 @type keyring: string
1095 @param keyring: the keyring name
1098 @return: the Keyring object for this keyring
1101 q = session.query(Keyring).filter_by(keyring_name=keyring)
1105 except NoResultFound:
1108 __all__.append('get_keyring')
1110 ################################################################################
1112 class KeyringACLMap(object):
1113 def __init__(self, *args, **kwargs):
1117 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1119 __all__.append('KeyringACLMap')
1121 ################################################################################
1123 class KnownChange(object):
1124 def __init__(self, *args, **kwargs):
1128 return '<KnownChange %s>' % self.changesname
1130 __all__.append('KnownChange')
1133 def get_knownchange(filename, session=None):
1135 returns knownchange object for given C{filename}.
1137 @type archive: string
1138 @param archive: the name of the arhive
1140 @type session: Session
1141 @param session: Optional SQLA session object (a temporary one will be
1142 generated if not supplied)
1145 @return: Archive object for the given name (None if not present)
1148 q = session.query(KnownChange).filter_by(changesname=filename)
1152 except NoResultFound:
1155 __all__.append('get_knownchange')
1157 ################################################################################
1158 class Location(object):
1159 def __init__(self, *args, **kwargs):
1163 return '<Location %s (%s)>' % (self.path, self.location_id)
1165 __all__.append('Location')
1168 def get_location(location, component=None, archive=None, session=None):
1170 Returns Location object for the given combination of location, component
1173 @type location: string
1174 @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
1176 @type component: string
1177 @param component: the component name (if None, no restriction applied)
1179 @type archive: string
1180 @param archive_id: the archive name (if None, no restriction applied)
1182 @rtype: Location / None
1183 @return: Either a Location object or None if one can't be found
1186 q = session.query(Location).filter_by(path=location)
1188 if archive is not None:
1189 q = q.join(Archive).filter_by(archive_name=archive)
1191 if component is not None:
1192 q = q.join(Component).filter_by(component_name=component)
1196 except NoResultFound:
1199 __all__.append('get_location')
1201 ################################################################################
1203 class Maintainer(object):
1204 def __init__(self, *args, **kwargs):
1208 return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
1210 def get_split_maintainer(self):
1211 if not hasattr(self, 'name') or self.name is None:
1212 return ('', '', '', '')
1214 return fix_maintainer(self.name.strip())
1216 __all__.append('Maintainer')
1219 def get_or_set_maintainer(name, session=None):
1221 Returns Maintainer object for given maintainer name.
1223 If no matching maintainer name is found, a row is inserted.
1226 @param name: The maintainer name to add
1228 @type session: SQLAlchemy
1229 @param session: Optional SQL session object (a temporary one will be
1230 generated if not supplied). If not passed, a commit will be performed at
1231 the end of the function, otherwise the caller is responsible for commiting.
1232 A flush will be performed either way.
1235 @return: the Maintainer object for the given maintainer
1238 q = session.query(Maintainer).filter_by(name=name)
1241 except NoResultFound:
1242 maintainer = Maintainer()
1243 maintainer.name = name
1244 session.add(maintainer)
1245 session.commit_or_flush()
1250 __all__.append('get_or_set_maintainer')
1253 def get_maintainer(maintainer_id, session=None):
1255 Return the name of the maintainer behind C{maintainer_id} or None if that
1256 maintainer_id is invalid.
1258 @type maintainer_id: int
1259 @param maintainer_id: the id of the maintainer
1262 @return: the Maintainer with this C{maintainer_id}
1265 return session.query(Maintainer).get(maintainer_id)
1267 __all__.append('get_maintainer')
1269 ################################################################################
1271 class NewComment(object):
1272 def __init__(self, *args, **kwargs):
1276 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1278 __all__.append('NewComment')
1281 def has_new_comment(package, version, session=None):
1283 Returns true if the given combination of C{package}, C{version} has a comment.
1285 @type package: string
1286 @param package: name of the package
1288 @type version: string
1289 @param version: package version
1291 @type session: Session
1292 @param session: Optional SQLA session object (a temporary one will be
1293 generated if not supplied)
1299 q = session.query(NewComment)
1300 q = q.filter_by(package=package)
1301 q = q.filter_by(version=version)
1303 return bool(q.count() > 0)
1305 __all__.append('has_new_comment')
1308 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1310 Returns (possibly empty) list of NewComment objects for the given
1313 @type package: string (optional)
1314 @param package: name of the package
1316 @type version: string (optional)
1317 @param version: package version
1319 @type comment_id: int (optional)
1320 @param comment_id: An id of a comment
1322 @type session: Session
1323 @param session: Optional SQLA session object (a temporary one will be
1324 generated if not supplied)
1327 @return: A (possibly empty) list of NewComment objects will be returned
1330 q = session.query(NewComment)
1331 if package is not None: q = q.filter_by(package=package)
1332 if version is not None: q = q.filter_by(version=version)
1333 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1337 __all__.append('get_new_comments')
1339 ################################################################################
1341 class Override(object):
1342 def __init__(self, *args, **kwargs):
1346 return '<Override %s (%s)>' % (self.package, self.suite_id)
1348 __all__.append('Override')
1351 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1353 Returns Override object for the given parameters
1355 @type package: string
1356 @param package: The name of the package
1358 @type suite: string, list or None
1359 @param suite: The name of the suite (or suites if a list) to limit to. If
1360 None, don't limit. Defaults to None.
1362 @type component: string, list or None
1363 @param component: The name of the component (or components if a list) to
1364 limit to. If None, don't limit. Defaults to None.
1366 @type overridetype: string, list or None
1367 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1368 limit to. If None, don't limit. Defaults to None.
1370 @type session: Session
1371 @param session: Optional SQLA session object (a temporary one will be
1372 generated if not supplied)
1375 @return: A (possibly empty) list of Override objects will be returned
1378 q = session.query(Override)
1379 q = q.filter_by(package=package)
1381 if suite is not None:
1382 if not isinstance(suite, list): suite = [suite]
1383 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1385 if component is not None:
1386 if not isinstance(component, list): component = [component]
1387 q = q.join(Component).filter(Component.component_name.in_(component))
1389 if overridetype is not None:
1390 if not isinstance(overridetype, list): overridetype = [overridetype]
1391 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1395 __all__.append('get_override')
1398 ################################################################################
1400 class OverrideType(object):
1401 def __init__(self, *args, **kwargs):
1405 return '<OverrideType %s>' % self.overridetype
1407 __all__.append('OverrideType')
1410 def get_override_type(override_type, session=None):
1412 Returns OverrideType object for given C{override type}.
1414 @type override_type: string
1415 @param override_type: The name of the override type
1417 @type session: Session
1418 @param session: Optional SQLA session object (a temporary one will be
1419 generated if not supplied)
1422 @return: the database id for the given override type
1425 q = session.query(OverrideType).filter_by(overridetype=override_type)
1429 except NoResultFound:
1432 __all__.append('get_override_type')
1434 ################################################################################
1436 class PendingContentAssociation(object):
1437 def __init__(self, *args, **kwargs):
1441 return '<PendingContentAssociation %s>' % self.pca_id
1443 __all__.append('PendingContentAssociation')
1445 def insert_pending_content_paths(package, fullpaths, session=None):
1447 Make sure given paths are temporarily associated with given
1451 @param package: the package to associate with should have been read in from the binary control file
1452 @type fullpaths: list
1453 @param fullpaths: the list of paths of the file being associated with the binary
1454 @type session: SQLAlchemy session
1455 @param session: Optional SQLAlchemy session. If this is passed, the caller
1456 is responsible for ensuring a transaction has begun and committing the
1457 results or rolling back based on the result code. If not passed, a commit
1458 will be performed at the end of the function
1460 @return: True upon success, False if there is a problem
1463 privatetrans = False
1466 session = DBConn().session()
1470 arch = get_architecture(package['Architecture'], session)
1471 arch_id = arch.arch_id
1473 # Remove any already existing recorded files for this package
1474 q = session.query(PendingContentAssociation)
1475 q = q.filter_by(package=package['Package'])
1476 q = q.filter_by(version=package['Version'])
1477 q = q.filter_by(architecture=arch_id)
1482 for fullpath in fullpaths:
1483 (path, filename) = os.path.split(fullpath)
1485 if path.startswith( "./" ):
1488 filepath_id = get_or_set_contents_path_id(path, session)
1489 filename_id = get_or_set_contents_file_id(filename, session)
1491 pathcache[fullpath] = (filepath_id, filename_id)
1493 for fullpath, dat in pathcache.items():
1494 pca = PendingContentAssociation()
1495 pca.package = package['Package']
1496 pca.version = package['Version']
1497 pca.filepath_id = dat[0]
1498 pca.filename_id = dat[1]
1499 pca.architecture = arch_id
1502 # Only commit if we set up the session ourself
1510 except Exception, e:
1511 traceback.print_exc()
1513 # Only rollback if we set up the session ourself
1520 __all__.append('insert_pending_content_paths')
1522 ################################################################################
1524 class Priority(object):
1525 def __init__(self, *args, **kwargs):
1528 def __eq__(self, val):
1529 if isinstance(val, str):
1530 return (self.priority == val)
1531 # This signals to use the normal comparison operator
1532 return NotImplemented
1534 def __ne__(self, val):
1535 if isinstance(val, str):
1536 return (self.priority != val)
1537 # This signals to use the normal comparison operator
1538 return NotImplemented
1541 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
1543 __all__.append('Priority')
1546 def get_priority(priority, session=None):
1548 Returns Priority object for given C{priority name}.
1550 @type priority: string
1551 @param priority: The name of the priority
1553 @type session: Session
1554 @param session: Optional SQLA session object (a temporary one will be
1555 generated if not supplied)
1558 @return: Priority object for the given priority
1561 q = session.query(Priority).filter_by(priority=priority)
1565 except NoResultFound:
1568 __all__.append('get_priority')
1571 def get_priorities(session=None):
1573 Returns dictionary of priority names -> id mappings
1575 @type session: Session
1576 @param session: Optional SQL session object (a temporary one will be
1577 generated if not supplied)
1580 @return: dictionary of priority names -> id mappings
1584 q = session.query(Priority)
1586 ret[x.priority] = x.priority_id
1590 __all__.append('get_priorities')
1592 ################################################################################
1594 class Queue(object):
1595 def __init__(self, *args, **kwargs):
1599 return '<Queue %s>' % self.queue_name
1601 def autobuild_upload(self, changes, srcpath, session=None):
1603 Update queue_build database table used for incoming autobuild support.
1605 @type changes: Changes
1606 @param changes: changes object for the upload to process
1608 @type srcpath: string
1609 @param srcpath: path for the queue file entries/link destinations
1611 @type session: SQLAlchemy session
1612 @param session: Optional SQLAlchemy session. If this is passed, the
1613 caller is responsible for ensuring a transaction has begun and
1614 committing the results or rolling back based on the result code. If
1615 not passed, a commit will be performed at the end of the function,
1616 otherwise the caller is responsible for commiting.
1618 @rtype: NoneType or string
1619 @return: None if the operation failed, a string describing the error if not
1622 privatetrans = False
1624 session = DBConn().session()
1627 # TODO: Remove by moving queue config into the database
1630 for suitename in changes.changes["distribution"].keys():
1631 # TODO: Move into database as:
1632 # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build)
1633 # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink)
1634 # This also gets rid of the SecurityQueueBuild hack below
1635 if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"):
1639 s = get_suite(suitename, session)
1641 return "INTERNAL ERROR: Could not find suite %s" % suitename
1643 # TODO: Get from database as above
1644 dest_dir = conf["Dir::QueueBuild"]
1646 # TODO: Move into database as above
1647 if conf.FindB("Dinstall::SecurityQueueBuild"):
1648 dest_dir = os.path.join(dest_dir, suitename)
1650 for file_entry in changes.files.keys():
1651 src = os.path.join(srcpath, file_entry)
1652 dest = os.path.join(dest_dir, file_entry)
1654 # TODO: Move into database as above
1655 if conf.FindB("Dinstall::SecurityQueueBuild"):
1656 # Copy it since the original won't be readable by www-data
1658 utils.copy(src, dest)
1660 # Create a symlink to it
1661 os.symlink(src, dest)
1664 qb.suite_id = s.suite_id
1665 qb.queue_id = self.queue_id
1671 # If the .orig tarballs are in the pool, create a symlink to
1672 # them (if one doesn't already exist)
1673 for dsc_file in changes.dsc_files.keys():
1674 # Skip all files except orig tarballs
1675 from daklib.regexes import re_is_orig_source
1676 if not re_is_orig_source.match(dsc_file):
1678 # Skip orig files not identified in the pool
1679 if not (changes.orig_files.has_key(dsc_file) and
1680 changes.orig_files[dsc_file].has_key("id")):
1682 orig_file_id = changes.orig_files[dsc_file]["id"]
1683 dest = os.path.join(dest_dir, dsc_file)
1685 # If it doesn't exist, create a symlink
1686 if not os.path.exists(dest):
1687 q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id",
1688 {'id': orig_file_id})
1691 return "[INTERNAL ERROR] Couldn't find id %s in files table." % (orig_file_id)
1693 src = os.path.join(res[0], res[1])
1694 os.symlink(src, dest)
1696 # Add it to the list of packages for later processing by apt-ftparchive
1698 qb.suite_id = s.suite_id
1699 qb.queue_id = self.queue_id
1704 # If it does, update things to ensure it's not removed prematurely
1706 qb = get_queue_build(dest, s.suite_id, session)
1718 __all__.append('Queue')
1721 def get_or_set_queue(queuename, session=None):
1723 Returns Queue object for given C{queue name}, creating it if it does not
1726 @type queuename: string
1727 @param queuename: The name of the queue
1729 @type session: Session
1730 @param session: Optional SQLA session object (a temporary one will be
1731 generated if not supplied)
1734 @return: Queue object for the given queue
1737 q = session.query(Queue).filter_by(queue_name=queuename)
1741 except NoResultFound:
1743 queue.queue_name = queuename
1745 session.commit_or_flush()
1750 __all__.append('get_or_set_queue')
1752 ################################################################################
1754 class QueueBuild(object):
1755 def __init__(self, *args, **kwargs):
1759 return '<QueueBuild %s (%s)>' % (self.filename, self.queue_id)
1761 __all__.append('QueueBuild')
1764 def get_queue_build(filename, suite, session=None):
1766 Returns QueueBuild object for given C{filename} and C{suite}.
1768 @type filename: string
1769 @param filename: The name of the file
1771 @type suiteid: int or str
1772 @param suiteid: Suite name or ID
1774 @type session: Session
1775 @param session: Optional SQLA session object (a temporary one will be
1776 generated if not supplied)
1779 @return: Queue object for the given queue
1782 if isinstance(suite, int):
1783 q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite)
1785 q = session.query(QueueBuild).filter_by(filename=filename)
1786 q = q.join(Suite).filter_by(suite_name=suite)
1790 except NoResultFound:
1793 __all__.append('get_queue_build')
1795 ################################################################################
1797 class Section(object):
1798 def __init__(self, *args, **kwargs):
1801 def __eq__(self, val):
1802 if isinstance(val, str):
1803 return (self.section == val)
1804 # This signals to use the normal comparison operator
1805 return NotImplemented
1807 def __ne__(self, val):
1808 if isinstance(val, str):
1809 return (self.section != val)
1810 # This signals to use the normal comparison operator
1811 return NotImplemented
1814 return '<Section %s>' % self.section
1816 __all__.append('Section')
1819 def get_section(section, session=None):
1821 Returns Section object for given C{section name}.
1823 @type section: string
1824 @param section: The name of the section
1826 @type session: Session
1827 @param session: Optional SQLA session object (a temporary one will be
1828 generated if not supplied)
1831 @return: Section object for the given section name
1834 q = session.query(Section).filter_by(section=section)
1838 except NoResultFound:
1841 __all__.append('get_section')
1844 def get_sections(session=None):
1846 Returns dictionary of section names -> id mappings
1848 @type session: Session
1849 @param session: Optional SQL session object (a temporary one will be
1850 generated if not supplied)
1853 @return: dictionary of section names -> id mappings
1857 q = session.query(Section)
1859 ret[x.section] = x.section_id
1863 __all__.append('get_sections')
1865 ################################################################################
1867 class DBSource(object):
1868 def __init__(self, *args, **kwargs):
1872 return '<DBSource %s (%s)>' % (self.source, self.version)
1874 __all__.append('DBSource')
1877 def source_exists(source, source_version, suites = ["any"], session=None):
1879 Ensure that source exists somewhere in the archive for the binary
1880 upload being processed.
1881 1. exact match => 1.0-3
1882 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1884 @type package: string
1885 @param package: package source name
1887 @type source_version: string
1888 @param source_version: expected source version
1891 @param suites: list of suites to check in, default I{any}
1893 @type session: Session
1894 @param session: Optional SQLA session object (a temporary one will be
1895 generated if not supplied)
1898 @return: returns 1 if a source with expected version is found, otherwise 0
1905 for suite in suites:
1906 q = session.query(DBSource).filter_by(source=source)
1908 # source must exist in suite X, or in some other suite that's
1909 # mapped to X, recursively... silent-maps are counted too,
1910 # unreleased-maps aren't.
1911 maps = cnf.ValueList("SuiteMappings")[:]
1913 maps = [ m.split() for m in maps ]
1914 maps = [ (x[1], x[2]) for x in maps
1915 if x[0] == "map" or x[0] == "silent-map" ]
1918 if x[1] in s and x[0] not in s:
1921 q = q.join(SrcAssociation).join(Suite)
1922 q = q.filter(Suite.suite_name.in_(s))
1924 # Reduce the query results to a list of version numbers
1925 ql = [ j.version for j in q.all() ]
1928 if source_version in ql:
1932 from daklib.regexes import re_bin_only_nmu
1933 orig_source_version = re_bin_only_nmu.sub('', source_version)
1934 if orig_source_version in ql:
1937 # No source found so return not ok
1942 __all__.append('source_exists')
1945 def get_suites_source_in(source, session=None):
1947 Returns list of Suite objects which given C{source} name is in
1950 @param source: DBSource package name to search for
1953 @return: list of Suite objects for the given source
1956 return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
1958 __all__.append('get_suites_source_in')
1961 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
1963 Returns list of DBSource objects for given C{source} name and other parameters
1966 @param source: DBSource package name to search for
1968 @type source: str or None
1969 @param source: DBSource version name to search for or None if not applicable
1971 @type dm_upload_allowed: bool
1972 @param dm_upload_allowed: If None, no effect. If True or False, only
1973 return packages with that dm_upload_allowed setting
1975 @type session: Session
1976 @param session: Optional SQL session object (a temporary one will be
1977 generated if not supplied)
1980 @return: list of DBSource objects for the given name (may be empty)
1983 q = session.query(DBSource).filter_by(source=source)
1985 if version is not None:
1986 q = q.filter_by(version=version)
1988 if dm_upload_allowed is not None:
1989 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
1993 __all__.append('get_sources_from_name')
1996 def get_source_in_suite(source, suite, session=None):
1998 Returns list of DBSource objects for a combination of C{source} and C{suite}.
2000 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2001 - B{suite} - a suite name, eg. I{unstable}
2003 @type source: string
2004 @param source: source package name
2007 @param suite: the suite name
2010 @return: the version for I{source} in I{suite}
2014 q = session.query(SrcAssociation)
2015 q = q.join('source').filter_by(source=source)
2016 q = q.join('suite').filter_by(suite_name=suite)
2019 return q.one().source
2020 except NoResultFound:
2023 __all__.append('get_source_in_suite')
2025 ################################################################################
2028 def add_dsc_to_db(u, filename, session=None):
2029 entry = u.pkg.files[filename]
2032 source.source = u.pkg.dsc["source"]
2033 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2034 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2035 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2036 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2037 source.install_date = datetime.now().date()
2039 dsc_component = entry["component"]
2040 dsc_location_id = entry["location id"]
2042 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2044 # Set up a new poolfile if necessary
2045 if not entry.has_key("files id") or not entry["files id"]:
2046 filename = entry["pool name"] + filename
2047 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2048 entry["files id"] = poolfile.file_id
2050 source.poolfile_id = entry["files id"]
2054 for suite_name in u.pkg.changes["distribution"].keys():
2055 sa = SrcAssociation()
2056 sa.source_id = source.source_id
2057 sa.suite_id = get_suite(suite_name).suite_id
2062 # Add the source files to the DB (files and dsc_files)
2064 dscfile.source_id = source.source_id
2065 dscfile.poolfile_id = entry["files id"]
2066 session.add(dscfile)
2068 for dsc_file, dentry in u.pkg.dsc_files.items():
2070 df.source_id = source.source_id
2072 # If the .orig tarball is already in the pool, it's
2073 # files id is stored in dsc_files by check_dsc().
2074 files_id = dentry.get("files id", None)
2076 # Find the entry in the files hash
2077 # TODO: Bail out here properly
2079 for f, e in u.pkg.files.items():
2084 if files_id is None:
2085 filename = dfentry["pool name"] + dsc_file
2087 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2088 # FIXME: needs to check for -1/-2 and or handle exception
2089 if found and obj is not None:
2090 files_id = obj.file_id
2092 # If still not found, add it
2093 if files_id is None:
2094 # HACK: Force sha1sum etc into dentry
2095 dentry["sha1sum"] = dfentry["sha1sum"]
2096 dentry["sha256sum"] = dfentry["sha256sum"]
2097 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2098 files_id = poolfile.file_id
2100 df.poolfile_id = files_id
2105 # Add the src_uploaders to the DB
2106 uploader_ids = [source.maintainer_id]
2107 if u.pkg.dsc.has_key("uploaders"):
2108 for up in u.pkg.dsc["uploaders"].split(","):
2110 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2113 for up in uploader_ids:
2114 if added_ids.has_key(up):
2115 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
2121 su.maintainer_id = up
2122 su.source_id = source.source_id
2127 return dsc_component, dsc_location_id
2129 __all__.append('add_dsc_to_db')
2132 def add_deb_to_db(u, filename, session=None):
2134 Contrary to what you might expect, this routine deals with both
2135 debs and udebs. That info is in 'dbtype', whilst 'type' is
2136 'deb' for both of them
2139 entry = u.pkg.files[filename]
2142 bin.package = entry["package"]
2143 bin.version = entry["version"]
2144 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2145 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2146 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2147 bin.binarytype = entry["dbtype"]
2150 filename = entry["pool name"] + filename
2151 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2152 if not entry.get("location id", None):
2153 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], utils.where_am_i(), session).location_id
2155 if not entry.get("files id", None):
2156 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2157 entry["files id"] = poolfile.file_id
2159 bin.poolfile_id = entry["files id"]
2162 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2163 if len(bin_sources) != 1:
2164 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2165 (bin.package, bin.version, bin.architecture.arch_string,
2166 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2168 bin.source_id = bin_sources[0].source_id
2170 # Add and flush object so it has an ID
2174 # Add BinAssociations
2175 for suite_name in u.pkg.changes["distribution"].keys():
2176 ba = BinAssociation()
2177 ba.binary_id = bin.binary_id
2178 ba.suite_id = get_suite(suite_name).suite_id
2183 # Deal with contents - disabled for now
2184 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2186 # print "REJECT\nCould not determine contents of package %s" % bin.package
2187 # session.rollback()
2188 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2190 __all__.append('add_deb_to_db')
2192 ################################################################################
2194 class SourceACL(object):
2195 def __init__(self, *args, **kwargs):
2199 return '<SourceACL %s>' % self.source_acl_id
2201 __all__.append('SourceACL')
2203 ################################################################################
2205 class SrcAssociation(object):
2206 def __init__(self, *args, **kwargs):
2210 return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
2212 __all__.append('SrcAssociation')
2214 ################################################################################
2216 class SrcFormat(object):
2217 def __init__(self, *args, **kwargs):
2221 return '<SrcFormat %s>' % (self.format_name)
2223 __all__.append('SrcFormat')
2225 ################################################################################
2227 class SrcUploader(object):
2228 def __init__(self, *args, **kwargs):
2232 return '<SrcUploader %s>' % self.uploader_id
2234 __all__.append('SrcUploader')
2236 ################################################################################
2238 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2239 ('SuiteID', 'suite_id'),
2240 ('Version', 'version'),
2241 ('Origin', 'origin'),
2243 ('Description', 'description'),
2244 ('Untouchable', 'untouchable'),
2245 ('Announce', 'announce'),
2246 ('Codename', 'codename'),
2247 ('OverrideCodename', 'overridecodename'),
2248 ('ValidTime', 'validtime'),
2249 ('Priority', 'priority'),
2250 ('NotAutomatic', 'notautomatic'),
2251 ('CopyChanges', 'copychanges'),
2252 ('CopyDotDak', 'copydotdak'),
2253 ('CommentsDir', 'commentsdir'),
2254 ('OverrideSuite', 'overridesuite'),
2255 ('ChangelogBase', 'changelogbase')]
2258 class Suite(object):
2259 def __init__(self, *args, **kwargs):
2263 return '<Suite %s>' % self.suite_name
2265 def __eq__(self, val):
2266 if isinstance(val, str):
2267 return (self.suite_name == val)
2268 # This signals to use the normal comparison operator
2269 return NotImplemented
2271 def __ne__(self, val):
2272 if isinstance(val, str):
2273 return (self.suite_name != val)
2274 # This signals to use the normal comparison operator
2275 return NotImplemented
2279 for disp, field in SUITE_FIELDS:
2280 val = getattr(self, field, None)
2282 ret.append("%s: %s" % (disp, val))
2284 return "\n".join(ret)
2286 __all__.append('Suite')
2289 def get_suite_architecture(suite, architecture, session=None):
2291 Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
2295 @param suite: Suite name to search for
2297 @type architecture: str
2298 @param architecture: Architecture name to search for
2300 @type session: Session
2301 @param session: Optional SQL session object (a temporary one will be
2302 generated if not supplied)
2304 @rtype: SuiteArchitecture
2305 @return: the SuiteArchitecture object or None
2308 q = session.query(SuiteArchitecture)
2309 q = q.join(Architecture).filter_by(arch_string=architecture)
2310 q = q.join(Suite).filter_by(suite_name=suite)
2314 except NoResultFound:
2317 __all__.append('get_suite_architecture')
2320 def get_suite(suite, session=None):
2322 Returns Suite object for given C{suite name}.
2325 @param suite: The name of the suite
2327 @type session: Session
2328 @param session: Optional SQLA session object (a temporary one will be
2329 generated if not supplied)
2332 @return: Suite object for the requested suite name (None if not present)
2335 q = session.query(Suite).filter_by(suite_name=suite)
2339 except NoResultFound:
2342 __all__.append('get_suite')
2344 ################################################################################
2346 class SuiteArchitecture(object):
2347 def __init__(self, *args, **kwargs):
2351 return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
2353 __all__.append('SuiteArchitecture')
2356 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2358 Returns list of Architecture objects for given C{suite} name
2361 @param source: Suite name to search for
2363 @type skipsrc: boolean
2364 @param skipsrc: Whether to skip returning the 'source' architecture entry
2367 @type skipall: boolean
2368 @param skipall: Whether to skip returning the 'all' architecture entry
2371 @type session: Session
2372 @param session: Optional SQL session object (a temporary one will be
2373 generated if not supplied)
2376 @return: list of Architecture objects for the given name (may be empty)
2379 q = session.query(Architecture)
2380 q = q.join(SuiteArchitecture)
2381 q = q.join(Suite).filter_by(suite_name=suite)
2384 q = q.filter(Architecture.arch_string != 'source')
2387 q = q.filter(Architecture.arch_string != 'all')
2389 q = q.order_by('arch_string')
2393 __all__.append('get_suite_architectures')
2395 ################################################################################
2397 class SuiteSrcFormat(object):
2398 def __init__(self, *args, **kwargs):
2402 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2404 __all__.append('SuiteSrcFormat')
2407 def get_suite_src_formats(suite, session=None):
2409 Returns list of allowed SrcFormat for C{suite}.
2412 @param suite: Suite name to search for
2414 @type session: Session
2415 @param session: Optional SQL session object (a temporary one will be
2416 generated if not supplied)
2419 @return: the list of allowed source formats for I{suite}
2422 q = session.query(SrcFormat)
2423 q = q.join(SuiteSrcFormat)
2424 q = q.join(Suite).filter_by(suite_name=suite)
2425 q = q.order_by('format_name')
2429 __all__.append('get_suite_src_formats')
2431 ################################################################################
2434 def __init__(self, *args, **kwargs):
2437 def __eq__(self, val):
2438 if isinstance(val, str):
2439 return (self.uid == val)
2440 # This signals to use the normal comparison operator
2441 return NotImplemented
2443 def __ne__(self, val):
2444 if isinstance(val, str):
2445 return (self.uid != val)
2446 # This signals to use the normal comparison operator
2447 return NotImplemented
2450 return '<Uid %s (%s)>' % (self.uid, self.name)
2452 __all__.append('Uid')
2455 def add_database_user(uidname, session=None):
2457 Adds a database user
2459 @type uidname: string
2460 @param uidname: The uid of the user to add
2462 @type session: SQLAlchemy
2463 @param session: Optional SQL session object (a temporary one will be
2464 generated if not supplied). If not passed, a commit will be performed at
2465 the end of the function, otherwise the caller is responsible for commiting.
2468 @return: the uid object for the given uidname
2471 session.execute("CREATE USER :uid", {'uid': uidname})
2472 session.commit_or_flush()
2474 __all__.append('add_database_user')
2477 def get_or_set_uid(uidname, session=None):
2479 Returns uid object for given uidname.
2481 If no matching uidname is found, a row is inserted.
2483 @type uidname: string
2484 @param uidname: The uid to add
2486 @type session: SQLAlchemy
2487 @param session: Optional SQL session object (a temporary one will be
2488 generated if not supplied). If not passed, a commit will be performed at
2489 the end of the function, otherwise the caller is responsible for commiting.
2492 @return: the uid object for the given uidname
2495 q = session.query(Uid).filter_by(uid=uidname)
2499 except NoResultFound:
2503 session.commit_or_flush()
2508 __all__.append('get_or_set_uid')
2511 def get_uid_from_fingerprint(fpr, session=None):
2512 q = session.query(Uid)
2513 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2517 except NoResultFound:
2520 __all__.append('get_uid_from_fingerprint')
2522 ################################################################################
2524 class UploadBlock(object):
2525 def __init__(self, *args, **kwargs):
2529 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2531 __all__.append('UploadBlock')
2533 ################################################################################
2535 class DBConn(Singleton):
2537 database module init.
2539 def __init__(self, *args, **kwargs):
2540 super(DBConn, self).__init__(*args, **kwargs)
2542 def _startup(self, *args, **kwargs):
2544 if kwargs.has_key('debug'):
2548 def __setuptables(self):
2549 self.tbl_architecture = Table('architecture', self.db_meta, autoload=True)
2550 self.tbl_archive = Table('archive', self.db_meta, autoload=True)
2551 self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True)
2552 self.tbl_binaries = Table('binaries', self.db_meta, autoload=True)
2553 self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True)
2554 self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True)
2555 self.tbl_component = Table('component', self.db_meta, autoload=True)
2556 self.tbl_config = Table('config', self.db_meta, autoload=True)
2557 self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True)
2558 self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True)
2559 self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True)
2560 self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True)
2561 self.tbl_files = Table('files', self.db_meta, autoload=True)
2562 self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True)
2563 self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True)
2564 self.tbl_known_changes = Table('known_changes', self.db_meta, autoload=True)
2565 self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True)
2566 self.tbl_location = Table('location', self.db_meta, autoload=True)
2567 self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True)
2568 self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True)
2569 self.tbl_override = Table('override', self.db_meta, autoload=True)
2570 self.tbl_override_type = Table('override_type', self.db_meta, autoload=True)
2571 self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True)
2572 self.tbl_priority = Table('priority', self.db_meta, autoload=True)
2573 self.tbl_queue = Table('queue', self.db_meta, autoload=True)
2574 self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True)
2575 self.tbl_section = Table('section', self.db_meta, autoload=True)
2576 self.tbl_source = Table('source', self.db_meta, autoload=True)
2577 self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True)
2578 self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True)
2579 self.tbl_src_format = Table('src_format', self.db_meta, autoload=True)
2580 self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True)
2581 self.tbl_suite = Table('suite', self.db_meta, autoload=True)
2582 self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True)
2583 self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True)
2584 self.tbl_uid = Table('uid', self.db_meta, autoload=True)
2585 self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True)
2587 def __setupmappers(self):
2588 mapper(Architecture, self.tbl_architecture,
2589 properties = dict(arch_id = self.tbl_architecture.c.id))
2591 mapper(Archive, self.tbl_archive,
2592 properties = dict(archive_id = self.tbl_archive.c.id,
2593 archive_name = self.tbl_archive.c.name))
2595 mapper(BinAssociation, self.tbl_bin_associations,
2596 properties = dict(ba_id = self.tbl_bin_associations.c.id,
2597 suite_id = self.tbl_bin_associations.c.suite,
2598 suite = relation(Suite),
2599 binary_id = self.tbl_bin_associations.c.bin,
2600 binary = relation(DBBinary)))
2603 mapper(DBBinary, self.tbl_binaries,
2604 properties = dict(binary_id = self.tbl_binaries.c.id,
2605 package = self.tbl_binaries.c.package,
2606 version = self.tbl_binaries.c.version,
2607 maintainer_id = self.tbl_binaries.c.maintainer,
2608 maintainer = relation(Maintainer),
2609 source_id = self.tbl_binaries.c.source,
2610 source = relation(DBSource),
2611 arch_id = self.tbl_binaries.c.architecture,
2612 architecture = relation(Architecture),
2613 poolfile_id = self.tbl_binaries.c.file,
2614 poolfile = relation(PoolFile),
2615 binarytype = self.tbl_binaries.c.type,
2616 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2617 fingerprint = relation(Fingerprint),
2618 install_date = self.tbl_binaries.c.install_date,
2619 binassociations = relation(BinAssociation,
2620 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
2622 mapper(BinaryACL, self.tbl_binary_acl,
2623 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2625 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2626 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2627 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2628 architecture = relation(Architecture)))
2630 mapper(Component, self.tbl_component,
2631 properties = dict(component_id = self.tbl_component.c.id,
2632 component_name = self.tbl_component.c.name))
2634 mapper(DBConfig, self.tbl_config,
2635 properties = dict(config_id = self.tbl_config.c.id))
2637 mapper(DSCFile, self.tbl_dsc_files,
2638 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2639 source_id = self.tbl_dsc_files.c.source,
2640 source = relation(DBSource),
2641 poolfile_id = self.tbl_dsc_files.c.file,
2642 poolfile = relation(PoolFile)))
2644 mapper(PoolFile, self.tbl_files,
2645 properties = dict(file_id = self.tbl_files.c.id,
2646 filesize = self.tbl_files.c.size,
2647 location_id = self.tbl_files.c.location,
2648 location = relation(Location)))
2650 mapper(Fingerprint, self.tbl_fingerprint,
2651 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2652 uid_id = self.tbl_fingerprint.c.uid,
2653 uid = relation(Uid),
2654 keyring_id = self.tbl_fingerprint.c.keyring,
2655 keyring = relation(Keyring),
2656 source_acl = relation(SourceACL),
2657 binary_acl = relation(BinaryACL)))
2659 mapper(Keyring, self.tbl_keyrings,
2660 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2661 keyring_id = self.tbl_keyrings.c.id))
2663 mapper(KnownChange, self.tbl_known_changes,
2664 properties = dict(known_change_id = self.tbl_known_changes.c.id))
2666 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
2667 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
2668 keyring = relation(Keyring, backref="keyring_acl_map"),
2669 architecture = relation(Architecture)))
2671 mapper(Location, self.tbl_location,
2672 properties = dict(location_id = self.tbl_location.c.id,
2673 component_id = self.tbl_location.c.component,
2674 component = relation(Component),
2675 archive_id = self.tbl_location.c.archive,
2676 archive = relation(Archive),
2677 archive_type = self.tbl_location.c.type))
2679 mapper(Maintainer, self.tbl_maintainer,
2680 properties = dict(maintainer_id = self.tbl_maintainer.c.id))
2682 mapper(NewComment, self.tbl_new_comments,
2683 properties = dict(comment_id = self.tbl_new_comments.c.id))
2685 mapper(Override, self.tbl_override,
2686 properties = dict(suite_id = self.tbl_override.c.suite,
2687 suite = relation(Suite),
2688 component_id = self.tbl_override.c.component,
2689 component = relation(Component),
2690 priority_id = self.tbl_override.c.priority,
2691 priority = relation(Priority),
2692 section_id = self.tbl_override.c.section,
2693 section = relation(Section),
2694 overridetype_id = self.tbl_override.c.type,
2695 overridetype = relation(OverrideType)))
2697 mapper(OverrideType, self.tbl_override_type,
2698 properties = dict(overridetype = self.tbl_override_type.c.type,
2699 overridetype_id = self.tbl_override_type.c.id))
2701 mapper(Priority, self.tbl_priority,
2702 properties = dict(priority_id = self.tbl_priority.c.id))
2704 mapper(Queue, self.tbl_queue,
2705 properties = dict(queue_id = self.tbl_queue.c.id))
2707 mapper(QueueBuild, self.tbl_queue_build,
2708 properties = dict(suite_id = self.tbl_queue_build.c.suite,
2709 queue_id = self.tbl_queue_build.c.queue,
2710 queue = relation(Queue, backref='queuebuild')))
2712 mapper(Section, self.tbl_section,
2713 properties = dict(section_id = self.tbl_section.c.id))
2715 mapper(DBSource, self.tbl_source,
2716 properties = dict(source_id = self.tbl_source.c.id,
2717 version = self.tbl_source.c.version,
2718 maintainer_id = self.tbl_source.c.maintainer,
2719 maintainer = relation(Maintainer,
2720 primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
2721 poolfile_id = self.tbl_source.c.file,
2722 poolfile = relation(PoolFile),
2723 fingerprint_id = self.tbl_source.c.sig_fpr,
2724 fingerprint = relation(Fingerprint),
2725 changedby_id = self.tbl_source.c.changedby,
2726 changedby = relation(Maintainer,
2727 primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
2728 srcfiles = relation(DSCFile,
2729 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
2730 srcassociations = relation(SrcAssociation,
2731 primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)),
2732 srcuploaders = relation(SrcUploader)))
2734 mapper(SourceACL, self.tbl_source_acl,
2735 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
2737 mapper(SrcAssociation, self.tbl_src_associations,
2738 properties = dict(sa_id = self.tbl_src_associations.c.id,
2739 suite_id = self.tbl_src_associations.c.suite,
2740 suite = relation(Suite),
2741 source_id = self.tbl_src_associations.c.source,
2742 source = relation(DBSource)))
2744 mapper(SrcFormat, self.tbl_src_format,
2745 properties = dict(src_format_id = self.tbl_src_format.c.id,
2746 format_name = self.tbl_src_format.c.format_name))
2748 mapper(SrcUploader, self.tbl_src_uploaders,
2749 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
2750 source_id = self.tbl_src_uploaders.c.source,
2751 source = relation(DBSource,
2752 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
2753 maintainer_id = self.tbl_src_uploaders.c.maintainer,
2754 maintainer = relation(Maintainer,
2755 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
2757 mapper(Suite, self.tbl_suite,
2758 properties = dict(suite_id = self.tbl_suite.c.id,
2759 policy_queue = relation(Queue)))
2761 mapper(SuiteArchitecture, self.tbl_suite_architectures,
2762 properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
2763 suite = relation(Suite, backref='suitearchitectures'),
2764 arch_id = self.tbl_suite_architectures.c.architecture,
2765 architecture = relation(Architecture)))
2767 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
2768 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
2769 suite = relation(Suite, backref='suitesrcformats'),
2770 src_format_id = self.tbl_suite_src_formats.c.src_format,
2771 src_format = relation(SrcFormat)))
2773 mapper(Uid, self.tbl_uid,
2774 properties = dict(uid_id = self.tbl_uid.c.id,
2775 fingerprint = relation(Fingerprint)))
2777 mapper(UploadBlock, self.tbl_upload_blocks,
2778 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
2779 fingerprint = relation(Fingerprint, backref="uploadblocks"),
2780 uid = relation(Uid, backref="uploadblocks")))
2782 ## Connection functions
2783 def __createconn(self):
2784 from config import Config
2788 connstr = "postgres://%s" % cnf["DB::Host"]
2789 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2790 connstr += ":%s" % cnf["DB::Port"]
2791 connstr += "/%s" % cnf["DB::Name"]
2794 connstr = "postgres:///%s" % cnf["DB::Name"]
2795 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2796 connstr += "?port=%s" % cnf["DB::Port"]
2798 self.db_pg = create_engine(connstr, echo=self.debug)
2799 self.db_meta = MetaData()
2800 self.db_meta.bind = self.db_pg
2801 self.db_smaker = sessionmaker(bind=self.db_pg,
2805 self.__setuptables()
2806 self.__setupmappers()
2809 return self.db_smaker()
2811 __all__.append('DBConn')