5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
40 from datetime import datetime
42 from inspect import getargspec
45 from sqlalchemy import create_engine, Table, MetaData
46 from sqlalchemy.orm import sessionmaker, mapper, relation
47 from sqlalchemy import types as sqltypes
49 # Don't remove this, we re-export the exceptions to scripts which import us
50 from sqlalchemy.exc import *
51 from sqlalchemy.orm.exc import NoResultFound
53 # Only import Config until Queue stuff is changed to store its config
55 from config import Config
56 from singleton import Singleton
57 from textutils import fix_maintainer
59 ################################################################################
61 # Patch in support for the debversion field type so that it works during
64 class DebVersion(sqltypes.Text):
65 def get_col_spec(self):
68 sa_major_version = sqlalchemy.__version__[0:3]
69 if sa_major_version == "0.5":
70 from sqlalchemy.databases import postgres
71 postgres.ischema_names['debversion'] = DebVersion
73 raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
75 ################################################################################
77 __all__ = ['IntegrityError', 'SQLAlchemyError']
79 ################################################################################
81 def session_wrapper(fn):
83 Wrapper around common ".., session=None):" handling. If the wrapped
84 function is called without passing 'session', we create a local one
85 and destroy it when the function ends.
87 Also attaches a commit_or_flush method to the session; if we created a
88 local session, this is a synonym for session.commit(), otherwise it is a
89 synonym for session.flush().
92 def wrapped(*args, **kwargs):
93 private_transaction = False
95 # Find the session object
96 session = kwargs.get('session')
99 if len(args) <= len(getargspec(fn)[0]) - 1:
100 # No session specified as last argument or in kwargs
101 private_transaction = True
102 session = kwargs['session'] = DBConn().session()
104 # Session is last argument in args
108 session = args[-1] = DBConn().session()
109 private_transaction = True
111 if private_transaction:
112 session.commit_or_flush = session.commit
114 session.commit_or_flush = session.flush
117 return fn(*args, **kwargs)
119 if private_transaction:
120 # We created a session; close it.
123 wrapped.__doc__ = fn.__doc__
124 wrapped.func_name = fn.func_name
128 __all__.append('session_wrapper')
130 ################################################################################
132 class Architecture(object):
133 def __init__(self, *args, **kwargs):
136 def __eq__(self, val):
137 if isinstance(val, str):
138 return (self.arch_string== val)
139 # This signals to use the normal comparison operator
140 return NotImplemented
142 def __ne__(self, val):
143 if isinstance(val, str):
144 return (self.arch_string != val)
145 # This signals to use the normal comparison operator
146 return NotImplemented
149 return '<Architecture %s>' % self.arch_string
151 __all__.append('Architecture')
154 def get_architecture(architecture, session=None):
156 Returns database id for given C{architecture}.
158 @type architecture: string
159 @param architecture: The name of the architecture
161 @type session: Session
162 @param session: Optional SQLA session object (a temporary one will be
163 generated if not supplied)
166 @return: Architecture object for the given arch (None if not present)
169 q = session.query(Architecture).filter_by(arch_string=architecture)
173 except NoResultFound:
176 __all__.append('get_architecture')
179 def get_architecture_suites(architecture, session=None):
181 Returns list of Suite objects for given C{architecture} name
184 @param source: Architecture name to search for
186 @type session: Session
187 @param session: Optional SQL session object (a temporary one will be
188 generated if not supplied)
191 @return: list of Suite objects for the given name (may be empty)
194 q = session.query(Suite)
195 q = q.join(SuiteArchitecture)
196 q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
202 __all__.append('get_architecture_suites')
204 ################################################################################
206 class Archive(object):
207 def __init__(self, *args, **kwargs):
211 return '<Archive %s>' % self.archive_name
213 __all__.append('Archive')
216 def get_archive(archive, session=None):
218 returns database id for given C{archive}.
220 @type archive: string
221 @param archive: the name of the arhive
223 @type session: Session
224 @param session: Optional SQLA session object (a temporary one will be
225 generated if not supplied)
228 @return: Archive object for the given name (None if not present)
231 archive = archive.lower()
233 q = session.query(Archive).filter_by(archive_name=archive)
237 except NoResultFound:
240 __all__.append('get_archive')
242 ################################################################################
244 class BinAssociation(object):
245 def __init__(self, *args, **kwargs):
249 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
251 __all__.append('BinAssociation')
253 ################################################################################
255 class BinContents(object):
256 def __init__(self, *args, **kwargs):
260 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
262 __all__.append('BinContents')
264 ################################################################################
266 class DBBinary(object):
267 def __init__(self, *args, **kwargs):
271 return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
273 __all__.append('DBBinary')
276 def get_suites_binary_in(package, session=None):
278 Returns list of Suite objects which given C{package} name is in
281 @param source: DBBinary package name to search for
284 @return: list of Suite objects for the given package
287 return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
289 __all__.append('get_suites_binary_in')
292 def get_binary_from_id(binary_id, session=None):
294 Returns DBBinary object for given C{id}
297 @param binary_id: Id of the required binary
299 @type session: Session
300 @param session: Optional SQLA session object (a temporary one will be
301 generated if not supplied)
304 @return: DBBinary object for the given binary (None if not present)
307 q = session.query(DBBinary).filter_by(binary_id=binary_id)
311 except NoResultFound:
314 __all__.append('get_binary_from_id')
317 def get_binaries_from_name(package, version=None, architecture=None, session=None):
319 Returns list of DBBinary objects for given C{package} name
322 @param package: DBBinary package name to search for
324 @type version: str or None
325 @param version: Version to search for (or None)
327 @type package: str, list or None
328 @param package: Architectures to limit to (or None if no limit)
330 @type session: Session
331 @param session: Optional SQL session object (a temporary one will be
332 generated if not supplied)
335 @return: list of DBBinary objects for the given name (may be empty)
338 q = session.query(DBBinary).filter_by(package=package)
340 if version is not None:
341 q = q.filter_by(version=version)
343 if architecture is not None:
344 if not isinstance(architecture, list):
345 architecture = [architecture]
346 q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
352 __all__.append('get_binaries_from_name')
355 def get_binaries_from_source_id(source_id, session=None):
357 Returns list of DBBinary objects for given C{source_id}
360 @param source_id: source_id to search for
362 @type session: Session
363 @param session: Optional SQL session object (a temporary one will be
364 generated if not supplied)
367 @return: list of DBBinary objects for the given name (may be empty)
370 return session.query(DBBinary).filter_by(source_id=source_id).all()
372 __all__.append('get_binaries_from_source_id')
375 def get_binary_from_name_suite(package, suitename, session=None):
376 ### For dak examine-package
377 ### XXX: Doesn't use object API yet
379 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
380 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
381 WHERE b.package=:package
383 AND fi.location = l.id
384 AND l.component = c.id
387 AND su.suite_name=:suitename
388 ORDER BY b.version DESC"""
390 return session.execute(sql, {'package': package, 'suitename': suitename})
392 __all__.append('get_binary_from_name_suite')
395 def get_binary_components(package, suitename, arch, session=None):
396 # Check for packages that have moved from one component to another
397 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
398 WHERE b.package=:package AND s.suite_name=:suitename
399 AND (a.arch_string = :arch OR a.arch_string = 'all')
400 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
401 AND f.location = l.id
402 AND l.component = c.id
405 vals = {'package': package, 'suitename': suitename, 'arch': arch}
407 return session.execute(query, vals)
409 __all__.append('get_binary_components')
411 ################################################################################
413 class BinaryACL(object):
414 def __init__(self, *args, **kwargs):
418 return '<BinaryACL %s>' % self.binary_acl_id
420 __all__.append('BinaryACL')
422 ################################################################################
424 class BinaryACLMap(object):
425 def __init__(self, *args, **kwargs):
429 return '<BinaryACLMap %s>' % self.binary_acl_map_id
431 __all__.append('BinaryACLMap')
433 ################################################################################
435 class Component(object):
436 def __init__(self, *args, **kwargs):
439 def __eq__(self, val):
440 if isinstance(val, str):
441 return (self.component_name == val)
442 # This signals to use the normal comparison operator
443 return NotImplemented
445 def __ne__(self, val):
446 if isinstance(val, str):
447 return (self.component_name != val)
448 # This signals to use the normal comparison operator
449 return NotImplemented
452 return '<Component %s>' % self.component_name
455 __all__.append('Component')
458 def get_component(component, session=None):
460 Returns database id for given C{component}.
462 @type component: string
463 @param component: The name of the override type
466 @return: the database id for the given component
469 component = component.lower()
471 q = session.query(Component).filter_by(component_name=component)
475 except NoResultFound:
478 __all__.append('get_component')
480 ################################################################################
482 class DBConfig(object):
483 def __init__(self, *args, **kwargs):
487 return '<DBConfig %s>' % self.name
489 __all__.append('DBConfig')
491 ################################################################################
494 def get_or_set_contents_file_id(filename, session=None):
496 Returns database id for given filename.
498 If no matching file is found, a row is inserted.
500 @type filename: string
501 @param filename: The filename
502 @type session: SQLAlchemy
503 @param session: Optional SQL session object (a temporary one will be
504 generated if not supplied). If not passed, a commit will be performed at
505 the end of the function, otherwise the caller is responsible for commiting.
508 @return: the database id for the given component
511 q = session.query(ContentFilename).filter_by(filename=filename)
514 ret = q.one().cafilename_id
515 except NoResultFound:
516 cf = ContentFilename()
517 cf.filename = filename
519 session.commit_or_flush()
520 ret = cf.cafilename_id
524 __all__.append('get_or_set_contents_file_id')
527 def get_contents(suite, overridetype, section=None, session=None):
529 Returns contents for a suite / overridetype combination, limiting
530 to a section if not None.
533 @param suite: Suite object
535 @type overridetype: OverrideType
536 @param overridetype: OverrideType object
538 @type section: Section
539 @param section: Optional section object to limit results to
541 @type session: SQLAlchemy
542 @param session: Optional SQL session object (a temporary one will be
543 generated if not supplied)
546 @return: ResultsProxy object set up to return tuples of (filename, section,
550 # find me all of the contents for a given suite
551 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
555 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
556 JOIN content_file_names n ON (c.filename=n.id)
557 JOIN binaries b ON (b.id=c.binary_pkg)
558 JOIN override o ON (o.package=b.package)
559 JOIN section s ON (s.id=o.section)
560 WHERE o.suite = :suiteid AND o.type = :overridetypeid
561 AND b.type=:overridetypename"""
563 vals = {'suiteid': suite.suite_id,
564 'overridetypeid': overridetype.overridetype_id,
565 'overridetypename': overridetype.overridetype}
567 if section is not None:
568 contents_q += " AND s.id = :sectionid"
569 vals['sectionid'] = section.section_id
571 contents_q += " ORDER BY fn"
573 return session.execute(contents_q, vals)
575 __all__.append('get_contents')
577 ################################################################################
579 class ContentFilepath(object):
580 def __init__(self, *args, **kwargs):
584 return '<ContentFilepath %s>' % self.filepath
586 __all__.append('ContentFilepath')
589 def get_or_set_contents_path_id(filepath, session=None):
591 Returns database id for given path.
593 If no matching file is found, a row is inserted.
595 @type filename: string
596 @param filename: The filepath
597 @type session: SQLAlchemy
598 @param session: Optional SQL session object (a temporary one will be
599 generated if not supplied). If not passed, a commit will be performed at
600 the end of the function, otherwise the caller is responsible for commiting.
603 @return: the database id for the given path
606 q = session.query(ContentFilepath).filter_by(filepath=filepath)
609 ret = q.one().cafilepath_id
610 except NoResultFound:
611 cf = ContentFilepath()
612 cf.filepath = filepath
614 session.commit_or_flush()
615 ret = cf.cafilepath_id
619 __all__.append('get_or_set_contents_path_id')
621 ################################################################################
623 class ContentAssociation(object):
624 def __init__(self, *args, **kwargs):
628 return '<ContentAssociation %s>' % self.ca_id
630 __all__.append('ContentAssociation')
632 def insert_content_paths(binary_id, fullpaths, session=None):
634 Make sure given path is associated with given binary id
637 @param binary_id: the id of the binary
638 @type fullpaths: list
639 @param fullpaths: the list of paths of the file being associated with the binary
640 @type session: SQLAlchemy session
641 @param session: Optional SQLAlchemy session. If this is passed, the caller
642 is responsible for ensuring a transaction has begun and committing the
643 results or rolling back based on the result code. If not passed, a commit
644 will be performed at the end of the function, otherwise the caller is
645 responsible for commiting.
647 @return: True upon success
652 session = DBConn().session()
658 for fullpath in fullpaths:
659 if fullpath.startswith( './' ):
660 fullpath = fullpath[2:]
662 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} )
670 traceback.print_exc()
672 # Only rollback if we set up the session ourself
679 __all__.append('insert_content_paths')
681 ################################################################################
683 class DSCFile(object):
684 def __init__(self, *args, **kwargs):
688 return '<DSCFile %s>' % self.dscfile_id
690 __all__.append('DSCFile')
693 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
695 Returns a list of DSCFiles which may be empty
697 @type dscfile_id: int (optional)
698 @param dscfile_id: the dscfile_id of the DSCFiles to find
700 @type source_id: int (optional)
701 @param source_id: the source id related to the DSCFiles to find
703 @type poolfile_id: int (optional)
704 @param poolfile_id: the poolfile id related to the DSCFiles to find
707 @return: Possibly empty list of DSCFiles
710 q = session.query(DSCFile)
712 if dscfile_id is not None:
713 q = q.filter_by(dscfile_id=dscfile_id)
715 if source_id is not None:
716 q = q.filter_by(source_id=source_id)
718 if poolfile_id is not None:
719 q = q.filter_by(poolfile_id=poolfile_id)
723 __all__.append('get_dscfiles')
725 ################################################################################
727 class PoolFile(object):
728 def __init__(self, *args, **kwargs):
732 return '<PoolFile %s>' % self.filename
734 __all__.append('PoolFile')
737 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
740 (ValidFileFound [boolean or None], PoolFile object or None)
742 @type filename: string
743 @param filename: the filename of the file to check against the DB
746 @param filesize: the size of the file to check against the DB
749 @param md5sum: the md5sum of the file to check against the DB
751 @type location_id: int
752 @param location_id: the id of the location to look in
755 @return: Tuple of length 2.
756 If more than one file found with that name:
758 If valid pool file found: (True, PoolFile object)
759 If valid pool file not found:
760 (False, None) if no file found
761 (False, PoolFile object) if file found with size/md5sum mismatch
764 q = session.query(PoolFile).filter_by(filename=filename)
765 q = q.join(Location).filter_by(location_id=location_id)
775 if obj.md5sum != md5sum or obj.filesize != int(filesize):
783 __all__.append('check_poolfile')
786 def get_poolfile_by_id(file_id, session=None):
788 Returns a PoolFile objects or None for the given id
791 @param file_id: the id of the file to look for
793 @rtype: PoolFile or None
794 @return: either the PoolFile object or None
797 q = session.query(PoolFile).filter_by(file_id=file_id)
801 except NoResultFound:
804 __all__.append('get_poolfile_by_id')
808 def get_poolfile_by_name(filename, location_id=None, session=None):
810 Returns an array of PoolFile objects for the given filename and
811 (optionally) location_id
813 @type filename: string
814 @param filename: the filename of the file to check against the DB
816 @type location_id: int
817 @param location_id: the id of the location to look in (optional)
820 @return: array of PoolFile objects
823 q = session.query(PoolFile).filter_by(filename=filename)
825 if location_id is not None:
826 q = q.join(Location).filter_by(location_id=location_id)
830 __all__.append('get_poolfile_by_name')
833 def get_poolfile_like_name(filename, session=None):
835 Returns an array of PoolFile objects which are like the given name
837 @type filename: string
838 @param filename: the filename of the file to check against the DB
841 @return: array of PoolFile objects
844 # TODO: There must be a way of properly using bind parameters with %FOO%
845 q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
849 __all__.append('get_poolfile_like_name')
852 def add_poolfile(filename, datadict, location_id, session=None):
854 Add a new file to the pool
856 @type filename: string
857 @param filename: filename
860 @param datadict: dict with needed data
862 @type location_id: int
863 @param location_id: database id of the location
866 @return: the PoolFile object created
868 poolfile = PoolFile()
869 poolfile.filename = filename
870 poolfile.filesize = datadict["size"]
871 poolfile.md5sum = datadict["md5sum"]
872 poolfile.sha1sum = datadict["sha1sum"]
873 poolfile.sha256sum = datadict["sha256sum"]
874 poolfile.location_id = location_id
876 session.add(poolfile)
877 # Flush to get a file id (NB: This is not a commit)
882 __all__.append('add_poolfile')
884 ################################################################################
886 class Fingerprint(object):
887 def __init__(self, *args, **kwargs):
891 return '<Fingerprint %s>' % self.fingerprint
893 __all__.append('Fingerprint')
896 def get_fingerprint(fpr, session=None):
898 Returns Fingerprint object for given fpr.
901 @param fpr: The fpr to find / add
903 @type session: SQLAlchemy
904 @param session: Optional SQL session object (a temporary one will be
905 generated if not supplied).
908 @return: the Fingerprint object for the given fpr or None
911 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
915 except NoResultFound:
920 __all__.append('get_fingerprint')
923 def get_or_set_fingerprint(fpr, session=None):
925 Returns Fingerprint object for given fpr.
927 If no matching fpr is found, a row is inserted.
930 @param fpr: The fpr to find / add
932 @type session: SQLAlchemy
933 @param session: Optional SQL session object (a temporary one will be
934 generated if not supplied). If not passed, a commit will be performed at
935 the end of the function, otherwise the caller is responsible for commiting.
936 A flush will be performed either way.
939 @return: the Fingerprint object for the given fpr
942 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
946 except NoResultFound:
947 fingerprint = Fingerprint()
948 fingerprint.fingerprint = fpr
949 session.add(fingerprint)
950 session.commit_or_flush()
955 __all__.append('get_or_set_fingerprint')
957 ################################################################################
959 # Helper routine for Keyring class
960 def get_ldap_name(entry):
962 for k in ["cn", "mn", "sn"]:
964 if ret and ret[0] != "" and ret[0] != "-":
966 return " ".join(name)
968 ################################################################################
970 class Keyring(object):
971 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
972 " --with-colons --fingerprint --fingerprint"
977 def __init__(self, *args, **kwargs):
981 return '<Keyring %s>' % self.keyring_name
983 def de_escape_gpg_str(self, txt):
984 esclist = re.split(r'(\\x..)', txt)
985 for x in range(1,len(esclist),2):
986 esclist[x] = "%c" % (int(esclist[x][2:],16))
987 return "".join(esclist)
989 def load_keys(self, keyring):
992 if not self.keyring_id:
993 raise Exception('Must be initialized with database information')
995 k = os.popen(self.gpg_invocation % keyring, "r")
999 for line in k.xreadlines():
1000 field = line.split(":")
1001 if field[0] == "pub":
1003 (name, addr) = email.Utils.parseaddr(field[9])
1004 name = re.sub(r"\s*[(].*[)]", "", name)
1005 if name == "" or addr == "" or "@" not in addr:
1007 addr = "invalid-uid"
1008 name = self.de_escape_gpg_str(name)
1009 self.keys[key] = {"email": addr}
1011 self.keys[key]["name"] = name
1012 self.keys[key]["aliases"] = [name]
1013 self.keys[key]["fingerprints"] = []
1015 elif key and field[0] == "sub" and len(field) >= 12:
1016 signingkey = ("s" in field[11])
1017 elif key and field[0] == "uid":
1018 (name, addr) = email.Utils.parseaddr(field[9])
1019 if name and name not in self.keys[key]["aliases"]:
1020 self.keys[key]["aliases"].append(name)
1021 elif signingkey and field[0] == "fpr":
1022 self.keys[key]["fingerprints"].append(field[9])
1023 self.fpr_lookup[field[9]] = key
1025 def import_users_from_ldap(self, session):
1029 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1030 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1032 l = ldap.open(LDAPServer)
1033 l.simple_bind_s("","")
1034 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1035 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1036 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1038 ldap_fin_uid_id = {}
1045 uid = entry["uid"][0]
1046 name = get_ldap_name(entry)
1047 fingerprints = entry["keyFingerPrint"]
1049 for f in fingerprints:
1050 key = self.fpr_lookup.get(f, None)
1051 if key not in self.keys:
1053 self.keys[key]["uid"] = uid
1057 keyid = get_or_set_uid(uid, session).uid_id
1058 byuid[keyid] = (uid, name)
1059 byname[uid] = (keyid, name)
1061 return (byname, byuid)
1063 def generate_users_from_keyring(self, format, session):
1067 for x in self.keys.keys():
1068 if self.keys[x]["email"] == "invalid-uid":
1070 self.keys[x]["uid"] = format % "invalid-uid"
1072 uid = format % self.keys[x]["email"]
1073 keyid = get_or_set_uid(uid, session).uid_id
1074 byuid[keyid] = (uid, self.keys[x]["name"])
1075 byname[uid] = (keyid, self.keys[x]["name"])
1076 self.keys[x]["uid"] = uid
1079 uid = format % "invalid-uid"
1080 keyid = get_or_set_uid(uid, session).uid_id
1081 byuid[keyid] = (uid, "ungeneratable user id")
1082 byname[uid] = (keyid, "ungeneratable user id")
1084 return (byname, byuid)
1086 __all__.append('Keyring')
1089 def get_keyring(keyring, session=None):
1091 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1092 If C{keyring} already has an entry, simply return the existing Keyring
1094 @type keyring: string
1095 @param keyring: the keyring name
1098 @return: the Keyring object for this keyring
1101 q = session.query(Keyring).filter_by(keyring_name=keyring)
1105 except NoResultFound:
1108 __all__.append('get_keyring')
1110 ################################################################################
1112 class KeyringACLMap(object):
1113 def __init__(self, *args, **kwargs):
1117 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1119 __all__.append('KeyringACLMap')
1121 ################################################################################
1123 class KnownChange(object):
1124 def __init__(self, *args, **kwargs):
1128 return '<KnownChange %s>' % self.changesname
1130 __all__.append('KnownChange')
1133 def get_knownchange(filename, session=None):
1135 returns knownchange object for given C{filename}.
1137 @type archive: string
1138 @param archive: the name of the arhive
1140 @type session: Session
1141 @param session: Optional SQLA session object (a temporary one will be
1142 generated if not supplied)
1145 @return: Archive object for the given name (None if not present)
1148 q = session.query(KnownChange).filter_by(changesname=filename)
1152 except NoResultFound:
1155 __all__.append('get_knownchange')
1157 ################################################################################
1158 class Location(object):
1159 def __init__(self, *args, **kwargs):
1163 return '<Location %s (%s)>' % (self.path, self.location_id)
1165 __all__.append('Location')
1168 def get_location(location, component=None, archive=None, session=None):
1170 Returns Location object for the given combination of location, component
1173 @type location: string
1174 @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
1176 @type component: string
1177 @param component: the component name (if None, no restriction applied)
1179 @type archive: string
1180 @param archive_id: the archive name (if None, no restriction applied)
1182 @rtype: Location / None
1183 @return: Either a Location object or None if one can't be found
1186 q = session.query(Location).filter_by(path=location)
1188 if archive is not None:
1189 q = q.join(Archive).filter_by(archive_name=archive)
1191 if component is not None:
1192 q = q.join(Component).filter_by(component_name=component)
1196 except NoResultFound:
1199 __all__.append('get_location')
1201 ################################################################################
1203 class Maintainer(object):
1204 def __init__(self, *args, **kwargs):
1208 return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
1210 def get_split_maintainer(self):
1211 if not hasattr(self, 'name') or self.name is None:
1212 return ('', '', '', '')
1214 return fix_maintainer(self.name.strip())
1216 __all__.append('Maintainer')
1219 def get_or_set_maintainer(name, session=None):
1221 Returns Maintainer object for given maintainer name.
1223 If no matching maintainer name is found, a row is inserted.
1226 @param name: The maintainer name to add
1228 @type session: SQLAlchemy
1229 @param session: Optional SQL session object (a temporary one will be
1230 generated if not supplied). If not passed, a commit will be performed at
1231 the end of the function, otherwise the caller is responsible for commiting.
1232 A flush will be performed either way.
1235 @return: the Maintainer object for the given maintainer
1238 q = session.query(Maintainer).filter_by(name=name)
1241 except NoResultFound:
1242 maintainer = Maintainer()
1243 maintainer.name = name
1244 session.add(maintainer)
1245 session.commit_or_flush()
1250 __all__.append('get_or_set_maintainer')
1253 def get_maintainer(maintainer_id, session=None):
1255 Return the name of the maintainer behind C{maintainer_id} or None if that
1256 maintainer_id is invalid.
1258 @type maintainer_id: int
1259 @param maintainer_id: the id of the maintainer
1262 @return: the Maintainer with this C{maintainer_id}
1265 return session.query(Maintainer).get(maintainer_id)
1267 __all__.append('get_maintainer')
1269 ################################################################################
1271 class NewComment(object):
1272 def __init__(self, *args, **kwargs):
1276 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1278 __all__.append('NewComment')
1281 def has_new_comment(package, version, session=None):
1283 Returns true if the given combination of C{package}, C{version} has a comment.
1285 @type package: string
1286 @param package: name of the package
1288 @type version: string
1289 @param version: package version
1291 @type session: Session
1292 @param session: Optional SQLA session object (a temporary one will be
1293 generated if not supplied)
1299 q = session.query(NewComment)
1300 q = q.filter_by(package=package)
1301 q = q.filter_by(version=version)
1303 return bool(q.count() > 0)
1305 __all__.append('has_new_comment')
1308 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1310 Returns (possibly empty) list of NewComment objects for the given
1313 @type package: string (optional)
1314 @param package: name of the package
1316 @type version: string (optional)
1317 @param version: package version
1319 @type comment_id: int (optional)
1320 @param comment_id: An id of a comment
1322 @type session: Session
1323 @param session: Optional SQLA session object (a temporary one will be
1324 generated if not supplied)
1327 @return: A (possibly empty) list of NewComment objects will be returned
1330 q = session.query(NewComment)
1331 if package is not None: q = q.filter_by(package=package)
1332 if version is not None: q = q.filter_by(version=version)
1333 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1337 __all__.append('get_new_comments')
1339 ################################################################################
1341 class Override(object):
1342 def __init__(self, *args, **kwargs):
1346 return '<Override %s (%s)>' % (self.package, self.suite_id)
1348 __all__.append('Override')
1351 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1353 Returns Override object for the given parameters
1355 @type package: string
1356 @param package: The name of the package
1358 @type suite: string, list or None
1359 @param suite: The name of the suite (or suites if a list) to limit to. If
1360 None, don't limit. Defaults to None.
1362 @type component: string, list or None
1363 @param component: The name of the component (or components if a list) to
1364 limit to. If None, don't limit. Defaults to None.
1366 @type overridetype: string, list or None
1367 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1368 limit to. If None, don't limit. Defaults to None.
1370 @type session: Session
1371 @param session: Optional SQLA session object (a temporary one will be
1372 generated if not supplied)
1375 @return: A (possibly empty) list of Override objects will be returned
1378 q = session.query(Override)
1379 q = q.filter_by(package=package)
1381 if suite is not None:
1382 if not isinstance(suite, list): suite = [suite]
1383 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1385 if component is not None:
1386 if not isinstance(component, list): component = [component]
1387 q = q.join(Component).filter(Component.component_name.in_(component))
1389 if overridetype is not None:
1390 if not isinstance(overridetype, list): overridetype = [overridetype]
1391 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1395 __all__.append('get_override')
1398 ################################################################################
1400 class OverrideType(object):
1401 def __init__(self, *args, **kwargs):
1405 return '<OverrideType %s>' % self.overridetype
1407 __all__.append('OverrideType')
1410 def get_override_type(override_type, session=None):
1412 Returns OverrideType object for given C{override type}.
1414 @type override_type: string
1415 @param override_type: The name of the override type
1417 @type session: Session
1418 @param session: Optional SQLA session object (a temporary one will be
1419 generated if not supplied)
1422 @return: the database id for the given override type
1425 q = session.query(OverrideType).filter_by(overridetype=override_type)
1429 except NoResultFound:
1432 __all__.append('get_override_type')
1434 ################################################################################
1436 class PendingContentAssociation(object):
1437 def __init__(self, *args, **kwargs):
1441 return '<PendingContentAssociation %s>' % self.pca_id
1443 __all__.append('PendingContentAssociation')
1445 def insert_pending_content_paths(package, fullpaths, session=None):
1447 Make sure given paths are temporarily associated with given
1451 @param package: the package to associate with should have been read in from the binary control file
1452 @type fullpaths: list
1453 @param fullpaths: the list of paths of the file being associated with the binary
1454 @type session: SQLAlchemy session
1455 @param session: Optional SQLAlchemy session. If this is passed, the caller
1456 is responsible for ensuring a transaction has begun and committing the
1457 results or rolling back based on the result code. If not passed, a commit
1458 will be performed at the end of the function
1460 @return: True upon success, False if there is a problem
1463 privatetrans = False
1466 session = DBConn().session()
1470 arch = get_architecture(package['Architecture'], session)
1471 arch_id = arch.arch_id
1473 # Remove any already existing recorded files for this package
1474 q = session.query(PendingContentAssociation)
1475 q = q.filter_by(package=package['Package'])
1476 q = q.filter_by(version=package['Version'])
1477 q = q.filter_by(architecture=arch_id)
1482 for fullpath in fullpaths:
1483 (path, filename) = os.path.split(fullpath)
1485 if path.startswith( "./" ):
1488 filepath_id = get_or_set_contents_path_id(path, session)
1489 filename_id = get_or_set_contents_file_id(filename, session)
1491 pathcache[fullpath] = (filepath_id, filename_id)
1493 for fullpath, dat in pathcache.items():
1494 pca = PendingContentAssociation()
1495 pca.package = package['Package']
1496 pca.version = package['Version']
1497 pca.filepath_id = dat[0]
1498 pca.filename_id = dat[1]
1499 pca.architecture = arch_id
1502 # Only commit if we set up the session ourself
1510 except Exception, e:
1511 traceback.print_exc()
1513 # Only rollback if we set up the session ourself
1520 __all__.append('insert_pending_content_paths')
1522 ################################################################################
1524 class Priority(object):
1525 def __init__(self, *args, **kwargs):
1528 def __eq__(self, val):
1529 if isinstance(val, str):
1530 return (self.priority == val)
1531 # This signals to use the normal comparison operator
1532 return NotImplemented
1534 def __ne__(self, val):
1535 if isinstance(val, str):
1536 return (self.priority != val)
1537 # This signals to use the normal comparison operator
1538 return NotImplemented
1541 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
1543 __all__.append('Priority')
1546 def get_priority(priority, session=None):
1548 Returns Priority object for given C{priority name}.
1550 @type priority: string
1551 @param priority: The name of the priority
1553 @type session: Session
1554 @param session: Optional SQLA session object (a temporary one will be
1555 generated if not supplied)
1558 @return: Priority object for the given priority
1561 q = session.query(Priority).filter_by(priority=priority)
1565 except NoResultFound:
1568 __all__.append('get_priority')
1571 def get_priorities(session=None):
1573 Returns dictionary of priority names -> id mappings
1575 @type session: Session
1576 @param session: Optional SQL session object (a temporary one will be
1577 generated if not supplied)
1580 @return: dictionary of priority names -> id mappings
1584 q = session.query(Priority)
1586 ret[x.priority] = x.priority_id
1590 __all__.append('get_priorities')
1592 ################################################################################
1594 class Queue(object):
1595 def __init__(self, *args, **kwargs):
1599 return '<Queue %s>' % self.queue_name
1601 def autobuild_upload(self, changes, srcpath, session=None):
1603 Update queue_build database table used for incoming autobuild support.
1605 @type changes: Changes
1606 @param changes: changes object for the upload to process
1608 @type srcpath: string
1609 @param srcpath: path for the queue file entries/link destinations
1611 @type session: SQLAlchemy session
1612 @param session: Optional SQLAlchemy session. If this is passed, the
1613 caller is responsible for ensuring a transaction has begun and
1614 committing the results or rolling back based on the result code. If
1615 not passed, a commit will be performed at the end of the function,
1616 otherwise the caller is responsible for commiting.
1618 @rtype: NoneType or string
1619 @return: None if the operation failed, a string describing the error if not
1622 privatetrans = False
1624 session = DBConn().session()
1627 # TODO: Remove by moving queue config into the database
1630 for suitename in changes.changes["distribution"].keys():
1631 # TODO: Move into database as:
1632 # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build)
1633 # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink)
1634 # This also gets rid of the SecurityQueueBuild hack below
1635 if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"):
1639 s = get_suite(suitename, session)
1641 return "INTERNAL ERROR: Could not find suite %s" % suitename
1643 # TODO: Get from database as above
1644 dest_dir = conf["Dir::QueueBuild"]
1646 # TODO: Move into database as above
1647 if conf.FindB("Dinstall::SecurityQueueBuild"):
1648 dest_dir = os.path.join(dest_dir, suitename)
1650 for file_entry in changes.files.keys():
1651 src = os.path.join(srcpath, file_entry)
1652 dest = os.path.join(dest_dir, file_entry)
1654 # TODO: Move into database as above
1655 if conf.FindB("Dinstall::SecurityQueueBuild"):
1656 # Copy it since the original won't be readable by www-data
1658 utils.copy(src, dest)
1660 # Create a symlink to it
1661 os.symlink(src, dest)
1664 qb.suite_id = s.suite_id
1665 qb.queue_id = self.queue_id
1671 # If the .orig tarballs are in the pool, create a symlink to
1672 # them (if one doesn't already exist)
1673 for dsc_file in changes.dsc_files.keys():
1674 # Skip all files except orig tarballs
1675 from daklib.regexes import re_is_orig_source
1676 if not re_is_orig_source.match(dsc_file):
1678 # Skip orig files not identified in the pool
1679 if not (changes.orig_files.has_key(dsc_file) and
1680 changes.orig_files[dsc_file].has_key("id")):
1682 orig_file_id = changes.orig_files[dsc_file]["id"]
1683 dest = os.path.join(dest_dir, dsc_file)
1685 # If it doesn't exist, create a symlink
1686 if not os.path.exists(dest):
1687 q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id",
1688 {'id': orig_file_id})
1691 return "[INTERNAL ERROR] Couldn't find id %s in files table." % (orig_file_id)
1693 src = os.path.join(res[0], res[1])
1694 os.symlink(src, dest)
1696 # Add it to the list of packages for later processing by apt-ftparchive
1698 qb.suite_id = s.suite_id
1699 qb.queue_id = self.queue_id
1704 # If it does, update things to ensure it's not removed prematurely
1706 qb = get_queue_build(dest, s.suite_id, session)
1718 __all__.append('Queue')
1721 def get_or_set_queue(queuename, session=None):
1723 Returns Queue object for given C{queue name}, creating it if it does not
1726 @type queuename: string
1727 @param queuename: The name of the queue
1729 @type session: Session
1730 @param session: Optional SQLA session object (a temporary one will be
1731 generated if not supplied)
1734 @return: Queue object for the given queue
1737 q = session.query(Queue).filter_by(queue_name=queuename)
1741 except NoResultFound:
1743 queue.queue_name = queuename
1745 session.commit_or_flush()
1750 __all__.append('get_or_set_queue')
1752 ################################################################################
1754 class QueueBuild(object):
1755 def __init__(self, *args, **kwargs):
1759 return '<QueueBuild %s (%s)>' % (self.filename, self.queue_id)
1761 __all__.append('QueueBuild')
1764 def get_queue_build(filename, suite, session=None):
1766 Returns QueueBuild object for given C{filename} and C{suite}.
1768 @type filename: string
1769 @param filename: The name of the file
1771 @type suiteid: int or str
1772 @param suiteid: Suite name or ID
1774 @type session: Session
1775 @param session: Optional SQLA session object (a temporary one will be
1776 generated if not supplied)
1779 @return: Queue object for the given queue
1782 if isinstance(suite, int):
1783 q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite)
1785 q = session.query(QueueBuild).filter_by(filename=filename)
1786 q = q.join(Suite).filter_by(suite_name=suite)
1790 except NoResultFound:
1793 __all__.append('get_queue_build')
1795 ################################################################################
1797 class Section(object):
1798 def __init__(self, *args, **kwargs):
1801 def __eq__(self, val):
1802 if isinstance(val, str):
1803 return (self.section == val)
1804 # This signals to use the normal comparison operator
1805 return NotImplemented
1807 def __ne__(self, val):
1808 if isinstance(val, str):
1809 return (self.section != val)
1810 # This signals to use the normal comparison operator
1811 return NotImplemented
1814 return '<Section %s>' % self.section
1816 __all__.append('Section')
1819 def get_section(section, session=None):
1821 Returns Section object for given C{section name}.
1823 @type section: string
1824 @param section: The name of the section
1826 @type session: Session
1827 @param session: Optional SQLA session object (a temporary one will be
1828 generated if not supplied)
1831 @return: Section object for the given section name
1834 q = session.query(Section).filter_by(section=section)
1838 except NoResultFound:
1841 __all__.append('get_section')
1844 def get_sections(session=None):
1846 Returns dictionary of section names -> id mappings
1848 @type session: Session
1849 @param session: Optional SQL session object (a temporary one will be
1850 generated if not supplied)
1853 @return: dictionary of section names -> id mappings
1857 q = session.query(Section)
1859 ret[x.section] = x.section_id
1863 __all__.append('get_sections')
1865 ################################################################################
1867 class DBSource(object):
1868 def __init__(self, *args, **kwargs):
1872 return '<DBSource %s (%s)>' % (self.source, self.version)
1874 __all__.append('DBSource')
1877 def source_exists(source, source_version, suites = ["any"], session=None):
1879 Ensure that source exists somewhere in the archive for the binary
1880 upload being processed.
1881 1. exact match => 1.0-3
1882 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1884 @type package: string
1885 @param package: package source name
1887 @type source_version: string
1888 @param source_version: expected source version
1891 @param suites: list of suites to check in, default I{any}
1893 @type session: Session
1894 @param session: Optional SQLA session object (a temporary one will be
1895 generated if not supplied)
1898 @return: returns 1 if a source with expected version is found, otherwise 0
1905 for suite in suites:
1906 q = session.query(DBSource).filter_by(source=source)
1908 # source must exist in suite X, or in some other suite that's
1909 # mapped to X, recursively... silent-maps are counted too,
1910 # unreleased-maps aren't.
1911 maps = cnf.ValueList("SuiteMappings")[:]
1913 maps = [ m.split() for m in maps ]
1914 maps = [ (x[1], x[2]) for x in maps
1915 if x[0] == "map" or x[0] == "silent-map" ]
1918 if x[1] in s and x[0] not in s:
1921 q = q.join(SrcAssociation).join(Suite)
1922 q = q.filter(Suite.suite_name.in_(s))
1924 # Reduce the query results to a list of version numbers
1925 ql = [ j.version for j in q.all() ]
1928 if source_version in ql:
1932 from daklib.regexes import re_bin_only_nmu
1933 orig_source_version = re_bin_only_nmu.sub('', source_version)
1934 if orig_source_version in ql:
1937 # No source found so return not ok
1942 __all__.append('source_exists')
1945 def get_suites_source_in(source, session=None):
1947 Returns list of Suite objects which given C{source} name is in
1950 @param source: DBSource package name to search for
1953 @return: list of Suite objects for the given source
1956 return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
1958 __all__.append('get_suites_source_in')
1961 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
1963 Returns list of DBSource objects for given C{source} name and other parameters
1966 @param source: DBSource package name to search for
1968 @type source: str or None
1969 @param source: DBSource version name to search for or None if not applicable
1971 @type dm_upload_allowed: bool
1972 @param dm_upload_allowed: If None, no effect. If True or False, only
1973 return packages with that dm_upload_allowed setting
1975 @type session: Session
1976 @param session: Optional SQL session object (a temporary one will be
1977 generated if not supplied)
1980 @return: list of DBSource objects for the given name (may be empty)
1983 q = session.query(DBSource).filter_by(source=source)
1985 if version is not None:
1986 q = q.filter_by(version=version)
1988 if dm_upload_allowed is not None:
1989 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
1993 __all__.append('get_sources_from_name')
1996 def get_source_in_suite(source, suite, session=None):
1998 Returns list of DBSource objects for a combination of C{source} and C{suite}.
2000 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2001 - B{suite} - a suite name, eg. I{unstable}
2003 @type source: string
2004 @param source: source package name
2007 @param suite: the suite name
2010 @return: the version for I{source} in I{suite}
2014 q = session.query(SrcAssociation)
2015 q = q.join('source').filter_by(source=source)
2016 q = q.join('suite').filter_by(suite_name=suite)
2019 return q.one().source
2020 except NoResultFound:
2023 __all__.append('get_source_in_suite')
2025 ################################################################################
2028 def add_dsc_to_db(u, filename, session=None):
2029 entry = u.pkg.files[filename]
2032 source.source = u.pkg.dsc["source"]
2033 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2034 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2035 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2036 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2037 source.install_date = datetime.now().date()
2039 dsc_component = entry["component"]
2040 dsc_location_id = entry["location id"]
2042 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2044 # Set up a new poolfile if necessary
2045 if not entry.has_key("files id") or not entry["files id"]:
2046 filename = entry["pool name"] + filename
2047 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2049 entry["files id"] = poolfile.file_id
2051 source.poolfile_id = entry["files id"]
2055 for suite_name in u.pkg.changes["distribution"].keys():
2056 sa = SrcAssociation()
2057 sa.source_id = source.source_id
2058 sa.suite_id = get_suite(suite_name).suite_id
2063 # Add the source files to the DB (files and dsc_files)
2065 dscfile.source_id = source.source_id
2066 dscfile.poolfile_id = entry["files id"]
2067 session.add(dscfile)
2069 for dsc_file, dentry in u.pkg.dsc_files.items():
2071 df.source_id = source.source_id
2073 # If the .orig tarball is already in the pool, it's
2074 # files id is stored in dsc_files by check_dsc().
2075 files_id = dentry.get("files id", None)
2077 # Find the entry in the files hash
2078 # TODO: Bail out here properly
2080 for f, e in u.pkg.files.items():
2085 if files_id is None:
2086 filename = dfentry["pool name"] + dsc_file
2088 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2089 # FIXME: needs to check for -1/-2 and or handle exception
2090 if found and obj is not None:
2091 files_id = obj.file_id
2093 # If still not found, add it
2094 if files_id is None:
2095 # HACK: Force sha1sum etc into dentry
2096 dentry["sha1sum"] = dfentry["sha1sum"]
2097 dentry["sha256sum"] = dfentry["sha256sum"]
2098 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2099 files_id = poolfile.file_id
2101 df.poolfile_id = files_id
2106 # Add the src_uploaders to the DB
2107 uploader_ids = [source.maintainer_id]
2108 if u.pkg.dsc.has_key("uploaders"):
2109 for up in u.pkg.dsc["uploaders"].split(","):
2111 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2114 for up in uploader_ids:
2115 if added_ids.has_key(up):
2116 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
2122 su.maintainer_id = up
2123 su.source_id = source.source_id
2128 return dsc_component, dsc_location_id
2130 __all__.append('add_dsc_to_db')
2133 def add_deb_to_db(u, filename, session=None):
2135 Contrary to what you might expect, this routine deals with both
2136 debs and udebs. That info is in 'dbtype', whilst 'type' is
2137 'deb' for both of them
2140 entry = u.pkg.files[filename]
2143 bin.package = entry["package"]
2144 bin.version = entry["version"]
2145 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2146 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2147 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2148 bin.binarytype = entry["dbtype"]
2151 filename = entry["pool name"] + filename
2152 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2153 if not entry.get("location id", None):
2154 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], utils.where_am_i(), session).location_id
2156 if not entry.get("files id", None):
2157 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2158 entry["files id"] = poolfile.file_id
2160 bin.poolfile_id = entry["files id"]
2163 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2164 if len(bin_sources) != 1:
2165 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2166 (bin.package, bin.version, bin.architecture.arch_string,
2167 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2169 bin.source_id = bin_sources[0].source_id
2171 # Add and flush object so it has an ID
2175 # Add BinAssociations
2176 for suite_name in u.pkg.changes["distribution"].keys():
2177 ba = BinAssociation()
2178 ba.binary_id = bin.binary_id
2179 ba.suite_id = get_suite(suite_name).suite_id
2184 # Deal with contents - disabled for now
2185 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2187 # print "REJECT\nCould not determine contents of package %s" % bin.package
2188 # session.rollback()
2189 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2191 __all__.append('add_deb_to_db')
2193 ################################################################################
2195 class SourceACL(object):
2196 def __init__(self, *args, **kwargs):
2200 return '<SourceACL %s>' % self.source_acl_id
2202 __all__.append('SourceACL')
2204 ################################################################################
2206 class SrcAssociation(object):
2207 def __init__(self, *args, **kwargs):
2211 return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
2213 __all__.append('SrcAssociation')
2215 ################################################################################
2217 class SrcFormat(object):
2218 def __init__(self, *args, **kwargs):
2222 return '<SrcFormat %s>' % (self.format_name)
2224 __all__.append('SrcFormat')
2226 ################################################################################
2228 class SrcUploader(object):
2229 def __init__(self, *args, **kwargs):
2233 return '<SrcUploader %s>' % self.uploader_id
2235 __all__.append('SrcUploader')
2237 ################################################################################
2239 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2240 ('SuiteID', 'suite_id'),
2241 ('Version', 'version'),
2242 ('Origin', 'origin'),
2244 ('Description', 'description'),
2245 ('Untouchable', 'untouchable'),
2246 ('Announce', 'announce'),
2247 ('Codename', 'codename'),
2248 ('OverrideCodename', 'overridecodename'),
2249 ('ValidTime', 'validtime'),
2250 ('Priority', 'priority'),
2251 ('NotAutomatic', 'notautomatic'),
2252 ('CopyChanges', 'copychanges'),
2253 ('CopyDotDak', 'copydotdak'),
2254 ('CommentsDir', 'commentsdir'),
2255 ('OverrideSuite', 'overridesuite'),
2256 ('ChangelogBase', 'changelogbase')]
2259 class Suite(object):
2260 def __init__(self, *args, **kwargs):
2264 return '<Suite %s>' % self.suite_name
2266 def __eq__(self, val):
2267 if isinstance(val, str):
2268 return (self.suite_name == val)
2269 # This signals to use the normal comparison operator
2270 return NotImplemented
2272 def __ne__(self, val):
2273 if isinstance(val, str):
2274 return (self.suite_name != val)
2275 # This signals to use the normal comparison operator
2276 return NotImplemented
2280 for disp, field in SUITE_FIELDS:
2281 val = getattr(self, field, None)
2283 ret.append("%s: %s" % (disp, val))
2285 return "\n".join(ret)
2287 __all__.append('Suite')
2290 def get_suite_architecture(suite, architecture, session=None):
2292 Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
2296 @param suite: Suite name to search for
2298 @type architecture: str
2299 @param architecture: Architecture name to search for
2301 @type session: Session
2302 @param session: Optional SQL session object (a temporary one will be
2303 generated if not supplied)
2305 @rtype: SuiteArchitecture
2306 @return: the SuiteArchitecture object or None
2309 q = session.query(SuiteArchitecture)
2310 q = q.join(Architecture).filter_by(arch_string=architecture)
2311 q = q.join(Suite).filter_by(suite_name=suite)
2315 except NoResultFound:
2318 __all__.append('get_suite_architecture')
2321 def get_suite(suite, session=None):
2323 Returns Suite object for given C{suite name}.
2326 @param suite: The name of the suite
2328 @type session: Session
2329 @param session: Optional SQLA session object (a temporary one will be
2330 generated if not supplied)
2333 @return: Suite object for the requested suite name (None if not present)
2336 q = session.query(Suite).filter_by(suite_name=suite)
2340 except NoResultFound:
2343 __all__.append('get_suite')
2345 ################################################################################
2347 class SuiteArchitecture(object):
2348 def __init__(self, *args, **kwargs):
2352 return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
2354 __all__.append('SuiteArchitecture')
2357 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2359 Returns list of Architecture objects for given C{suite} name
2362 @param source: Suite name to search for
2364 @type skipsrc: boolean
2365 @param skipsrc: Whether to skip returning the 'source' architecture entry
2368 @type skipall: boolean
2369 @param skipall: Whether to skip returning the 'all' architecture entry
2372 @type session: Session
2373 @param session: Optional SQL session object (a temporary one will be
2374 generated if not supplied)
2377 @return: list of Architecture objects for the given name (may be empty)
2380 q = session.query(Architecture)
2381 q = q.join(SuiteArchitecture)
2382 q = q.join(Suite).filter_by(suite_name=suite)
2385 q = q.filter(Architecture.arch_string != 'source')
2388 q = q.filter(Architecture.arch_string != 'all')
2390 q = q.order_by('arch_string')
2394 __all__.append('get_suite_architectures')
2396 ################################################################################
2398 class SuiteSrcFormat(object):
2399 def __init__(self, *args, **kwargs):
2403 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2405 __all__.append('SuiteSrcFormat')
2408 def get_suite_src_formats(suite, session=None):
2410 Returns list of allowed SrcFormat for C{suite}.
2413 @param suite: Suite name to search for
2415 @type session: Session
2416 @param session: Optional SQL session object (a temporary one will be
2417 generated if not supplied)
2420 @return: the list of allowed source formats for I{suite}
2423 q = session.query(SrcFormat)
2424 q = q.join(SuiteSrcFormat)
2425 q = q.join(Suite).filter_by(suite_name=suite)
2426 q = q.order_by('format_name')
2430 __all__.append('get_suite_src_formats')
2432 ################################################################################
2435 def __init__(self, *args, **kwargs):
2438 def __eq__(self, val):
2439 if isinstance(val, str):
2440 return (self.uid == val)
2441 # This signals to use the normal comparison operator
2442 return NotImplemented
2444 def __ne__(self, val):
2445 if isinstance(val, str):
2446 return (self.uid != val)
2447 # This signals to use the normal comparison operator
2448 return NotImplemented
2451 return '<Uid %s (%s)>' % (self.uid, self.name)
2453 __all__.append('Uid')
2456 def add_database_user(uidname, session=None):
2458 Adds a database user
2460 @type uidname: string
2461 @param uidname: The uid of the user to add
2463 @type session: SQLAlchemy
2464 @param session: Optional SQL session object (a temporary one will be
2465 generated if not supplied). If not passed, a commit will be performed at
2466 the end of the function, otherwise the caller is responsible for commiting.
2469 @return: the uid object for the given uidname
2472 session.execute("CREATE USER :uid", {'uid': uidname})
2473 session.commit_or_flush()
2475 __all__.append('add_database_user')
2478 def get_or_set_uid(uidname, session=None):
2480 Returns uid object for given uidname.
2482 If no matching uidname is found, a row is inserted.
2484 @type uidname: string
2485 @param uidname: The uid to add
2487 @type session: SQLAlchemy
2488 @param session: Optional SQL session object (a temporary one will be
2489 generated if not supplied). If not passed, a commit will be performed at
2490 the end of the function, otherwise the caller is responsible for commiting.
2493 @return: the uid object for the given uidname
2496 q = session.query(Uid).filter_by(uid=uidname)
2500 except NoResultFound:
2504 session.commit_or_flush()
2509 __all__.append('get_or_set_uid')
2512 def get_uid_from_fingerprint(fpr, session=None):
2513 q = session.query(Uid)
2514 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2518 except NoResultFound:
2521 __all__.append('get_uid_from_fingerprint')
2523 ################################################################################
2525 class UploadBlock(object):
2526 def __init__(self, *args, **kwargs):
2530 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2532 __all__.append('UploadBlock')
2534 ################################################################################
2536 class DBConn(Singleton):
2538 database module init.
2540 def __init__(self, *args, **kwargs):
2541 super(DBConn, self).__init__(*args, **kwargs)
2543 def _startup(self, *args, **kwargs):
2545 if kwargs.has_key('debug'):
2549 def __setuptables(self):
2550 self.tbl_architecture = Table('architecture', self.db_meta, autoload=True)
2551 self.tbl_archive = Table('archive', self.db_meta, autoload=True)
2552 self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True)
2553 self.tbl_binaries = Table('binaries', self.db_meta, autoload=True)
2554 self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True)
2555 self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True)
2556 self.tbl_component = Table('component', self.db_meta, autoload=True)
2557 self.tbl_config = Table('config', self.db_meta, autoload=True)
2558 self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True)
2559 self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True)
2560 self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True)
2561 self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True)
2562 self.tbl_files = Table('files', self.db_meta, autoload=True)
2563 self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True)
2564 self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True)
2565 self.tbl_known_changes = Table('known_changes', self.db_meta, autoload=True)
2566 self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True)
2567 self.tbl_location = Table('location', self.db_meta, autoload=True)
2568 self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True)
2569 self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True)
2570 self.tbl_override = Table('override', self.db_meta, autoload=True)
2571 self.tbl_override_type = Table('override_type', self.db_meta, autoload=True)
2572 self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True)
2573 self.tbl_priority = Table('priority', self.db_meta, autoload=True)
2574 self.tbl_queue = Table('queue', self.db_meta, autoload=True)
2575 self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True)
2576 self.tbl_section = Table('section', self.db_meta, autoload=True)
2577 self.tbl_source = Table('source', self.db_meta, autoload=True)
2578 self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True)
2579 self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True)
2580 self.tbl_src_format = Table('src_format', self.db_meta, autoload=True)
2581 self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True)
2582 self.tbl_suite = Table('suite', self.db_meta, autoload=True)
2583 self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True)
2584 self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True)
2585 self.tbl_uid = Table('uid', self.db_meta, autoload=True)
2586 self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True)
2588 def __setupmappers(self):
2589 mapper(Architecture, self.tbl_architecture,
2590 properties = dict(arch_id = self.tbl_architecture.c.id))
2592 mapper(Archive, self.tbl_archive,
2593 properties = dict(archive_id = self.tbl_archive.c.id,
2594 archive_name = self.tbl_archive.c.name))
2596 mapper(BinAssociation, self.tbl_bin_associations,
2597 properties = dict(ba_id = self.tbl_bin_associations.c.id,
2598 suite_id = self.tbl_bin_associations.c.suite,
2599 suite = relation(Suite),
2600 binary_id = self.tbl_bin_associations.c.bin,
2601 binary = relation(DBBinary)))
2604 mapper(DBBinary, self.tbl_binaries,
2605 properties = dict(binary_id = self.tbl_binaries.c.id,
2606 package = self.tbl_binaries.c.package,
2607 version = self.tbl_binaries.c.version,
2608 maintainer_id = self.tbl_binaries.c.maintainer,
2609 maintainer = relation(Maintainer),
2610 source_id = self.tbl_binaries.c.source,
2611 source = relation(DBSource),
2612 arch_id = self.tbl_binaries.c.architecture,
2613 architecture = relation(Architecture),
2614 poolfile_id = self.tbl_binaries.c.file,
2615 poolfile = relation(PoolFile),
2616 binarytype = self.tbl_binaries.c.type,
2617 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2618 fingerprint = relation(Fingerprint),
2619 install_date = self.tbl_binaries.c.install_date,
2620 binassociations = relation(BinAssociation,
2621 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
2623 mapper(BinaryACL, self.tbl_binary_acl,
2624 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2626 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2627 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2628 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2629 architecture = relation(Architecture)))
2631 mapper(Component, self.tbl_component,
2632 properties = dict(component_id = self.tbl_component.c.id,
2633 component_name = self.tbl_component.c.name))
2635 mapper(DBConfig, self.tbl_config,
2636 properties = dict(config_id = self.tbl_config.c.id))
2638 mapper(DSCFile, self.tbl_dsc_files,
2639 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2640 source_id = self.tbl_dsc_files.c.source,
2641 source = relation(DBSource),
2642 poolfile_id = self.tbl_dsc_files.c.file,
2643 poolfile = relation(PoolFile)))
2645 mapper(PoolFile, self.tbl_files,
2646 properties = dict(file_id = self.tbl_files.c.id,
2647 filesize = self.tbl_files.c.size,
2648 location_id = self.tbl_files.c.location,
2649 location = relation(Location)))
2651 mapper(Fingerprint, self.tbl_fingerprint,
2652 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2653 uid_id = self.tbl_fingerprint.c.uid,
2654 uid = relation(Uid),
2655 keyring_id = self.tbl_fingerprint.c.keyring,
2656 keyring = relation(Keyring),
2657 source_acl = relation(SourceACL),
2658 binary_acl = relation(BinaryACL)))
2660 mapper(Keyring, self.tbl_keyrings,
2661 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2662 keyring_id = self.tbl_keyrings.c.id))
2664 mapper(KnownChange, self.tbl_known_changes,
2665 properties = dict(known_change_id = self.tbl_known_changes.c.id))
2667 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
2668 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
2669 keyring = relation(Keyring, backref="keyring_acl_map"),
2670 architecture = relation(Architecture)))
2672 mapper(Location, self.tbl_location,
2673 properties = dict(location_id = self.tbl_location.c.id,
2674 component_id = self.tbl_location.c.component,
2675 component = relation(Component),
2676 archive_id = self.tbl_location.c.archive,
2677 archive = relation(Archive),
2678 archive_type = self.tbl_location.c.type))
2680 mapper(Maintainer, self.tbl_maintainer,
2681 properties = dict(maintainer_id = self.tbl_maintainer.c.id))
2683 mapper(NewComment, self.tbl_new_comments,
2684 properties = dict(comment_id = self.tbl_new_comments.c.id))
2686 mapper(Override, self.tbl_override,
2687 properties = dict(suite_id = self.tbl_override.c.suite,
2688 suite = relation(Suite),
2689 component_id = self.tbl_override.c.component,
2690 component = relation(Component),
2691 priority_id = self.tbl_override.c.priority,
2692 priority = relation(Priority),
2693 section_id = self.tbl_override.c.section,
2694 section = relation(Section),
2695 overridetype_id = self.tbl_override.c.type,
2696 overridetype = relation(OverrideType)))
2698 mapper(OverrideType, self.tbl_override_type,
2699 properties = dict(overridetype = self.tbl_override_type.c.type,
2700 overridetype_id = self.tbl_override_type.c.id))
2702 mapper(Priority, self.tbl_priority,
2703 properties = dict(priority_id = self.tbl_priority.c.id))
2705 mapper(Queue, self.tbl_queue,
2706 properties = dict(queue_id = self.tbl_queue.c.id))
2708 mapper(QueueBuild, self.tbl_queue_build,
2709 properties = dict(suite_id = self.tbl_queue_build.c.suite,
2710 queue_id = self.tbl_queue_build.c.queue,
2711 queue = relation(Queue, backref='queuebuild')))
2713 mapper(Section, self.tbl_section,
2714 properties = dict(section_id = self.tbl_section.c.id))
2716 mapper(DBSource, self.tbl_source,
2717 properties = dict(source_id = self.tbl_source.c.id,
2718 version = self.tbl_source.c.version,
2719 maintainer_id = self.tbl_source.c.maintainer,
2720 maintainer = relation(Maintainer,
2721 primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
2722 poolfile_id = self.tbl_source.c.file,
2723 poolfile = relation(PoolFile),
2724 fingerprint_id = self.tbl_source.c.sig_fpr,
2725 fingerprint = relation(Fingerprint),
2726 changedby_id = self.tbl_source.c.changedby,
2727 changedby = relation(Maintainer,
2728 primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
2729 srcfiles = relation(DSCFile,
2730 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
2731 srcassociations = relation(SrcAssociation,
2732 primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)),
2733 srcuploaders = relation(SrcUploader)))
2735 mapper(SourceACL, self.tbl_source_acl,
2736 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
2738 mapper(SrcAssociation, self.tbl_src_associations,
2739 properties = dict(sa_id = self.tbl_src_associations.c.id,
2740 suite_id = self.tbl_src_associations.c.suite,
2741 suite = relation(Suite),
2742 source_id = self.tbl_src_associations.c.source,
2743 source = relation(DBSource)))
2745 mapper(SrcFormat, self.tbl_src_format,
2746 properties = dict(src_format_id = self.tbl_src_format.c.id,
2747 format_name = self.tbl_src_format.c.format_name))
2749 mapper(SrcUploader, self.tbl_src_uploaders,
2750 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
2751 source_id = self.tbl_src_uploaders.c.source,
2752 source = relation(DBSource,
2753 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
2754 maintainer_id = self.tbl_src_uploaders.c.maintainer,
2755 maintainer = relation(Maintainer,
2756 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
2758 mapper(Suite, self.tbl_suite,
2759 properties = dict(suite_id = self.tbl_suite.c.id,
2760 policy_queue = relation(Queue)))
2762 mapper(SuiteArchitecture, self.tbl_suite_architectures,
2763 properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
2764 suite = relation(Suite, backref='suitearchitectures'),
2765 arch_id = self.tbl_suite_architectures.c.architecture,
2766 architecture = relation(Architecture)))
2768 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
2769 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
2770 suite = relation(Suite, backref='suitesrcformats'),
2771 src_format_id = self.tbl_suite_src_formats.c.src_format,
2772 src_format = relation(SrcFormat)))
2774 mapper(Uid, self.tbl_uid,
2775 properties = dict(uid_id = self.tbl_uid.c.id,
2776 fingerprint = relation(Fingerprint)))
2778 mapper(UploadBlock, self.tbl_upload_blocks,
2779 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
2780 fingerprint = relation(Fingerprint, backref="uploadblocks"),
2781 uid = relation(Uid, backref="uploadblocks")))
2783 ## Connection functions
2784 def __createconn(self):
2785 from config import Config
2789 connstr = "postgres://%s" % cnf["DB::Host"]
2790 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2791 connstr += ":%s" % cnf["DB::Port"]
2792 connstr += "/%s" % cnf["DB::Name"]
2795 connstr = "postgres:///%s" % cnf["DB::Name"]
2796 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2797 connstr += "?port=%s" % cnf["DB::Port"]
2799 self.db_pg = create_engine(connstr, echo=self.debug)
2800 self.db_meta = MetaData()
2801 self.db_meta.bind = self.db_pg
2802 self.db_smaker = sessionmaker(bind=self.db_pg,
2806 self.__setuptables()
2807 self.__setupmappers()
2810 return self.db_smaker()
2812 __all__.append('DBConn')