5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
41 from inspect import getargspec
44 from sqlalchemy import create_engine, Table, MetaData
45 from sqlalchemy.orm import sessionmaker, mapper, relation
46 from sqlalchemy import types as sqltypes
48 # Don't remove this, we re-export the exceptions to scripts which import us
49 from sqlalchemy.exc import *
50 from sqlalchemy.orm.exc import NoResultFound
52 # Only import Config until Queue stuff is changed to store its config
54 from config import Config
55 from singleton import Singleton
56 from textutils import fix_maintainer
58 ################################################################################
60 # Patch in support for the debversion field type so that it works during
63 class DebVersion(sqltypes.Text):
64 def get_col_spec(self):
67 sa_major_version = sqlalchemy.__version__[0:3]
68 if sa_major_version == "0.5":
69 from sqlalchemy.databases import postgres
70 postgres.ischema_names['debversion'] = DebVersion
72 raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
74 ################################################################################
76 __all__ = ['IntegrityError', 'SQLAlchemyError']
78 ################################################################################
80 def session_wrapper(fn):
82 Wrapper around common ".., session=None):" handling. If the wrapped
83 function is called without passing 'session', we create a local one
84 and destroy it when the function ends.
86 Also attaches a commit_or_flush method to the session; if we created a
87 local session, this is a synonym for session.commit(), otherwise it is a
88 synonym for session.flush().
91 def wrapped(*args, **kwargs):
92 private_transaction = False
94 # Find the session object
95 session = kwargs.get('session')
98 if len(args) <= len(getargspec(fn)[0]) - 1:
99 # No session specified as last argument or in kwargs
100 private_transaction = True
101 session = kwargs['session'] = DBConn().session()
103 # Session is last argument in args
107 session = args[-1] = DBConn().session()
108 private_transaction = True
110 if private_transaction:
111 session.commit_or_flush = session.commit
113 session.commit_or_flush = session.flush
116 return fn(*args, **kwargs)
118 if private_transaction:
119 # We created a session; close it.
122 wrapped.__doc__ = fn.__doc__
123 wrapped.func_name = fn.func_name
127 __all__.append('session_wrapper')
129 ################################################################################
131 class Architecture(object):
132 def __init__(self, *args, **kwargs):
135 def __eq__(self, val):
136 if isinstance(val, str):
137 return (self.arch_string== val)
138 # This signals to use the normal comparison operator
139 return NotImplemented
141 def __ne__(self, val):
142 if isinstance(val, str):
143 return (self.arch_string != val)
144 # This signals to use the normal comparison operator
145 return NotImplemented
148 return '<Architecture %s>' % self.arch_string
150 __all__.append('Architecture')
153 def get_architecture(architecture, session=None):
155 Returns database id for given C{architecture}.
157 @type architecture: string
158 @param architecture: The name of the architecture
160 @type session: Session
161 @param session: Optional SQLA session object (a temporary one will be
162 generated if not supplied)
165 @return: Architecture object for the given arch (None if not present)
168 q = session.query(Architecture).filter_by(arch_string=architecture)
172 except NoResultFound:
175 __all__.append('get_architecture')
178 def get_architecture_suites(architecture, session=None):
180 Returns list of Suite objects for given C{architecture} name
183 @param source: Architecture name to search for
185 @type session: Session
186 @param session: Optional SQL session object (a temporary one will be
187 generated if not supplied)
190 @return: list of Suite objects for the given name (may be empty)
193 q = session.query(Suite)
194 q = q.join(SuiteArchitecture)
195 q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
201 __all__.append('get_architecture_suites')
203 ################################################################################
205 class Archive(object):
206 def __init__(self, *args, **kwargs):
210 return '<Archive %s>' % self.archive_name
212 __all__.append('Archive')
215 def get_archive(archive, session=None):
217 returns database id for given C{archive}.
219 @type archive: string
220 @param archive: the name of the arhive
222 @type session: Session
223 @param session: Optional SQLA session object (a temporary one will be
224 generated if not supplied)
227 @return: Archive object for the given name (None if not present)
230 archive = archive.lower()
232 q = session.query(Archive).filter_by(archive_name=archive)
236 except NoResultFound:
239 __all__.append('get_archive')
241 ################################################################################
243 class BinAssociation(object):
244 def __init__(self, *args, **kwargs):
248 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
250 __all__.append('BinAssociation')
252 ################################################################################
254 class BinContents(object):
255 def __init__(self, *args, **kwargs):
259 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
261 __all__.append('BinContents')
263 ################################################################################
265 class DBBinary(object):
266 def __init__(self, *args, **kwargs):
270 return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
272 __all__.append('DBBinary')
275 def get_suites_binary_in(package, session=None):
277 Returns list of Suite objects which given C{package} name is in
280 @param source: DBBinary package name to search for
283 @return: list of Suite objects for the given package
286 return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
288 __all__.append('get_suites_binary_in')
291 def get_binary_from_id(binary_id, session=None):
293 Returns DBBinary object for given C{id}
296 @param binary_id: Id of the required binary
298 @type session: Session
299 @param session: Optional SQLA session object (a temporary one will be
300 generated if not supplied)
303 @return: DBBinary object for the given binary (None if not present)
306 q = session.query(DBBinary).filter_by(binary_id=binary_id)
310 except NoResultFound:
313 __all__.append('get_binary_from_id')
316 def get_binaries_from_name(package, version=None, architecture=None, session=None):
318 Returns list of DBBinary objects for given C{package} name
321 @param package: DBBinary package name to search for
323 @type version: str or None
324 @param version: Version to search for (or None)
326 @type package: str, list or None
327 @param package: Architectures to limit to (or None if no limit)
329 @type session: Session
330 @param session: Optional SQL session object (a temporary one will be
331 generated if not supplied)
334 @return: list of DBBinary objects for the given name (may be empty)
337 q = session.query(DBBinary).filter_by(package=package)
339 if version is not None:
340 q = q.filter_by(version=version)
342 if architecture is not None:
343 if not isinstance(architecture, list):
344 architecture = [architecture]
345 q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
351 __all__.append('get_binaries_from_name')
354 def get_binaries_from_source_id(source_id, session=None):
356 Returns list of DBBinary objects for given C{source_id}
359 @param source_id: source_id to search for
361 @type session: Session
362 @param session: Optional SQL session object (a temporary one will be
363 generated if not supplied)
366 @return: list of DBBinary objects for the given name (may be empty)
369 return session.query(DBBinary).filter_by(source_id=source_id).all()
371 __all__.append('get_binaries_from_source_id')
374 def get_binary_from_name_suite(package, suitename, session=None):
375 ### For dak examine-package
376 ### XXX: Doesn't use object API yet
378 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
379 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
380 WHERE b.package=:package
382 AND fi.location = l.id
383 AND l.component = c.id
386 AND su.suite_name=:suitename
387 ORDER BY b.version DESC"""
389 return session.execute(sql, {'package': package, 'suitename': suitename})
391 __all__.append('get_binary_from_name_suite')
394 def get_binary_components(package, suitename, arch, session=None):
395 # Check for packages that have moved from one component to another
396 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
397 WHERE b.package=:package AND s.suite_name=:suitename
398 AND (a.arch_string = :arch OR a.arch_string = 'all')
399 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
400 AND f.location = l.id
401 AND l.component = c.id
404 vals = {'package': package, 'suitename': suitename, 'arch': arch}
406 return session.execute(query, vals)
408 __all__.append('get_binary_components')
410 ################################################################################
412 class BinaryACL(object):
413 def __init__(self, *args, **kwargs):
417 return '<BinaryACL %s>' % self.binary_acl_id
419 __all__.append('BinaryACL')
421 ################################################################################
423 class BinaryACLMap(object):
424 def __init__(self, *args, **kwargs):
428 return '<BinaryACLMap %s>' % self.binary_acl_map_id
430 __all__.append('BinaryACLMap')
432 ################################################################################
434 class Component(object):
435 def __init__(self, *args, **kwargs):
438 def __eq__(self, val):
439 if isinstance(val, str):
440 return (self.component_name == val)
441 # This signals to use the normal comparison operator
442 return NotImplemented
444 def __ne__(self, val):
445 if isinstance(val, str):
446 return (self.component_name != val)
447 # This signals to use the normal comparison operator
448 return NotImplemented
451 return '<Component %s>' % self.component_name
454 __all__.append('Component')
457 def get_component(component, session=None):
459 Returns database id for given C{component}.
461 @type component: string
462 @param component: The name of the override type
465 @return: the database id for the given component
468 component = component.lower()
470 q = session.query(Component).filter_by(component_name=component)
474 except NoResultFound:
477 __all__.append('get_component')
479 ################################################################################
481 class DBConfig(object):
482 def __init__(self, *args, **kwargs):
486 return '<DBConfig %s>' % self.name
488 __all__.append('DBConfig')
490 ################################################################################
493 def get_or_set_contents_file_id(filename, session=None):
495 Returns database id for given filename.
497 If no matching file is found, a row is inserted.
499 @type filename: string
500 @param filename: The filename
501 @type session: SQLAlchemy
502 @param session: Optional SQL session object (a temporary one will be
503 generated if not supplied). If not passed, a commit will be performed at
504 the end of the function, otherwise the caller is responsible for commiting.
507 @return: the database id for the given component
510 q = session.query(ContentFilename).filter_by(filename=filename)
513 ret = q.one().cafilename_id
514 except NoResultFound:
515 cf = ContentFilename()
516 cf.filename = filename
518 session.commit_or_flush()
519 ret = cf.cafilename_id
523 __all__.append('get_or_set_contents_file_id')
526 def get_contents(suite, overridetype, section=None, session=None):
528 Returns contents for a suite / overridetype combination, limiting
529 to a section if not None.
532 @param suite: Suite object
534 @type overridetype: OverrideType
535 @param overridetype: OverrideType object
537 @type section: Section
538 @param section: Optional section object to limit results to
540 @type session: SQLAlchemy
541 @param session: Optional SQL session object (a temporary one will be
542 generated if not supplied)
545 @return: ResultsProxy object set up to return tuples of (filename, section,
549 # find me all of the contents for a given suite
550 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
554 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
555 JOIN content_file_names n ON (c.filename=n.id)
556 JOIN binaries b ON (b.id=c.binary_pkg)
557 JOIN override o ON (o.package=b.package)
558 JOIN section s ON (s.id=o.section)
559 WHERE o.suite = :suiteid AND o.type = :overridetypeid
560 AND b.type=:overridetypename"""
562 vals = {'suiteid': suite.suite_id,
563 'overridetypeid': overridetype.overridetype_id,
564 'overridetypename': overridetype.overridetype}
566 if section is not None:
567 contents_q += " AND s.id = :sectionid"
568 vals['sectionid'] = section.section_id
570 contents_q += " ORDER BY fn"
572 return session.execute(contents_q, vals)
574 __all__.append('get_contents')
576 ################################################################################
578 class ContentFilepath(object):
579 def __init__(self, *args, **kwargs):
583 return '<ContentFilepath %s>' % self.filepath
585 __all__.append('ContentFilepath')
588 def get_or_set_contents_path_id(filepath, session=None):
590 Returns database id for given path.
592 If no matching file is found, a row is inserted.
594 @type filename: string
595 @param filename: The filepath
596 @type session: SQLAlchemy
597 @param session: Optional SQL session object (a temporary one will be
598 generated if not supplied). If not passed, a commit will be performed at
599 the end of the function, otherwise the caller is responsible for commiting.
602 @return: the database id for the given path
605 q = session.query(ContentFilepath).filter_by(filepath=filepath)
608 ret = q.one().cafilepath_id
609 except NoResultFound:
610 cf = ContentFilepath()
611 cf.filepath = filepath
613 session.commit_or_flush()
614 ret = cf.cafilepath_id
618 __all__.append('get_or_set_contents_path_id')
620 ################################################################################
622 class ContentAssociation(object):
623 def __init__(self, *args, **kwargs):
627 return '<ContentAssociation %s>' % self.ca_id
629 __all__.append('ContentAssociation')
631 def insert_content_paths(binary_id, fullpaths, session=None):
633 Make sure given path is associated with given binary id
636 @param binary_id: the id of the binary
637 @type fullpaths: list
638 @param fullpaths: the list of paths of the file being associated with the binary
639 @type session: SQLAlchemy session
640 @param session: Optional SQLAlchemy session. If this is passed, the caller
641 is responsible for ensuring a transaction has begun and committing the
642 results or rolling back based on the result code. If not passed, a commit
643 will be performed at the end of the function, otherwise the caller is
644 responsible for commiting.
646 @return: True upon success
651 session = DBConn().session()
657 for fullpath in fullpaths:
658 if fullpath.startswith( './' ):
659 fullpath = fullpath[2:]
661 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} )
669 traceback.print_exc()
671 # Only rollback if we set up the session ourself
678 __all__.append('insert_content_paths')
680 ################################################################################
682 class DSCFile(object):
683 def __init__(self, *args, **kwargs):
687 return '<DSCFile %s>' % self.dscfile_id
689 __all__.append('DSCFile')
692 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
694 Returns a list of DSCFiles which may be empty
696 @type dscfile_id: int (optional)
697 @param dscfile_id: the dscfile_id of the DSCFiles to find
699 @type source_id: int (optional)
700 @param source_id: the source id related to the DSCFiles to find
702 @type poolfile_id: int (optional)
703 @param poolfile_id: the poolfile id related to the DSCFiles to find
706 @return: Possibly empty list of DSCFiles
709 q = session.query(DSCFile)
711 if dscfile_id is not None:
712 q = q.filter_by(dscfile_id=dscfile_id)
714 if source_id is not None:
715 q = q.filter_by(source_id=source_id)
717 if poolfile_id is not None:
718 q = q.filter_by(poolfile_id=poolfile_id)
722 __all__.append('get_dscfiles')
724 ################################################################################
726 class PoolFile(object):
727 def __init__(self, *args, **kwargs):
731 return '<PoolFile %s>' % self.filename
733 __all__.append('PoolFile')
736 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
739 (ValidFileFound [boolean or None], PoolFile object or None)
741 @type filename: string
742 @param filename: the filename of the file to check against the DB
745 @param filesize: the size of the file to check against the DB
748 @param md5sum: the md5sum of the file to check against the DB
750 @type location_id: int
751 @param location_id: the id of the location to look in
754 @return: Tuple of length 2.
755 If more than one file found with that name:
757 If valid pool file found: (True, PoolFile object)
758 If valid pool file not found:
759 (False, None) if no file found
760 (False, PoolFile object) if file found with size/md5sum mismatch
763 q = session.query(PoolFile).filter_by(filename=filename)
764 q = q.join(Location).filter_by(location_id=location_id)
774 if obj.md5sum != md5sum or obj.filesize != int(filesize):
782 __all__.append('check_poolfile')
785 def get_poolfile_by_id(file_id, session=None):
787 Returns a PoolFile objects or None for the given id
790 @param file_id: the id of the file to look for
792 @rtype: PoolFile or None
793 @return: either the PoolFile object or None
796 q = session.query(PoolFile).filter_by(file_id=file_id)
800 except NoResultFound:
803 __all__.append('get_poolfile_by_id')
807 def get_poolfile_by_name(filename, location_id=None, session=None):
809 Returns an array of PoolFile objects for the given filename and
810 (optionally) location_id
812 @type filename: string
813 @param filename: the filename of the file to check against the DB
815 @type location_id: int
816 @param location_id: the id of the location to look in (optional)
819 @return: array of PoolFile objects
822 q = session.query(PoolFile).filter_by(filename=filename)
824 if location_id is not None:
825 q = q.join(Location).filter_by(location_id=location_id)
829 __all__.append('get_poolfile_by_name')
832 def get_poolfile_like_name(filename, session=None):
834 Returns an array of PoolFile objects which are like the given name
836 @type filename: string
837 @param filename: the filename of the file to check against the DB
840 @return: array of PoolFile objects
843 # TODO: There must be a way of properly using bind parameters with %FOO%
844 q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
848 __all__.append('get_poolfile_like_name')
851 def add_poolfile(filename, datadict, location_id, session=None):
853 Add a new file to the pool
855 @type filename: string
856 @param filename: filename
859 @param datadict: dict with needed data
861 @type location_id: int
862 @param location_id: database id of the location
865 @return: the PoolFile object created
867 poolfile = PoolFile()
868 poolfile.filename = filename
869 poolfile.filesize = datadict["size"]
870 poolfile.md5sum = datadict["md5sum"]
871 poolfile.sha1sum = datadict["sha1sum"]
872 poolfile.sha256sum = datadict["sha256sum"]
873 poolfile.location_id = location_id
875 session.add(poolfile)
876 # Flush to get a file id (NB: This is not a commit)
881 __all__.append('add_poolfile')
883 ################################################################################
885 class Fingerprint(object):
886 def __init__(self, *args, **kwargs):
890 return '<Fingerprint %s>' % self.fingerprint
892 __all__.append('Fingerprint')
895 def get_fingerprint(fpr, session=None):
897 Returns Fingerprint object for given fpr.
900 @param fpr: The fpr to find / add
902 @type session: SQLAlchemy
903 @param session: Optional SQL session object (a temporary one will be
904 generated if not supplied).
907 @return: the Fingerprint object for the given fpr or None
910 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
914 except NoResultFound:
919 __all__.append('get_fingerprint')
922 def get_or_set_fingerprint(fpr, session=None):
924 Returns Fingerprint object for given fpr.
926 If no matching fpr is found, a row is inserted.
929 @param fpr: The fpr to find / add
931 @type session: SQLAlchemy
932 @param session: Optional SQL session object (a temporary one will be
933 generated if not supplied). If not passed, a commit will be performed at
934 the end of the function, otherwise the caller is responsible for commiting.
935 A flush will be performed either way.
938 @return: the Fingerprint object for the given fpr
941 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
945 except NoResultFound:
946 fingerprint = Fingerprint()
947 fingerprint.fingerprint = fpr
948 session.add(fingerprint)
949 session.commit_or_flush()
954 __all__.append('get_or_set_fingerprint')
956 ################################################################################
958 # Helper routine for Keyring class
959 def get_ldap_name(entry):
961 for k in ["cn", "mn", "sn"]:
963 if ret and ret[0] != "" and ret[0] != "-":
965 return " ".join(name)
967 ################################################################################
969 class Keyring(object):
970 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
971 " --with-colons --fingerprint --fingerprint"
976 def __init__(self, *args, **kwargs):
980 return '<Keyring %s>' % self.keyring_name
982 def de_escape_gpg_str(self, txt):
983 esclist = re.split(r'(\\x..)', txt)
984 for x in range(1,len(esclist),2):
985 esclist[x] = "%c" % (int(esclist[x][2:],16))
986 return "".join(esclist)
988 def load_keys(self, keyring):
991 if not self.keyring_id:
992 raise Exception('Must be initialized with database information')
994 k = os.popen(self.gpg_invocation % keyring, "r")
998 for line in k.xreadlines():
999 field = line.split(":")
1000 if field[0] == "pub":
1002 (name, addr) = email.Utils.parseaddr(field[9])
1003 name = re.sub(r"\s*[(].*[)]", "", name)
1004 if name == "" or addr == "" or "@" not in addr:
1006 addr = "invalid-uid"
1007 name = self.de_escape_gpg_str(name)
1008 self.keys[key] = {"email": addr}
1010 self.keys[key]["name"] = name
1011 self.keys[key]["aliases"] = [name]
1012 self.keys[key]["fingerprints"] = []
1014 elif key and field[0] == "sub" and len(field) >= 12:
1015 signingkey = ("s" in field[11])
1016 elif key and field[0] == "uid":
1017 (name, addr) = email.Utils.parseaddr(field[9])
1018 if name and name not in self.keys[key]["aliases"]:
1019 self.keys[key]["aliases"].append(name)
1020 elif signingkey and field[0] == "fpr":
1021 self.keys[key]["fingerprints"].append(field[9])
1022 self.fpr_lookup[field[9]] = key
1024 def import_users_from_ldap(self, session):
1028 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1029 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1031 l = ldap.open(LDAPServer)
1032 l.simple_bind_s("","")
1033 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1034 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1035 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1037 ldap_fin_uid_id = {}
1044 uid = entry["uid"][0]
1045 name = get_ldap_name(entry)
1046 fingerprints = entry["keyFingerPrint"]
1048 for f in fingerprints:
1049 key = self.fpr_lookup.get(f, None)
1050 if key not in self.keys:
1052 self.keys[key]["uid"] = uid
1056 keyid = get_or_set_uid(uid, session).uid_id
1057 byuid[keyid] = (uid, name)
1058 byname[uid] = (keyid, name)
1060 return (byname, byuid)
1062 def generate_users_from_keyring(self, format, session):
1066 for x in self.keys.keys():
1067 if self.keys[x]["email"] == "invalid-uid":
1069 self.keys[x]["uid"] = format % "invalid-uid"
1071 uid = format % self.keys[x]["email"]
1072 keyid = get_or_set_uid(uid, session).uid_id
1073 byuid[keyid] = (uid, self.keys[x]["name"])
1074 byname[uid] = (keyid, self.keys[x]["name"])
1075 self.keys[x]["uid"] = uid
1078 uid = format % "invalid-uid"
1079 keyid = get_or_set_uid(uid, session).uid_id
1080 byuid[keyid] = (uid, "ungeneratable user id")
1081 byname[uid] = (keyid, "ungeneratable user id")
1083 return (byname, byuid)
1085 __all__.append('Keyring')
1088 def get_keyring(keyring, session=None):
1090 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1091 If C{keyring} already has an entry, simply return the existing Keyring
1093 @type keyring: string
1094 @param keyring: the keyring name
1097 @return: the Keyring object for this keyring
1100 q = session.query(Keyring).filter_by(keyring_name=keyring)
1104 except NoResultFound:
1107 __all__.append('get_keyring')
1109 ################################################################################
1111 class KeyringACLMap(object):
1112 def __init__(self, *args, **kwargs):
1116 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1118 __all__.append('KeyringACLMap')
1120 ################################################################################
1122 class KnownChange(object):
1123 def __init__(self, *args, **kwargs):
1127 return '<KnownChange %s>' % self.changesname
1129 __all__.append('KnownChange')
1132 def get_knownchange(filename, session=None):
1134 returns knownchange object for given C{filename}.
1136 @type archive: string
1137 @param archive: the name of the arhive
1139 @type session: Session
1140 @param session: Optional SQLA session object (a temporary one will be
1141 generated if not supplied)
1144 @return: Archive object for the given name (None if not present)
1147 q = session.query(KnownChange).filter_by(changesname=filename)
1151 except NoResultFound:
1154 __all__.append('get_knownchange')
1156 ################################################################################
1157 class Location(object):
1158 def __init__(self, *args, **kwargs):
1162 return '<Location %s (%s)>' % (self.path, self.location_id)
1164 __all__.append('Location')
1167 def get_location(location, component=None, archive=None, session=None):
1169 Returns Location object for the given combination of location, component
1172 @type location: string
1173 @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
1175 @type component: string
1176 @param component: the component name (if None, no restriction applied)
1178 @type archive: string
1179 @param archive_id: the archive name (if None, no restriction applied)
1181 @rtype: Location / None
1182 @return: Either a Location object or None if one can't be found
1185 q = session.query(Location).filter_by(path=location)
1187 if archive is not None:
1188 q = q.join(Archive).filter_by(archive_name=archive)
1190 if component is not None:
1191 q = q.join(Component).filter_by(component_name=component)
1195 except NoResultFound:
1198 __all__.append('get_location')
1200 ################################################################################
1202 class Maintainer(object):
1203 def __init__(self, *args, **kwargs):
1207 return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
1209 def get_split_maintainer(self):
1210 if not hasattr(self, 'name') or self.name is None:
1211 return ('', '', '', '')
1213 return fix_maintainer(self.name.strip())
1215 __all__.append('Maintainer')
1218 def get_or_set_maintainer(name, session=None):
1220 Returns Maintainer object for given maintainer name.
1222 If no matching maintainer name is found, a row is inserted.
1225 @param name: The maintainer name to add
1227 @type session: SQLAlchemy
1228 @param session: Optional SQL session object (a temporary one will be
1229 generated if not supplied). If not passed, a commit will be performed at
1230 the end of the function, otherwise the caller is responsible for commiting.
1231 A flush will be performed either way.
1234 @return: the Maintainer object for the given maintainer
1237 q = session.query(Maintainer).filter_by(name=name)
1240 except NoResultFound:
1241 maintainer = Maintainer()
1242 maintainer.name = name
1243 session.add(maintainer)
1244 session.commit_or_flush()
1249 __all__.append('get_or_set_maintainer')
1252 def get_maintainer(maintainer_id, session=None):
1254 Return the name of the maintainer behind C{maintainer_id} or None if that
1255 maintainer_id is invalid.
1257 @type maintainer_id: int
1258 @param maintainer_id: the id of the maintainer
1261 @return: the Maintainer with this C{maintainer_id}
1264 return session.query(Maintainer).get(maintainer_id)
1266 __all__.append('get_maintainer')
1268 ################################################################################
1270 class NewComment(object):
1271 def __init__(self, *args, **kwargs):
1275 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1277 __all__.append('NewComment')
1280 def has_new_comment(package, version, session=None):
1282 Returns true if the given combination of C{package}, C{version} has a comment.
1284 @type package: string
1285 @param package: name of the package
1287 @type version: string
1288 @param version: package version
1290 @type session: Session
1291 @param session: Optional SQLA session object (a temporary one will be
1292 generated if not supplied)
1298 q = session.query(NewComment)
1299 q = q.filter_by(package=package)
1300 q = q.filter_by(version=version)
1302 return bool(q.count() > 0)
1304 __all__.append('has_new_comment')
1307 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1309 Returns (possibly empty) list of NewComment objects for the given
1312 @type package: string (optional)
1313 @param package: name of the package
1315 @type version: string (optional)
1316 @param version: package version
1318 @type comment_id: int (optional)
1319 @param comment_id: An id of a comment
1321 @type session: Session
1322 @param session: Optional SQLA session object (a temporary one will be
1323 generated if not supplied)
1326 @return: A (possibly empty) list of NewComment objects will be returned
1329 q = session.query(NewComment)
1330 if package is not None: q = q.filter_by(package=package)
1331 if version is not None: q = q.filter_by(version=version)
1332 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1336 __all__.append('get_new_comments')
1338 ################################################################################
1340 class Override(object):
1341 def __init__(self, *args, **kwargs):
1345 return '<Override %s (%s)>' % (self.package, self.suite_id)
1347 __all__.append('Override')
1350 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1352 Returns Override object for the given parameters
1354 @type package: string
1355 @param package: The name of the package
1357 @type suite: string, list or None
1358 @param suite: The name of the suite (or suites if a list) to limit to. If
1359 None, don't limit. Defaults to None.
1361 @type component: string, list or None
1362 @param component: The name of the component (or components if a list) to
1363 limit to. If None, don't limit. Defaults to None.
1365 @type overridetype: string, list or None
1366 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1367 limit to. If None, don't limit. Defaults to None.
1369 @type session: Session
1370 @param session: Optional SQLA session object (a temporary one will be
1371 generated if not supplied)
1374 @return: A (possibly empty) list of Override objects will be returned
1377 q = session.query(Override)
1378 q = q.filter_by(package=package)
1380 if suite is not None:
1381 if not isinstance(suite, list): suite = [suite]
1382 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1384 if component is not None:
1385 if not isinstance(component, list): component = [component]
1386 q = q.join(Component).filter(Component.component_name.in_(component))
1388 if overridetype is not None:
1389 if not isinstance(overridetype, list): overridetype = [overridetype]
1390 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1394 __all__.append('get_override')
1397 ################################################################################
1399 class OverrideType(object):
1400 def __init__(self, *args, **kwargs):
1404 return '<OverrideType %s>' % self.overridetype
1406 __all__.append('OverrideType')
1409 def get_override_type(override_type, session=None):
1411 Returns OverrideType object for given C{override type}.
1413 @type override_type: string
1414 @param override_type: The name of the override type
1416 @type session: Session
1417 @param session: Optional SQLA session object (a temporary one will be
1418 generated if not supplied)
1421 @return: the database id for the given override type
1424 q = session.query(OverrideType).filter_by(overridetype=override_type)
1428 except NoResultFound:
1431 __all__.append('get_override_type')
1433 ################################################################################
1435 class PendingContentAssociation(object):
1436 def __init__(self, *args, **kwargs):
1440 return '<PendingContentAssociation %s>' % self.pca_id
1442 __all__.append('PendingContentAssociation')
1444 def insert_pending_content_paths(package, fullpaths, session=None):
1446 Make sure given paths are temporarily associated with given
1450 @param package: the package to associate with should have been read in from the binary control file
1451 @type fullpaths: list
1452 @param fullpaths: the list of paths of the file being associated with the binary
1453 @type session: SQLAlchemy session
1454 @param session: Optional SQLAlchemy session. If this is passed, the caller
1455 is responsible for ensuring a transaction has begun and committing the
1456 results or rolling back based on the result code. If not passed, a commit
1457 will be performed at the end of the function
1459 @return: True upon success, False if there is a problem
1462 privatetrans = False
1465 session = DBConn().session()
1469 arch = get_architecture(package['Architecture'], session)
1470 arch_id = arch.arch_id
1472 # Remove any already existing recorded files for this package
1473 q = session.query(PendingContentAssociation)
1474 q = q.filter_by(package=package['Package'])
1475 q = q.filter_by(version=package['Version'])
1476 q = q.filter_by(architecture=arch_id)
1481 for fullpath in fullpaths:
1482 (path, filename) = os.path.split(fullpath)
1484 if path.startswith( "./" ):
1487 filepath_id = get_or_set_contents_path_id(path, session)
1488 filename_id = get_or_set_contents_file_id(filename, session)
1490 pathcache[fullpath] = (filepath_id, filename_id)
1492 for fullpath, dat in pathcache.items():
1493 pca = PendingContentAssociation()
1494 pca.package = package['Package']
1495 pca.version = package['Version']
1496 pca.filepath_id = dat[0]
1497 pca.filename_id = dat[1]
1498 pca.architecture = arch_id
1501 # Only commit if we set up the session ourself
1509 except Exception, e:
1510 traceback.print_exc()
1512 # Only rollback if we set up the session ourself
1519 __all__.append('insert_pending_content_paths')
1521 ################################################################################
1523 class Priority(object):
1524 def __init__(self, *args, **kwargs):
1527 def __eq__(self, val):
1528 if isinstance(val, str):
1529 return (self.priority == val)
1530 # This signals to use the normal comparison operator
1531 return NotImplemented
1533 def __ne__(self, val):
1534 if isinstance(val, str):
1535 return (self.priority != val)
1536 # This signals to use the normal comparison operator
1537 return NotImplemented
1540 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
1542 __all__.append('Priority')
1545 def get_priority(priority, session=None):
1547 Returns Priority object for given C{priority name}.
1549 @type priority: string
1550 @param priority: The name of the priority
1552 @type session: Session
1553 @param session: Optional SQLA session object (a temporary one will be
1554 generated if not supplied)
1557 @return: Priority object for the given priority
1560 q = session.query(Priority).filter_by(priority=priority)
1564 except NoResultFound:
1567 __all__.append('get_priority')
1570 def get_priorities(session=None):
1572 Returns dictionary of priority names -> id mappings
1574 @type session: Session
1575 @param session: Optional SQL session object (a temporary one will be
1576 generated if not supplied)
1579 @return: dictionary of priority names -> id mappings
1583 q = session.query(Priority)
1585 ret[x.priority] = x.priority_id
1589 __all__.append('get_priorities')
1591 ################################################################################
1593 class Queue(object):
1594 def __init__(self, *args, **kwargs):
1598 return '<Queue %s>' % self.queue_name
1600 def autobuild_upload(self, changes, srcpath, session=None):
1602 Update queue_build database table used for incoming autobuild support.
1604 @type changes: Changes
1605 @param changes: changes object for the upload to process
1607 @type srcpath: string
1608 @param srcpath: path for the queue file entries/link destinations
1610 @type session: SQLAlchemy session
1611 @param session: Optional SQLAlchemy session. If this is passed, the
1612 caller is responsible for ensuring a transaction has begun and
1613 committing the results or rolling back based on the result code. If
1614 not passed, a commit will be performed at the end of the function,
1615 otherwise the caller is responsible for commiting.
1617 @rtype: NoneType or string
1618 @return: None if the operation failed, a string describing the error if not
1621 privatetrans = False
1623 session = DBConn().session()
1626 # TODO: Remove by moving queue config into the database
1629 for suitename in changes.changes["distribution"].keys():
1630 # TODO: Move into database as:
1631 # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build)
1632 # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink)
1633 # This also gets rid of the SecurityQueueBuild hack below
1634 if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"):
1638 s = get_suite(suitename, session)
1640 return "INTERNAL ERROR: Could not find suite %s" % suitename
1642 # TODO: Get from database as above
1643 dest_dir = conf["Dir::QueueBuild"]
1645 # TODO: Move into database as above
1646 if conf.FindB("Dinstall::SecurityQueueBuild"):
1647 dest_dir = os.path.join(dest_dir, suitename)
1649 for file_entry in changes.files.keys():
1650 src = os.path.join(srcpath, file_entry)
1651 dest = os.path.join(dest_dir, file_entry)
1653 # TODO: Move into database as above
1654 if conf.FindB("Dinstall::SecurityQueueBuild"):
1655 # Copy it since the original won't be readable by www-data
1657 utils.copy(src, dest)
1659 # Create a symlink to it
1660 os.symlink(src, dest)
1663 qb.suite_id = s.suite_id
1664 qb.queue_id = self.queue_id
1670 # If the .orig tarballs are in the pool, create a symlink to
1671 # them (if one doesn't already exist)
1672 for dsc_file in changes.dsc_files.keys():
1673 # Skip all files except orig tarballs
1674 from daklib.regexes import re_is_orig_source
1675 if not re_is_orig_source.match(dsc_file):
1677 # Skip orig files not identified in the pool
1678 if not (changes.orig_files.has_key(dsc_file) and
1679 changes.orig_files[dsc_file].has_key("id")):
1681 orig_file_id = changes.orig_files[dsc_file]["id"]
1682 dest = os.path.join(dest_dir, dsc_file)
1684 # If it doesn't exist, create a symlink
1685 if not os.path.exists(dest):
1686 q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id",
1687 {'id': orig_file_id})
1690 return "[INTERNAL ERROR] Couldn't find id %s in files table." % (orig_file_id)
1692 src = os.path.join(res[0], res[1])
1693 os.symlink(src, dest)
1695 # Add it to the list of packages for later processing by apt-ftparchive
1697 qb.suite_id = s.suite_id
1698 qb.queue_id = self.queue_id
1703 # If it does, update things to ensure it's not removed prematurely
1705 qb = get_queue_build(dest, s.suite_id, session)
1717 __all__.append('Queue')
1720 def get_or_set_queue(queuename, session=None):
1722 Returns Queue object for given C{queue name}, creating it if it does not
1725 @type queuename: string
1726 @param queuename: The name of the queue
1728 @type session: Session
1729 @param session: Optional SQLA session object (a temporary one will be
1730 generated if not supplied)
1733 @return: Queue object for the given queue
1736 q = session.query(Queue).filter_by(queue_name=queuename)
1740 except NoResultFound:
1742 queue.queue_name = queuename
1744 session.commit_or_flush()
1749 __all__.append('get_or_set_queue')
1751 ################################################################################
1753 class QueueBuild(object):
1754 def __init__(self, *args, **kwargs):
1758 return '<QueueBuild %s (%s)>' % (self.filename, self.queue_id)
1760 __all__.append('QueueBuild')
1763 def get_queue_build(filename, suite, session=None):
1765 Returns QueueBuild object for given C{filename} and C{suite}.
1767 @type filename: string
1768 @param filename: The name of the file
1770 @type suiteid: int or str
1771 @param suiteid: Suite name or ID
1773 @type session: Session
1774 @param session: Optional SQLA session object (a temporary one will be
1775 generated if not supplied)
1778 @return: Queue object for the given queue
1781 if isinstance(suite, int):
1782 q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite)
1784 q = session.query(QueueBuild).filter_by(filename=filename)
1785 q = q.join(Suite).filter_by(suite_name=suite)
1789 except NoResultFound:
1792 __all__.append('get_queue_build')
1794 ################################################################################
1796 class Section(object):
1797 def __init__(self, *args, **kwargs):
1800 def __eq__(self, val):
1801 if isinstance(val, str):
1802 return (self.section == val)
1803 # This signals to use the normal comparison operator
1804 return NotImplemented
1806 def __ne__(self, val):
1807 if isinstance(val, str):
1808 return (self.section != val)
1809 # This signals to use the normal comparison operator
1810 return NotImplemented
1813 return '<Section %s>' % self.section
1815 __all__.append('Section')
1818 def get_section(section, session=None):
1820 Returns Section object for given C{section name}.
1822 @type section: string
1823 @param section: The name of the section
1825 @type session: Session
1826 @param session: Optional SQLA session object (a temporary one will be
1827 generated if not supplied)
1830 @return: Section object for the given section name
1833 q = session.query(Section).filter_by(section=section)
1837 except NoResultFound:
1840 __all__.append('get_section')
1843 def get_sections(session=None):
1845 Returns dictionary of section names -> id mappings
1847 @type session: Session
1848 @param session: Optional SQL session object (a temporary one will be
1849 generated if not supplied)
1852 @return: dictionary of section names -> id mappings
1856 q = session.query(Section)
1858 ret[x.section] = x.section_id
1862 __all__.append('get_sections')
1864 ################################################################################
1866 class DBSource(object):
1867 def __init__(self, *args, **kwargs):
1871 return '<DBSource %s (%s)>' % (self.source, self.version)
1873 __all__.append('DBSource')
1876 def source_exists(source, source_version, suites = ["any"], session=None):
1878 Ensure that source exists somewhere in the archive for the binary
1879 upload being processed.
1880 1. exact match => 1.0-3
1881 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1883 @type package: string
1884 @param package: package source name
1886 @type source_version: string
1887 @param source_version: expected source version
1890 @param suites: list of suites to check in, default I{any}
1892 @type session: Session
1893 @param session: Optional SQLA session object (a temporary one will be
1894 generated if not supplied)
1897 @return: returns 1 if a source with expected version is found, otherwise 0
1904 for suite in suites:
1905 q = session.query(DBSource).filter_by(source=source)
1907 # source must exist in suite X, or in some other suite that's
1908 # mapped to X, recursively... silent-maps are counted too,
1909 # unreleased-maps aren't.
1910 maps = cnf.ValueList("SuiteMappings")[:]
1912 maps = [ m.split() for m in maps ]
1913 maps = [ (x[1], x[2]) for x in maps
1914 if x[0] == "map" or x[0] == "silent-map" ]
1917 if x[1] in s and x[0] not in s:
1920 q = q.join(SrcAssociation).join(Suite)
1921 q = q.filter(Suite.suite_name.in_(s))
1923 # Reduce the query results to a list of version numbers
1924 ql = [ j.version for j in q.all() ]
1927 if source_version in ql:
1931 from daklib.regexes import re_bin_only_nmu
1932 orig_source_version = re_bin_only_nmu.sub('', source_version)
1933 if orig_source_version in ql:
1936 # No source found so return not ok
1941 __all__.append('source_exists')
1944 def get_suites_source_in(source, session=None):
1946 Returns list of Suite objects which given C{source} name is in
1949 @param source: DBSource package name to search for
1952 @return: list of Suite objects for the given source
1955 return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
1957 __all__.append('get_suites_source_in')
1960 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
1962 Returns list of DBSource objects for given C{source} name and other parameters
1965 @param source: DBSource package name to search for
1967 @type source: str or None
1968 @param source: DBSource version name to search for or None if not applicable
1970 @type dm_upload_allowed: bool
1971 @param dm_upload_allowed: If None, no effect. If True or False, only
1972 return packages with that dm_upload_allowed setting
1974 @type session: Session
1975 @param session: Optional SQL session object (a temporary one will be
1976 generated if not supplied)
1979 @return: list of DBSource objects for the given name (may be empty)
1982 q = session.query(DBSource).filter_by(source=source)
1984 if version is not None:
1985 q = q.filter_by(version=version)
1987 if dm_upload_allowed is not None:
1988 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
1992 __all__.append('get_sources_from_name')
1995 def get_source_in_suite(source, suite, session=None):
1997 Returns list of DBSource objects for a combination of C{source} and C{suite}.
1999 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2000 - B{suite} - a suite name, eg. I{unstable}
2002 @type source: string
2003 @param source: source package name
2006 @param suite: the suite name
2009 @return: the version for I{source} in I{suite}
2013 q = session.query(SrcAssociation)
2014 q = q.join('source').filter_by(source=source)
2015 q = q.join('suite').filter_by(suite_name=suite)
2018 return q.one().source
2019 except NoResultFound:
2022 __all__.append('get_source_in_suite')
2024 ################################################################################
2027 def add_dsc_to_db(u, filename, session=None):
2028 entry = u.pkg.files[filename]
2031 source.source = u.pkg.dsc["source"]
2032 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2033 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2034 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2035 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2036 source.install_date = datetime.now().date()
2038 dsc_component = entry["component"]
2039 dsc_location_id = entry["location id"]
2041 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2043 # Set up a new poolfile if necessary
2044 if not entry.has_key("files id") or not entry["files id"]:
2045 filename = entry["pool name"] + filename
2046 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2047 entry["files id"] = poolfile.file_id
2049 source.poolfile_id = entry["files id"]
2053 for suite_name in u.pkg.changes["distribution"].keys():
2054 sa = SrcAssociation()
2055 sa.source_id = source.source_id
2056 sa.suite_id = get_suite(suite_name).suite_id
2061 # Add the source files to the DB (files and dsc_files)
2063 dscfile.source_id = source.source_id
2064 dscfile.poolfile_id = entry["files id"]
2065 session.add(dscfile)
2067 for dsc_file, dentry in u.pkg.dsc_files.items():
2069 df.source_id = source.source_id
2071 # If the .orig tarball is already in the pool, it's
2072 # files id is stored in dsc_files by check_dsc().
2073 files_id = dentry.get("files id", None)
2075 # Find the entry in the files hash
2076 # TODO: Bail out here properly
2078 for f, e in u.pkg.files.items():
2083 if files_id is None:
2084 filename = dfentry["pool name"] + dsc_file
2086 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2087 # FIXME: needs to check for -1/-2 and or handle exception
2088 if found and obj is not None:
2089 files_id = obj.file_id
2091 # If still not found, add it
2092 if files_id is None:
2093 # HACK: Force sha1sum etc into dentry
2094 dentry["sha1sum"] = dfentry["sha1sum"]
2095 dentry["sha256sum"] = dfentry["sha256sum"]
2096 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2097 files_id = poolfile.file_id
2099 df.poolfile_id = files_id
2104 # Add the src_uploaders to the DB
2105 uploader_ids = [source.maintainer_id]
2106 if u.pkg.dsc.has_key("uploaders"):
2107 for up in u.pkg.dsc["uploaders"].split(","):
2109 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2112 for up in uploader_ids:
2113 if added_ids.has_key(up):
2114 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
2120 su.maintainer_id = up
2121 su.source_id = source.source_id
2126 return dsc_component, dsc_location_id
2128 __all__.append('add_dsc_to_db')
2131 def add_deb_to_db(u, filename, session=None):
2133 Contrary to what you might expect, this routine deals with both
2134 debs and udebs. That info is in 'dbtype', whilst 'type' is
2135 'deb' for both of them
2138 entry = u.pkg.files[filename]
2141 bin.package = entry["package"]
2142 bin.version = entry["version"]
2143 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2144 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2145 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2146 bin.binarytype = entry["dbtype"]
2149 filename = entry["pool name"] + filename
2150 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2151 if not entry.get("location id", None):
2152 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], utils.where_am_i(), session).location_id
2154 if not entry.get("files id", None):
2155 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2156 entry["files id"] = poolfile.file_id
2158 bin.poolfile_id = entry["files id"]
2161 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2162 if len(bin_sources) != 1:
2163 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2164 (bin.package, bin.version, bin.architecture.arch_string,
2165 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2167 bin.source_id = bin_sources[0].source_id
2169 # Add and flush object so it has an ID
2173 # Add BinAssociations
2174 for suite_name in u.pkg.changes["distribution"].keys():
2175 ba = BinAssociation()
2176 ba.binary_id = bin.binary_id
2177 ba.suite_id = get_suite(suite_name).suite_id
2182 # Deal with contents - disabled for now
2183 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2185 # print "REJECT\nCould not determine contents of package %s" % bin.package
2186 # session.rollback()
2187 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2189 __all__.append('add_deb_to_db')
2191 ################################################################################
2193 class SourceACL(object):
2194 def __init__(self, *args, **kwargs):
2198 return '<SourceACL %s>' % self.source_acl_id
2200 __all__.append('SourceACL')
2202 ################################################################################
2204 class SrcAssociation(object):
2205 def __init__(self, *args, **kwargs):
2209 return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
2211 __all__.append('SrcAssociation')
2213 ################################################################################
2215 class SrcFormat(object):
2216 def __init__(self, *args, **kwargs):
2220 return '<SrcFormat %s>' % (self.format_name)
2222 __all__.append('SrcFormat')
2224 ################################################################################
2226 class SrcUploader(object):
2227 def __init__(self, *args, **kwargs):
2231 return '<SrcUploader %s>' % self.uploader_id
2233 __all__.append('SrcUploader')
2235 ################################################################################
2237 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2238 ('SuiteID', 'suite_id'),
2239 ('Version', 'version'),
2240 ('Origin', 'origin'),
2242 ('Description', 'description'),
2243 ('Untouchable', 'untouchable'),
2244 ('Announce', 'announce'),
2245 ('Codename', 'codename'),
2246 ('OverrideCodename', 'overridecodename'),
2247 ('ValidTime', 'validtime'),
2248 ('Priority', 'priority'),
2249 ('NotAutomatic', 'notautomatic'),
2250 ('CopyChanges', 'copychanges'),
2251 ('CopyDotDak', 'copydotdak'),
2252 ('CommentsDir', 'commentsdir'),
2253 ('OverrideSuite', 'overridesuite'),
2254 ('ChangelogBase', 'changelogbase')]
2257 class Suite(object):
2258 def __init__(self, *args, **kwargs):
2262 return '<Suite %s>' % self.suite_name
2264 def __eq__(self, val):
2265 if isinstance(val, str):
2266 return (self.suite_name == val)
2267 # This signals to use the normal comparison operator
2268 return NotImplemented
2270 def __ne__(self, val):
2271 if isinstance(val, str):
2272 return (self.suite_name != val)
2273 # This signals to use the normal comparison operator
2274 return NotImplemented
2278 for disp, field in SUITE_FIELDS:
2279 val = getattr(self, field, None)
2281 ret.append("%s: %s" % (disp, val))
2283 return "\n".join(ret)
2285 __all__.append('Suite')
2288 def get_suite_architecture(suite, architecture, session=None):
2290 Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
2294 @param suite: Suite name to search for
2296 @type architecture: str
2297 @param architecture: Architecture name to search for
2299 @type session: Session
2300 @param session: Optional SQL session object (a temporary one will be
2301 generated if not supplied)
2303 @rtype: SuiteArchitecture
2304 @return: the SuiteArchitecture object or None
2307 q = session.query(SuiteArchitecture)
2308 q = q.join(Architecture).filter_by(arch_string=architecture)
2309 q = q.join(Suite).filter_by(suite_name=suite)
2313 except NoResultFound:
2316 __all__.append('get_suite_architecture')
2319 def get_suite(suite, session=None):
2321 Returns Suite object for given C{suite name}.
2324 @param suite: The name of the suite
2326 @type session: Session
2327 @param session: Optional SQLA session object (a temporary one will be
2328 generated if not supplied)
2331 @return: Suite object for the requested suite name (None if not present)
2334 q = session.query(Suite).filter_by(suite_name=suite)
2338 except NoResultFound:
2341 __all__.append('get_suite')
2343 ################################################################################
2345 class SuiteArchitecture(object):
2346 def __init__(self, *args, **kwargs):
2350 return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
2352 __all__.append('SuiteArchitecture')
2355 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2357 Returns list of Architecture objects for given C{suite} name
2360 @param source: Suite name to search for
2362 @type skipsrc: boolean
2363 @param skipsrc: Whether to skip returning the 'source' architecture entry
2366 @type skipall: boolean
2367 @param skipall: Whether to skip returning the 'all' architecture entry
2370 @type session: Session
2371 @param session: Optional SQL session object (a temporary one will be
2372 generated if not supplied)
2375 @return: list of Architecture objects for the given name (may be empty)
2378 q = session.query(Architecture)
2379 q = q.join(SuiteArchitecture)
2380 q = q.join(Suite).filter_by(suite_name=suite)
2383 q = q.filter(Architecture.arch_string != 'source')
2386 q = q.filter(Architecture.arch_string != 'all')
2388 q = q.order_by('arch_string')
2392 __all__.append('get_suite_architectures')
2394 ################################################################################
2396 class SuiteSrcFormat(object):
2397 def __init__(self, *args, **kwargs):
2401 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2403 __all__.append('SuiteSrcFormat')
2406 def get_suite_src_formats(suite, session=None):
2408 Returns list of allowed SrcFormat for C{suite}.
2411 @param suite: Suite name to search for
2413 @type session: Session
2414 @param session: Optional SQL session object (a temporary one will be
2415 generated if not supplied)
2418 @return: the list of allowed source formats for I{suite}
2421 q = session.query(SrcFormat)
2422 q = q.join(SuiteSrcFormat)
2423 q = q.join(Suite).filter_by(suite_name=suite)
2424 q = q.order_by('format_name')
2428 __all__.append('get_suite_src_formats')
2430 ################################################################################
2433 def __init__(self, *args, **kwargs):
2436 def __eq__(self, val):
2437 if isinstance(val, str):
2438 return (self.uid == val)
2439 # This signals to use the normal comparison operator
2440 return NotImplemented
2442 def __ne__(self, val):
2443 if isinstance(val, str):
2444 return (self.uid != val)
2445 # This signals to use the normal comparison operator
2446 return NotImplemented
2449 return '<Uid %s (%s)>' % (self.uid, self.name)
2451 __all__.append('Uid')
2454 def add_database_user(uidname, session=None):
2456 Adds a database user
2458 @type uidname: string
2459 @param uidname: The uid of the user to add
2461 @type session: SQLAlchemy
2462 @param session: Optional SQL session object (a temporary one will be
2463 generated if not supplied). If not passed, a commit will be performed at
2464 the end of the function, otherwise the caller is responsible for commiting.
2467 @return: the uid object for the given uidname
2470 session.execute("CREATE USER :uid", {'uid': uidname})
2471 session.commit_or_flush()
2473 __all__.append('add_database_user')
2476 def get_or_set_uid(uidname, session=None):
2478 Returns uid object for given uidname.
2480 If no matching uidname is found, a row is inserted.
2482 @type uidname: string
2483 @param uidname: The uid to add
2485 @type session: SQLAlchemy
2486 @param session: Optional SQL session object (a temporary one will be
2487 generated if not supplied). If not passed, a commit will be performed at
2488 the end of the function, otherwise the caller is responsible for commiting.
2491 @return: the uid object for the given uidname
2494 q = session.query(Uid).filter_by(uid=uidname)
2498 except NoResultFound:
2502 session.commit_or_flush()
2507 __all__.append('get_or_set_uid')
2510 def get_uid_from_fingerprint(fpr, session=None):
2511 q = session.query(Uid)
2512 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2516 except NoResultFound:
2519 __all__.append('get_uid_from_fingerprint')
2521 ################################################################################
2523 class UploadBlock(object):
2524 def __init__(self, *args, **kwargs):
2528 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2530 __all__.append('UploadBlock')
2532 ################################################################################
2534 class DBConn(Singleton):
2536 database module init.
2538 def __init__(self, *args, **kwargs):
2539 super(DBConn, self).__init__(*args, **kwargs)
2541 def _startup(self, *args, **kwargs):
2543 if kwargs.has_key('debug'):
2547 def __setuptables(self):
2548 self.tbl_architecture = Table('architecture', self.db_meta, autoload=True)
2549 self.tbl_archive = Table('archive', self.db_meta, autoload=True)
2550 self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True)
2551 self.tbl_binaries = Table('binaries', self.db_meta, autoload=True)
2552 self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True)
2553 self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True)
2554 self.tbl_component = Table('component', self.db_meta, autoload=True)
2555 self.tbl_config = Table('config', self.db_meta, autoload=True)
2556 self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True)
2557 self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True)
2558 self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True)
2559 self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True)
2560 self.tbl_files = Table('files', self.db_meta, autoload=True)
2561 self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True)
2562 self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True)
2563 self.tbl_known_changes = Table('known_changes', self.db_meta, autoload=True)
2564 self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True)
2565 self.tbl_location = Table('location', self.db_meta, autoload=True)
2566 self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True)
2567 self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True)
2568 self.tbl_override = Table('override', self.db_meta, autoload=True)
2569 self.tbl_override_type = Table('override_type', self.db_meta, autoload=True)
2570 self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True)
2571 self.tbl_priority = Table('priority', self.db_meta, autoload=True)
2572 self.tbl_queue = Table('queue', self.db_meta, autoload=True)
2573 self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True)
2574 self.tbl_section = Table('section', self.db_meta, autoload=True)
2575 self.tbl_source = Table('source', self.db_meta, autoload=True)
2576 self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True)
2577 self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True)
2578 self.tbl_src_format = Table('src_format', self.db_meta, autoload=True)
2579 self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True)
2580 self.tbl_suite = Table('suite', self.db_meta, autoload=True)
2581 self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True)
2582 self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True)
2583 self.tbl_uid = Table('uid', self.db_meta, autoload=True)
2584 self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True)
2586 def __setupmappers(self):
2587 mapper(Architecture, self.tbl_architecture,
2588 properties = dict(arch_id = self.tbl_architecture.c.id))
2590 mapper(Archive, self.tbl_archive,
2591 properties = dict(archive_id = self.tbl_archive.c.id,
2592 archive_name = self.tbl_archive.c.name))
2594 mapper(BinAssociation, self.tbl_bin_associations,
2595 properties = dict(ba_id = self.tbl_bin_associations.c.id,
2596 suite_id = self.tbl_bin_associations.c.suite,
2597 suite = relation(Suite),
2598 binary_id = self.tbl_bin_associations.c.bin,
2599 binary = relation(DBBinary)))
2602 mapper(DBBinary, self.tbl_binaries,
2603 properties = dict(binary_id = self.tbl_binaries.c.id,
2604 package = self.tbl_binaries.c.package,
2605 version = self.tbl_binaries.c.version,
2606 maintainer_id = self.tbl_binaries.c.maintainer,
2607 maintainer = relation(Maintainer),
2608 source_id = self.tbl_binaries.c.source,
2609 source = relation(DBSource),
2610 arch_id = self.tbl_binaries.c.architecture,
2611 architecture = relation(Architecture),
2612 poolfile_id = self.tbl_binaries.c.file,
2613 poolfile = relation(PoolFile),
2614 binarytype = self.tbl_binaries.c.type,
2615 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2616 fingerprint = relation(Fingerprint),
2617 install_date = self.tbl_binaries.c.install_date,
2618 binassociations = relation(BinAssociation,
2619 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
2621 mapper(BinaryACL, self.tbl_binary_acl,
2622 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2624 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2625 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2626 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2627 architecture = relation(Architecture)))
2629 mapper(Component, self.tbl_component,
2630 properties = dict(component_id = self.tbl_component.c.id,
2631 component_name = self.tbl_component.c.name))
2633 mapper(DBConfig, self.tbl_config,
2634 properties = dict(config_id = self.tbl_config.c.id))
2636 mapper(DSCFile, self.tbl_dsc_files,
2637 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2638 source_id = self.tbl_dsc_files.c.source,
2639 source = relation(DBSource),
2640 poolfile_id = self.tbl_dsc_files.c.file,
2641 poolfile = relation(PoolFile)))
2643 mapper(PoolFile, self.tbl_files,
2644 properties = dict(file_id = self.tbl_files.c.id,
2645 filesize = self.tbl_files.c.size,
2646 location_id = self.tbl_files.c.location,
2647 location = relation(Location)))
2649 mapper(Fingerprint, self.tbl_fingerprint,
2650 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2651 uid_id = self.tbl_fingerprint.c.uid,
2652 uid = relation(Uid),
2653 keyring_id = self.tbl_fingerprint.c.keyring,
2654 keyring = relation(Keyring),
2655 source_acl = relation(SourceACL),
2656 binary_acl = relation(BinaryACL)))
2658 mapper(Keyring, self.tbl_keyrings,
2659 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2660 keyring_id = self.tbl_keyrings.c.id))
2662 mapper(KnownChange, self.tbl_known_changes,
2663 properties = dict(known_change_id = self.tbl_known_changes.c.id))
2665 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
2666 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
2667 keyring = relation(Keyring, backref="keyring_acl_map"),
2668 architecture = relation(Architecture)))
2670 mapper(Location, self.tbl_location,
2671 properties = dict(location_id = self.tbl_location.c.id,
2672 component_id = self.tbl_location.c.component,
2673 component = relation(Component),
2674 archive_id = self.tbl_location.c.archive,
2675 archive = relation(Archive),
2676 archive_type = self.tbl_location.c.type))
2678 mapper(Maintainer, self.tbl_maintainer,
2679 properties = dict(maintainer_id = self.tbl_maintainer.c.id))
2681 mapper(NewComment, self.tbl_new_comments,
2682 properties = dict(comment_id = self.tbl_new_comments.c.id))
2684 mapper(Override, self.tbl_override,
2685 properties = dict(suite_id = self.tbl_override.c.suite,
2686 suite = relation(Suite),
2687 component_id = self.tbl_override.c.component,
2688 component = relation(Component),
2689 priority_id = self.tbl_override.c.priority,
2690 priority = relation(Priority),
2691 section_id = self.tbl_override.c.section,
2692 section = relation(Section),
2693 overridetype_id = self.tbl_override.c.type,
2694 overridetype = relation(OverrideType)))
2696 mapper(OverrideType, self.tbl_override_type,
2697 properties = dict(overridetype = self.tbl_override_type.c.type,
2698 overridetype_id = self.tbl_override_type.c.id))
2700 mapper(Priority, self.tbl_priority,
2701 properties = dict(priority_id = self.tbl_priority.c.id))
2703 mapper(Queue, self.tbl_queue,
2704 properties = dict(queue_id = self.tbl_queue.c.id))
2706 mapper(QueueBuild, self.tbl_queue_build,
2707 properties = dict(suite_id = self.tbl_queue_build.c.suite,
2708 queue_id = self.tbl_queue_build.c.queue,
2709 queue = relation(Queue, backref='queuebuild')))
2711 mapper(Section, self.tbl_section,
2712 properties = dict(section_id = self.tbl_section.c.id))
2714 mapper(DBSource, self.tbl_source,
2715 properties = dict(source_id = self.tbl_source.c.id,
2716 version = self.tbl_source.c.version,
2717 maintainer_id = self.tbl_source.c.maintainer,
2718 maintainer = relation(Maintainer,
2719 primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
2720 poolfile_id = self.tbl_source.c.file,
2721 poolfile = relation(PoolFile),
2722 fingerprint_id = self.tbl_source.c.sig_fpr,
2723 fingerprint = relation(Fingerprint),
2724 changedby_id = self.tbl_source.c.changedby,
2725 changedby = relation(Maintainer,
2726 primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
2727 srcfiles = relation(DSCFile,
2728 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
2729 srcassociations = relation(SrcAssociation,
2730 primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)),
2731 srcuploaders = relation(SrcUploader)))
2733 mapper(SourceACL, self.tbl_source_acl,
2734 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
2736 mapper(SrcAssociation, self.tbl_src_associations,
2737 properties = dict(sa_id = self.tbl_src_associations.c.id,
2738 suite_id = self.tbl_src_associations.c.suite,
2739 suite = relation(Suite),
2740 source_id = self.tbl_src_associations.c.source,
2741 source = relation(DBSource)))
2743 mapper(SrcFormat, self.tbl_src_format,
2744 properties = dict(src_format_id = self.tbl_src_format.c.id,
2745 format_name = self.tbl_src_format.c.format_name))
2747 mapper(SrcUploader, self.tbl_src_uploaders,
2748 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
2749 source_id = self.tbl_src_uploaders.c.source,
2750 source = relation(DBSource,
2751 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
2752 maintainer_id = self.tbl_src_uploaders.c.maintainer,
2753 maintainer = relation(Maintainer,
2754 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
2756 mapper(Suite, self.tbl_suite,
2757 properties = dict(suite_id = self.tbl_suite.c.id,
2758 policy_queue = relation(Queue)))
2760 mapper(SuiteArchitecture, self.tbl_suite_architectures,
2761 properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
2762 suite = relation(Suite, backref='suitearchitectures'),
2763 arch_id = self.tbl_suite_architectures.c.architecture,
2764 architecture = relation(Architecture)))
2766 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
2767 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
2768 suite = relation(Suite, backref='suitesrcformats'),
2769 src_format_id = self.tbl_suite_src_formats.c.src_format,
2770 src_format = relation(SrcFormat)))
2772 mapper(Uid, self.tbl_uid,
2773 properties = dict(uid_id = self.tbl_uid.c.id,
2774 fingerprint = relation(Fingerprint)))
2776 mapper(UploadBlock, self.tbl_upload_blocks,
2777 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
2778 fingerprint = relation(Fingerprint, backref="uploadblocks"),
2779 uid = relation(Uid, backref="uploadblocks")))
2781 ## Connection functions
2782 def __createconn(self):
2783 from config import Config
2787 connstr = "postgres://%s" % cnf["DB::Host"]
2788 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2789 connstr += ":%s" % cnf["DB::Port"]
2790 connstr += "/%s" % cnf["DB::Name"]
2793 connstr = "postgres:///%s" % cnf["DB::Name"]
2794 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2795 connstr += "?port=%s" % cnf["DB::Port"]
2797 self.db_pg = create_engine(connstr, echo=self.debug)
2798 self.db_meta = MetaData()
2799 self.db_meta.bind = self.db_pg
2800 self.db_smaker = sessionmaker(bind=self.db_pg,
2804 self.__setuptables()
2805 self.__setupmappers()
2808 return self.db_smaker()
2810 __all__.append('DBConn')