5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
40 from datetime import datetime
42 from inspect import getargspec
45 from sqlalchemy import create_engine, Table, MetaData
46 from sqlalchemy.orm import sessionmaker, mapper, relation
47 from sqlalchemy import types as sqltypes
49 # Don't remove this, we re-export the exceptions to scripts which import us
50 from sqlalchemy.exc import *
51 from sqlalchemy.orm.exc import NoResultFound
53 from config import Config
54 from singleton import Singleton
55 from textutils import fix_maintainer
57 ################################################################################
59 # Patch in support for the debversion field type so that it works during
62 class DebVersion(sqltypes.Text):
63 def get_col_spec(self):
66 sa_major_version = sqlalchemy.__version__[0:3]
67 if sa_major_version == "0.5":
68 from sqlalchemy.databases import postgres
69 postgres.ischema_names['debversion'] = DebVersion
71 raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
73 ################################################################################
75 __all__ = ['IntegrityError', 'SQLAlchemyError']
77 ################################################################################
79 def session_wrapper(fn):
81 Wrapper around common ".., session=None):" handling. If the wrapped
82 function is called without passing 'session', we create a local one
83 and destroy it when the function ends.
85 Also attaches a commit_or_flush method to the session; if we created a
86 local session, this is a synonym for session.commit(), otherwise it is a
87 synonym for session.flush().
90 def wrapped(*args, **kwargs):
91 private_transaction = False
93 # Find the session object
94 session = kwargs.get('session')
97 if len(args) <= len(getargspec(fn)[0]) - 1:
98 # No session specified as last argument or in kwargs
99 private_transaction = True
100 session = kwargs['session'] = DBConn().session()
102 # Session is last argument in args
106 session = args[-1] = DBConn().session()
107 private_transaction = True
109 if private_transaction:
110 session.commit_or_flush = session.commit
112 session.commit_or_flush = session.flush
115 return fn(*args, **kwargs)
117 if private_transaction:
118 # We created a session; close it.
121 wrapped.__doc__ = fn.__doc__
122 wrapped.func_name = fn.func_name
126 __all__.append('session_wrapper')
128 ################################################################################
130 class Architecture(object):
131 def __init__(self, *args, **kwargs):
134 def __eq__(self, val):
135 if isinstance(val, str):
136 return (self.arch_string== val)
137 # This signals to use the normal comparison operator
138 return NotImplemented
140 def __ne__(self, val):
141 if isinstance(val, str):
142 return (self.arch_string != val)
143 # This signals to use the normal comparison operator
144 return NotImplemented
147 return '<Architecture %s>' % self.arch_string
149 __all__.append('Architecture')
152 def get_architecture(architecture, session=None):
154 Returns database id for given C{architecture}.
156 @type architecture: string
157 @param architecture: The name of the architecture
159 @type session: Session
160 @param session: Optional SQLA session object (a temporary one will be
161 generated if not supplied)
164 @return: Architecture object for the given arch (None if not present)
167 q = session.query(Architecture).filter_by(arch_string=architecture)
171 except NoResultFound:
174 __all__.append('get_architecture')
177 def get_architecture_suites(architecture, session=None):
179 Returns list of Suite objects for given C{architecture} name
182 @param source: Architecture name to search for
184 @type session: Session
185 @param session: Optional SQL session object (a temporary one will be
186 generated if not supplied)
189 @return: list of Suite objects for the given name (may be empty)
192 q = session.query(Suite)
193 q = q.join(SuiteArchitecture)
194 q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
200 __all__.append('get_architecture_suites')
202 ################################################################################
204 class Archive(object):
205 def __init__(self, *args, **kwargs):
209 return '<Archive %s>' % self.archive_name
211 __all__.append('Archive')
214 def get_archive(archive, session=None):
216 returns database id for given C{archive}.
218 @type archive: string
219 @param archive: the name of the arhive
221 @type session: Session
222 @param session: Optional SQLA session object (a temporary one will be
223 generated if not supplied)
226 @return: Archive object for the given name (None if not present)
229 archive = archive.lower()
231 q = session.query(Archive).filter_by(archive_name=archive)
235 except NoResultFound:
238 __all__.append('get_archive')
240 ################################################################################
242 class BinAssociation(object):
243 def __init__(self, *args, **kwargs):
247 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
249 __all__.append('BinAssociation')
251 ################################################################################
253 class BinContents(object):
254 def __init__(self, *args, **kwargs):
258 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
260 __all__.append('BinContents')
262 ################################################################################
264 class DBBinary(object):
265 def __init__(self, *args, **kwargs):
269 return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
271 __all__.append('DBBinary')
274 def get_suites_binary_in(package, session=None):
276 Returns list of Suite objects which given C{package} name is in
279 @param source: DBBinary package name to search for
282 @return: list of Suite objects for the given package
285 return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
287 __all__.append('get_suites_binary_in')
290 def get_binary_from_id(binary_id, session=None):
292 Returns DBBinary object for given C{id}
295 @param binary_id: Id of the required binary
297 @type session: Session
298 @param session: Optional SQLA session object (a temporary one will be
299 generated if not supplied)
302 @return: DBBinary object for the given binary (None if not present)
305 q = session.query(DBBinary).filter_by(binary_id=binary_id)
309 except NoResultFound:
312 __all__.append('get_binary_from_id')
315 def get_binaries_from_name(package, version=None, architecture=None, session=None):
317 Returns list of DBBinary objects for given C{package} name
320 @param package: DBBinary package name to search for
322 @type version: str or None
323 @param version: Version to search for (or None)
325 @type package: str, list or None
326 @param package: Architectures to limit to (or None if no limit)
328 @type session: Session
329 @param session: Optional SQL session object (a temporary one will be
330 generated if not supplied)
333 @return: list of DBBinary objects for the given name (may be empty)
336 q = session.query(DBBinary).filter_by(package=package)
338 if version is not None:
339 q = q.filter_by(version=version)
341 if architecture is not None:
342 if not isinstance(architecture, list):
343 architecture = [architecture]
344 q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
350 __all__.append('get_binaries_from_name')
353 def get_binaries_from_source_id(source_id, session=None):
355 Returns list of DBBinary objects for given C{source_id}
358 @param source_id: source_id to search for
360 @type session: Session
361 @param session: Optional SQL session object (a temporary one will be
362 generated if not supplied)
365 @return: list of DBBinary objects for the given name (may be empty)
368 return session.query(DBBinary).filter_by(source_id=source_id).all()
370 __all__.append('get_binaries_from_source_id')
373 def get_binary_from_name_suite(package, suitename, session=None):
374 ### For dak examine-package
375 ### XXX: Doesn't use object API yet
377 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
378 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
379 WHERE b.package=:package
381 AND fi.location = l.id
382 AND l.component = c.id
385 AND su.suite_name=:suitename
386 ORDER BY b.version DESC"""
388 return session.execute(sql, {'package': package, 'suitename': suitename})
390 __all__.append('get_binary_from_name_suite')
393 def get_binary_components(package, suitename, arch, session=None):
394 # Check for packages that have moved from one component to another
395 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
396 WHERE b.package=:package AND s.suite_name=:suitename
397 AND (a.arch_string = :arch OR a.arch_string = 'all')
398 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
399 AND f.location = l.id
400 AND l.component = c.id
403 vals = {'package': package, 'suitename': suitename, 'arch': arch}
405 return session.execute(query, vals)
407 __all__.append('get_binary_components')
409 ################################################################################
411 class BinaryACL(object):
412 def __init__(self, *args, **kwargs):
416 return '<BinaryACL %s>' % self.binary_acl_id
418 __all__.append('BinaryACL')
420 ################################################################################
422 class BinaryACLMap(object):
423 def __init__(self, *args, **kwargs):
427 return '<BinaryACLMap %s>' % self.binary_acl_map_id
429 __all__.append('BinaryACLMap')
431 ################################################################################
433 class BuildQueue(object):
434 def __init__(self, *args, **kwargs):
438 return '<Queue %s>' % self.queue_name
440 def add_file_from_pool(self, poolfile):
441 """Copies a file into the pool. Assumes that the PoolFile object is
442 attached to the same SQLAlchemy session as the Queue object is.
444 The caller is responsible for committing after calling this function."""
445 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
447 # Check if we have a file of this name or this ID already
448 for f in self.queuefiles:
449 if f.fileid is not None and f.fileid == poolfile.file_id or \
450 f.poolfile.filename == poolfile_basename:
451 # In this case, update the BuildQueueFile entry so we
452 # don't remove it too early
453 f.lastused = datetime.now()
454 DBConn().session().object_session(pf).add(f)
457 # Prepare BuildQueueFile object
458 qf = BuildQueueFile()
459 qf.queue_id = self.queue_id
460 qf.lastused = datetime.now()
461 qf.filename = poolfile_basename
463 targetpath = qf.fullpath
464 queuepath = os.path.join(self.path, poolfile_basename)
467 if self.copy_pool_files:
468 # We need to copy instead of symlink
470 utils.copy(targetfile, queuepath)
471 # NULL in the fileid field implies a copy
474 os.symlink(targetfile, queuepath)
475 qf.fileid = poolfile.file_id
479 # Get the same session as the PoolFile is using and add the qf to it
480 DBConn().session().object_session(poolfile).add(qf)
485 __all__.append('BuildQueue')
488 def get_queue(queuename, session=None):
490 Returns Queue object for given C{queue name}, creating it if it does not
493 @type queuename: string
494 @param queuename: The name of the queue
496 @type session: Session
497 @param session: Optional SQLA session object (a temporary one will be
498 generated if not supplied)
501 @return: Queue object for the given queue
504 q = session.query(Queue).filter_by(queue_name=queuename)
508 except NoResultFound:
511 __all__.append('get_queue')
513 ################################################################################
515 class BuildQueueFile(object):
516 def __init__(self, *args, **kwargs):
520 return '<BuildQueueFile %s (%s)>' % (self.filename, self.queue_id)
522 __all__.append('BuildQueueFile')
524 ################################################################################
526 class ChangePendingBinary(object):
527 def __init__(self, *args, **kwargs):
531 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
533 __all__.append('ChangePendingBinary')
535 ################################################################################
537 class ChangePendingFile(object):
538 def __init__(self, *args, **kwargs):
542 return '<ChangePendingFile %s>' % self.change_pending_file_id
544 __all__.append('ChangePendingFile')
546 ################################################################################
548 class ChangePendingSource(object):
549 def __init__(self, *args, **kwargs):
553 return '<ChangePendingSource %s>' % self.change_pending_source_id
555 __all__.append('ChangePendingSource')
557 ################################################################################
559 class Component(object):
560 def __init__(self, *args, **kwargs):
563 def __eq__(self, val):
564 if isinstance(val, str):
565 return (self.component_name == val)
566 # This signals to use the normal comparison operator
567 return NotImplemented
569 def __ne__(self, val):
570 if isinstance(val, str):
571 return (self.component_name != val)
572 # This signals to use the normal comparison operator
573 return NotImplemented
576 return '<Component %s>' % self.component_name
579 __all__.append('Component')
582 def get_component(component, session=None):
584 Returns database id for given C{component}.
586 @type component: string
587 @param component: The name of the override type
590 @return: the database id for the given component
593 component = component.lower()
595 q = session.query(Component).filter_by(component_name=component)
599 except NoResultFound:
602 __all__.append('get_component')
604 ################################################################################
606 class DBConfig(object):
607 def __init__(self, *args, **kwargs):
611 return '<DBConfig %s>' % self.name
613 __all__.append('DBConfig')
615 ################################################################################
618 def get_or_set_contents_file_id(filename, session=None):
620 Returns database id for given filename.
622 If no matching file is found, a row is inserted.
624 @type filename: string
625 @param filename: The filename
626 @type session: SQLAlchemy
627 @param session: Optional SQL session object (a temporary one will be
628 generated if not supplied). If not passed, a commit will be performed at
629 the end of the function, otherwise the caller is responsible for commiting.
632 @return: the database id for the given component
635 q = session.query(ContentFilename).filter_by(filename=filename)
638 ret = q.one().cafilename_id
639 except NoResultFound:
640 cf = ContentFilename()
641 cf.filename = filename
643 session.commit_or_flush()
644 ret = cf.cafilename_id
648 __all__.append('get_or_set_contents_file_id')
651 def get_contents(suite, overridetype, section=None, session=None):
653 Returns contents for a suite / overridetype combination, limiting
654 to a section if not None.
657 @param suite: Suite object
659 @type overridetype: OverrideType
660 @param overridetype: OverrideType object
662 @type section: Section
663 @param section: Optional section object to limit results to
665 @type session: SQLAlchemy
666 @param session: Optional SQL session object (a temporary one will be
667 generated if not supplied)
670 @return: ResultsProxy object set up to return tuples of (filename, section,
674 # find me all of the contents for a given suite
675 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
679 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
680 JOIN content_file_names n ON (c.filename=n.id)
681 JOIN binaries b ON (b.id=c.binary_pkg)
682 JOIN override o ON (o.package=b.package)
683 JOIN section s ON (s.id=o.section)
684 WHERE o.suite = :suiteid AND o.type = :overridetypeid
685 AND b.type=:overridetypename"""
687 vals = {'suiteid': suite.suite_id,
688 'overridetypeid': overridetype.overridetype_id,
689 'overridetypename': overridetype.overridetype}
691 if section is not None:
692 contents_q += " AND s.id = :sectionid"
693 vals['sectionid'] = section.section_id
695 contents_q += " ORDER BY fn"
697 return session.execute(contents_q, vals)
699 __all__.append('get_contents')
701 ################################################################################
703 class ContentFilepath(object):
704 def __init__(self, *args, **kwargs):
708 return '<ContentFilepath %s>' % self.filepath
710 __all__.append('ContentFilepath')
713 def get_or_set_contents_path_id(filepath, session=None):
715 Returns database id for given path.
717 If no matching file is found, a row is inserted.
719 @type filename: string
720 @param filename: The filepath
721 @type session: SQLAlchemy
722 @param session: Optional SQL session object (a temporary one will be
723 generated if not supplied). If not passed, a commit will be performed at
724 the end of the function, otherwise the caller is responsible for commiting.
727 @return: the database id for the given path
730 q = session.query(ContentFilepath).filter_by(filepath=filepath)
733 ret = q.one().cafilepath_id
734 except NoResultFound:
735 cf = ContentFilepath()
736 cf.filepath = filepath
738 session.commit_or_flush()
739 ret = cf.cafilepath_id
743 __all__.append('get_or_set_contents_path_id')
745 ################################################################################
747 class ContentAssociation(object):
748 def __init__(self, *args, **kwargs):
752 return '<ContentAssociation %s>' % self.ca_id
754 __all__.append('ContentAssociation')
756 def insert_content_paths(binary_id, fullpaths, session=None):
758 Make sure given path is associated with given binary id
761 @param binary_id: the id of the binary
762 @type fullpaths: list
763 @param fullpaths: the list of paths of the file being associated with the binary
764 @type session: SQLAlchemy session
765 @param session: Optional SQLAlchemy session. If this is passed, the caller
766 is responsible for ensuring a transaction has begun and committing the
767 results or rolling back based on the result code. If not passed, a commit
768 will be performed at the end of the function, otherwise the caller is
769 responsible for commiting.
771 @return: True upon success
776 session = DBConn().session()
782 for fullpath in fullpaths:
783 if fullpath.startswith( './' ):
784 fullpath = fullpath[2:]
786 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} )
794 traceback.print_exc()
796 # Only rollback if we set up the session ourself
803 __all__.append('insert_content_paths')
805 ################################################################################
807 class DSCFile(object):
808 def __init__(self, *args, **kwargs):
812 return '<DSCFile %s>' % self.dscfile_id
814 __all__.append('DSCFile')
817 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
819 Returns a list of DSCFiles which may be empty
821 @type dscfile_id: int (optional)
822 @param dscfile_id: the dscfile_id of the DSCFiles to find
824 @type source_id: int (optional)
825 @param source_id: the source id related to the DSCFiles to find
827 @type poolfile_id: int (optional)
828 @param poolfile_id: the poolfile id related to the DSCFiles to find
831 @return: Possibly empty list of DSCFiles
834 q = session.query(DSCFile)
836 if dscfile_id is not None:
837 q = q.filter_by(dscfile_id=dscfile_id)
839 if source_id is not None:
840 q = q.filter_by(source_id=source_id)
842 if poolfile_id is not None:
843 q = q.filter_by(poolfile_id=poolfile_id)
847 __all__.append('get_dscfiles')
849 ################################################################################
851 class PoolFile(object):
852 def __init__(self, *args, **kwargs):
856 return '<PoolFile %s>' % self.filename
860 return os.path.join(self.location.path, self.filename)
862 __all__.append('PoolFile')
865 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
868 (ValidFileFound [boolean or None], PoolFile object or None)
870 @type filename: string
871 @param filename: the filename of the file to check against the DB
874 @param filesize: the size of the file to check against the DB
877 @param md5sum: the md5sum of the file to check against the DB
879 @type location_id: int
880 @param location_id: the id of the location to look in
883 @return: Tuple of length 2.
884 If more than one file found with that name:
886 If valid pool file found: (True, PoolFile object)
887 If valid pool file not found:
888 (False, None) if no file found
889 (False, PoolFile object) if file found with size/md5sum mismatch
892 q = session.query(PoolFile).filter_by(filename=filename)
893 q = q.join(Location).filter_by(location_id=location_id)
903 if obj.md5sum != md5sum or obj.filesize != int(filesize):
911 __all__.append('check_poolfile')
914 def get_poolfile_by_id(file_id, session=None):
916 Returns a PoolFile objects or None for the given id
919 @param file_id: the id of the file to look for
921 @rtype: PoolFile or None
922 @return: either the PoolFile object or None
925 q = session.query(PoolFile).filter_by(file_id=file_id)
929 except NoResultFound:
932 __all__.append('get_poolfile_by_id')
936 def get_poolfile_by_name(filename, location_id=None, session=None):
938 Returns an array of PoolFile objects for the given filename and
939 (optionally) location_id
941 @type filename: string
942 @param filename: the filename of the file to check against the DB
944 @type location_id: int
945 @param location_id: the id of the location to look in (optional)
948 @return: array of PoolFile objects
951 q = session.query(PoolFile).filter_by(filename=filename)
953 if location_id is not None:
954 q = q.join(Location).filter_by(location_id=location_id)
958 __all__.append('get_poolfile_by_name')
961 def get_poolfile_like_name(filename, session=None):
963 Returns an array of PoolFile objects which are like the given name
965 @type filename: string
966 @param filename: the filename of the file to check against the DB
969 @return: array of PoolFile objects
972 # TODO: There must be a way of properly using bind parameters with %FOO%
973 q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
977 __all__.append('get_poolfile_like_name')
980 def add_poolfile(filename, datadict, location_id, session=None):
982 Add a new file to the pool
984 @type filename: string
985 @param filename: filename
988 @param datadict: dict with needed data
990 @type location_id: int
991 @param location_id: database id of the location
994 @return: the PoolFile object created
996 poolfile = PoolFile()
997 poolfile.filename = filename
998 poolfile.filesize = datadict["size"]
999 poolfile.md5sum = datadict["md5sum"]
1000 poolfile.sha1sum = datadict["sha1sum"]
1001 poolfile.sha256sum = datadict["sha256sum"]
1002 poolfile.location_id = location_id
1004 session.add(poolfile)
1005 # Flush to get a file id (NB: This is not a commit)
1010 __all__.append('add_poolfile')
1012 ################################################################################
1014 class Fingerprint(object):
1015 def __init__(self, *args, **kwargs):
1019 return '<Fingerprint %s>' % self.fingerprint
1021 __all__.append('Fingerprint')
1024 def get_fingerprint(fpr, session=None):
1026 Returns Fingerprint object for given fpr.
1029 @param fpr: The fpr to find / add
1031 @type session: SQLAlchemy
1032 @param session: Optional SQL session object (a temporary one will be
1033 generated if not supplied).
1036 @return: the Fingerprint object for the given fpr or None
1039 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1043 except NoResultFound:
1048 __all__.append('get_fingerprint')
1051 def get_or_set_fingerprint(fpr, session=None):
1053 Returns Fingerprint object for given fpr.
1055 If no matching fpr is found, a row is inserted.
1058 @param fpr: The fpr to find / add
1060 @type session: SQLAlchemy
1061 @param session: Optional SQL session object (a temporary one will be
1062 generated if not supplied). If not passed, a commit will be performed at
1063 the end of the function, otherwise the caller is responsible for commiting.
1064 A flush will be performed either way.
1067 @return: the Fingerprint object for the given fpr
1070 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1074 except NoResultFound:
1075 fingerprint = Fingerprint()
1076 fingerprint.fingerprint = fpr
1077 session.add(fingerprint)
1078 session.commit_or_flush()
1083 __all__.append('get_or_set_fingerprint')
1085 ################################################################################
1087 # Helper routine for Keyring class
1088 def get_ldap_name(entry):
1090 for k in ["cn", "mn", "sn"]:
1092 if ret and ret[0] != "" and ret[0] != "-":
1094 return " ".join(name)
1096 ################################################################################
1098 class Keyring(object):
1099 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1100 " --with-colons --fingerprint --fingerprint"
1105 def __init__(self, *args, **kwargs):
1109 return '<Keyring %s>' % self.keyring_name
1111 def de_escape_gpg_str(self, txt):
1112 esclist = re.split(r'(\\x..)', txt)
1113 for x in range(1,len(esclist),2):
1114 esclist[x] = "%c" % (int(esclist[x][2:],16))
1115 return "".join(esclist)
1117 def load_keys(self, keyring):
1120 if not self.keyring_id:
1121 raise Exception('Must be initialized with database information')
1123 k = os.popen(self.gpg_invocation % keyring, "r")
1127 for line in k.xreadlines():
1128 field = line.split(":")
1129 if field[0] == "pub":
1131 (name, addr) = email.Utils.parseaddr(field[9])
1132 name = re.sub(r"\s*[(].*[)]", "", name)
1133 if name == "" or addr == "" or "@" not in addr:
1135 addr = "invalid-uid"
1136 name = self.de_escape_gpg_str(name)
1137 self.keys[key] = {"email": addr}
1139 self.keys[key]["name"] = name
1140 self.keys[key]["aliases"] = [name]
1141 self.keys[key]["fingerprints"] = []
1143 elif key and field[0] == "sub" and len(field) >= 12:
1144 signingkey = ("s" in field[11])
1145 elif key and field[0] == "uid":
1146 (name, addr) = email.Utils.parseaddr(field[9])
1147 if name and name not in self.keys[key]["aliases"]:
1148 self.keys[key]["aliases"].append(name)
1149 elif signingkey and field[0] == "fpr":
1150 self.keys[key]["fingerprints"].append(field[9])
1151 self.fpr_lookup[field[9]] = key
1153 def import_users_from_ldap(self, session):
1157 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1158 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1160 l = ldap.open(LDAPServer)
1161 l.simple_bind_s("","")
1162 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1163 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1164 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1166 ldap_fin_uid_id = {}
1173 uid = entry["uid"][0]
1174 name = get_ldap_name(entry)
1175 fingerprints = entry["keyFingerPrint"]
1177 for f in fingerprints:
1178 key = self.fpr_lookup.get(f, None)
1179 if key not in self.keys:
1181 self.keys[key]["uid"] = uid
1185 keyid = get_or_set_uid(uid, session).uid_id
1186 byuid[keyid] = (uid, name)
1187 byname[uid] = (keyid, name)
1189 return (byname, byuid)
1191 def generate_users_from_keyring(self, format, session):
1195 for x in self.keys.keys():
1196 if self.keys[x]["email"] == "invalid-uid":
1198 self.keys[x]["uid"] = format % "invalid-uid"
1200 uid = format % self.keys[x]["email"]
1201 keyid = get_or_set_uid(uid, session).uid_id
1202 byuid[keyid] = (uid, self.keys[x]["name"])
1203 byname[uid] = (keyid, self.keys[x]["name"])
1204 self.keys[x]["uid"] = uid
1207 uid = format % "invalid-uid"
1208 keyid = get_or_set_uid(uid, session).uid_id
1209 byuid[keyid] = (uid, "ungeneratable user id")
1210 byname[uid] = (keyid, "ungeneratable user id")
1212 return (byname, byuid)
1214 __all__.append('Keyring')
1217 def get_keyring(keyring, session=None):
1219 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1220 If C{keyring} already has an entry, simply return the existing Keyring
1222 @type keyring: string
1223 @param keyring: the keyring name
1226 @return: the Keyring object for this keyring
1229 q = session.query(Keyring).filter_by(keyring_name=keyring)
1233 except NoResultFound:
1236 __all__.append('get_keyring')
1238 ################################################################################
1240 class KeyringACLMap(object):
1241 def __init__(self, *args, **kwargs):
1245 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1247 __all__.append('KeyringACLMap')
1249 ################################################################################
1251 class DBChange(object):
1252 def __init__(self, *args, **kwargs):
1256 return '<DBChange %s>' % self.changesname
1258 __all__.append('DBChange')
1261 def get_dbchange(filename, session=None):
1263 returns DBChange object for given C{filename}.
1265 @type archive: string
1266 @param archive: the name of the arhive
1268 @type session: Session
1269 @param session: Optional SQLA session object (a temporary one will be
1270 generated if not supplied)
1273 @return: Archive object for the given name (None if not present)
1276 q = session.query(DBChange).filter_by(changesname=filename)
1280 except NoResultFound:
1283 __all__.append('get_dbchange')
1285 ################################################################################
1287 class Location(object):
1288 def __init__(self, *args, **kwargs):
1292 return '<Location %s (%s)>' % (self.path, self.location_id)
1294 __all__.append('Location')
1297 def get_location(location, component=None, archive=None, session=None):
1299 Returns Location object for the given combination of location, component
1302 @type location: string
1303 @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
1305 @type component: string
1306 @param component: the component name (if None, no restriction applied)
1308 @type archive: string
1309 @param archive_id: the archive name (if None, no restriction applied)
1311 @rtype: Location / None
1312 @return: Either a Location object or None if one can't be found
1315 q = session.query(Location).filter_by(path=location)
1317 if archive is not None:
1318 q = q.join(Archive).filter_by(archive_name=archive)
1320 if component is not None:
1321 q = q.join(Component).filter_by(component_name=component)
1325 except NoResultFound:
1328 __all__.append('get_location')
1330 ################################################################################
1332 class Maintainer(object):
1333 def __init__(self, *args, **kwargs):
1337 return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
1339 def get_split_maintainer(self):
1340 if not hasattr(self, 'name') or self.name is None:
1341 return ('', '', '', '')
1343 return fix_maintainer(self.name.strip())
1345 __all__.append('Maintainer')
1348 def get_or_set_maintainer(name, session=None):
1350 Returns Maintainer object for given maintainer name.
1352 If no matching maintainer name is found, a row is inserted.
1355 @param name: The maintainer name to add
1357 @type session: SQLAlchemy
1358 @param session: Optional SQL session object (a temporary one will be
1359 generated if not supplied). If not passed, a commit will be performed at
1360 the end of the function, otherwise the caller is responsible for commiting.
1361 A flush will be performed either way.
1364 @return: the Maintainer object for the given maintainer
1367 q = session.query(Maintainer).filter_by(name=name)
1370 except NoResultFound:
1371 maintainer = Maintainer()
1372 maintainer.name = name
1373 session.add(maintainer)
1374 session.commit_or_flush()
1379 __all__.append('get_or_set_maintainer')
1382 def get_maintainer(maintainer_id, session=None):
1384 Return the name of the maintainer behind C{maintainer_id} or None if that
1385 maintainer_id is invalid.
1387 @type maintainer_id: int
1388 @param maintainer_id: the id of the maintainer
1391 @return: the Maintainer with this C{maintainer_id}
1394 return session.query(Maintainer).get(maintainer_id)
1396 __all__.append('get_maintainer')
1398 ################################################################################
1400 class NewComment(object):
1401 def __init__(self, *args, **kwargs):
1405 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1407 __all__.append('NewComment')
1410 def has_new_comment(package, version, session=None):
1412 Returns true if the given combination of C{package}, C{version} has a comment.
1414 @type package: string
1415 @param package: name of the package
1417 @type version: string
1418 @param version: package version
1420 @type session: Session
1421 @param session: Optional SQLA session object (a temporary one will be
1422 generated if not supplied)
1428 q = session.query(NewComment)
1429 q = q.filter_by(package=package)
1430 q = q.filter_by(version=version)
1432 return bool(q.count() > 0)
1434 __all__.append('has_new_comment')
1437 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1439 Returns (possibly empty) list of NewComment objects for the given
1442 @type package: string (optional)
1443 @param package: name of the package
1445 @type version: string (optional)
1446 @param version: package version
1448 @type comment_id: int (optional)
1449 @param comment_id: An id of a comment
1451 @type session: Session
1452 @param session: Optional SQLA session object (a temporary one will be
1453 generated if not supplied)
1456 @return: A (possibly empty) list of NewComment objects will be returned
1459 q = session.query(NewComment)
1460 if package is not None: q = q.filter_by(package=package)
1461 if version is not None: q = q.filter_by(version=version)
1462 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1466 __all__.append('get_new_comments')
1468 ################################################################################
1470 class Override(object):
1471 def __init__(self, *args, **kwargs):
1475 return '<Override %s (%s)>' % (self.package, self.suite_id)
1477 __all__.append('Override')
1480 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1482 Returns Override object for the given parameters
1484 @type package: string
1485 @param package: The name of the package
1487 @type suite: string, list or None
1488 @param suite: The name of the suite (or suites if a list) to limit to. If
1489 None, don't limit. Defaults to None.
1491 @type component: string, list or None
1492 @param component: The name of the component (or components if a list) to
1493 limit to. If None, don't limit. Defaults to None.
1495 @type overridetype: string, list or None
1496 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1497 limit to. If None, don't limit. Defaults to None.
1499 @type session: Session
1500 @param session: Optional SQLA session object (a temporary one will be
1501 generated if not supplied)
1504 @return: A (possibly empty) list of Override objects will be returned
1507 q = session.query(Override)
1508 q = q.filter_by(package=package)
1510 if suite is not None:
1511 if not isinstance(suite, list): suite = [suite]
1512 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1514 if component is not None:
1515 if not isinstance(component, list): component = [component]
1516 q = q.join(Component).filter(Component.component_name.in_(component))
1518 if overridetype is not None:
1519 if not isinstance(overridetype, list): overridetype = [overridetype]
1520 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1524 __all__.append('get_override')
1527 ################################################################################
1529 class OverrideType(object):
1530 def __init__(self, *args, **kwargs):
1534 return '<OverrideType %s>' % self.overridetype
1536 __all__.append('OverrideType')
1539 def get_override_type(override_type, session=None):
1541 Returns OverrideType object for given C{override type}.
1543 @type override_type: string
1544 @param override_type: The name of the override type
1546 @type session: Session
1547 @param session: Optional SQLA session object (a temporary one will be
1548 generated if not supplied)
1551 @return: the database id for the given override type
1554 q = session.query(OverrideType).filter_by(overridetype=override_type)
1558 except NoResultFound:
1561 __all__.append('get_override_type')
1563 ################################################################################
1565 class PendingContentAssociation(object):
1566 def __init__(self, *args, **kwargs):
1570 return '<PendingContentAssociation %s>' % self.pca_id
1572 __all__.append('PendingContentAssociation')
1574 def insert_pending_content_paths(package, fullpaths, session=None):
1576 Make sure given paths are temporarily associated with given
1580 @param package: the package to associate with should have been read in from the binary control file
1581 @type fullpaths: list
1582 @param fullpaths: the list of paths of the file being associated with the binary
1583 @type session: SQLAlchemy session
1584 @param session: Optional SQLAlchemy session. If this is passed, the caller
1585 is responsible for ensuring a transaction has begun and committing the
1586 results or rolling back based on the result code. If not passed, a commit
1587 will be performed at the end of the function
1589 @return: True upon success, False if there is a problem
1592 privatetrans = False
1595 session = DBConn().session()
1599 arch = get_architecture(package['Architecture'], session)
1600 arch_id = arch.arch_id
1602 # Remove any already existing recorded files for this package
1603 q = session.query(PendingContentAssociation)
1604 q = q.filter_by(package=package['Package'])
1605 q = q.filter_by(version=package['Version'])
1606 q = q.filter_by(architecture=arch_id)
1611 for fullpath in fullpaths:
1612 (path, filename) = os.path.split(fullpath)
1614 if path.startswith( "./" ):
1617 filepath_id = get_or_set_contents_path_id(path, session)
1618 filename_id = get_or_set_contents_file_id(filename, session)
1620 pathcache[fullpath] = (filepath_id, filename_id)
1622 for fullpath, dat in pathcache.items():
1623 pca = PendingContentAssociation()
1624 pca.package = package['Package']
1625 pca.version = package['Version']
1626 pca.filepath_id = dat[0]
1627 pca.filename_id = dat[1]
1628 pca.architecture = arch_id
1631 # Only commit if we set up the session ourself
1639 except Exception, e:
1640 traceback.print_exc()
1642 # Only rollback if we set up the session ourself
1649 __all__.append('insert_pending_content_paths')
1651 ################################################################################
1653 class PolicyQueue(object):
1654 def __init__(self, *args, **kwargs):
1658 return '<PolicyQueue %s>' % self.queue_name
1660 __all__.append('PolicyQueue')
1662 ################################################################################
1664 class Priority(object):
1665 def __init__(self, *args, **kwargs):
1668 def __eq__(self, val):
1669 if isinstance(val, str):
1670 return (self.priority == val)
1671 # This signals to use the normal comparison operator
1672 return NotImplemented
1674 def __ne__(self, val):
1675 if isinstance(val, str):
1676 return (self.priority != val)
1677 # This signals to use the normal comparison operator
1678 return NotImplemented
1681 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
1683 __all__.append('Priority')
1686 def get_priority(priority, session=None):
1688 Returns Priority object for given C{priority name}.
1690 @type priority: string
1691 @param priority: The name of the priority
1693 @type session: Session
1694 @param session: Optional SQLA session object (a temporary one will be
1695 generated if not supplied)
1698 @return: Priority object for the given priority
1701 q = session.query(Priority).filter_by(priority=priority)
1705 except NoResultFound:
1708 __all__.append('get_priority')
1711 def get_priorities(session=None):
1713 Returns dictionary of priority names -> id mappings
1715 @type session: Session
1716 @param session: Optional SQL session object (a temporary one will be
1717 generated if not supplied)
1720 @return: dictionary of priority names -> id mappings
1724 q = session.query(Priority)
1726 ret[x.priority] = x.priority_id
1730 __all__.append('get_priorities')
1732 ################################################################################
1734 class Section(object):
1735 def __init__(self, *args, **kwargs):
1738 def __eq__(self, val):
1739 if isinstance(val, str):
1740 return (self.section == val)
1741 # This signals to use the normal comparison operator
1742 return NotImplemented
1744 def __ne__(self, val):
1745 if isinstance(val, str):
1746 return (self.section != val)
1747 # This signals to use the normal comparison operator
1748 return NotImplemented
1751 return '<Section %s>' % self.section
1753 __all__.append('Section')
1756 def get_section(section, session=None):
1758 Returns Section object for given C{section name}.
1760 @type section: string
1761 @param section: The name of the section
1763 @type session: Session
1764 @param session: Optional SQLA session object (a temporary one will be
1765 generated if not supplied)
1768 @return: Section object for the given section name
1771 q = session.query(Section).filter_by(section=section)
1775 except NoResultFound:
1778 __all__.append('get_section')
1781 def get_sections(session=None):
1783 Returns dictionary of section names -> id mappings
1785 @type session: Session
1786 @param session: Optional SQL session object (a temporary one will be
1787 generated if not supplied)
1790 @return: dictionary of section names -> id mappings
1794 q = session.query(Section)
1796 ret[x.section] = x.section_id
1800 __all__.append('get_sections')
1802 ################################################################################
1804 class DBSource(object):
1805 def __init__(self, *args, **kwargs):
1809 return '<DBSource %s (%s)>' % (self.source, self.version)
1811 __all__.append('DBSource')
1814 def source_exists(source, source_version, suites = ["any"], session=None):
1816 Ensure that source exists somewhere in the archive for the binary
1817 upload being processed.
1818 1. exact match => 1.0-3
1819 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1821 @type package: string
1822 @param package: package source name
1824 @type source_version: string
1825 @param source_version: expected source version
1828 @param suites: list of suites to check in, default I{any}
1830 @type session: Session
1831 @param session: Optional SQLA session object (a temporary one will be
1832 generated if not supplied)
1835 @return: returns 1 if a source with expected version is found, otherwise 0
1842 for suite in suites:
1843 q = session.query(DBSource).filter_by(source=source)
1845 # source must exist in suite X, or in some other suite that's
1846 # mapped to X, recursively... silent-maps are counted too,
1847 # unreleased-maps aren't.
1848 maps = cnf.ValueList("SuiteMappings")[:]
1850 maps = [ m.split() for m in maps ]
1851 maps = [ (x[1], x[2]) for x in maps
1852 if x[0] == "map" or x[0] == "silent-map" ]
1855 if x[1] in s and x[0] not in s:
1858 q = q.join(SrcAssociation).join(Suite)
1859 q = q.filter(Suite.suite_name.in_(s))
1861 # Reduce the query results to a list of version numbers
1862 ql = [ j.version for j in q.all() ]
1865 if source_version in ql:
1869 from daklib.regexes import re_bin_only_nmu
1870 orig_source_version = re_bin_only_nmu.sub('', source_version)
1871 if orig_source_version in ql:
1874 # No source found so return not ok
1879 __all__.append('source_exists')
1882 def get_suites_source_in(source, session=None):
1884 Returns list of Suite objects which given C{source} name is in
1887 @param source: DBSource package name to search for
1890 @return: list of Suite objects for the given source
1893 return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
1895 __all__.append('get_suites_source_in')
1898 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
1900 Returns list of DBSource objects for given C{source} name and other parameters
1903 @param source: DBSource package name to search for
1905 @type source: str or None
1906 @param source: DBSource version name to search for or None if not applicable
1908 @type dm_upload_allowed: bool
1909 @param dm_upload_allowed: If None, no effect. If True or False, only
1910 return packages with that dm_upload_allowed setting
1912 @type session: Session
1913 @param session: Optional SQL session object (a temporary one will be
1914 generated if not supplied)
1917 @return: list of DBSource objects for the given name (may be empty)
1920 q = session.query(DBSource).filter_by(source=source)
1922 if version is not None:
1923 q = q.filter_by(version=version)
1925 if dm_upload_allowed is not None:
1926 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
1930 __all__.append('get_sources_from_name')
1933 def get_source_in_suite(source, suite, session=None):
1935 Returns list of DBSource objects for a combination of C{source} and C{suite}.
1937 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
1938 - B{suite} - a suite name, eg. I{unstable}
1940 @type source: string
1941 @param source: source package name
1944 @param suite: the suite name
1947 @return: the version for I{source} in I{suite}
1951 q = session.query(SrcAssociation)
1952 q = q.join('source').filter_by(source=source)
1953 q = q.join('suite').filter_by(suite_name=suite)
1956 return q.one().source
1957 except NoResultFound:
1960 __all__.append('get_source_in_suite')
1962 ################################################################################
1965 def add_dsc_to_db(u, filename, session=None):
1966 entry = u.pkg.files[filename]
1970 source.source = u.pkg.dsc["source"]
1971 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
1972 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
1973 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
1974 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
1975 source.install_date = datetime.now().date()
1977 dsc_component = entry["component"]
1978 dsc_location_id = entry["location id"]
1980 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
1982 # Set up a new poolfile if necessary
1983 if not entry.has_key("files id") or not entry["files id"]:
1984 filename = entry["pool name"] + filename
1985 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
1987 pfs.append(poolfile)
1988 entry["files id"] = poolfile.file_id
1990 source.poolfile_id = entry["files id"]
1994 for suite_name in u.pkg.changes["distribution"].keys():
1995 sa = SrcAssociation()
1996 sa.source_id = source.source_id
1997 sa.suite_id = get_suite(suite_name).suite_id
2002 # Add the source files to the DB (files and dsc_files)
2004 dscfile.source_id = source.source_id
2005 dscfile.poolfile_id = entry["files id"]
2006 session.add(dscfile)
2008 for dsc_file, dentry in u.pkg.dsc_files.items():
2010 df.source_id = source.source_id
2012 # If the .orig tarball is already in the pool, it's
2013 # files id is stored in dsc_files by check_dsc().
2014 files_id = dentry.get("files id", None)
2016 # Find the entry in the files hash
2017 # TODO: Bail out here properly
2019 for f, e in u.pkg.files.items():
2024 if files_id is None:
2025 filename = dfentry["pool name"] + dsc_file
2027 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2028 # FIXME: needs to check for -1/-2 and or handle exception
2029 if found and obj is not None:
2030 files_id = obj.file_id
2033 # If still not found, add it
2034 if files_id is None:
2035 # HACK: Force sha1sum etc into dentry
2036 dentry["sha1sum"] = dfentry["sha1sum"]
2037 dentry["sha256sum"] = dfentry["sha256sum"]
2038 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2039 pfs.append(poolfile)
2040 files_id = poolfile.file_id
2042 df.poolfile_id = files_id
2047 # Add the src_uploaders to the DB
2048 uploader_ids = [source.maintainer_id]
2049 if u.pkg.dsc.has_key("uploaders"):
2050 for up in u.pkg.dsc["uploaders"].split(","):
2052 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2055 for up in uploader_ids:
2056 if added_ids.has_key(up):
2057 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
2063 su.maintainer_id = up
2064 su.source_id = source.source_id
2069 return dsc_component, dsc_location_id, pfs
2071 __all__.append('add_dsc_to_db')
2074 def add_deb_to_db(u, filename, session=None):
2076 Contrary to what you might expect, this routine deals with both
2077 debs and udebs. That info is in 'dbtype', whilst 'type' is
2078 'deb' for both of them
2081 entry = u.pkg.files[filename]
2084 bin.package = entry["package"]
2085 bin.version = entry["version"]
2086 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2087 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2088 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2089 bin.binarytype = entry["dbtype"]
2092 filename = entry["pool name"] + filename
2093 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2094 if not entry.get("location id", None):
2095 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2097 if entry.get("files id", None):
2098 poolfile = get_poolfile_by_id(bin.poolfile_id)
2099 bin.poolfile_id = entry["files id"]
2101 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2102 bin.poolfile_id = entry["files id"] = poolfile.file_id
2105 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2106 if len(bin_sources) != 1:
2107 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2108 (bin.package, bin.version, bin.architecture.arch_string,
2109 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2111 bin.source_id = bin_sources[0].source_id
2113 # Add and flush object so it has an ID
2117 # Add BinAssociations
2118 for suite_name in u.pkg.changes["distribution"].keys():
2119 ba = BinAssociation()
2120 ba.binary_id = bin.binary_id
2121 ba.suite_id = get_suite(suite_name).suite_id
2126 # Deal with contents - disabled for now
2127 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2129 # print "REJECT\nCould not determine contents of package %s" % bin.package
2130 # session.rollback()
2131 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2135 __all__.append('add_deb_to_db')
2137 ################################################################################
2139 class SourceACL(object):
2140 def __init__(self, *args, **kwargs):
2144 return '<SourceACL %s>' % self.source_acl_id
2146 __all__.append('SourceACL')
2148 ################################################################################
2150 class SrcAssociation(object):
2151 def __init__(self, *args, **kwargs):
2155 return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
2157 __all__.append('SrcAssociation')
2159 ################################################################################
2161 class SrcFormat(object):
2162 def __init__(self, *args, **kwargs):
2166 return '<SrcFormat %s>' % (self.format_name)
2168 __all__.append('SrcFormat')
2170 ################################################################################
2172 class SrcUploader(object):
2173 def __init__(self, *args, **kwargs):
2177 return '<SrcUploader %s>' % self.uploader_id
2179 __all__.append('SrcUploader')
2181 ################################################################################
2183 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2184 ('SuiteID', 'suite_id'),
2185 ('Version', 'version'),
2186 ('Origin', 'origin'),
2188 ('Description', 'description'),
2189 ('Untouchable', 'untouchable'),
2190 ('Announce', 'announce'),
2191 ('Codename', 'codename'),
2192 ('OverrideCodename', 'overridecodename'),
2193 ('ValidTime', 'validtime'),
2194 ('Priority', 'priority'),
2195 ('NotAutomatic', 'notautomatic'),
2196 ('CopyChanges', 'copychanges'),
2197 ('CopyDotDak', 'copydotdak'),
2198 ('CommentsDir', 'commentsdir'),
2199 ('OverrideSuite', 'overridesuite'),
2200 ('ChangelogBase', 'changelogbase')]
2203 class Suite(object):
2204 def __init__(self, *args, **kwargs):
2208 return '<Suite %s>' % self.suite_name
2210 def __eq__(self, val):
2211 if isinstance(val, str):
2212 return (self.suite_name == val)
2213 # This signals to use the normal comparison operator
2214 return NotImplemented
2216 def __ne__(self, val):
2217 if isinstance(val, str):
2218 return (self.suite_name != val)
2219 # This signals to use the normal comparison operator
2220 return NotImplemented
2224 for disp, field in SUITE_FIELDS:
2225 val = getattr(self, field, None)
2227 ret.append("%s: %s" % (disp, val))
2229 return "\n".join(ret)
2231 __all__.append('Suite')
2234 def get_suite_architecture(suite, architecture, session=None):
2236 Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
2240 @param suite: Suite name to search for
2242 @type architecture: str
2243 @param architecture: Architecture name to search for
2245 @type session: Session
2246 @param session: Optional SQL session object (a temporary one will be
2247 generated if not supplied)
2249 @rtype: SuiteArchitecture
2250 @return: the SuiteArchitecture object or None
2253 q = session.query(SuiteArchitecture)
2254 q = q.join(Architecture).filter_by(arch_string=architecture)
2255 q = q.join(Suite).filter_by(suite_name=suite)
2259 except NoResultFound:
2262 __all__.append('get_suite_architecture')
2265 def get_suite(suite, session=None):
2267 Returns Suite object for given C{suite name}.
2270 @param suite: The name of the suite
2272 @type session: Session
2273 @param session: Optional SQLA session object (a temporary one will be
2274 generated if not supplied)
2277 @return: Suite object for the requested suite name (None if not present)
2280 q = session.query(Suite).filter_by(suite_name=suite)
2284 except NoResultFound:
2287 __all__.append('get_suite')
2289 ################################################################################
2291 class SuiteArchitecture(object):
2292 def __init__(self, *args, **kwargs):
2296 return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
2298 __all__.append('SuiteArchitecture')
2301 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2303 Returns list of Architecture objects for given C{suite} name
2306 @param source: Suite name to search for
2308 @type skipsrc: boolean
2309 @param skipsrc: Whether to skip returning the 'source' architecture entry
2312 @type skipall: boolean
2313 @param skipall: Whether to skip returning the 'all' architecture entry
2316 @type session: Session
2317 @param session: Optional SQL session object (a temporary one will be
2318 generated if not supplied)
2321 @return: list of Architecture objects for the given name (may be empty)
2324 q = session.query(Architecture)
2325 q = q.join(SuiteArchitecture)
2326 q = q.join(Suite).filter_by(suite_name=suite)
2329 q = q.filter(Architecture.arch_string != 'source')
2332 q = q.filter(Architecture.arch_string != 'all')
2334 q = q.order_by('arch_string')
2338 __all__.append('get_suite_architectures')
2340 ################################################################################
2342 class SuiteSrcFormat(object):
2343 def __init__(self, *args, **kwargs):
2347 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2349 __all__.append('SuiteSrcFormat')
2352 def get_suite_src_formats(suite, session=None):
2354 Returns list of allowed SrcFormat for C{suite}.
2357 @param suite: Suite name to search for
2359 @type session: Session
2360 @param session: Optional SQL session object (a temporary one will be
2361 generated if not supplied)
2364 @return: the list of allowed source formats for I{suite}
2367 q = session.query(SrcFormat)
2368 q = q.join(SuiteSrcFormat)
2369 q = q.join(Suite).filter_by(suite_name=suite)
2370 q = q.order_by('format_name')
2374 __all__.append('get_suite_src_formats')
2376 ################################################################################
2379 def __init__(self, *args, **kwargs):
2382 def __eq__(self, val):
2383 if isinstance(val, str):
2384 return (self.uid == val)
2385 # This signals to use the normal comparison operator
2386 return NotImplemented
2388 def __ne__(self, val):
2389 if isinstance(val, str):
2390 return (self.uid != val)
2391 # This signals to use the normal comparison operator
2392 return NotImplemented
2395 return '<Uid %s (%s)>' % (self.uid, self.name)
2397 __all__.append('Uid')
2400 def add_database_user(uidname, session=None):
2402 Adds a database user
2404 @type uidname: string
2405 @param uidname: The uid of the user to add
2407 @type session: SQLAlchemy
2408 @param session: Optional SQL session object (a temporary one will be
2409 generated if not supplied). If not passed, a commit will be performed at
2410 the end of the function, otherwise the caller is responsible for commiting.
2413 @return: the uid object for the given uidname
2416 session.execute("CREATE USER :uid", {'uid': uidname})
2417 session.commit_or_flush()
2419 __all__.append('add_database_user')
2422 def get_or_set_uid(uidname, session=None):
2424 Returns uid object for given uidname.
2426 If no matching uidname is found, a row is inserted.
2428 @type uidname: string
2429 @param uidname: The uid to add
2431 @type session: SQLAlchemy
2432 @param session: Optional SQL session object (a temporary one will be
2433 generated if not supplied). If not passed, a commit will be performed at
2434 the end of the function, otherwise the caller is responsible for commiting.
2437 @return: the uid object for the given uidname
2440 q = session.query(Uid).filter_by(uid=uidname)
2444 except NoResultFound:
2448 session.commit_or_flush()
2453 __all__.append('get_or_set_uid')
2456 def get_uid_from_fingerprint(fpr, session=None):
2457 q = session.query(Uid)
2458 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2462 except NoResultFound:
2465 __all__.append('get_uid_from_fingerprint')
2467 ################################################################################
2469 class UploadBlock(object):
2470 def __init__(self, *args, **kwargs):
2474 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2476 __all__.append('UploadBlock')
2478 ################################################################################
2480 class DBConn(Singleton):
2482 database module init.
2484 def __init__(self, *args, **kwargs):
2485 super(DBConn, self).__init__(*args, **kwargs)
2487 def _startup(self, *args, **kwargs):
2489 if kwargs.has_key('debug'):
2493 def __setuptables(self):
2494 self.tbl_architecture = Table('architecture', self.db_meta, autoload=True)
2495 self.tbl_archive = Table('archive', self.db_meta, autoload=True)
2496 self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True)
2497 self.tbl_binaries = Table('binaries', self.db_meta, autoload=True)
2498 self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True)
2499 self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True)
2500 self.tbl_build_queue = Table('build_queue', self.db_meta, autoload=True)
2501 self.tbl_build_queue_files = Table('build_queue_files', self.db_meta, autoload=True)
2502 self.tbl_component = Table('component', self.db_meta, autoload=True)
2503 self.tbl_config = Table('config', self.db_meta, autoload=True)
2504 self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True)
2505 self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True)
2506 self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True)
2507 self.tbl_changes_pending_binary = Table('changes_pending_binaries', self.db_meta, autoload=True)
2508 self.tbl_changes_pending_files = Table('changes_pending_files', self.db_meta, autoload=True)
2509 self.tbl_changes_pending_files_map = Table('changes_pending_files_map', self.db_meta, autoload=True)
2510 self.tbl_changes_pending_source = Table('changes_pending_source', self.db_meta, autoload=True)
2511 self.tbl_changes_pending_source_files = Table('changes_pending_source_files', self.db_meta, autoload=True)
2512 self.tbl_changes_pool_files = Table('changes_pool_files', self.db_meta, autoload=True)
2513 self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True)
2514 self.tbl_files = Table('files', self.db_meta, autoload=True)
2515 self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True)
2516 self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True)
2517 self.tbl_changes = Table('changes', self.db_meta, autoload=True)
2518 self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True)
2519 self.tbl_location = Table('location', self.db_meta, autoload=True)
2520 self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True)
2521 self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True)
2522 self.tbl_override = Table('override', self.db_meta, autoload=True)
2523 self.tbl_override_type = Table('override_type', self.db_meta, autoload=True)
2524 self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True)
2525 self.tbl_policy_queue = Table('policy_queue', self.db_meta, autoload=True)
2526 self.tbl_priority = Table('priority', self.db_meta, autoload=True)
2527 self.tbl_section = Table('section', self.db_meta, autoload=True)
2528 self.tbl_source = Table('source', self.db_meta, autoload=True)
2529 self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True)
2530 self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True)
2531 self.tbl_src_format = Table('src_format', self.db_meta, autoload=True)
2532 self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True)
2533 self.tbl_suite = Table('suite', self.db_meta, autoload=True)
2534 self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True)
2535 self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True)
2536 self.tbl_suite_build_queue_copy = Table('suite_build_queue_copy', self.db_meta, autoload=True)
2537 self.tbl_uid = Table('uid', self.db_meta, autoload=True)
2538 self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True)
2540 def __setupmappers(self):
2541 mapper(Architecture, self.tbl_architecture,
2542 properties = dict(arch_id = self.tbl_architecture.c.id))
2544 mapper(Archive, self.tbl_archive,
2545 properties = dict(archive_id = self.tbl_archive.c.id,
2546 archive_name = self.tbl_archive.c.name))
2548 mapper(BinAssociation, self.tbl_bin_associations,
2549 properties = dict(ba_id = self.tbl_bin_associations.c.id,
2550 suite_id = self.tbl_bin_associations.c.suite,
2551 suite = relation(Suite),
2552 binary_id = self.tbl_bin_associations.c.bin,
2553 binary = relation(DBBinary)))
2555 mapper(BuildQueue, self.tbl_build_queue,
2556 properties = dict(queue_id = self.tbl_build_queue.c.id))
2558 mapper(BuildQueueFile, self.tbl_build_queue_files,
2559 properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
2560 poolfile = relation(PoolFile, backref='buildqueueinstances')))
2562 mapper(DBBinary, self.tbl_binaries,
2563 properties = dict(binary_id = self.tbl_binaries.c.id,
2564 package = self.tbl_binaries.c.package,
2565 version = self.tbl_binaries.c.version,
2566 maintainer_id = self.tbl_binaries.c.maintainer,
2567 maintainer = relation(Maintainer),
2568 source_id = self.tbl_binaries.c.source,
2569 source = relation(DBSource),
2570 arch_id = self.tbl_binaries.c.architecture,
2571 architecture = relation(Architecture),
2572 poolfile_id = self.tbl_binaries.c.file,
2573 poolfile = relation(PoolFile),
2574 binarytype = self.tbl_binaries.c.type,
2575 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2576 fingerprint = relation(Fingerprint),
2577 install_date = self.tbl_binaries.c.install_date,
2578 binassociations = relation(BinAssociation,
2579 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
2581 mapper(BinaryACL, self.tbl_binary_acl,
2582 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2584 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2585 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2586 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2587 architecture = relation(Architecture)))
2589 mapper(Component, self.tbl_component,
2590 properties = dict(component_id = self.tbl_component.c.id,
2591 component_name = self.tbl_component.c.name))
2593 mapper(DBConfig, self.tbl_config,
2594 properties = dict(config_id = self.tbl_config.c.id))
2596 mapper(DSCFile, self.tbl_dsc_files,
2597 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2598 source_id = self.tbl_dsc_files.c.source,
2599 source = relation(DBSource),
2600 poolfile_id = self.tbl_dsc_files.c.file,
2601 poolfile = relation(PoolFile)))
2603 mapper(PoolFile, self.tbl_files,
2604 properties = dict(file_id = self.tbl_files.c.id,
2605 filesize = self.tbl_files.c.size,
2606 location_id = self.tbl_files.c.location,
2607 location = relation(Location)))
2609 mapper(Fingerprint, self.tbl_fingerprint,
2610 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2611 uid_id = self.tbl_fingerprint.c.uid,
2612 uid = relation(Uid),
2613 keyring_id = self.tbl_fingerprint.c.keyring,
2614 keyring = relation(Keyring),
2615 source_acl = relation(SourceACL),
2616 binary_acl = relation(BinaryACL)))
2618 mapper(Keyring, self.tbl_keyrings,
2619 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2620 keyring_id = self.tbl_keyrings.c.id))
2622 mapper(DBChange, self.tbl_changes,
2623 properties = dict(change_id = self.tbl_changes.c.id,
2624 poolfiles = relation(PoolFile,
2625 secondary=self.tbl_changes_pool_files,
2626 backref="changeslinks"),
2627 files = relation(ChangePendingFile,
2628 secondary=self.tbl_changes_pending_files_map,
2629 backref="changesfile"),
2630 in_queue_id = self.tbl_changes.c.in_queue,
2631 in_queue = relation(PolicyQueue,
2632 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
2633 approved_for_id = self.tbl_changes.c.approved_for))
2635 mapper(ChangePendingBinary, self.tbl_changes_pending_binary,
2636 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binary.c.id))
2638 mapper(ChangePendingFile, self.tbl_changes_pending_files,
2639 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id))
2641 mapper(ChangePendingSource, self.tbl_changes_pending_source,
2642 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
2643 change = relation(DBChange),
2644 maintainer = relation(Maintainer,
2645 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
2646 changedby = relation(Maintainer,
2647 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
2648 fingerprint = relation(Fingerprint),
2649 source_files = relation(ChangePendingFile,
2650 secondary=self.tbl_changes_pending_source_files,
2651 backref="pending_sources")))
2652 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
2653 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
2654 keyring = relation(Keyring, backref="keyring_acl_map"),
2655 architecture = relation(Architecture)))
2657 mapper(Location, self.tbl_location,
2658 properties = dict(location_id = self.tbl_location.c.id,
2659 component_id = self.tbl_location.c.component,
2660 component = relation(Component),
2661 archive_id = self.tbl_location.c.archive,
2662 archive = relation(Archive),
2663 archive_type = self.tbl_location.c.type))
2665 mapper(Maintainer, self.tbl_maintainer,
2666 properties = dict(maintainer_id = self.tbl_maintainer.c.id))
2668 mapper(NewComment, self.tbl_new_comments,
2669 properties = dict(comment_id = self.tbl_new_comments.c.id))
2671 mapper(Override, self.tbl_override,
2672 properties = dict(suite_id = self.tbl_override.c.suite,
2673 suite = relation(Suite),
2674 component_id = self.tbl_override.c.component,
2675 component = relation(Component),
2676 priority_id = self.tbl_override.c.priority,
2677 priority = relation(Priority),
2678 section_id = self.tbl_override.c.section,
2679 section = relation(Section),
2680 overridetype_id = self.tbl_override.c.type,
2681 overridetype = relation(OverrideType)))
2683 mapper(OverrideType, self.tbl_override_type,
2684 properties = dict(overridetype = self.tbl_override_type.c.type,
2685 overridetype_id = self.tbl_override_type.c.id))
2687 mapper(PolicyQueue, self.tbl_policy_queue,
2688 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
2690 mapper(Priority, self.tbl_priority,
2691 properties = dict(priority_id = self.tbl_priority.c.id))
2693 mapper(Section, self.tbl_section,
2694 properties = dict(section_id = self.tbl_section.c.id))
2696 mapper(DBSource, self.tbl_source,
2697 properties = dict(source_id = self.tbl_source.c.id,
2698 version = self.tbl_source.c.version,
2699 maintainer_id = self.tbl_source.c.maintainer,
2700 maintainer = relation(Maintainer,
2701 primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
2702 poolfile_id = self.tbl_source.c.file,
2703 poolfile = relation(PoolFile),
2704 fingerprint_id = self.tbl_source.c.sig_fpr,
2705 fingerprint = relation(Fingerprint),
2706 changedby_id = self.tbl_source.c.changedby,
2707 changedby = relation(Maintainer,
2708 primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
2709 srcfiles = relation(DSCFile,
2710 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
2711 srcassociations = relation(SrcAssociation,
2712 primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)),
2713 srcuploaders = relation(SrcUploader)))
2715 mapper(SourceACL, self.tbl_source_acl,
2716 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
2718 mapper(SrcAssociation, self.tbl_src_associations,
2719 properties = dict(sa_id = self.tbl_src_associations.c.id,
2720 suite_id = self.tbl_src_associations.c.suite,
2721 suite = relation(Suite),
2722 source_id = self.tbl_src_associations.c.source,
2723 source = relation(DBSource)))
2725 mapper(SrcFormat, self.tbl_src_format,
2726 properties = dict(src_format_id = self.tbl_src_format.c.id,
2727 format_name = self.tbl_src_format.c.format_name))
2729 mapper(SrcUploader, self.tbl_src_uploaders,
2730 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
2731 source_id = self.tbl_src_uploaders.c.source,
2732 source = relation(DBSource,
2733 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
2734 maintainer_id = self.tbl_src_uploaders.c.maintainer,
2735 maintainer = relation(Maintainer,
2736 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
2738 mapper(Suite, self.tbl_suite,
2739 properties = dict(suite_id = self.tbl_suite.c.id,
2740 policy_queue = relation(PolicyQueue),
2741 copy_queues = relation(BuildQueue, secondary=self.tbl_suite_build_queue_copy)))
2743 mapper(SuiteArchitecture, self.tbl_suite_architectures,
2744 properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
2745 suite = relation(Suite, backref='suitearchitectures'),
2746 arch_id = self.tbl_suite_architectures.c.architecture,
2747 architecture = relation(Architecture)))
2749 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
2750 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
2751 suite = relation(Suite, backref='suitesrcformats'),
2752 src_format_id = self.tbl_suite_src_formats.c.src_format,
2753 src_format = relation(SrcFormat)))
2755 mapper(Uid, self.tbl_uid,
2756 properties = dict(uid_id = self.tbl_uid.c.id,
2757 fingerprint = relation(Fingerprint)))
2759 mapper(UploadBlock, self.tbl_upload_blocks,
2760 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
2761 fingerprint = relation(Fingerprint, backref="uploadblocks"),
2762 uid = relation(Uid, backref="uploadblocks")))
2764 ## Connection functions
2765 def __createconn(self):
2766 from config import Config
2770 connstr = "postgres://%s" % cnf["DB::Host"]
2771 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2772 connstr += ":%s" % cnf["DB::Port"]
2773 connstr += "/%s" % cnf["DB::Name"]
2776 connstr = "postgres:///%s" % cnf["DB::Name"]
2777 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2778 connstr += "?port=%s" % cnf["DB::Port"]
2780 self.db_pg = create_engine(connstr, echo=self.debug)
2781 self.db_meta = MetaData()
2782 self.db_meta.bind = self.db_pg
2783 self.db_smaker = sessionmaker(bind=self.db_pg,
2787 self.__setuptables()
2788 self.__setupmappers()
2791 return self.db_smaker()
2793 __all__.append('DBConn')