5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
7 @copyright: 2008-2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Mike O'Connor <stew@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # < mhy> I need a funny comment
30 # < sgran> two peanuts were walking down a dark street
31 # < sgran> one was a-salted
32 # * mhy looks up the definition of "funny"
34 ################################################################################
40 from datetime import datetime
42 from inspect import getargspec
45 from sqlalchemy import create_engine, Table, MetaData
46 from sqlalchemy.orm import sessionmaker, mapper, relation
47 from sqlalchemy import types as sqltypes
49 # Don't remove this, we re-export the exceptions to scripts which import us
50 from sqlalchemy.exc import *
51 from sqlalchemy.orm.exc import NoResultFound
53 # Only import Config until Queue stuff is changed to store its config
55 from config import Config
56 from textutils import fix_maintainer
58 ################################################################################
60 # Patch in support for the debversion field type so that it works during
63 class DebVersion(sqltypes.Text):
64 def get_col_spec(self):
67 sa_major_version = sqlalchemy.__version__[0:3]
68 if sa_major_version == "0.5":
69 from sqlalchemy.databases import postgres
70 postgres.ischema_names['debversion'] = DebVersion
72 raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
74 ################################################################################
76 __all__ = ['IntegrityError', 'SQLAlchemyError']
78 ################################################################################
80 def session_wrapper(fn):
82 Wrapper around common ".., session=None):" handling. If the wrapped
83 function is called without passing 'session', we create a local one
84 and destroy it when the function ends.
86 Also attaches a commit_or_flush method to the session; if we created a
87 local session, this is a synonym for session.commit(), otherwise it is a
88 synonym for session.flush().
91 def wrapped(*args, **kwargs):
92 private_transaction = False
94 # Find the session object
95 session = kwargs.get('session')
98 if len(args) <= len(getargspec(fn)[0]) - 1:
99 # No session specified as last argument or in kwargs
100 private_transaction = True
101 session = kwargs['session'] = DBConn().session()
103 # Session is last argument in args
107 session = args[-1] = DBConn().session()
108 private_transaction = True
110 if private_transaction:
111 session.commit_or_flush = session.commit
113 session.commit_or_flush = session.flush
116 return fn(*args, **kwargs)
118 if private_transaction:
119 # We created a session; close it.
122 wrapped.__doc__ = fn.__doc__
123 wrapped.func_name = fn.func_name
127 __all__.append('session_wrapper')
129 ################################################################################
131 class Architecture(object):
132 def __init__(self, *args, **kwargs):
135 def __eq__(self, val):
136 if isinstance(val, str):
137 return (self.arch_string== val)
138 # This signals to use the normal comparison operator
139 return NotImplemented
141 def __ne__(self, val):
142 if isinstance(val, str):
143 return (self.arch_string != val)
144 # This signals to use the normal comparison operator
145 return NotImplemented
148 return '<Architecture %s>' % self.arch_string
150 __all__.append('Architecture')
153 def get_architecture(architecture, session=None):
155 Returns database id for given C{architecture}.
157 @type architecture: string
158 @param architecture: The name of the architecture
160 @type session: Session
161 @param session: Optional SQLA session object (a temporary one will be
162 generated if not supplied)
165 @return: Architecture object for the given arch (None if not present)
168 q = session.query(Architecture).filter_by(arch_string=architecture)
172 except NoResultFound:
175 __all__.append('get_architecture')
178 def get_architecture_suites(architecture, session=None):
180 Returns list of Suite objects for given C{architecture} name
183 @param source: Architecture name to search for
185 @type session: Session
186 @param session: Optional SQL session object (a temporary one will be
187 generated if not supplied)
190 @return: list of Suite objects for the given name (may be empty)
193 q = session.query(Suite)
194 q = q.join(SuiteArchitecture)
195 q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
201 __all__.append('get_architecture_suites')
203 ################################################################################
205 class Archive(object):
206 def __init__(self, *args, **kwargs):
210 return '<Archive %s>' % self.archive_name
212 __all__.append('Archive')
215 def get_archive(archive, session=None):
217 returns database id for given C{archive}.
219 @type archive: string
220 @param archive: the name of the arhive
222 @type session: Session
223 @param session: Optional SQLA session object (a temporary one will be
224 generated if not supplied)
227 @return: Archive object for the given name (None if not present)
230 archive = archive.lower()
232 q = session.query(Archive).filter_by(archive_name=archive)
236 except NoResultFound:
239 __all__.append('get_archive')
241 ################################################################################
243 class BinAssociation(object):
244 def __init__(self, *args, **kwargs):
248 return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
250 __all__.append('BinAssociation')
252 ################################################################################
254 class BinContents(object):
255 def __init__(self, *args, **kwargs):
259 return '<BinContents (%s, %s)>' % (self.binary, self.filename)
261 __all__.append('BinContents')
263 ################################################################################
265 class DBBinary(object):
266 def __init__(self, *args, **kwargs):
270 return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
272 __all__.append('DBBinary')
275 def get_suites_binary_in(package, session=None):
277 Returns list of Suite objects which given C{package} name is in
280 @param source: DBBinary package name to search for
283 @return: list of Suite objects for the given package
286 return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
288 __all__.append('get_suites_binary_in')
291 def get_binary_from_id(binary_id, session=None):
293 Returns DBBinary object for given C{id}
296 @param binary_id: Id of the required binary
298 @type session: Session
299 @param session: Optional SQLA session object (a temporary one will be
300 generated if not supplied)
303 @return: DBBinary object for the given binary (None if not present)
306 q = session.query(DBBinary).filter_by(binary_id=binary_id)
310 except NoResultFound:
313 __all__.append('get_binary_from_id')
316 def get_binaries_from_name(package, version=None, architecture=None, session=None):
318 Returns list of DBBinary objects for given C{package} name
321 @param package: DBBinary package name to search for
323 @type version: str or None
324 @param version: Version to search for (or None)
326 @type package: str, list or None
327 @param package: Architectures to limit to (or None if no limit)
329 @type session: Session
330 @param session: Optional SQL session object (a temporary one will be
331 generated if not supplied)
334 @return: list of DBBinary objects for the given name (may be empty)
337 q = session.query(DBBinary).filter_by(package=package)
339 if version is not None:
340 q = q.filter_by(version=version)
342 if architecture is not None:
343 if not isinstance(architecture, list):
344 architecture = [architecture]
345 q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
351 __all__.append('get_binaries_from_name')
354 def get_binaries_from_source_id(source_id, session=None):
356 Returns list of DBBinary objects for given C{source_id}
359 @param source_id: source_id to search for
361 @type session: Session
362 @param session: Optional SQL session object (a temporary one will be
363 generated if not supplied)
366 @return: list of DBBinary objects for the given name (may be empty)
369 return session.query(DBBinary).filter_by(source_id=source_id).all()
371 __all__.append('get_binaries_from_source_id')
374 def get_binary_from_name_suite(package, suitename, session=None):
375 ### For dak examine-package
376 ### XXX: Doesn't use object API yet
378 sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
379 FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
380 WHERE b.package=:package
382 AND fi.location = l.id
383 AND l.component = c.id
386 AND su.suite_name=:suitename
387 ORDER BY b.version DESC"""
389 return session.execute(sql, {'package': package, 'suitename': suitename})
391 __all__.append('get_binary_from_name_suite')
394 def get_binary_components(package, suitename, arch, session=None):
395 # Check for packages that have moved from one component to another
396 query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
397 WHERE b.package=:package AND s.suite_name=:suitename
398 AND (a.arch_string = :arch OR a.arch_string = 'all')
399 AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
400 AND f.location = l.id
401 AND l.component = c.id
404 vals = {'package': package, 'suitename': suitename, 'arch': arch}
406 return session.execute(query, vals)
408 __all__.append('get_binary_components')
410 ################################################################################
412 class BinaryACL(object):
413 def __init__(self, *args, **kwargs):
417 return '<BinaryACL %s>' % self.binary_acl_id
419 __all__.append('BinaryACL')
421 ################################################################################
423 class BinaryACLMap(object):
424 def __init__(self, *args, **kwargs):
428 return '<BinaryACLMap %s>' % self.binary_acl_map_id
430 __all__.append('BinaryACLMap')
432 ################################################################################
434 class BuildQueue(object):
435 def __init__(self, *args, **kwargs):
439 return '<BuildQueue %s>' % self.queue_name
441 def add_file_from_pool(self, poolfile):
442 """Copies a file into the pool. Assumes that the PoolFile object is
443 attached to the same SQLAlchemy session as the Queue object is.
445 The caller is responsible for committing after calling this function."""
446 poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
448 # Check if we have a file of this name or this ID already
449 for f in self.queuefiles:
450 if f.fileid is not None and f.fileid == poolfile.file_id or \
451 f.poolfile.filename == poolfile_basename:
452 # In this case, update the BuildQueueFile entry so we
453 # don't remove it too early
454 f.lastused = datetime.now()
455 DBConn().session().object_session(poolfile).add(f)
458 # Prepare BuildQueueFile object
459 qf = BuildQueueFile()
460 qf.build_queue_id = self.queue_id
461 qf.lastused = datetime.now()
462 qf.filename = poolfile_basename
464 targetpath = poolfile.fullpath
465 queuepath = os.path.join(self.path, poolfile_basename)
469 # We need to copy instead of symlink
471 utils.copy(targetpath, queuepath)
472 # NULL in the fileid field implies a copy
475 os.symlink(targetpath, queuepath)
476 qf.fileid = poolfile.file_id
480 # Get the same session as the PoolFile is using and add the qf to it
481 DBConn().session().object_session(poolfile).add(qf)
486 __all__.append('BuildQueue')
489 def get_build_queue(queuename, session=None):
491 Returns BuildQueue object for given C{queue name}, creating it if it does not
494 @type queuename: string
495 @param queuename: The name of the queue
497 @type session: Session
498 @param session: Optional SQLA session object (a temporary one will be
499 generated if not supplied)
502 @return: BuildQueue object for the given queue
505 q = session.query(BuildQueue).filter_by(queue_name=queuename)
509 except NoResultFound:
512 __all__.append('get_build_queue')
514 ################################################################################
516 class BuildQueueFile(object):
517 def __init__(self, *args, **kwargs):
521 return '<BuildQueueFile %s (%s)>' % (self.filename, self.queue_id)
523 __all__.append('BuildQueueFile')
525 ################################################################################
527 class ChangePendingBinary(object):
528 def __init__(self, *args, **kwargs):
532 return '<ChangePendingBinary %s>' % self.change_pending_binary_id
534 __all__.append('ChangePendingBinary')
536 ################################################################################
538 class ChangePendingFile(object):
539 def __init__(self, *args, **kwargs):
543 return '<ChangePendingFile %s>' % self.change_pending_file_id
545 __all__.append('ChangePendingFile')
547 ################################################################################
549 class ChangePendingSource(object):
550 def __init__(self, *args, **kwargs):
554 return '<ChangePendingSource %s>' % self.change_pending_source_id
556 __all__.append('ChangePendingSource')
558 ################################################################################
560 class Component(object):
561 def __init__(self, *args, **kwargs):
564 def __eq__(self, val):
565 if isinstance(val, str):
566 return (self.component_name == val)
567 # This signals to use the normal comparison operator
568 return NotImplemented
570 def __ne__(self, val):
571 if isinstance(val, str):
572 return (self.component_name != val)
573 # This signals to use the normal comparison operator
574 return NotImplemented
577 return '<Component %s>' % self.component_name
580 __all__.append('Component')
583 def get_component(component, session=None):
585 Returns database id for given C{component}.
587 @type component: string
588 @param component: The name of the override type
591 @return: the database id for the given component
594 component = component.lower()
596 q = session.query(Component).filter_by(component_name=component)
600 except NoResultFound:
603 __all__.append('get_component')
605 ################################################################################
607 class DBConfig(object):
608 def __init__(self, *args, **kwargs):
612 return '<DBConfig %s>' % self.name
614 __all__.append('DBConfig')
616 ################################################################################
619 def get_or_set_contents_file_id(filename, session=None):
621 Returns database id for given filename.
623 If no matching file is found, a row is inserted.
625 @type filename: string
626 @param filename: The filename
627 @type session: SQLAlchemy
628 @param session: Optional SQL session object (a temporary one will be
629 generated if not supplied). If not passed, a commit will be performed at
630 the end of the function, otherwise the caller is responsible for commiting.
633 @return: the database id for the given component
636 q = session.query(ContentFilename).filter_by(filename=filename)
639 ret = q.one().cafilename_id
640 except NoResultFound:
641 cf = ContentFilename()
642 cf.filename = filename
644 session.commit_or_flush()
645 ret = cf.cafilename_id
649 __all__.append('get_or_set_contents_file_id')
652 def get_contents(suite, overridetype, section=None, session=None):
654 Returns contents for a suite / overridetype combination, limiting
655 to a section if not None.
658 @param suite: Suite object
660 @type overridetype: OverrideType
661 @param overridetype: OverrideType object
663 @type section: Section
664 @param section: Optional section object to limit results to
666 @type session: SQLAlchemy
667 @param session: Optional SQL session object (a temporary one will be
668 generated if not supplied)
671 @return: ResultsProxy object set up to return tuples of (filename, section,
675 # find me all of the contents for a given suite
676 contents_q = """SELECT (p.path||'/'||n.file) AS fn,
680 FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
681 JOIN content_file_names n ON (c.filename=n.id)
682 JOIN binaries b ON (b.id=c.binary_pkg)
683 JOIN override o ON (o.package=b.package)
684 JOIN section s ON (s.id=o.section)
685 WHERE o.suite = :suiteid AND o.type = :overridetypeid
686 AND b.type=:overridetypename"""
688 vals = {'suiteid': suite.suite_id,
689 'overridetypeid': overridetype.overridetype_id,
690 'overridetypename': overridetype.overridetype}
692 if section is not None:
693 contents_q += " AND s.id = :sectionid"
694 vals['sectionid'] = section.section_id
696 contents_q += " ORDER BY fn"
698 return session.execute(contents_q, vals)
700 __all__.append('get_contents')
702 ################################################################################
704 class ContentFilepath(object):
705 def __init__(self, *args, **kwargs):
709 return '<ContentFilepath %s>' % self.filepath
711 __all__.append('ContentFilepath')
714 def get_or_set_contents_path_id(filepath, session=None):
716 Returns database id for given path.
718 If no matching file is found, a row is inserted.
720 @type filename: string
721 @param filename: The filepath
722 @type session: SQLAlchemy
723 @param session: Optional SQL session object (a temporary one will be
724 generated if not supplied). If not passed, a commit will be performed at
725 the end of the function, otherwise the caller is responsible for commiting.
728 @return: the database id for the given path
731 q = session.query(ContentFilepath).filter_by(filepath=filepath)
734 ret = q.one().cafilepath_id
735 except NoResultFound:
736 cf = ContentFilepath()
737 cf.filepath = filepath
739 session.commit_or_flush()
740 ret = cf.cafilepath_id
744 __all__.append('get_or_set_contents_path_id')
746 ################################################################################
748 class ContentAssociation(object):
749 def __init__(self, *args, **kwargs):
753 return '<ContentAssociation %s>' % self.ca_id
755 __all__.append('ContentAssociation')
757 def insert_content_paths(binary_id, fullpaths, session=None):
759 Make sure given path is associated with given binary id
762 @param binary_id: the id of the binary
763 @type fullpaths: list
764 @param fullpaths: the list of paths of the file being associated with the binary
765 @type session: SQLAlchemy session
766 @param session: Optional SQLAlchemy session. If this is passed, the caller
767 is responsible for ensuring a transaction has begun and committing the
768 results or rolling back based on the result code. If not passed, a commit
769 will be performed at the end of the function, otherwise the caller is
770 responsible for commiting.
772 @return: True upon success
777 session = DBConn().session()
784 def generate_path_dicts():
785 for fullpath in fullpaths:
786 if fullpath.startswith( './' ):
787 fullpath = fullpath[2:]
789 yield {'fulename':fullpath, 'id': binary_id }
791 session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
792 generate_path_dicts() )
800 traceback.print_exc()
802 # Only rollback if we set up the session ourself
809 __all__.append('insert_content_paths')
811 ################################################################################
813 class DSCFile(object):
814 def __init__(self, *args, **kwargs):
818 return '<DSCFile %s>' % self.dscfile_id
820 __all__.append('DSCFile')
823 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
825 Returns a list of DSCFiles which may be empty
827 @type dscfile_id: int (optional)
828 @param dscfile_id: the dscfile_id of the DSCFiles to find
830 @type source_id: int (optional)
831 @param source_id: the source id related to the DSCFiles to find
833 @type poolfile_id: int (optional)
834 @param poolfile_id: the poolfile id related to the DSCFiles to find
837 @return: Possibly empty list of DSCFiles
840 q = session.query(DSCFile)
842 if dscfile_id is not None:
843 q = q.filter_by(dscfile_id=dscfile_id)
845 if source_id is not None:
846 q = q.filter_by(source_id=source_id)
848 if poolfile_id is not None:
849 q = q.filter_by(poolfile_id=poolfile_id)
853 __all__.append('get_dscfiles')
855 ################################################################################
857 class PoolFile(object):
858 def __init__(self, *args, **kwargs):
862 return '<PoolFile %s>' % self.filename
866 return os.path.join(self.location.path, self.filename)
868 __all__.append('PoolFile')
871 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
874 (ValidFileFound [boolean or None], PoolFile object or None)
876 @type filename: string
877 @param filename: the filename of the file to check against the DB
880 @param filesize: the size of the file to check against the DB
883 @param md5sum: the md5sum of the file to check against the DB
885 @type location_id: int
886 @param location_id: the id of the location to look in
889 @return: Tuple of length 2.
890 If more than one file found with that name:
892 If valid pool file found: (True, PoolFile object)
893 If valid pool file not found:
894 (False, None) if no file found
895 (False, PoolFile object) if file found with size/md5sum mismatch
898 q = session.query(PoolFile).filter_by(filename=filename)
899 q = q.join(Location).filter_by(location_id=location_id)
909 if obj.md5sum != md5sum or obj.filesize != int(filesize):
917 __all__.append('check_poolfile')
920 def get_poolfile_by_id(file_id, session=None):
922 Returns a PoolFile objects or None for the given id
925 @param file_id: the id of the file to look for
927 @rtype: PoolFile or None
928 @return: either the PoolFile object or None
931 q = session.query(PoolFile).filter_by(file_id=file_id)
935 except NoResultFound:
938 __all__.append('get_poolfile_by_id')
942 def get_poolfile_by_name(filename, location_id=None, session=None):
944 Returns an array of PoolFile objects for the given filename and
945 (optionally) location_id
947 @type filename: string
948 @param filename: the filename of the file to check against the DB
950 @type location_id: int
951 @param location_id: the id of the location to look in (optional)
954 @return: array of PoolFile objects
957 q = session.query(PoolFile).filter_by(filename=filename)
959 if location_id is not None:
960 q = q.join(Location).filter_by(location_id=location_id)
964 __all__.append('get_poolfile_by_name')
967 def get_poolfile_like_name(filename, session=None):
969 Returns an array of PoolFile objects which are like the given name
971 @type filename: string
972 @param filename: the filename of the file to check against the DB
975 @return: array of PoolFile objects
978 # TODO: There must be a way of properly using bind parameters with %FOO%
979 q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
983 __all__.append('get_poolfile_like_name')
986 def add_poolfile(filename, datadict, location_id, session=None):
988 Add a new file to the pool
990 @type filename: string
991 @param filename: filename
994 @param datadict: dict with needed data
996 @type location_id: int
997 @param location_id: database id of the location
1000 @return: the PoolFile object created
1002 poolfile = PoolFile()
1003 poolfile.filename = filename
1004 poolfile.filesize = datadict["size"]
1005 poolfile.md5sum = datadict["md5sum"]
1006 poolfile.sha1sum = datadict["sha1sum"]
1007 poolfile.sha256sum = datadict["sha256sum"]
1008 poolfile.location_id = location_id
1010 session.add(poolfile)
1011 # Flush to get a file id (NB: This is not a commit)
1016 __all__.append('add_poolfile')
1018 ################################################################################
1020 class Fingerprint(object):
1021 def __init__(self, *args, **kwargs):
1025 return '<Fingerprint %s>' % self.fingerprint
1027 __all__.append('Fingerprint')
1030 def get_fingerprint(fpr, session=None):
1032 Returns Fingerprint object for given fpr.
1035 @param fpr: The fpr to find / add
1037 @type session: SQLAlchemy
1038 @param session: Optional SQL session object (a temporary one will be
1039 generated if not supplied).
1042 @return: the Fingerprint object for the given fpr or None
1045 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1049 except NoResultFound:
1054 __all__.append('get_fingerprint')
1057 def get_or_set_fingerprint(fpr, session=None):
1059 Returns Fingerprint object for given fpr.
1061 If no matching fpr is found, a row is inserted.
1064 @param fpr: The fpr to find / add
1066 @type session: SQLAlchemy
1067 @param session: Optional SQL session object (a temporary one will be
1068 generated if not supplied). If not passed, a commit will be performed at
1069 the end of the function, otherwise the caller is responsible for commiting.
1070 A flush will be performed either way.
1073 @return: the Fingerprint object for the given fpr
1076 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1080 except NoResultFound:
1081 fingerprint = Fingerprint()
1082 fingerprint.fingerprint = fpr
1083 session.add(fingerprint)
1084 session.commit_or_flush()
1089 __all__.append('get_or_set_fingerprint')
1091 ################################################################################
1093 # Helper routine for Keyring class
1094 def get_ldap_name(entry):
1096 for k in ["cn", "mn", "sn"]:
1098 if ret and ret[0] != "" and ret[0] != "-":
1100 return " ".join(name)
1102 ################################################################################
1104 class Keyring(object):
1105 gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1106 " --with-colons --fingerprint --fingerprint"
1111 def __init__(self, *args, **kwargs):
1115 return '<Keyring %s>' % self.keyring_name
1117 def de_escape_gpg_str(self, txt):
1118 esclist = re.split(r'(\\x..)', txt)
1119 for x in range(1,len(esclist),2):
1120 esclist[x] = "%c" % (int(esclist[x][2:],16))
1121 return "".join(esclist)
1123 def load_keys(self, keyring):
1126 if not self.keyring_id:
1127 raise Exception('Must be initialized with database information')
1129 k = os.popen(self.gpg_invocation % keyring, "r")
1133 for line in k.xreadlines():
1134 field = line.split(":")
1135 if field[0] == "pub":
1137 (name, addr) = email.Utils.parseaddr(field[9])
1138 name = re.sub(r"\s*[(].*[)]", "", name)
1139 if name == "" or addr == "" or "@" not in addr:
1141 addr = "invalid-uid"
1142 name = self.de_escape_gpg_str(name)
1143 self.keys[key] = {"email": addr}
1145 self.keys[key]["name"] = name
1146 self.keys[key]["aliases"] = [name]
1147 self.keys[key]["fingerprints"] = []
1149 elif key and field[0] == "sub" and len(field) >= 12:
1150 signingkey = ("s" in field[11])
1151 elif key and field[0] == "uid":
1152 (name, addr) = email.Utils.parseaddr(field[9])
1153 if name and name not in self.keys[key]["aliases"]:
1154 self.keys[key]["aliases"].append(name)
1155 elif signingkey and field[0] == "fpr":
1156 self.keys[key]["fingerprints"].append(field[9])
1157 self.fpr_lookup[field[9]] = key
1159 def import_users_from_ldap(self, session):
1163 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1164 LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1166 l = ldap.open(LDAPServer)
1167 l.simple_bind_s("","")
1168 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1169 "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1170 ["uid", "keyfingerprint", "cn", "mn", "sn"])
1172 ldap_fin_uid_id = {}
1179 uid = entry["uid"][0]
1180 name = get_ldap_name(entry)
1181 fingerprints = entry["keyFingerPrint"]
1183 for f in fingerprints:
1184 key = self.fpr_lookup.get(f, None)
1185 if key not in self.keys:
1187 self.keys[key]["uid"] = uid
1191 keyid = get_or_set_uid(uid, session).uid_id
1192 byuid[keyid] = (uid, name)
1193 byname[uid] = (keyid, name)
1195 return (byname, byuid)
1197 def generate_users_from_keyring(self, format, session):
1201 for x in self.keys.keys():
1202 if self.keys[x]["email"] == "invalid-uid":
1204 self.keys[x]["uid"] = format % "invalid-uid"
1206 uid = format % self.keys[x]["email"]
1207 keyid = get_or_set_uid(uid, session).uid_id
1208 byuid[keyid] = (uid, self.keys[x]["name"])
1209 byname[uid] = (keyid, self.keys[x]["name"])
1210 self.keys[x]["uid"] = uid
1213 uid = format % "invalid-uid"
1214 keyid = get_or_set_uid(uid, session).uid_id
1215 byuid[keyid] = (uid, "ungeneratable user id")
1216 byname[uid] = (keyid, "ungeneratable user id")
1218 return (byname, byuid)
1220 __all__.append('Keyring')
1223 def get_keyring(keyring, session=None):
1225 If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1226 If C{keyring} already has an entry, simply return the existing Keyring
1228 @type keyring: string
1229 @param keyring: the keyring name
1232 @return: the Keyring object for this keyring
1235 q = session.query(Keyring).filter_by(keyring_name=keyring)
1239 except NoResultFound:
1242 __all__.append('get_keyring')
1244 ################################################################################
1246 class KeyringACLMap(object):
1247 def __init__(self, *args, **kwargs):
1251 return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1253 __all__.append('KeyringACLMap')
1255 ################################################################################
1257 class DBChange(object):
1258 def __init__(self, *args, **kwargs):
1262 return '<DBChange %s>' % self.changesname
1264 __all__.append('DBChange')
1267 def get_dbchange(filename, session=None):
1269 returns DBChange object for given C{filename}.
1271 @type archive: string
1272 @param archive: the name of the arhive
1274 @type session: Session
1275 @param session: Optional SQLA session object (a temporary one will be
1276 generated if not supplied)
1279 @return: Archive object for the given name (None if not present)
1282 q = session.query(DBChange).filter_by(changesname=filename)
1286 except NoResultFound:
1289 __all__.append('get_dbchange')
1291 ################################################################################
1293 class Location(object):
1294 def __init__(self, *args, **kwargs):
1298 return '<Location %s (%s)>' % (self.path, self.location_id)
1300 __all__.append('Location')
1303 def get_location(location, component=None, archive=None, session=None):
1305 Returns Location object for the given combination of location, component
1308 @type location: string
1309 @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
1311 @type component: string
1312 @param component: the component name (if None, no restriction applied)
1314 @type archive: string
1315 @param archive_id: the archive name (if None, no restriction applied)
1317 @rtype: Location / None
1318 @return: Either a Location object or None if one can't be found
1321 q = session.query(Location).filter_by(path=location)
1323 if archive is not None:
1324 q = q.join(Archive).filter_by(archive_name=archive)
1326 if component is not None:
1327 q = q.join(Component).filter_by(component_name=component)
1331 except NoResultFound:
1334 __all__.append('get_location')
1336 ################################################################################
1338 class Maintainer(object):
1339 def __init__(self, *args, **kwargs):
1343 return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
1345 def get_split_maintainer(self):
1346 if not hasattr(self, 'name') or self.name is None:
1347 return ('', '', '', '')
1349 return fix_maintainer(self.name.strip())
1351 __all__.append('Maintainer')
1354 def get_or_set_maintainer(name, session=None):
1356 Returns Maintainer object for given maintainer name.
1358 If no matching maintainer name is found, a row is inserted.
1361 @param name: The maintainer name to add
1363 @type session: SQLAlchemy
1364 @param session: Optional SQL session object (a temporary one will be
1365 generated if not supplied). If not passed, a commit will be performed at
1366 the end of the function, otherwise the caller is responsible for commiting.
1367 A flush will be performed either way.
1370 @return: the Maintainer object for the given maintainer
1373 q = session.query(Maintainer).filter_by(name=name)
1376 except NoResultFound:
1377 maintainer = Maintainer()
1378 maintainer.name = name
1379 session.add(maintainer)
1380 session.commit_or_flush()
1385 __all__.append('get_or_set_maintainer')
1388 def get_maintainer(maintainer_id, session=None):
1390 Return the name of the maintainer behind C{maintainer_id} or None if that
1391 maintainer_id is invalid.
1393 @type maintainer_id: int
1394 @param maintainer_id: the id of the maintainer
1397 @return: the Maintainer with this C{maintainer_id}
1400 return session.query(Maintainer).get(maintainer_id)
1402 __all__.append('get_maintainer')
1404 ################################################################################
1406 class NewComment(object):
1407 def __init__(self, *args, **kwargs):
1411 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1413 __all__.append('NewComment')
1416 def has_new_comment(package, version, session=None):
1418 Returns true if the given combination of C{package}, C{version} has a comment.
1420 @type package: string
1421 @param package: name of the package
1423 @type version: string
1424 @param version: package version
1426 @type session: Session
1427 @param session: Optional SQLA session object (a temporary one will be
1428 generated if not supplied)
1434 q = session.query(NewComment)
1435 q = q.filter_by(package=package)
1436 q = q.filter_by(version=version)
1438 return bool(q.count() > 0)
1440 __all__.append('has_new_comment')
1443 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1445 Returns (possibly empty) list of NewComment objects for the given
1448 @type package: string (optional)
1449 @param package: name of the package
1451 @type version: string (optional)
1452 @param version: package version
1454 @type comment_id: int (optional)
1455 @param comment_id: An id of a comment
1457 @type session: Session
1458 @param session: Optional SQLA session object (a temporary one will be
1459 generated if not supplied)
1462 @return: A (possibly empty) list of NewComment objects will be returned
1465 q = session.query(NewComment)
1466 if package is not None: q = q.filter_by(package=package)
1467 if version is not None: q = q.filter_by(version=version)
1468 if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1472 __all__.append('get_new_comments')
1474 ################################################################################
1476 class Override(object):
1477 def __init__(self, *args, **kwargs):
1481 return '<Override %s (%s)>' % (self.package, self.suite_id)
1483 __all__.append('Override')
1486 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1488 Returns Override object for the given parameters
1490 @type package: string
1491 @param package: The name of the package
1493 @type suite: string, list or None
1494 @param suite: The name of the suite (or suites if a list) to limit to. If
1495 None, don't limit. Defaults to None.
1497 @type component: string, list or None
1498 @param component: The name of the component (or components if a list) to
1499 limit to. If None, don't limit. Defaults to None.
1501 @type overridetype: string, list or None
1502 @param overridetype: The name of the overridetype (or overridetypes if a list) to
1503 limit to. If None, don't limit. Defaults to None.
1505 @type session: Session
1506 @param session: Optional SQLA session object (a temporary one will be
1507 generated if not supplied)
1510 @return: A (possibly empty) list of Override objects will be returned
1513 q = session.query(Override)
1514 q = q.filter_by(package=package)
1516 if suite is not None:
1517 if not isinstance(suite, list): suite = [suite]
1518 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1520 if component is not None:
1521 if not isinstance(component, list): component = [component]
1522 q = q.join(Component).filter(Component.component_name.in_(component))
1524 if overridetype is not None:
1525 if not isinstance(overridetype, list): overridetype = [overridetype]
1526 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1530 __all__.append('get_override')
1533 ################################################################################
1535 class OverrideType(object):
1536 def __init__(self, *args, **kwargs):
1540 return '<OverrideType %s>' % self.overridetype
1542 __all__.append('OverrideType')
1545 def get_override_type(override_type, session=None):
1547 Returns OverrideType object for given C{override type}.
1549 @type override_type: string
1550 @param override_type: The name of the override type
1552 @type session: Session
1553 @param session: Optional SQLA session object (a temporary one will be
1554 generated if not supplied)
1557 @return: the database id for the given override type
1560 q = session.query(OverrideType).filter_by(overridetype=override_type)
1564 except NoResultFound:
1567 __all__.append('get_override_type')
1569 ################################################################################
1571 class DebContents(object):
1572 def __init__(self, *args, **kwargs):
1576 return '<DebConetnts %s: %s>' % (self.package.package,self.file)
1578 __all__.append('DebContents')
1581 class UdebContents(object):
1582 def __init__(self, *args, **kwargs):
1586 return '<UdebConetnts %s: %s>' % (self.package.package,self.file)
1588 __all__.append('UdebContents')
1590 class PendingBinContents(object):
1591 def __init__(self, *args, **kwargs):
1595 return '<PendingBinContents %s>' % self.contents_id
1597 __all__.append('PendingBinContents')
1599 def insert_pending_content_paths(package,
1604 Make sure given paths are temporarily associated with given
1608 @param package: the package to associate with should have been read in from the binary control file
1609 @type fullpaths: list
1610 @param fullpaths: the list of paths of the file being associated with the binary
1611 @type session: SQLAlchemy session
1612 @param session: Optional SQLAlchemy session. If this is passed, the caller
1613 is responsible for ensuring a transaction has begun and committing the
1614 results or rolling back based on the result code. If not passed, a commit
1615 will be performed at the end of the function
1617 @return: True upon success, False if there is a problem
1620 privatetrans = False
1623 session = DBConn().session()
1627 arch = get_architecture(package['Architecture'], session)
1628 arch_id = arch.arch_id
1630 # Remove any already existing recorded files for this package
1631 q = session.query(PendingBinContents)
1632 q = q.filter_by(package=package['Package'])
1633 q = q.filter_by(version=package['Version'])
1634 q = q.filter_by(architecture=arch_id)
1637 for fullpath in fullpaths:
1639 if fullpath.startswith( "./" ):
1640 fullpath = fullpath[2:]
1642 pca = PendingBinContents()
1643 pca.package = package['Package']
1644 pca.version = package['Version']
1646 pca.architecture = arch_id
1649 pca.type = 8 # gross
1651 pca.type = 7 # also gross
1654 # Only commit if we set up the session ourself
1662 except Exception, e:
1663 traceback.print_exc()
1665 # Only rollback if we set up the session ourself
1672 __all__.append('insert_pending_content_paths')
1674 ################################################################################
1676 class PolicyQueue(object):
1677 def __init__(self, *args, **kwargs):
1681 return '<PolicyQueue %s>' % self.queue_name
1683 __all__.append('PolicyQueue')
1686 def get_policy_queue(queuename, session=None):
1688 Returns PolicyQueue object for given C{queue name}
1690 @type queuename: string
1691 @param queuename: The name of the queue
1693 @type session: Session
1694 @param session: Optional SQLA session object (a temporary one will be
1695 generated if not supplied)
1698 @return: PolicyQueue object for the given queue
1701 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
1705 except NoResultFound:
1708 __all__.append('get_policy_queue')
1710 ################################################################################
1712 class Priority(object):
1713 def __init__(self, *args, **kwargs):
1716 def __eq__(self, val):
1717 if isinstance(val, str):
1718 return (self.priority == val)
1719 # This signals to use the normal comparison operator
1720 return NotImplemented
1722 def __ne__(self, val):
1723 if isinstance(val, str):
1724 return (self.priority != val)
1725 # This signals to use the normal comparison operator
1726 return NotImplemented
1729 return '<Priority %s (%s)>' % (self.priority, self.priority_id)
1731 __all__.append('Priority')
1734 def get_priority(priority, session=None):
1736 Returns Priority object for given C{priority name}.
1738 @type priority: string
1739 @param priority: The name of the priority
1741 @type session: Session
1742 @param session: Optional SQLA session object (a temporary one will be
1743 generated if not supplied)
1746 @return: Priority object for the given priority
1749 q = session.query(Priority).filter_by(priority=priority)
1753 except NoResultFound:
1756 __all__.append('get_priority')
1759 def get_priorities(session=None):
1761 Returns dictionary of priority names -> id mappings
1763 @type session: Session
1764 @param session: Optional SQL session object (a temporary one will be
1765 generated if not supplied)
1768 @return: dictionary of priority names -> id mappings
1772 q = session.query(Priority)
1774 ret[x.priority] = x.priority_id
1778 __all__.append('get_priorities')
1780 ################################################################################
1782 class Section(object):
1783 def __init__(self, *args, **kwargs):
1786 def __eq__(self, val):
1787 if isinstance(val, str):
1788 return (self.section == val)
1789 # This signals to use the normal comparison operator
1790 return NotImplemented
1792 def __ne__(self, val):
1793 if isinstance(val, str):
1794 return (self.section != val)
1795 # This signals to use the normal comparison operator
1796 return NotImplemented
1799 return '<Section %s>' % self.section
1801 __all__.append('Section')
1804 def get_section(section, session=None):
1806 Returns Section object for given C{section name}.
1808 @type section: string
1809 @param section: The name of the section
1811 @type session: Session
1812 @param session: Optional SQLA session object (a temporary one will be
1813 generated if not supplied)
1816 @return: Section object for the given section name
1819 q = session.query(Section).filter_by(section=section)
1823 except NoResultFound:
1826 __all__.append('get_section')
1829 def get_sections(session=None):
1831 Returns dictionary of section names -> id mappings
1833 @type session: Session
1834 @param session: Optional SQL session object (a temporary one will be
1835 generated if not supplied)
1838 @return: dictionary of section names -> id mappings
1842 q = session.query(Section)
1844 ret[x.section] = x.section_id
1848 __all__.append('get_sections')
1850 ################################################################################
1852 class DBSource(object):
1853 def __init__(self, *args, **kwargs):
1857 return '<DBSource %s (%s)>' % (self.source, self.version)
1859 __all__.append('DBSource')
1862 def source_exists(source, source_version, suites = ["any"], session=None):
1864 Ensure that source exists somewhere in the archive for the binary
1865 upload being processed.
1866 1. exact match => 1.0-3
1867 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
1869 @type package: string
1870 @param package: package source name
1872 @type source_version: string
1873 @param source_version: expected source version
1876 @param suites: list of suites to check in, default I{any}
1878 @type session: Session
1879 @param session: Optional SQLA session object (a temporary one will be
1880 generated if not supplied)
1883 @return: returns 1 if a source with expected version is found, otherwise 0
1890 for suite in suites:
1891 q = session.query(DBSource).filter_by(source=source)
1893 # source must exist in suite X, or in some other suite that's
1894 # mapped to X, recursively... silent-maps are counted too,
1895 # unreleased-maps aren't.
1896 maps = cnf.ValueList("SuiteMappings")[:]
1898 maps = [ m.split() for m in maps ]
1899 maps = [ (x[1], x[2]) for x in maps
1900 if x[0] == "map" or x[0] == "silent-map" ]
1903 if x[1] in s and x[0] not in s:
1906 q = q.join(SrcAssociation).join(Suite)
1907 q = q.filter(Suite.suite_name.in_(s))
1909 # Reduce the query results to a list of version numbers
1910 ql = [ j.version for j in q.all() ]
1913 if source_version in ql:
1917 from daklib.regexes import re_bin_only_nmu
1918 orig_source_version = re_bin_only_nmu.sub('', source_version)
1919 if orig_source_version in ql:
1922 # No source found so return not ok
1927 __all__.append('source_exists')
1930 def get_suites_source_in(source, session=None):
1932 Returns list of Suite objects which given C{source} name is in
1935 @param source: DBSource package name to search for
1938 @return: list of Suite objects for the given source
1941 return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
1943 __all__.append('get_suites_source_in')
1946 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
1948 Returns list of DBSource objects for given C{source} name and other parameters
1951 @param source: DBSource package name to search for
1953 @type source: str or None
1954 @param source: DBSource version name to search for or None if not applicable
1956 @type dm_upload_allowed: bool
1957 @param dm_upload_allowed: If None, no effect. If True or False, only
1958 return packages with that dm_upload_allowed setting
1960 @type session: Session
1961 @param session: Optional SQL session object (a temporary one will be
1962 generated if not supplied)
1965 @return: list of DBSource objects for the given name (may be empty)
1968 q = session.query(DBSource).filter_by(source=source)
1970 if version is not None:
1971 q = q.filter_by(version=version)
1973 if dm_upload_allowed is not None:
1974 q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
1978 __all__.append('get_sources_from_name')
1981 def get_source_in_suite(source, suite, session=None):
1983 Returns list of DBSource objects for a combination of C{source} and C{suite}.
1985 - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
1986 - B{suite} - a suite name, eg. I{unstable}
1988 @type source: string
1989 @param source: source package name
1992 @param suite: the suite name
1995 @return: the version for I{source} in I{suite}
1999 q = session.query(SrcAssociation)
2000 q = q.join('source').filter_by(source=source)
2001 q = q.join('suite').filter_by(suite_name=suite)
2004 return q.one().source
2005 except NoResultFound:
2008 __all__.append('get_source_in_suite')
2010 ################################################################################
2013 def add_dsc_to_db(u, filename, session=None):
2014 entry = u.pkg.files[filename]
2018 source.source = u.pkg.dsc["source"]
2019 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2020 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2021 source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2022 source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2023 source.install_date = datetime.now().date()
2025 dsc_component = entry["component"]
2026 dsc_location_id = entry["location id"]
2028 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2030 # Set up a new poolfile if necessary
2031 if not entry.has_key("files id") or not entry["files id"]:
2032 filename = entry["pool name"] + filename
2033 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2035 pfs.append(poolfile)
2036 entry["files id"] = poolfile.file_id
2038 source.poolfile_id = entry["files id"]
2042 for suite_name in u.pkg.changes["distribution"].keys():
2043 sa = SrcAssociation()
2044 sa.source_id = source.source_id
2045 sa.suite_id = get_suite(suite_name).suite_id
2050 # Add the source files to the DB (files and dsc_files)
2052 dscfile.source_id = source.source_id
2053 dscfile.poolfile_id = entry["files id"]
2054 session.add(dscfile)
2056 for dsc_file, dentry in u.pkg.dsc_files.items():
2058 df.source_id = source.source_id
2060 # If the .orig tarball is already in the pool, it's
2061 # files id is stored in dsc_files by check_dsc().
2062 files_id = dentry.get("files id", None)
2064 # Find the entry in the files hash
2065 # TODO: Bail out here properly
2067 for f, e in u.pkg.files.items():
2072 if files_id is None:
2073 filename = dfentry["pool name"] + dsc_file
2075 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2076 # FIXME: needs to check for -1/-2 and or handle exception
2077 if found and obj is not None:
2078 files_id = obj.file_id
2081 # If still not found, add it
2082 if files_id is None:
2083 # HACK: Force sha1sum etc into dentry
2084 dentry["sha1sum"] = dfentry["sha1sum"]
2085 dentry["sha256sum"] = dfentry["sha256sum"]
2086 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2087 pfs.append(poolfile)
2088 files_id = poolfile.file_id
2090 poolfile = get_poolfile_by_id(files_id, session)
2091 if poolfile is None:
2092 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2093 pfs.append(poolfile)
2095 df.poolfile_id = files_id
2100 # Add the src_uploaders to the DB
2101 uploader_ids = [source.maintainer_id]
2102 if u.pkg.dsc.has_key("uploaders"):
2103 for up in u.pkg.dsc["uploaders"].split(","):
2105 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2108 for up in uploader_ids:
2109 if added_ids.has_key(up):
2110 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
2116 su.maintainer_id = up
2117 su.source_id = source.source_id
2122 return dsc_component, dsc_location_id, pfs
2124 __all__.append('add_dsc_to_db')
2127 def add_deb_to_db(u, filename, session=None):
2129 Contrary to what you might expect, this routine deals with both
2130 debs and udebs. That info is in 'dbtype', whilst 'type' is
2131 'deb' for both of them
2134 entry = u.pkg.files[filename]
2137 bin.package = entry["package"]
2138 bin.version = entry["version"]
2139 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2140 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2141 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2142 bin.binarytype = entry["dbtype"]
2145 filename = entry["pool name"] + filename
2146 fullpath = os.path.join(cnf["Dir::Pool"], filename)
2147 if not entry.get("location id", None):
2148 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2150 if entry.get("files id", None):
2151 poolfile = get_poolfile_by_id(bin.poolfile_id)
2152 bin.poolfile_id = entry["files id"]
2154 poolfile = add_poolfile(filename, entry, entry["location id"], session)
2155 bin.poolfile_id = entry["files id"] = poolfile.file_id
2158 bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2159 if len(bin_sources) != 1:
2160 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2161 (bin.package, bin.version, bin.architecture.arch_string,
2162 filename, bin.binarytype, u.pkg.changes["fingerprint"])
2164 bin.source_id = bin_sources[0].source_id
2166 # Add and flush object so it has an ID
2170 # Add BinAssociations
2171 for suite_name in u.pkg.changes["distribution"].keys():
2172 ba = BinAssociation()
2173 ba.binary_id = bin.binary_id
2174 ba.suite_id = get_suite(suite_name).suite_id
2179 # Deal with contents - disabled for now
2180 #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2182 # print "REJECT\nCould not determine contents of package %s" % bin.package
2183 # session.rollback()
2184 # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2188 __all__.append('add_deb_to_db')
2190 ################################################################################
2192 class SourceACL(object):
2193 def __init__(self, *args, **kwargs):
2197 return '<SourceACL %s>' % self.source_acl_id
2199 __all__.append('SourceACL')
2201 ################################################################################
2203 class SrcAssociation(object):
2204 def __init__(self, *args, **kwargs):
2208 return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
2210 __all__.append('SrcAssociation')
2212 ################################################################################
2214 class SrcFormat(object):
2215 def __init__(self, *args, **kwargs):
2219 return '<SrcFormat %s>' % (self.format_name)
2221 __all__.append('SrcFormat')
2223 ################################################################################
2225 class SrcUploader(object):
2226 def __init__(self, *args, **kwargs):
2230 return '<SrcUploader %s>' % self.uploader_id
2232 __all__.append('SrcUploader')
2234 ################################################################################
2236 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2237 ('SuiteID', 'suite_id'),
2238 ('Version', 'version'),
2239 ('Origin', 'origin'),
2241 ('Description', 'description'),
2242 ('Untouchable', 'untouchable'),
2243 ('Announce', 'announce'),
2244 ('Codename', 'codename'),
2245 ('OverrideCodename', 'overridecodename'),
2246 ('ValidTime', 'validtime'),
2247 ('Priority', 'priority'),
2248 ('NotAutomatic', 'notautomatic'),
2249 ('CopyChanges', 'copychanges'),
2250 ('CopyDotDak', 'copydotdak'),
2251 ('CommentsDir', 'commentsdir'),
2252 ('OverrideSuite', 'overridesuite'),
2253 ('ChangelogBase', 'changelogbase')]
2256 class Suite(object):
2257 def __init__(self, *args, **kwargs):
2261 return '<Suite %s>' % self.suite_name
2263 def __eq__(self, val):
2264 if isinstance(val, str):
2265 return (self.suite_name == val)
2266 # This signals to use the normal comparison operator
2267 return NotImplemented
2269 def __ne__(self, val):
2270 if isinstance(val, str):
2271 return (self.suite_name != val)
2272 # This signals to use the normal comparison operator
2273 return NotImplemented
2277 for disp, field in SUITE_FIELDS:
2278 val = getattr(self, field, None)
2280 ret.append("%s: %s" % (disp, val))
2282 return "\n".join(ret)
2284 __all__.append('Suite')
2287 def get_suite_architecture(suite, architecture, session=None):
2289 Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
2293 @param suite: Suite name to search for
2295 @type architecture: str
2296 @param architecture: Architecture name to search for
2298 @type session: Session
2299 @param session: Optional SQL session object (a temporary one will be
2300 generated if not supplied)
2302 @rtype: SuiteArchitecture
2303 @return: the SuiteArchitecture object or None
2306 q = session.query(SuiteArchitecture)
2307 q = q.join(Architecture).filter_by(arch_string=architecture)
2308 q = q.join(Suite).filter_by(suite_name=suite)
2312 except NoResultFound:
2315 __all__.append('get_suite_architecture')
2318 def get_suite(suite, session=None):
2320 Returns Suite object for given C{suite name}.
2323 @param suite: The name of the suite
2325 @type session: Session
2326 @param session: Optional SQLA session object (a temporary one will be
2327 generated if not supplied)
2330 @return: Suite object for the requested suite name (None if not present)
2333 q = session.query(Suite).filter_by(suite_name=suite)
2337 except NoResultFound:
2340 __all__.append('get_suite')
2342 ################################################################################
2344 class SuiteArchitecture(object):
2345 def __init__(self, *args, **kwargs):
2349 return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
2351 __all__.append('SuiteArchitecture')
2354 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2356 Returns list of Architecture objects for given C{suite} name
2359 @param source: Suite name to search for
2361 @type skipsrc: boolean
2362 @param skipsrc: Whether to skip returning the 'source' architecture entry
2365 @type skipall: boolean
2366 @param skipall: Whether to skip returning the 'all' architecture entry
2369 @type session: Session
2370 @param session: Optional SQL session object (a temporary one will be
2371 generated if not supplied)
2374 @return: list of Architecture objects for the given name (may be empty)
2377 q = session.query(Architecture)
2378 q = q.join(SuiteArchitecture)
2379 q = q.join(Suite).filter_by(suite_name=suite)
2382 q = q.filter(Architecture.arch_string != 'source')
2385 q = q.filter(Architecture.arch_string != 'all')
2387 q = q.order_by('arch_string')
2391 __all__.append('get_suite_architectures')
2393 ################################################################################
2395 class SuiteSrcFormat(object):
2396 def __init__(self, *args, **kwargs):
2400 return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2402 __all__.append('SuiteSrcFormat')
2405 def get_suite_src_formats(suite, session=None):
2407 Returns list of allowed SrcFormat for C{suite}.
2410 @param suite: Suite name to search for
2412 @type session: Session
2413 @param session: Optional SQL session object (a temporary one will be
2414 generated if not supplied)
2417 @return: the list of allowed source formats for I{suite}
2420 q = session.query(SrcFormat)
2421 q = q.join(SuiteSrcFormat)
2422 q = q.join(Suite).filter_by(suite_name=suite)
2423 q = q.order_by('format_name')
2427 __all__.append('get_suite_src_formats')
2429 ################################################################################
2432 def __init__(self, *args, **kwargs):
2435 def __eq__(self, val):
2436 if isinstance(val, str):
2437 return (self.uid == val)
2438 # This signals to use the normal comparison operator
2439 return NotImplemented
2441 def __ne__(self, val):
2442 if isinstance(val, str):
2443 return (self.uid != val)
2444 # This signals to use the normal comparison operator
2445 return NotImplemented
2448 return '<Uid %s (%s)>' % (self.uid, self.name)
2450 __all__.append('Uid')
2453 def add_database_user(uidname, session=None):
2455 Adds a database user
2457 @type uidname: string
2458 @param uidname: The uid of the user to add
2460 @type session: SQLAlchemy
2461 @param session: Optional SQL session object (a temporary one will be
2462 generated if not supplied). If not passed, a commit will be performed at
2463 the end of the function, otherwise the caller is responsible for commiting.
2466 @return: the uid object for the given uidname
2469 session.execute("CREATE USER :uid", {'uid': uidname})
2470 session.commit_or_flush()
2472 __all__.append('add_database_user')
2475 def get_or_set_uid(uidname, session=None):
2477 Returns uid object for given uidname.
2479 If no matching uidname is found, a row is inserted.
2481 @type uidname: string
2482 @param uidname: The uid to add
2484 @type session: SQLAlchemy
2485 @param session: Optional SQL session object (a temporary one will be
2486 generated if not supplied). If not passed, a commit will be performed at
2487 the end of the function, otherwise the caller is responsible for commiting.
2490 @return: the uid object for the given uidname
2493 q = session.query(Uid).filter_by(uid=uidname)
2497 except NoResultFound:
2501 session.commit_or_flush()
2506 __all__.append('get_or_set_uid')
2509 def get_uid_from_fingerprint(fpr, session=None):
2510 q = session.query(Uid)
2511 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2515 except NoResultFound:
2518 __all__.append('get_uid_from_fingerprint')
2520 ################################################################################
2522 class UploadBlock(object):
2523 def __init__(self, *args, **kwargs):
2527 return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2529 __all__.append('UploadBlock')
2531 ################################################################################
2533 class DBConn(object):
2535 database module init.
2539 def __init__(self, *args, **kwargs):
2540 self.__dict__ = self.__shared_state
2542 if not getattr(self, 'initialised', False):
2543 self.initialised = True
2544 self.debug = kwargs.has_key('debug')
2547 def __setuptables(self):
2557 'build_queue_files',
2560 'changes_pending_binaries',
2561 'changes_pending_files',
2562 'changes_pending_files_map',
2563 'changes_pending_source',
2564 'changes_pending_source_files',
2565 'changes_pool_files',
2578 'pending_bin_contents',
2588 'suite_architectures',
2589 'suite_src_formats',
2590 'suite_build_queue_copy',
2596 for table_name in tables:
2597 table = Table(table_name, self.db_meta, autoload=True)
2598 setattr(self, 'tbl_%s' % table_name, table)
2600 def __setupmappers(self):
2601 mapper(Architecture, self.tbl_architecture,
2602 properties = dict(arch_id = self.tbl_architecture.c.id))
2604 mapper(Archive, self.tbl_archive,
2605 properties = dict(archive_id = self.tbl_archive.c.id,
2606 archive_name = self.tbl_archive.c.name))
2608 mapper(BinAssociation, self.tbl_bin_associations,
2609 properties = dict(ba_id = self.tbl_bin_associations.c.id,
2610 suite_id = self.tbl_bin_associations.c.suite,
2611 suite = relation(Suite),
2612 binary_id = self.tbl_bin_associations.c.bin,
2613 binary = relation(DBBinary)))
2615 mapper(PendingBinContents, self.tbl_pending_bin_contents,
2616 properties = dict(contents_id =self.tbl_pending_bin_contents.c.id,
2617 filename = self.tbl_pending_bin_contents.c.filename,
2618 package = self.tbl_pending_bin_contents.c.package,
2619 version = self.tbl_pending_bin_contents.c.version,
2620 arch = self.tbl_pending_bin_contents.c.arch,
2621 otype = self.tbl_pending_bin_contents.c.type))
2623 mapper(DebContents, self.tbl_deb_contents,
2624 properties = dict(binary_id=self.tbl_deb_contents.c.binary_id,
2625 package=self.tbl_deb_contents.c.package,
2626 component=self.tbl_deb_contents.c.component,
2627 arch=self.tbl_deb_contents.c.arch,
2628 section=self.tbl_deb_contents.c.section,
2629 filename=self.tbl_deb_contents.c.filename))
2631 mapper(UdebContents, self.tbl_udeb_contents,
2632 properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id,
2633 package=self.tbl_udeb_contents.c.package,
2634 component=self.tbl_udeb_contents.c.component,
2635 arch=self.tbl_udeb_contents.c.arch,
2636 section=self.tbl_udeb_contents.c.section,
2637 filename=self.tbl_udeb_contents.c.filename))
2639 mapper(DBBinary, self.tbl_binaries,
2640 properties = dict(binary_id = self.tbl_binaries.c.id,
2641 package = self.tbl_binaries.c.package,
2642 version = self.tbl_binaries.c.version,
2643 maintainer_id = self.tbl_binaries.c.maintainer,
2644 maintainer = relation(Maintainer),
2645 source_id = self.tbl_binaries.c.source,
2646 source = relation(DBSource),
2647 arch_id = self.tbl_binaries.c.architecture,
2648 architecture = relation(Architecture),
2649 poolfile_id = self.tbl_binaries.c.file,
2650 poolfile = relation(PoolFile),
2651 binarytype = self.tbl_binaries.c.type,
2652 fingerprint_id = self.tbl_binaries.c.sig_fpr,
2653 fingerprint = relation(Fingerprint),
2654 install_date = self.tbl_binaries.c.install_date,
2655 binassociations = relation(BinAssociation,
2656 primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
2658 mapper(BinaryACL, self.tbl_binary_acl,
2659 properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
2661 mapper(BinaryACLMap, self.tbl_binary_acl_map,
2662 properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
2663 fingerprint = relation(Fingerprint, backref="binary_acl_map"),
2664 architecture = relation(Architecture)))
2666 mapper(Component, self.tbl_component,
2667 properties = dict(component_id = self.tbl_component.c.id,
2668 component_name = self.tbl_component.c.name))
2670 mapper(DBConfig, self.tbl_config,
2671 properties = dict(config_id = self.tbl_config.c.id))
2673 mapper(DSCFile, self.tbl_dsc_files,
2674 properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
2675 source_id = self.tbl_dsc_files.c.source,
2676 source = relation(DBSource),
2677 poolfile_id = self.tbl_dsc_files.c.file,
2678 poolfile = relation(PoolFile)))
2680 mapper(PoolFile, self.tbl_files,
2681 properties = dict(file_id = self.tbl_files.c.id,
2682 filesize = self.tbl_files.c.size,
2683 location_id = self.tbl_files.c.location,
2684 location = relation(Location)))
2686 mapper(Fingerprint, self.tbl_fingerprint,
2687 properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
2688 uid_id = self.tbl_fingerprint.c.uid,
2689 uid = relation(Uid),
2690 keyring_id = self.tbl_fingerprint.c.keyring,
2691 keyring = relation(Keyring),
2692 source_acl = relation(SourceACL),
2693 binary_acl = relation(BinaryACL)))
2695 mapper(Keyring, self.tbl_keyrings,
2696 properties = dict(keyring_name = self.tbl_keyrings.c.name,
2697 keyring_id = self.tbl_keyrings.c.id))
2699 mapper(DBChange, self.tbl_changes,
2700 properties = dict(change_id = self.tbl_changes.c.id,
2701 poolfiles = relation(PoolFile,
2702 secondary=self.tbl_changes_pool_files,
2703 backref="changeslinks"),
2704 files = relation(ChangePendingFile,
2705 secondary=self.tbl_changes_pending_files_map,
2706 backref="changesfile"),
2707 in_queue_id = self.tbl_changes.c.in_queue,
2708 in_queue = relation(PolicyQueue,
2709 primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
2710 approved_for_id = self.tbl_changes.c.approved_for))
2712 mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
2713 properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
2715 mapper(ChangePendingFile, self.tbl_changes_pending_files,
2716 properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id))
2718 mapper(ChangePendingSource, self.tbl_changes_pending_source,
2719 properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
2720 change = relation(DBChange),
2721 maintainer = relation(Maintainer,
2722 primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
2723 changedby = relation(Maintainer,
2724 primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
2725 fingerprint = relation(Fingerprint),
2726 source_files = relation(ChangePendingFile,
2727 secondary=self.tbl_changes_pending_source_files,
2728 backref="pending_sources")))
2729 files = relation(KnownChangePendingFile, backref="changesfile")))
2731 mapper(KnownChangePendingFile, self.tbl_changes_pending_files,
2732 properties = dict(known_change_pending_file_id = self.tbl_changes_pending_files.c.id))
2734 mapper(KeyringACLMap, self.tbl_keyring_acl_map,
2735 properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
2736 keyring = relation(Keyring, backref="keyring_acl_map"),
2737 architecture = relation(Architecture)))
2739 mapper(Location, self.tbl_location,
2740 properties = dict(location_id = self.tbl_location.c.id,
2741 component_id = self.tbl_location.c.component,
2742 component = relation(Component),
2743 archive_id = self.tbl_location.c.archive,
2744 archive = relation(Archive),
2745 archive_type = self.tbl_location.c.type))
2747 mapper(Maintainer, self.tbl_maintainer,
2748 properties = dict(maintainer_id = self.tbl_maintainer.c.id))
2750 mapper(NewComment, self.tbl_new_comments,
2751 properties = dict(comment_id = self.tbl_new_comments.c.id))
2753 mapper(Override, self.tbl_override,
2754 properties = dict(suite_id = self.tbl_override.c.suite,
2755 suite = relation(Suite),
2756 package = self.tbl_override.c.package,
2757 component_id = self.tbl_override.c.component,
2758 component = relation(Component),
2759 priority_id = self.tbl_override.c.priority,
2760 priority = relation(Priority),
2761 section_id = self.tbl_override.c.section,
2762 section = relation(Section),
2763 overridetype_id = self.tbl_override.c.type,
2764 overridetype = relation(OverrideType)))
2766 mapper(OverrideType, self.tbl_override_type,
2767 properties = dict(overridetype = self.tbl_override_type.c.type,
2768 overridetype_id = self.tbl_override_type.c.id))
2770 mapper(PolicyQueue, self.tbl_policy_queue,
2771 properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
2773 mapper(Priority, self.tbl_priority,
2774 properties = dict(priority_id = self.tbl_priority.c.id))
2776 mapper(Section, self.tbl_section,
2777 properties = dict(section_id = self.tbl_section.c.id,
2778 section=self.tbl_section.c.section))
2780 mapper(DBSource, self.tbl_source,
2781 properties = dict(source_id = self.tbl_source.c.id,
2782 version = self.tbl_source.c.version,
2783 maintainer_id = self.tbl_source.c.maintainer,
2784 maintainer = relation(Maintainer,
2785 primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
2786 poolfile_id = self.tbl_source.c.file,
2787 poolfile = relation(PoolFile),
2788 fingerprint_id = self.tbl_source.c.sig_fpr,
2789 fingerprint = relation(Fingerprint),
2790 changedby_id = self.tbl_source.c.changedby,
2791 changedby = relation(Maintainer,
2792 primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
2793 srcfiles = relation(DSCFile,
2794 primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
2795 srcassociations = relation(SrcAssociation,
2796 primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)),
2797 srcuploaders = relation(SrcUploader)))
2799 mapper(SourceACL, self.tbl_source_acl,
2800 properties = dict(source_acl_id = self.tbl_source_acl.c.id))
2802 mapper(SrcAssociation, self.tbl_src_associations,
2803 properties = dict(sa_id = self.tbl_src_associations.c.id,
2804 suite_id = self.tbl_src_associations.c.suite,
2805 suite = relation(Suite),
2806 source_id = self.tbl_src_associations.c.source,
2807 source = relation(DBSource)))
2809 mapper(SrcFormat, self.tbl_src_format,
2810 properties = dict(src_format_id = self.tbl_src_format.c.id,
2811 format_name = self.tbl_src_format.c.format_name))
2813 mapper(SrcUploader, self.tbl_src_uploaders,
2814 properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
2815 source_id = self.tbl_src_uploaders.c.source,
2816 source = relation(DBSource,
2817 primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
2818 maintainer_id = self.tbl_src_uploaders.c.maintainer,
2819 maintainer = relation(Maintainer,
2820 primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
2822 mapper(Suite, self.tbl_suite,
2823 properties = dict(suite_id = self.tbl_suite.c.id,
2824 policy_queue = relation(PolicyQueue),
2825 copy_queues = relation(BuildQueue, secondary=self.tbl_suite_build_queue_copy)))
2827 mapper(SuiteArchitecture, self.tbl_suite_architectures,
2828 properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
2829 suite = relation(Suite, backref='suitearchitectures'),
2830 arch_id = self.tbl_suite_architectures.c.architecture,
2831 architecture = relation(Architecture)))
2833 mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
2834 properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
2835 suite = relation(Suite, backref='suitesrcformats'),
2836 src_format_id = self.tbl_suite_src_formats.c.src_format,
2837 src_format = relation(SrcFormat)))
2839 mapper(Uid, self.tbl_uid,
2840 properties = dict(uid_id = self.tbl_uid.c.id,
2841 fingerprint = relation(Fingerprint)))
2843 mapper(UploadBlock, self.tbl_upload_blocks,
2844 properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
2845 fingerprint = relation(Fingerprint, backref="uploadblocks"),
2846 uid = relation(Uid, backref="uploadblocks")))
2848 ## Connection functions
2849 def __createconn(self):
2850 from config import Config
2854 connstr = "postgres://%s" % cnf["DB::Host"]
2855 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2856 connstr += ":%s" % cnf["DB::Port"]
2857 connstr += "/%s" % cnf["DB::Name"]
2860 connstr = "postgres:///%s" % cnf["DB::Name"]
2861 if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
2862 connstr += "?port=%s" % cnf["DB::Port"]
2864 self.db_pg = create_engine(connstr, echo=self.debug)
2865 self.db_meta = MetaData()
2866 self.db_meta.bind = self.db_pg
2867 self.db_smaker = sessionmaker(bind=self.db_pg,
2871 self.__setuptables()
2872 self.__setupmappers()
2875 return self.db_smaker()
2877 __all__.append('DBConn')