daklib/dbconn.py

   1 #!/usr/bin/python
   2
   3 """ DB access class
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2003, 2004, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2008-2009  Mark Hymers <mhy@debian.org>
   8 @copyright: 2009, 2010  Joerg Jaspert <joerg@debian.org>
   9 @copyright: 2009  Mike O'Connor <stew@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ################################################################################
  28
  29 # < mhy> I need a funny comment
  30 # < sgran> two peanuts were walking down a dark street
  31 # < sgran> one was a-salted
  32 #  * mhy looks up the definition of "funny"
  33
  34 ################################################################################
  35
  36 import os
  37 import re
  38 import psycopg2
  39 import traceback
  40 import commands
  41 from datetime import datetime, timedelta
  42 from errno import ENOENT
  43 from tempfile import mkstemp, mkdtemp
  44
  45 from inspect import getargspec
  46
  47 import sqlalchemy
  48 from sqlalchemy import create_engine, Table, MetaData
  49 from sqlalchemy.orm import sessionmaker, mapper, relation
  50 from sqlalchemy import types as sqltypes
  51
  52 # Don't remove this, we re-export the exceptions to scripts which import us
  53 from sqlalchemy.exc import *
  54 from sqlalchemy.orm.exc import NoResultFound
  55
  56 # Only import Config until Queue stuff is changed to store its config
  57 # in the database
  58 from config import Config
  59 from textutils import fix_maintainer
  60
  61 ################################################################################
  62
  63 # Patch in support for the debversion field type so that it works during
  64 # reflection
  65
  66 class DebVersion(sqltypes.Text):
  67     """
  68     Support the debversion type
  69     """
  70
  71     def get_col_spec(self):
  72         return "DEBVERSION"
  73
  74 sa_major_version = sqlalchemy.__version__[0:3]
  75 if sa_major_version == "0.5":
  76     from sqlalchemy.databases import postgres
  77     postgres.ischema_names['debversion'] = DebVersion
  78 else:
  79     raise Exception("dak isn't ported to SQLA versions != 0.5 yet.  See daklib/dbconn.py")
  80
  81 ################################################################################
  82
  83 __all__ = ['IntegrityError', 'SQLAlchemyError']
  84
  85 ################################################################################
  86
  87 def session_wrapper(fn):
  88     """
  89     Wrapper around common ".., session=None):" handling. If the wrapped
  90     function is called without passing 'session', we create a local one
  91     and destroy it when the function ends.
  92
  93     Also attaches a commit_or_flush method to the session; if we created a
  94     local session, this is a synonym for session.commit(), otherwise it is a
  95     synonym for session.flush().
  96     """
  97
  98     def wrapped(*args, **kwargs):
  99         private_transaction = False
 100
 101         # Find the session object
 102         session = kwargs.get('session')
 103
 104         if session is None:
 105             if len(args) <= len(getargspec(fn)[0]) - 1:
 106                 # No session specified as last argument or in kwargs
 107                 private_transaction = True
 108                 session = kwargs['session'] = DBConn().session()
 109             else:
 110                 # Session is last argument in args
 111                 session = args[-1]
 112                 if session is None:
 113                     args = list(args)
 114                     session = args[-1] = DBConn().session()
 115                     private_transaction = True
 116
 117         if private_transaction:
 118             session.commit_or_flush = session.commit
 119         else:
 120             session.commit_or_flush = session.flush
 121
 122         try:
 123             return fn(*args, **kwargs)
 124         finally:
 125             if private_transaction:
 126                 # We created a session; close it.
 127                 session.close()
 128
 129     wrapped.__doc__ = fn.__doc__
 130     wrapped.func_name = fn.func_name
 131
 132     return wrapped
 133
 134 __all__.append('session_wrapper')
 135
 136 ################################################################################
 137
 138 class Architecture(object):
 139     def __init__(self, *args, **kwargs):
 140         pass
 141
 142     def __eq__(self, val):
 143         if isinstance(val, str):
 144             return (self.arch_string== val)
 145         # This signals to use the normal comparison operator
 146         return NotImplemented
 147
 148     def __ne__(self, val):
 149         if isinstance(val, str):
 150             return (self.arch_string != val)
 151         # This signals to use the normal comparison operator
 152         return NotImplemented
 153
 154     def __repr__(self):
 155         return '<Architecture %s>' % self.arch_string
 156
 157 __all__.append('Architecture')
 158
 159 @session_wrapper
 160 def get_architecture(architecture, session=None):
 161     """
 162     Returns database id for given C{architecture}.
 163
 164     @type architecture: string
 165     @param architecture: The name of the architecture
 166
 167     @type session: Session
 168     @param session: Optional SQLA session object (a temporary one will be
 169     generated if not supplied)
 170
 171     @rtype: Architecture
 172     @return: Architecture object for the given arch (None if not present)
 173     """
 174
 175     q = session.query(Architecture).filter_by(arch_string=architecture)
 176
 177     try:
 178         return q.one()
 179     except NoResultFound:
 180         return None
 181
 182 __all__.append('get_architecture')
 183
 184 @session_wrapper
 185 def get_architecture_suites(architecture, session=None):
 186     """
 187     Returns list of Suite objects for given C{architecture} name
 188
 189     @type architecture: str
 190     @param architecture: Architecture name to search for
 191
 192     @type session: Session
 193     @param session: Optional SQL session object (a temporary one will be
 194     generated if not supplied)
 195
 196     @rtype: list
 197     @return: list of Suite objects for the given name (may be empty)
 198     """
 199
 200     q = session.query(Suite)
 201     q = q.join(SuiteArchitecture)
 202     q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
 203
 204     ret = q.all()
 205
 206     return ret
 207
 208 __all__.append('get_architecture_suites')
 209
 210 ################################################################################
 211
 212 class Archive(object):
 213     def __init__(self, *args, **kwargs):
 214         pass
 215
 216     def __repr__(self):
 217         return '<Archive %s>' % self.archive_name
 218
 219 __all__.append('Archive')
 220
 221 @session_wrapper
 222 def get_archive(archive, session=None):
 223     """
 224     returns database id for given C{archive}.
 225
 226     @type archive: string
 227     @param archive: the name of the arhive
 228
 229     @type session: Session
 230     @param session: Optional SQLA session object (a temporary one will be
 231     generated if not supplied)
 232
 233     @rtype: Archive
 234     @return: Archive object for the given name (None if not present)
 235
 236     """
 237     archive = archive.lower()
 238
 239     q = session.query(Archive).filter_by(archive_name=archive)
 240
 241     try:
 242         return q.one()
 243     except NoResultFound:
 244         return None
 245
 246 __all__.append('get_archive')
 247
 248 ################################################################################
 249
 250 class BinAssociation(object):
 251     def __init__(self, *args, **kwargs):
 252         pass
 253
 254     def __repr__(self):
 255         return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
 256
 257 __all__.append('BinAssociation')
 258
 259 ################################################################################
 260
 261 class BinContents(object):
 262     def __init__(self, *args, **kwargs):
 263         pass
 264
 265     def __repr__(self):
 266         return '<BinContents (%s, %s)>' % (self.binary, self.filename)
 267
 268 __all__.append('BinContents')
 269
 270 ################################################################################
 271
 272 class DBBinary(object):
 273     def __init__(self, *args, **kwargs):
 274         pass
 275
 276     def __repr__(self):
 277         return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
 278
 279 __all__.append('DBBinary')
 280
 281 @session_wrapper
 282 def get_suites_binary_in(package, session=None):
 283     """
 284     Returns list of Suite objects which given C{package} name is in
 285
 286     @type package: str
 287     @param package: DBBinary package name to search for
 288
 289     @rtype: list
 290     @return: list of Suite objects for the given package
 291     """
 292
 293     return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
 294
 295 __all__.append('get_suites_binary_in')
 296
 297 @session_wrapper
 298 def get_binary_from_id(binary_id, session=None):
 299     """
 300     Returns DBBinary object for given C{id}
 301
 302     @type binary_id: int
 303     @param binary_id: Id of the required binary
 304
 305     @type session: Session
 306     @param session: Optional SQLA session object (a temporary one will be
 307     generated if not supplied)
 308
 309     @rtype: DBBinary
 310     @return: DBBinary object for the given binary (None if not present)
 311     """
 312
 313     q = session.query(DBBinary).filter_by(binary_id=binary_id)
 314
 315     try:
 316         return q.one()
 317     except NoResultFound:
 318         return None
 319
 320 __all__.append('get_binary_from_id')
 321
 322 @session_wrapper
 323 def get_binaries_from_name(package, version=None, architecture=None, session=None):
 324     """
 325     Returns list of DBBinary objects for given C{package} name
 326
 327     @type package: str
 328     @param package: DBBinary package name to search for
 329
 330     @type version: str or None
 331     @param version: Version to search for (or None)
 332
 333     @type architecture: str, list or None
 334     @param architecture: Architectures to limit to (or None if no limit)
 335
 336     @type session: Session
 337     @param session: Optional SQL session object (a temporary one will be
 338     generated if not supplied)
 339
 340     @rtype: list
 341     @return: list of DBBinary objects for the given name (may be empty)
 342     """
 343
 344     q = session.query(DBBinary).filter_by(package=package)
 345
 346     if version is not None:
 347         q = q.filter_by(version=version)
 348
 349     if architecture is not None:
 350         if not isinstance(architecture, list):
 351             architecture = [architecture]
 352         q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
 353
 354     ret = q.all()
 355
 356     return ret
 357
 358 __all__.append('get_binaries_from_name')
 359
 360 @session_wrapper
 361 def get_binaries_from_source_id(source_id, session=None):
 362     """
 363     Returns list of DBBinary objects for given C{source_id}
 364
 365     @type source_id: int
 366     @param source_id: source_id to search for
 367
 368     @type session: Session
 369     @param session: Optional SQL session object (a temporary one will be
 370     generated if not supplied)
 371
 372     @rtype: list
 373     @return: list of DBBinary objects for the given name (may be empty)
 374     """
 375
 376     return session.query(DBBinary).filter_by(source_id=source_id).all()
 377
 378 __all__.append('get_binaries_from_source_id')
 379
 380 @session_wrapper
 381 def get_binary_from_name_suite(package, suitename, session=None):
 382     ### For dak examine-package
 383     ### XXX: Doesn't use object API yet
 384
 385     sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
 386              FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
 387              WHERE b.package='%(package)s'
 388                AND b.file = fi.id
 389                AND fi.location = l.id
 390                AND l.component = c.id
 391                AND ba.bin=b.id
 392                AND ba.suite = su.id
 393                AND su.suite_name %(suitename)s
 394           ORDER BY b.version DESC"""
 395
 396     return session.execute(sql % {'package': package, 'suitename': suitename})
 397
 398 __all__.append('get_binary_from_name_suite')
 399
 400 @session_wrapper
 401 def get_binary_components(package, suitename, arch, session=None):
 402     # Check for packages that have moved from one component to another
 403     query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
 404     WHERE b.package=:package AND s.suite_name=:suitename
 405       AND (a.arch_string = :arch OR a.arch_string = 'all')
 406       AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
 407       AND f.location = l.id
 408       AND l.component = c.id
 409       AND b.file = f.id"""
 410
 411     vals = {'package': package, 'suitename': suitename, 'arch': arch}
 412
 413     return session.execute(query, vals)
 414
 415 __all__.append('get_binary_components')
 416
 417 ################################################################################
 418
 419 class BinaryACL(object):
 420     def __init__(self, *args, **kwargs):
 421         pass
 422
 423     def __repr__(self):
 424         return '<BinaryACL %s>' % self.binary_acl_id
 425
 426 __all__.append('BinaryACL')
 427
 428 ################################################################################
 429
 430 class BinaryACLMap(object):
 431     def __init__(self, *args, **kwargs):
 432         pass
 433
 434     def __repr__(self):
 435         return '<BinaryACLMap %s>' % self.binary_acl_map_id
 436
 437 __all__.append('BinaryACLMap')
 438
 439 ################################################################################
 440
 441 MINIMAL_APT_CONF="""
 442 Dir
 443 {
 444    ArchiveDir "%(archivepath)s";
 445    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
 446    CacheDir "/srv/ftp-master.debian.org/database/";
 447 };
 448
 449 Default
 450 {
 451    Packages::Compress ". bzip2 gzip";
 452    Sources::Compress ". bzip2 gzip";
 453    DeLinkLimit 0;
 454    FileMode 0664;
 455 }
 456
 457 bindirectory "incoming"
 458 {
 459    Packages "Packages";
 460    Contents " ";
 461
 462    BinOverride "override.sid.all3";
 463    BinCacheDB "packages-accepted.db";
 464
 465    FileList "%(filelist)s";
 466
 467    PathPrefix "";
 468    Packages::Extensions ".deb .udeb";
 469 };
 470
 471 bindirectory "incoming/"
 472 {
 473    Sources "Sources";
 474    BinOverride "override.sid.all3";
 475    SrcOverride "override.sid.all3.src";
 476    FileList "%(filelist)s";
 477 };
 478 """
 479
 480 class BuildQueue(object):
 481     def __init__(self, *args, **kwargs):
 482         pass
 483
 484     def __repr__(self):
 485         return '<BuildQueue %s>' % self.queue_name
 486
 487     def write_metadata(self, starttime, force=False):
 488         # Do we write out metafiles?
 489         if not (force or self.generate_metadata):
 490             return
 491
 492         session = DBConn().session().object_session(self)
 493
 494         fl_fd = fl_name = ac_fd = ac_name = None
 495         tempdir = None
 496         arches = " ".join([ a.arch_string for a in session.query(Architecture).all() if a.arch_string != 'source' ])
 497         startdir = os.getcwd()
 498
 499         try:
 500             # Grab files we want to include
 501             newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all()
 502             # Write file list with newer files
 503             (fl_fd, fl_name) = mkstemp()
 504             for n in newer:
 505                 os.write(fl_fd, '%s\n' % n.fullpath)
 506             os.close(fl_fd)
 507
 508             # Write minimal apt.conf
 509             # TODO: Remove hardcoding from template
 510             (ac_fd, ac_name) = mkstemp()
 511             os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path,
 512                                                 'filelist': fl_name})
 513             os.close(ac_fd)
 514
 515             # Run apt-ftparchive generate
 516             os.chdir(os.path.dirname(ac_name))
 517             os.system('apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
 518
 519             # Run apt-ftparchive release
 520             # TODO: Eww - fix this
 521             bname = os.path.basename(self.path)
 522             os.chdir(self.path)
 523             os.chdir('..')
 524
 525             # We have to remove the Release file otherwise it'll be included in the
 526             # new one
 527             try:
 528                 os.unlink(os.path.join(bname, 'Release'))
 529             except OSError:
 530                 pass
 531
 532             os.system("""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="%s" release %s > Release""" % (self.origin, self.label, self.releasedescription, arches, bname))
 533
 534             # Crude hack with open and append, but this whole section is and should be redone.
 535             if self.notautomatic:
 536                 release=open("Release", "a")
 537                 release.write("NotAutomatic: yes")
 538                 release.close()
 539
 540             # Sign if necessary
 541             if self.signingkey:
 542                 cnf = Config()
 543                 keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
 544                 if cnf.has_key("Dinstall::SigningPubKeyring"):
 545                     keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
 546
 547                 os.system("gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey))
 548
 549             # Move the files if we got this far
 550             os.rename('Release', os.path.join(bname, 'Release'))
 551             if self.signingkey:
 552                 os.rename('Release.gpg', os.path.join(bname, 'Release.gpg'))
 553
 554         # Clean up any left behind files
 555         finally:
 556             os.chdir(startdir)
 557             if fl_fd:
 558                 try:
 559                     os.close(fl_fd)
 560                 except OSError:
 561                     pass
 562
 563             if fl_name:
 564                 try:
 565                     os.unlink(fl_name)
 566                 except OSError:
 567                     pass
 568
 569             if ac_fd:
 570                 try:
 571                     os.close(ac_fd)
 572                 except OSError:
 573                     pass
 574
 575             if ac_name:
 576                 try:
 577                     os.unlink(ac_name)
 578                 except OSError:
 579                     pass
 580
 581     def clean_and_update(self, starttime, Logger, dryrun=False):
 582         """WARNING: This routine commits for you"""
 583         session = DBConn().session().object_session(self)
 584
 585         if self.generate_metadata and not dryrun:
 586             self.write_metadata(starttime)
 587
 588         # Grab files older than our execution time
 589         older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
 590
 591         for o in older:
 592             killdb = False
 593             try:
 594                 if dryrun:
 595                     Logger.log(["I: Would have removed %s from the queue" % o.fullpath])
 596                 else:
 597                     Logger.log(["I: Removing %s from the queue" % o.fullpath])
 598                     os.unlink(o.fullpath)
 599                     killdb = True
 600             except OSError, e:
 601                 # If it wasn't there, don't worry
 602                 if e.errno == ENOENT:
 603                     killdb = True
 604                 else:
 605                     # TODO: Replace with proper logging call
 606                     Logger.log(["E: Could not remove %s" % o.fullpath])
 607
 608             if killdb:
 609                 session.delete(o)
 610
 611         session.commit()
 612
 613         for f in os.listdir(self.path):
 614             if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release'):
 615                 continue
 616
 617             try:
 618                 r = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter_by(filename = f).one()
 619             except NoResultFound:
 620                 fp = os.path.join(self.path, f)
 621                 if dryrun:
 622                     Logger.log(["I: Would remove unused link %s" % fp])
 623                 else:
 624                     Logger.log(["I: Removing unused link %s" % fp])
 625                     try:
 626                         os.unlink(fp)
 627                     except OSError:
 628                         Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath])
 629
 630     def add_file_from_pool(self, poolfile):
 631         """Copies a file into the pool.  Assumes that the PoolFile object is
 632         attached to the same SQLAlchemy session as the Queue object is.
 633
 634         The caller is responsible for committing after calling this function."""
 635         poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:]
 636
 637         # Check if we have a file of this name or this ID already
 638         for f in self.queuefiles:
 639             if f.fileid is not None and f.fileid == poolfile.file_id or \
 640                f.poolfile.filename == poolfile_basename:
 641                    # In this case, update the BuildQueueFile entry so we
 642                    # don't remove it too early
 643                    f.lastused = datetime.now()
 644                    DBConn().session().object_session(poolfile).add(f)
 645                    return f
 646
 647         # Prepare BuildQueueFile object
 648         qf = BuildQueueFile()
 649         qf.build_queue_id = self.queue_id
 650         qf.lastused = datetime.now()
 651         qf.filename = poolfile_basename
 652
 653         targetpath = poolfile.fullpath
 654         queuepath = os.path.join(self.path, poolfile_basename)
 655
 656         try:
 657             if self.copy_files:
 658                 # We need to copy instead of symlink
 659                 import utils
 660                 utils.copy(targetpath, queuepath)
 661                 # NULL in the fileid field implies a copy
 662                 qf.fileid = None
 663             else:
 664                 os.symlink(targetpath, queuepath)
 665                 qf.fileid = poolfile.file_id
 666         except OSError:
 667             return None
 668
 669         # Get the same session as the PoolFile is using and add the qf to it
 670         DBConn().session().object_session(poolfile).add(qf)
 671
 672         return qf
 673
 674
 675 __all__.append('BuildQueue')
 676
 677 @session_wrapper
 678 def get_build_queue(queuename, session=None):
 679     """
 680     Returns BuildQueue object for given C{queue name}, creating it if it does not
 681     exist.
 682
 683     @type queuename: string
 684     @param queuename: The name of the queue
 685
 686     @type session: Session
 687     @param session: Optional SQLA session object (a temporary one will be
 688     generated if not supplied)
 689
 690     @rtype: BuildQueue
 691     @return: BuildQueue object for the given queue
 692     """
 693
 694     q = session.query(BuildQueue).filter_by(queue_name=queuename)
 695
 696     try:
 697         return q.one()
 698     except NoResultFound:
 699         return None
 700
 701 __all__.append('get_build_queue')
 702
 703 ################################################################################
 704
 705 class BuildQueueFile(object):
 706     def __init__(self, *args, **kwargs):
 707         pass
 708
 709     def __repr__(self):
 710         return '<BuildQueueFile %s (%s)>' % (self.filename, self.build_queue_id)
 711
 712     @property
 713     def fullpath(self):
 714         return os.path.join(self.buildqueue.path, self.filename)
 715
 716
 717 __all__.append('BuildQueueFile')
 718
 719 ################################################################################
 720
 721 class ChangePendingBinary(object):
 722     def __init__(self, *args, **kwargs):
 723         pass
 724
 725     def __repr__(self):
 726         return '<ChangePendingBinary %s>' % self.change_pending_binary_id
 727
 728 __all__.append('ChangePendingBinary')
 729
 730 ################################################################################
 731
 732 class ChangePendingFile(object):
 733     def __init__(self, *args, **kwargs):
 734         pass
 735
 736     def __repr__(self):
 737         return '<ChangePendingFile %s>' % self.change_pending_file_id
 738
 739 __all__.append('ChangePendingFile')
 740
 741 ################################################################################
 742
 743 class ChangePendingSource(object):
 744     def __init__(self, *args, **kwargs):
 745         pass
 746
 747     def __repr__(self):
 748         return '<ChangePendingSource %s>' % self.change_pending_source_id
 749
 750 __all__.append('ChangePendingSource')
 751
 752 ################################################################################
 753
 754 class Component(object):
 755     def __init__(self, *args, **kwargs):
 756         pass
 757
 758     def __eq__(self, val):
 759         if isinstance(val, str):
 760             return (self.component_name == val)
 761         # This signals to use the normal comparison operator
 762         return NotImplemented
 763
 764     def __ne__(self, val):
 765         if isinstance(val, str):
 766             return (self.component_name != val)
 767         # This signals to use the normal comparison operator
 768         return NotImplemented
 769
 770     def __repr__(self):
 771         return '<Component %s>' % self.component_name
 772
 773
 774 __all__.append('Component')
 775
 776 @session_wrapper
 777 def get_component(component, session=None):
 778     """
 779     Returns database id for given C{component}.
 780
 781     @type component: string
 782     @param component: The name of the override type
 783
 784     @rtype: int
 785     @return: the database id for the given component
 786
 787     """
 788     component = component.lower()
 789
 790     q = session.query(Component).filter_by(component_name=component)
 791
 792     try:
 793         return q.one()
 794     except NoResultFound:
 795         return None
 796
 797 __all__.append('get_component')
 798
 799 ################################################################################
 800
 801 class DBConfig(object):
 802     def __init__(self, *args, **kwargs):
 803         pass
 804
 805     def __repr__(self):
 806         return '<DBConfig %s>' % self.name
 807
 808 __all__.append('DBConfig')
 809
 810 ################################################################################
 811
 812 @session_wrapper
 813 def get_or_set_contents_file_id(filename, session=None):
 814     """
 815     Returns database id for given filename.
 816
 817     If no matching file is found, a row is inserted.
 818
 819     @type filename: string
 820     @param filename: The filename
 821     @type session: SQLAlchemy
 822     @param session: Optional SQL session object (a temporary one will be
 823     generated if not supplied).  If not passed, a commit will be performed at
 824     the end of the function, otherwise the caller is responsible for commiting.
 825
 826     @rtype: int
 827     @return: the database id for the given component
 828     """
 829
 830     q = session.query(ContentFilename).filter_by(filename=filename)
 831
 832     try:
 833         ret = q.one().cafilename_id
 834     except NoResultFound:
 835         cf = ContentFilename()
 836         cf.filename = filename
 837         session.add(cf)
 838         session.commit_or_flush()
 839         ret = cf.cafilename_id
 840
 841     return ret
 842
 843 __all__.append('get_or_set_contents_file_id')
 844
 845 @session_wrapper
 846 def get_contents(suite, overridetype, section=None, session=None):
 847     """
 848     Returns contents for a suite / overridetype combination, limiting
 849     to a section if not None.
 850
 851     @type suite: Suite
 852     @param suite: Suite object
 853
 854     @type overridetype: OverrideType
 855     @param overridetype: OverrideType object
 856
 857     @type section: Section
 858     @param section: Optional section object to limit results to
 859
 860     @type session: SQLAlchemy
 861     @param session: Optional SQL session object (a temporary one will be
 862     generated if not supplied)
 863
 864     @rtype: ResultsProxy
 865     @return: ResultsProxy object set up to return tuples of (filename, section,
 866     package, arch_id)
 867     """
 868
 869     # find me all of the contents for a given suite
 870     contents_q = """SELECT (p.path||'/'||n.file) AS fn,
 871                             s.section,
 872                             b.package,
 873                             b.architecture
 874                    FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
 875                    JOIN content_file_names n ON (c.filename=n.id)
 876                    JOIN binaries b ON (b.id=c.binary_pkg)
 877                    JOIN override o ON (o.package=b.package)
 878                    JOIN section s ON (s.id=o.section)
 879                    WHERE o.suite = :suiteid AND o.type = :overridetypeid
 880                    AND b.type=:overridetypename"""
 881
 882     vals = {'suiteid': suite.suite_id,
 883             'overridetypeid': overridetype.overridetype_id,
 884             'overridetypename': overridetype.overridetype}
 885
 886     if section is not None:
 887         contents_q += " AND s.id = :sectionid"
 888         vals['sectionid'] = section.section_id
 889
 890     contents_q += " ORDER BY fn"
 891
 892     return session.execute(contents_q, vals)
 893
 894 __all__.append('get_contents')
 895
 896 ################################################################################
 897
 898 class ContentFilepath(object):
 899     def __init__(self, *args, **kwargs):
 900         pass
 901
 902     def __repr__(self):
 903         return '<ContentFilepath %s>' % self.filepath
 904
 905 __all__.append('ContentFilepath')
 906
 907 @session_wrapper
 908 def get_or_set_contents_path_id(filepath, session=None):
 909     """
 910     Returns database id for given path.
 911
 912     If no matching file is found, a row is inserted.
 913
 914     @type filepath: string
 915     @param filepath: The filepath
 916
 917     @type session: SQLAlchemy
 918     @param session: Optional SQL session object (a temporary one will be
 919     generated if not supplied).  If not passed, a commit will be performed at
 920     the end of the function, otherwise the caller is responsible for commiting.
 921
 922     @rtype: int
 923     @return: the database id for the given path
 924     """
 925
 926     q = session.query(ContentFilepath).filter_by(filepath=filepath)
 927
 928     try:
 929         ret = q.one().cafilepath_id
 930     except NoResultFound:
 931         cf = ContentFilepath()
 932         cf.filepath = filepath
 933         session.add(cf)
 934         session.commit_or_flush()
 935         ret = cf.cafilepath_id
 936
 937     return ret
 938
 939 __all__.append('get_or_set_contents_path_id')
 940
 941 ################################################################################
 942
 943 class ContentAssociation(object):
 944     def __init__(self, *args, **kwargs):
 945         pass
 946
 947     def __repr__(self):
 948         return '<ContentAssociation %s>' % self.ca_id
 949
 950 __all__.append('ContentAssociation')
 951
 952 def insert_content_paths(binary_id, fullpaths, session=None):
 953     """
 954     Make sure given path is associated with given binary id
 955
 956     @type binary_id: int
 957     @param binary_id: the id of the binary
 958     @type fullpaths: list
 959     @param fullpaths: the list of paths of the file being associated with the binary
 960     @type session: SQLAlchemy session
 961     @param session: Optional SQLAlchemy session.  If this is passed, the caller
 962     is responsible for ensuring a transaction has begun and committing the
 963     results or rolling back based on the result code.  If not passed, a commit
 964     will be performed at the end of the function, otherwise the caller is
 965     responsible for commiting.
 966
 967     @return: True upon success
 968     """
 969
 970     privatetrans = False
 971     if session is None:
 972         session = DBConn().session()
 973         privatetrans = True
 974
 975     try:
 976         # Insert paths
 977         def generate_path_dicts():
 978             for fullpath in fullpaths:
 979                 if fullpath.startswith( './' ):
 980                     fullpath = fullpath[2:]
 981
 982                 yield {'filename':fullpath, 'id': binary_id }
 983
 984         for d in generate_path_dicts():
 985             session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )",
 986                          d )
 987
 988         session.commit()
 989         if privatetrans:
 990             session.close()
 991         return True
 992
 993     except:
 994         traceback.print_exc()
 995
 996         # Only rollback if we set up the session ourself
 997         if privatetrans:
 998             session.rollback()
 999             session.close()
1000
1001         return False
1002
1003 __all__.append('insert_content_paths')
1004
1005 ################################################################################
1006
1007 class DSCFile(object):
1008     def __init__(self, *args, **kwargs):
1009         pass
1010
1011     def __repr__(self):
1012         return '<DSCFile %s>' % self.dscfile_id
1013
1014 __all__.append('DSCFile')
1015
1016 @session_wrapper
1017 def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
1018     """
1019     Returns a list of DSCFiles which may be empty
1020
1021     @type dscfile_id: int (optional)
1022     @param dscfile_id: the dscfile_id of the DSCFiles to find
1023
1024     @type source_id: int (optional)
1025     @param source_id: the source id related to the DSCFiles to find
1026
1027     @type poolfile_id: int (optional)
1028     @param poolfile_id: the poolfile id related to the DSCFiles to find
1029
1030     @rtype: list
1031     @return: Possibly empty list of DSCFiles
1032     """
1033
1034     q = session.query(DSCFile)
1035
1036     if dscfile_id is not None:
1037         q = q.filter_by(dscfile_id=dscfile_id)
1038
1039     if source_id is not None:
1040         q = q.filter_by(source_id=source_id)
1041
1042     if poolfile_id is not None:
1043         q = q.filter_by(poolfile_id=poolfile_id)
1044
1045     return q.all()
1046
1047 __all__.append('get_dscfiles')
1048
1049 ################################################################################
1050
1051 class PoolFile(object):
1052     def __init__(self, *args, **kwargs):
1053         pass
1054
1055     def __repr__(self):
1056         return '<PoolFile %s>' % self.filename
1057
1058     @property
1059     def fullpath(self):
1060         return os.path.join(self.location.path, self.filename)
1061
1062 __all__.append('PoolFile')
1063
1064 @session_wrapper
1065 def check_poolfile(filename, filesize, md5sum, location_id, session=None):
1066     """
1067     Returns a tuple:
1068     (ValidFileFound [boolean or None], PoolFile object or None)
1069
1070     @type filename: string
1071     @param filename: the filename of the file to check against the DB
1072
1073     @type filesize: int
1074     @param filesize: the size of the file to check against the DB
1075
1076     @type md5sum: string
1077     @param md5sum: the md5sum of the file to check against the DB
1078
1079     @type location_id: int
1080     @param location_id: the id of the location to look in
1081
1082     @rtype: tuple
1083     @return: Tuple of length 2.
1084                  - If more than one file found with that name: (C{None},  C{None})
1085                  - If valid pool file found: (C{True}, C{PoolFile object})
1086                  - If valid pool file not found:
1087                      - (C{False}, C{None}) if no file found
1088                      - (C{False}, C{PoolFile object}) if file found with size/md5sum mismatch
1089     """
1090
1091     q = session.query(PoolFile).filter_by(filename=filename)
1092     q = q.join(Location).filter_by(location_id=location_id)
1093
1094     ret = None
1095
1096     if q.count() > 1:
1097         ret = (None, None)
1098     elif q.count() < 1:
1099         ret = (False, None)
1100     else:
1101         obj = q.one()
1102         if obj.md5sum != md5sum or obj.filesize != int(filesize):
1103             ret = (False, obj)
1104
1105     if ret is None:
1106         ret = (True, obj)
1107
1108     return ret
1109
1110 __all__.append('check_poolfile')
1111
1112 @session_wrapper
1113 def get_poolfile_by_id(file_id, session=None):
1114     """
1115     Returns a PoolFile objects or None for the given id
1116
1117     @type file_id: int
1118     @param file_id: the id of the file to look for
1119
1120     @rtype: PoolFile or None
1121     @return: either the PoolFile object or None
1122     """
1123
1124     q = session.query(PoolFile).filter_by(file_id=file_id)
1125
1126     try:
1127         return q.one()
1128     except NoResultFound:
1129         return None
1130
1131 __all__.append('get_poolfile_by_id')
1132
1133
1134 @session_wrapper
1135 def get_poolfile_by_name(filename, location_id=None, session=None):
1136     """
1137     Returns an array of PoolFile objects for the given filename and
1138     (optionally) location_id
1139
1140     @type filename: string
1141     @param filename: the filename of the file to check against the DB
1142
1143     @type location_id: int
1144     @param location_id: the id of the location to look in (optional)
1145
1146     @rtype: array
1147     @return: array of PoolFile objects
1148     """
1149
1150     q = session.query(PoolFile).filter_by(filename=filename)
1151
1152     if location_id is not None:
1153         q = q.join(Location).filter_by(location_id=location_id)
1154
1155     return q.all()
1156
1157 __all__.append('get_poolfile_by_name')
1158
1159 @session_wrapper
1160 def get_poolfile_like_name(filename, session=None):
1161     """
1162     Returns an array of PoolFile objects which are like the given name
1163
1164     @type filename: string
1165     @param filename: the filename of the file to check against the DB
1166
1167     @rtype: array
1168     @return: array of PoolFile objects
1169     """
1170
1171     # TODO: There must be a way of properly using bind parameters with %FOO%
1172     q = session.query(PoolFile).filter(PoolFile.filename.like('%%/%s' % filename))
1173
1174     return q.all()
1175
1176 __all__.append('get_poolfile_like_name')
1177
1178 @session_wrapper
1179 def add_poolfile(filename, datadict, location_id, session=None):
1180     """
1181     Add a new file to the pool
1182
1183     @type filename: string
1184     @param filename: filename
1185
1186     @type datadict: dict
1187     @param datadict: dict with needed data
1188
1189     @type location_id: int
1190     @param location_id: database id of the location
1191
1192     @rtype: PoolFile
1193     @return: the PoolFile object created
1194     """
1195     poolfile = PoolFile()
1196     poolfile.filename = filename
1197     poolfile.filesize = datadict["size"]
1198     poolfile.md5sum = datadict["md5sum"]
1199     poolfile.sha1sum = datadict["sha1sum"]
1200     poolfile.sha256sum = datadict["sha256sum"]
1201     poolfile.location_id = location_id
1202
1203     session.add(poolfile)
1204     # Flush to get a file id (NB: This is not a commit)
1205     session.flush()
1206
1207     return poolfile
1208
1209 __all__.append('add_poolfile')
1210
1211 ################################################################################
1212
1213 class Fingerprint(object):
1214     def __init__(self, *args, **kwargs):
1215         pass
1216
1217     def __repr__(self):
1218         return '<Fingerprint %s>' % self.fingerprint
1219
1220 __all__.append('Fingerprint')
1221
1222 @session_wrapper
1223 def get_fingerprint(fpr, session=None):
1224     """
1225     Returns Fingerprint object for given fpr.
1226
1227     @type fpr: string
1228     @param fpr: The fpr to find / add
1229
1230     @type session: SQLAlchemy
1231     @param session: Optional SQL session object (a temporary one will be
1232     generated if not supplied).
1233
1234     @rtype: Fingerprint
1235     @return: the Fingerprint object for the given fpr or None
1236     """
1237
1238     q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1239
1240     try:
1241         ret = q.one()
1242     except NoResultFound:
1243         ret = None
1244
1245     return ret
1246
1247 __all__.append('get_fingerprint')
1248
1249 @session_wrapper
1250 def get_or_set_fingerprint(fpr, session=None):
1251     """
1252     Returns Fingerprint object for given fpr.
1253
1254     If no matching fpr is found, a row is inserted.
1255
1256     @type fpr: string
1257     @param fpr: The fpr to find / add
1258
1259     @type session: SQLAlchemy
1260     @param session: Optional SQL session object (a temporary one will be
1261     generated if not supplied).  If not passed, a commit will be performed at
1262     the end of the function, otherwise the caller is responsible for commiting.
1263     A flush will be performed either way.
1264
1265     @rtype: Fingerprint
1266     @return: the Fingerprint object for the given fpr
1267     """
1268
1269     q = session.query(Fingerprint).filter_by(fingerprint=fpr)
1270
1271     try:
1272         ret = q.one()
1273     except NoResultFound:
1274         fingerprint = Fingerprint()
1275         fingerprint.fingerprint = fpr
1276         session.add(fingerprint)
1277         session.commit_or_flush()
1278         ret = fingerprint
1279
1280     return ret
1281
1282 __all__.append('get_or_set_fingerprint')
1283
1284 ################################################################################
1285
1286 # Helper routine for Keyring class
1287 def get_ldap_name(entry):
1288     name = []
1289     for k in ["cn", "mn", "sn"]:
1290         ret = entry.get(k)
1291         if ret and ret[0] != "" and ret[0] != "-":
1292             name.append(ret[0])
1293     return " ".join(name)
1294
1295 ################################################################################
1296
1297 class Keyring(object):
1298     gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
1299                      " --with-colons --fingerprint --fingerprint"
1300
1301     keys = {}
1302     fpr_lookup = {}
1303
1304     def __init__(self, *args, **kwargs):
1305         pass
1306
1307     def __repr__(self):
1308         return '<Keyring %s>' % self.keyring_name
1309
1310     def de_escape_gpg_str(self, txt):
1311         esclist = re.split(r'(\\x..)', txt)
1312         for x in range(1,len(esclist),2):
1313             esclist[x] = "%c" % (int(esclist[x][2:],16))
1314         return "".join(esclist)
1315
1316     def load_keys(self, keyring):
1317         import email.Utils
1318
1319         if not self.keyring_id:
1320             raise Exception('Must be initialized with database information')
1321
1322         k = os.popen(self.gpg_invocation % keyring, "r")
1323         key = None
1324         signingkey = False
1325
1326         for line in k.xreadlines():
1327             field = line.split(":")
1328             if field[0] == "pub":
1329                 key = field[4]
1330                 (name, addr) = email.Utils.parseaddr(field[9])
1331                 name = re.sub(r"\s*[(].*[)]", "", name)
1332                 if name == "" or addr == "" or "@" not in addr:
1333                     name = field[9]
1334                     addr = "invalid-uid"
1335                 name = self.de_escape_gpg_str(name)
1336                 self.keys[key] = {"email": addr}
1337                 if name != "":
1338                     self.keys[key]["name"] = name
1339                 self.keys[key]["aliases"] = [name]
1340                 self.keys[key]["fingerprints"] = []
1341                 signingkey = True
1342             elif key and field[0] == "sub" and len(field) >= 12:
1343                 signingkey = ("s" in field[11])
1344             elif key and field[0] == "uid":
1345                 (name, addr) = email.Utils.parseaddr(field[9])
1346                 if name and name not in self.keys[key]["aliases"]:
1347                     self.keys[key]["aliases"].append(name)
1348             elif signingkey and field[0] == "fpr":
1349                 self.keys[key]["fingerprints"].append(field[9])
1350                 self.fpr_lookup[field[9]] = key
1351
1352     def import_users_from_ldap(self, session):
1353         import ldap
1354         cnf = Config()
1355
1356         LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
1357         LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
1358
1359         l = ldap.open(LDAPServer)
1360         l.simple_bind_s("","")
1361         Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
1362                "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
1363                ["uid", "keyfingerprint", "cn", "mn", "sn"])
1364
1365         ldap_fin_uid_id = {}
1366
1367         byuid = {}
1368         byname = {}
1369
1370         for i in Attrs:
1371             entry = i[1]
1372             uid = entry["uid"][0]
1373             name = get_ldap_name(entry)
1374             fingerprints = entry["keyFingerPrint"]
1375             keyid = None
1376             for f in fingerprints:
1377                 key = self.fpr_lookup.get(f, None)
1378                 if key not in self.keys:
1379                     continue
1380                 self.keys[key]["uid"] = uid
1381
1382                 if keyid != None:
1383                     continue
1384                 keyid = get_or_set_uid(uid, session).uid_id
1385                 byuid[keyid] = (uid, name)
1386                 byname[uid] = (keyid, name)
1387
1388         return (byname, byuid)
1389
1390     def generate_users_from_keyring(self, format, session):
1391         byuid = {}
1392         byname = {}
1393         any_invalid = False
1394         for x in self.keys.keys():
1395             if self.keys[x]["email"] == "invalid-uid":
1396                 any_invalid = True
1397                 self.keys[x]["uid"] = format % "invalid-uid"
1398             else:
1399                 uid = format % self.keys[x]["email"]
1400                 keyid = get_or_set_uid(uid, session).uid_id
1401                 byuid[keyid] = (uid, self.keys[x]["name"])
1402                 byname[uid] = (keyid, self.keys[x]["name"])
1403                 self.keys[x]["uid"] = uid
1404
1405         if any_invalid:
1406             uid = format % "invalid-uid"
1407             keyid = get_or_set_uid(uid, session).uid_id
1408             byuid[keyid] = (uid, "ungeneratable user id")
1409             byname[uid] = (keyid, "ungeneratable user id")
1410
1411         return (byname, byuid)
1412
1413 __all__.append('Keyring')
1414
1415 @session_wrapper
1416 def get_keyring(keyring, session=None):
1417     """
1418     If C{keyring} does not have an entry in the C{keyrings} table yet, return None
1419     If C{keyring} already has an entry, simply return the existing Keyring
1420
1421     @type keyring: string
1422     @param keyring: the keyring name
1423
1424     @rtype: Keyring
1425     @return: the Keyring object for this keyring
1426     """
1427
1428     q = session.query(Keyring).filter_by(keyring_name=keyring)
1429
1430     try:
1431         return q.one()
1432     except NoResultFound:
1433         return None
1434
1435 __all__.append('get_keyring')
1436
1437 ################################################################################
1438
1439 class KeyringACLMap(object):
1440     def __init__(self, *args, **kwargs):
1441         pass
1442
1443     def __repr__(self):
1444         return '<KeyringACLMap %s>' % self.keyring_acl_map_id
1445
1446 __all__.append('KeyringACLMap')
1447
1448 ################################################################################
1449
1450 class DBChange(object):
1451     def __init__(self, *args, **kwargs):
1452         pass
1453
1454     def __repr__(self):
1455         return '<DBChange %s>' % self.changesname
1456
1457     def clean_from_queue(self):
1458         session = DBConn().session().object_session(self)
1459
1460         # Remove changes_pool_files entries
1461         self.poolfiles = []
1462
1463         # Remove changes_pending_files references
1464         self.files = []
1465
1466         # Clear out of queue
1467         self.in_queue = None
1468         self.approved_for_id = None
1469
1470 __all__.append('DBChange')
1471
1472 @session_wrapper
1473 def get_dbchange(filename, session=None):
1474     """
1475     returns DBChange object for given C{filename}.
1476
1477     @type filename: string
1478     @param filename: the name of the file
1479
1480     @type session: Session
1481     @param session: Optional SQLA session object (a temporary one will be
1482     generated if not supplied)
1483
1484     @rtype: DBChange
1485     @return:  DBChange object for the given filename (C{None} if not present)
1486
1487     """
1488     q = session.query(DBChange).filter_by(changesname=filename)
1489
1490     try:
1491         return q.one()
1492     except NoResultFound:
1493         return None
1494
1495 __all__.append('get_dbchange')
1496
1497 ################################################################################
1498
1499 class Location(object):
1500     def __init__(self, *args, **kwargs):
1501         pass
1502
1503     def __repr__(self):
1504         return '<Location %s (%s)>' % (self.path, self.location_id)
1505
1506 __all__.append('Location')
1507
1508 @session_wrapper
1509 def get_location(location, component=None, archive=None, session=None):
1510     """
1511     Returns Location object for the given combination of location, component
1512     and archive
1513
1514     @type location: string
1515     @param location: the path of the location, e.g. I{/srv/ftp-master.debian.org/ftp/pool/}
1516
1517     @type component: string
1518     @param component: the component name (if None, no restriction applied)
1519
1520     @type archive: string
1521     @param archive: the archive name (if None, no restriction applied)
1522
1523     @rtype: Location / None
1524     @return: Either a Location object or None if one can't be found
1525     """
1526
1527     q = session.query(Location).filter_by(path=location)
1528
1529     if archive is not None:
1530         q = q.join(Archive).filter_by(archive_name=archive)
1531
1532     if component is not None:
1533         q = q.join(Component).filter_by(component_name=component)
1534
1535     try:
1536         return q.one()
1537     except NoResultFound:
1538         return None
1539
1540 __all__.append('get_location')
1541
1542 ################################################################################
1543
1544 class Maintainer(object):
1545     def __init__(self, *args, **kwargs):
1546         pass
1547
1548     def __repr__(self):
1549         return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
1550
1551     def get_split_maintainer(self):
1552         if not hasattr(self, 'name') or self.name is None:
1553             return ('', '', '', '')
1554
1555         return fix_maintainer(self.name.strip())
1556
1557 __all__.append('Maintainer')
1558
1559 @session_wrapper
1560 def get_or_set_maintainer(name, session=None):
1561     """
1562     Returns Maintainer object for given maintainer name.
1563
1564     If no matching maintainer name is found, a row is inserted.
1565
1566     @type name: string
1567     @param name: The maintainer name to add
1568
1569     @type session: SQLAlchemy
1570     @param session: Optional SQL session object (a temporary one will be
1571     generated if not supplied).  If not passed, a commit will be performed at
1572     the end of the function, otherwise the caller is responsible for commiting.
1573     A flush will be performed either way.
1574
1575     @rtype: Maintainer
1576     @return: the Maintainer object for the given maintainer
1577     """
1578
1579     q = session.query(Maintainer).filter_by(name=name)
1580     try:
1581         ret = q.one()
1582     except NoResultFound:
1583         maintainer = Maintainer()
1584         maintainer.name = name
1585         session.add(maintainer)
1586         session.commit_or_flush()
1587         ret = maintainer
1588
1589     return ret
1590
1591 __all__.append('get_or_set_maintainer')
1592
1593 @session_wrapper
1594 def get_maintainer(maintainer_id, session=None):
1595     """
1596     Return the name of the maintainer behind C{maintainer_id} or None if that
1597     maintainer_id is invalid.
1598
1599     @type maintainer_id: int
1600     @param maintainer_id: the id of the maintainer
1601
1602     @rtype: Maintainer
1603     @return: the Maintainer with this C{maintainer_id}
1604     """
1605
1606     return session.query(Maintainer).get(maintainer_id)
1607
1608 __all__.append('get_maintainer')
1609
1610 ################################################################################
1611
1612 class NewComment(object):
1613     def __init__(self, *args, **kwargs):
1614         pass
1615
1616     def __repr__(self):
1617         return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1618
1619 __all__.append('NewComment')
1620
1621 @session_wrapper
1622 def has_new_comment(package, version, session=None):
1623     """
1624     Returns true if the given combination of C{package}, C{version} has a comment.
1625
1626     @type package: string
1627     @param package: name of the package
1628
1629     @type version: string
1630     @param version: package version
1631
1632     @type session: Session
1633     @param session: Optional SQLA session object (a temporary one will be
1634     generated if not supplied)
1635
1636     @rtype: boolean
1637     @return: true/false
1638     """
1639
1640     q = session.query(NewComment)
1641     q = q.filter_by(package=package)
1642     q = q.filter_by(version=version)
1643
1644     return bool(q.count() > 0)
1645
1646 __all__.append('has_new_comment')
1647
1648 @session_wrapper
1649 def get_new_comments(package=None, version=None, comment_id=None, session=None):
1650     """
1651     Returns (possibly empty) list of NewComment objects for the given
1652     parameters
1653
1654     @type package: string (optional)
1655     @param package: name of the package
1656
1657     @type version: string (optional)
1658     @param version: package version
1659
1660     @type comment_id: int (optional)
1661     @param comment_id: An id of a comment
1662
1663     @type session: Session
1664     @param session: Optional SQLA session object (a temporary one will be
1665     generated if not supplied)
1666
1667     @rtype: list
1668     @return: A (possibly empty) list of NewComment objects will be returned
1669     """
1670
1671     q = session.query(NewComment)
1672     if package is not None: q = q.filter_by(package=package)
1673     if version is not None: q = q.filter_by(version=version)
1674     if comment_id is not None: q = q.filter_by(comment_id=comment_id)
1675
1676     return q.all()
1677
1678 __all__.append('get_new_comments')
1679
1680 ################################################################################
1681
1682 class Override(object):
1683     def __init__(self, *args, **kwargs):
1684         pass
1685
1686     def __repr__(self):
1687         return '<Override %s (%s)>' % (self.package, self.suite_id)
1688
1689 __all__.append('Override')
1690
1691 @session_wrapper
1692 def get_override(package, suite=None, component=None, overridetype=None, session=None):
1693     """
1694     Returns Override object for the given parameters
1695
1696     @type package: string
1697     @param package: The name of the package
1698
1699     @type suite: string, list or None
1700     @param suite: The name of the suite (or suites if a list) to limit to.  If
1701                   None, don't limit.  Defaults to None.
1702
1703     @type component: string, list or None
1704     @param component: The name of the component (or components if a list) to
1705                       limit to.  If None, don't limit.  Defaults to None.
1706
1707     @type overridetype: string, list or None
1708     @param overridetype: The name of the overridetype (or overridetypes if a list) to
1709                          limit to.  If None, don't limit.  Defaults to None.
1710
1711     @type session: Session
1712     @param session: Optional SQLA session object (a temporary one will be
1713     generated if not supplied)
1714
1715     @rtype: list
1716     @return: A (possibly empty) list of Override objects will be returned
1717     """
1718
1719     q = session.query(Override)
1720     q = q.filter_by(package=package)
1721
1722     if suite is not None:
1723         if not isinstance(suite, list): suite = [suite]
1724         q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1725
1726     if component is not None:
1727         if not isinstance(component, list): component = [component]
1728         q = q.join(Component).filter(Component.component_name.in_(component))
1729
1730     if overridetype is not None:
1731         if not isinstance(overridetype, list): overridetype = [overridetype]
1732         q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1733
1734     return q.all()
1735
1736 __all__.append('get_override')
1737
1738
1739 ################################################################################
1740
1741 class OverrideType(object):
1742     def __init__(self, *args, **kwargs):
1743         pass
1744
1745     def __repr__(self):
1746         return '<OverrideType %s>' % self.overridetype
1747
1748 __all__.append('OverrideType')
1749
1750 @session_wrapper
1751 def get_override_type(override_type, session=None):
1752     """
1753     Returns OverrideType object for given C{override type}.
1754
1755     @type override_type: string
1756     @param override_type: The name of the override type
1757
1758     @type session: Session
1759     @param session: Optional SQLA session object (a temporary one will be
1760     generated if not supplied)
1761
1762     @rtype: int
1763     @return: the database id for the given override type
1764     """
1765
1766     q = session.query(OverrideType).filter_by(overridetype=override_type)
1767
1768     try:
1769         return q.one()
1770     except NoResultFound:
1771         return None
1772
1773 __all__.append('get_override_type')
1774
1775 ################################################################################
1776
1777 class DebContents(object):
1778     def __init__(self, *args, **kwargs):
1779         pass
1780
1781     def __repr__(self):
1782         return '<DebConetnts %s: %s>' % (self.package.package,self.file)
1783
1784 __all__.append('DebContents')
1785
1786
1787 class UdebContents(object):
1788     def __init__(self, *args, **kwargs):
1789         pass
1790
1791     def __repr__(self):
1792         return '<UdebConetnts %s: %s>' % (self.package.package,self.file)
1793
1794 __all__.append('UdebContents')
1795
1796 class PendingBinContents(object):
1797     def __init__(self, *args, **kwargs):
1798         pass
1799
1800     def __repr__(self):
1801         return '<PendingBinContents %s>' % self.contents_id
1802
1803 __all__.append('PendingBinContents')
1804
1805 def insert_pending_content_paths(package,
1806                                  is_udeb,
1807                                  fullpaths,
1808                                  session=None):
1809     """
1810     Make sure given paths are temporarily associated with given
1811     package
1812
1813     @type package: dict
1814     @param package: the package to associate with should have been read in from the binary control file
1815     @type fullpaths: list
1816     @param fullpaths: the list of paths of the file being associated with the binary
1817     @type session: SQLAlchemy session
1818     @param session: Optional SQLAlchemy session.  If this is passed, the caller
1819     is responsible for ensuring a transaction has begun and committing the
1820     results or rolling back based on the result code.  If not passed, a commit
1821     will be performed at the end of the function
1822
1823     @return: True upon success, False if there is a problem
1824     """
1825
1826     privatetrans = False
1827
1828     if session is None:
1829         session = DBConn().session()
1830         privatetrans = True
1831
1832     try:
1833         arch = get_architecture(package['Architecture'], session)
1834         arch_id = arch.arch_id
1835
1836         # Remove any already existing recorded files for this package
1837         q = session.query(PendingBinContents)
1838         q = q.filter_by(package=package['Package'])
1839         q = q.filter_by(version=package['Version'])
1840         q = q.filter_by(architecture=arch_id)
1841         q.delete()
1842
1843         for fullpath in fullpaths:
1844
1845             if fullpath.startswith( "./" ):
1846                 fullpath = fullpath[2:]
1847
1848             pca = PendingBinContents()
1849             pca.package = package['Package']
1850             pca.version = package['Version']
1851             pca.file = fullpath
1852             pca.architecture = arch_id
1853
1854             if isudeb:
1855                 pca.type = 8 # gross
1856             else:
1857                 pca.type = 7 # also gross
1858             session.add(pca)
1859
1860         # Only commit if we set up the session ourself
1861         if privatetrans:
1862             session.commit()
1863             session.close()
1864         else:
1865             session.flush()
1866
1867         return True
1868     except Exception, e:
1869         traceback.print_exc()
1870
1871         # Only rollback if we set up the session ourself
1872         if privatetrans:
1873             session.rollback()
1874             session.close()
1875
1876         return False
1877
1878 __all__.append('insert_pending_content_paths')
1879
1880 ################################################################################
1881
1882 class PolicyQueue(object):
1883     def __init__(self, *args, **kwargs):
1884         pass
1885
1886     def __repr__(self):
1887         return '<PolicyQueue %s>' % self.queue_name
1888
1889 __all__.append('PolicyQueue')
1890
1891 @session_wrapper
1892 def get_policy_queue(queuename, session=None):
1893     """
1894     Returns PolicyQueue object for given C{queue name}
1895
1896     @type queuename: string
1897     @param queuename: The name of the queue
1898
1899     @type session: Session
1900     @param session: Optional SQLA session object (a temporary one will be
1901     generated if not supplied)
1902
1903     @rtype: PolicyQueue
1904     @return: PolicyQueue object for the given queue
1905     """
1906
1907     q = session.query(PolicyQueue).filter_by(queue_name=queuename)
1908
1909     try:
1910         return q.one()
1911     except NoResultFound:
1912         return None
1913
1914 __all__.append('get_policy_queue')
1915
1916 ################################################################################
1917
1918 class Priority(object):
1919     def __init__(self, *args, **kwargs):
1920         pass
1921
1922     def __eq__(self, val):
1923         if isinstance(val, str):
1924             return (self.priority == val)
1925         # This signals to use the normal comparison operator
1926         return NotImplemented
1927
1928     def __ne__(self, val):
1929         if isinstance(val, str):
1930             return (self.priority != val)
1931         # This signals to use the normal comparison operator
1932         return NotImplemented
1933
1934     def __repr__(self):
1935         return '<Priority %s (%s)>' % (self.priority, self.priority_id)
1936
1937 __all__.append('Priority')
1938
1939 @session_wrapper
1940 def get_priority(priority, session=None):
1941     """
1942     Returns Priority object for given C{priority name}.
1943
1944     @type priority: string
1945     @param priority: The name of the priority
1946
1947     @type session: Session
1948     @param session: Optional SQLA session object (a temporary one will be
1949     generated if not supplied)
1950
1951     @rtype: Priority
1952     @return: Priority object for the given priority
1953     """
1954
1955     q = session.query(Priority).filter_by(priority=priority)
1956
1957     try:
1958         return q.one()
1959     except NoResultFound:
1960         return None
1961
1962 __all__.append('get_priority')
1963
1964 @session_wrapper
1965 def get_priorities(session=None):
1966     """
1967     Returns dictionary of priority names -> id mappings
1968
1969     @type session: Session
1970     @param session: Optional SQL session object (a temporary one will be
1971     generated if not supplied)
1972
1973     @rtype: dictionary
1974     @return: dictionary of priority names -> id mappings
1975     """
1976
1977     ret = {}
1978     q = session.query(Priority)
1979     for x in q.all():
1980         ret[x.priority] = x.priority_id
1981
1982     return ret
1983
1984 __all__.append('get_priorities')
1985
1986 ################################################################################
1987
1988 class Section(object):
1989     def __init__(self, *args, **kwargs):
1990         pass
1991
1992     def __eq__(self, val):
1993         if isinstance(val, str):
1994             return (self.section == val)
1995         # This signals to use the normal comparison operator
1996         return NotImplemented
1997
1998     def __ne__(self, val):
1999         if isinstance(val, str):
2000             return (self.section != val)
2001         # This signals to use the normal comparison operator
2002         return NotImplemented
2003
2004     def __repr__(self):
2005         return '<Section %s>' % self.section
2006
2007 __all__.append('Section')
2008
2009 @session_wrapper
2010 def get_section(section, session=None):
2011     """
2012     Returns Section object for given C{section name}.
2013
2014     @type section: string
2015     @param section: The name of the section
2016
2017     @type session: Session
2018     @param session: Optional SQLA session object (a temporary one will be
2019     generated if not supplied)
2020
2021     @rtype: Section
2022     @return: Section object for the given section name
2023     """
2024
2025     q = session.query(Section).filter_by(section=section)
2026
2027     try:
2028         return q.one()
2029     except NoResultFound:
2030         return None
2031
2032 __all__.append('get_section')
2033
2034 @session_wrapper
2035 def get_sections(session=None):
2036     """
2037     Returns dictionary of section names -> id mappings
2038
2039     @type session: Session
2040     @param session: Optional SQL session object (a temporary one will be
2041     generated if not supplied)
2042
2043     @rtype: dictionary
2044     @return: dictionary of section names -> id mappings
2045     """
2046
2047     ret = {}
2048     q = session.query(Section)
2049     for x in q.all():
2050         ret[x.section] = x.section_id
2051
2052     return ret
2053
2054 __all__.append('get_sections')
2055
2056 ################################################################################
2057
2058 class DBSource(object):
2059     def __init__(self, *args, **kwargs):
2060         pass
2061
2062     def __repr__(self):
2063         return '<DBSource %s (%s)>' % (self.source, self.version)
2064
2065 __all__.append('DBSource')
2066
2067 @session_wrapper
2068 def source_exists(source, source_version, suites = ["any"], session=None):
2069     """
2070     Ensure that source exists somewhere in the archive for the binary
2071     upload being processed.
2072       1. exact match     => 1.0-3
2073       2. bin-only NMU    => 1.0-3+b1 , 1.0-3.1+b1
2074
2075     @type source: string
2076     @param source: source name
2077
2078     @type source_version: string
2079     @param source_version: expected source version
2080
2081     @type suites: list
2082     @param suites: list of suites to check in, default I{any}
2083
2084     @type session: Session
2085     @param session: Optional SQLA session object (a temporary one will be
2086     generated if not supplied)
2087
2088     @rtype: int
2089     @return: returns 1 if a source with expected version is found, otherwise 0
2090
2091     """
2092
2093     cnf = Config()
2094     ret = 1
2095
2096     for suite in suites:
2097         q = session.query(DBSource).filter_by(source=source)
2098         if suite != "any":
2099             # source must exist in suite X, or in some other suite that's
2100             # mapped to X, recursively... silent-maps are counted too,
2101             # unreleased-maps aren't.
2102             maps = cnf.ValueList("SuiteMappings")[:]
2103             maps.reverse()
2104             maps = [ m.split() for m in maps ]
2105             maps = [ (x[1], x[2]) for x in maps
2106                             if x[0] == "map" or x[0] == "silent-map" ]
2107             s = [suite]
2108             for x in maps:
2109                 if x[1] in s and x[0] not in s:
2110                     s.append(x[0])
2111
2112             q = q.join(SrcAssociation).join(Suite)
2113             q = q.filter(Suite.suite_name.in_(s))
2114
2115         # Reduce the query results to a list of version numbers
2116         ql = [ j.version for j in q.all() ]
2117
2118         # Try (1)
2119         if source_version in ql:
2120             continue
2121
2122         # Try (2)
2123         from daklib.regexes import re_bin_only_nmu
2124         orig_source_version = re_bin_only_nmu.sub('', source_version)
2125         if orig_source_version in ql:
2126             continue
2127
2128         # No source found so return not ok
2129         ret = 0
2130
2131     return ret
2132
2133 __all__.append('source_exists')
2134
2135 @session_wrapper
2136 def get_suites_source_in(source, session=None):
2137     """
2138     Returns list of Suite objects which given C{source} name is in
2139
2140     @type source: str
2141     @param source: DBSource package name to search for
2142
2143     @rtype: list
2144     @return: list of Suite objects for the given source
2145     """
2146
2147     return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
2148
2149 __all__.append('get_suites_source_in')
2150
2151 @session_wrapper
2152 def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
2153     """
2154     Returns list of DBSource objects for given C{source} name and other parameters
2155
2156     @type source: str
2157     @param source: DBSource package name to search for
2158
2159     @type version: str or None
2160     @param version: DBSource version name to search for or None if not applicable
2161
2162     @type dm_upload_allowed: bool
2163     @param dm_upload_allowed: If None, no effect.  If True or False, only
2164     return packages with that dm_upload_allowed setting
2165
2166     @type session: Session
2167     @param session: Optional SQL session object (a temporary one will be
2168     generated if not supplied)
2169
2170     @rtype: list
2171     @return: list of DBSource objects for the given name (may be empty)
2172     """
2173
2174     q = session.query(DBSource).filter_by(source=source)
2175
2176     if version is not None:
2177         q = q.filter_by(version=version)
2178
2179     if dm_upload_allowed is not None:
2180         q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
2181
2182     return q.all()
2183
2184 __all__.append('get_sources_from_name')
2185
2186 @session_wrapper
2187 def get_source_in_suite(source, suite, session=None):
2188     """
2189     Returns list of DBSource objects for a combination of C{source} and C{suite}.
2190
2191       - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
2192       - B{suite} - a suite name, eg. I{unstable}
2193
2194     @type source: string
2195     @param source: source package name
2196
2197     @type suite: string
2198     @param suite: the suite name
2199
2200     @rtype: string
2201     @return: the version for I{source} in I{suite}
2202
2203     """
2204
2205     q = session.query(SrcAssociation)
2206     q = q.join('source').filter_by(source=source)
2207     q = q.join('suite').filter_by(suite_name=suite)
2208
2209     try:
2210         return q.one().source
2211     except NoResultFound:
2212         return None
2213
2214 __all__.append('get_source_in_suite')
2215
2216 ################################################################################
2217
2218 @session_wrapper
2219 def add_dsc_to_db(u, filename, session=None):
2220     entry = u.pkg.files[filename]
2221     source = DBSource()
2222     pfs = []
2223
2224     source.source = u.pkg.dsc["source"]
2225     source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
2226     source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
2227     source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
2228     source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2229     source.install_date = datetime.now().date()
2230
2231     dsc_component = entry["component"]
2232     dsc_location_id = entry["location id"]
2233
2234     source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
2235
2236     # Set up a new poolfile if necessary
2237     if not entry.has_key("files id") or not entry["files id"]:
2238         filename = entry["pool name"] + filename
2239         poolfile = add_poolfile(filename, entry, dsc_location_id, session)
2240         session.flush()
2241         pfs.append(poolfile)
2242         entry["files id"] = poolfile.file_id
2243
2244     source.poolfile_id = entry["files id"]
2245     session.add(source)
2246     session.flush()
2247
2248     for suite_name in u.pkg.changes["distribution"].keys():
2249         sa = SrcAssociation()
2250         sa.source_id = source.source_id
2251         sa.suite_id = get_suite(suite_name).suite_id
2252         session.add(sa)
2253
2254     session.flush()
2255
2256     # Add the source files to the DB (files and dsc_files)
2257     dscfile = DSCFile()
2258     dscfile.source_id = source.source_id
2259     dscfile.poolfile_id = entry["files id"]
2260     session.add(dscfile)
2261
2262     for dsc_file, dentry in u.pkg.dsc_files.items():
2263         df = DSCFile()
2264         df.source_id = source.source_id
2265
2266         # If the .orig tarball is already in the pool, it's
2267         # files id is stored in dsc_files by check_dsc().
2268         files_id = dentry.get("files id", None)
2269
2270         # Find the entry in the files hash
2271         # TODO: Bail out here properly
2272         dfentry = None
2273         for f, e in u.pkg.files.items():
2274             if f == dsc_file:
2275                 dfentry = e
2276                 break
2277
2278         if files_id is None:
2279             filename = dfentry["pool name"] + dsc_file
2280
2281             (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
2282             # FIXME: needs to check for -1/-2 and or handle exception
2283             if found and obj is not None:
2284                 files_id = obj.file_id
2285                 pfs.append(obj)
2286
2287             # If still not found, add it
2288             if files_id is None:
2289                 # HACK: Force sha1sum etc into dentry
2290                 dentry["sha1sum"] = dfentry["sha1sum"]
2291                 dentry["sha256sum"] = dfentry["sha256sum"]
2292                 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
2293                 pfs.append(poolfile)
2294                 files_id = poolfile.file_id
2295         else:
2296             poolfile = get_poolfile_by_id(files_id, session)
2297             if poolfile is None:
2298                 utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id)
2299             pfs.append(poolfile)
2300
2301         df.poolfile_id = files_id
2302         session.add(df)
2303
2304     session.flush()
2305
2306     # Add the src_uploaders to the DB
2307     uploader_ids = [source.maintainer_id]
2308     if u.pkg.dsc.has_key("uploaders"):
2309         for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"):
2310             up = up.strip()
2311             uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
2312
2313     added_ids = {}
2314     for up_id in uploader_ids:
2315         if added_ids.has_key(up_id):
2316             import utils
2317             utils.warn("Already saw uploader %s for source %s" % (up_id, source.source))
2318             continue
2319
2320         added_ids[up_id]=1
2321
2322         su = SrcUploader()
2323         su.maintainer_id = up_id
2324         su.source_id = source.source_id
2325         session.add(su)
2326
2327     session.flush()
2328
2329     return source, dsc_component, dsc_location_id, pfs
2330
2331 __all__.append('add_dsc_to_db')
2332
2333 @session_wrapper
2334 def add_deb_to_db(u, filename, session=None):
2335     """
2336     Contrary to what you might expect, this routine deals with both
2337     debs and udebs.  That info is in 'dbtype', whilst 'type' is
2338     'deb' for both of them
2339     """
2340     cnf = Config()
2341     entry = u.pkg.files[filename]
2342
2343     bin = DBBinary()
2344     bin.package = entry["package"]
2345     bin.version = entry["version"]
2346     bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
2347     bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
2348     bin.arch_id = get_architecture(entry["architecture"], session).arch_id
2349     bin.binarytype = entry["dbtype"]
2350
2351     # Find poolfile id
2352     filename = entry["pool name"] + filename
2353     fullpath = os.path.join(cnf["Dir::Pool"], filename)
2354     if not entry.get("location id", None):
2355         entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
2356
2357     if entry.get("files id", None):
2358         poolfile = get_poolfile_by_id(bin.poolfile_id)
2359         bin.poolfile_id = entry["files id"]
2360     else:
2361         poolfile = add_poolfile(filename, entry, entry["location id"], session)
2362         bin.poolfile_id = entry["files id"] = poolfile.file_id
2363
2364     # Find source id
2365     bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
2366     if len(bin_sources) != 1:
2367         raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
2368                                   (bin.package, bin.version, bin.architecture.arch_string,
2369                                    filename, bin.binarytype, u.pkg.changes["fingerprint"])
2370
2371     bin.source_id = bin_sources[0].source_id
2372
2373     # Add and flush object so it has an ID
2374     session.add(bin)
2375     session.flush()
2376
2377     # Add BinAssociations
2378     for suite_name in u.pkg.changes["distribution"].keys():
2379         ba = BinAssociation()
2380         ba.binary_id = bin.binary_id
2381         ba.suite_id = get_suite(suite_name).suite_id
2382         session.add(ba)
2383
2384     session.flush()
2385
2386     # Deal with contents - disabled for now
2387     #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
2388     #if not contents:
2389     #    print "REJECT\nCould not determine contents of package %s" % bin.package
2390     #    session.rollback()
2391     #    raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
2392
2393     return poolfile
2394
2395 __all__.append('add_deb_to_db')
2396
2397 ################################################################################
2398
2399 class SourceACL(object):
2400     def __init__(self, *args, **kwargs):
2401         pass
2402
2403     def __repr__(self):
2404         return '<SourceACL %s>' % self.source_acl_id
2405
2406 __all__.append('SourceACL')
2407
2408 ################################################################################
2409
2410 class SrcAssociation(object):
2411     def __init__(self, *args, **kwargs):
2412         pass
2413
2414     def __repr__(self):
2415         return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
2416
2417 __all__.append('SrcAssociation')
2418
2419 ################################################################################
2420
2421 class SrcFormat(object):
2422     def __init__(self, *args, **kwargs):
2423         pass
2424
2425     def __repr__(self):
2426         return '<SrcFormat %s>' % (self.format_name)
2427
2428 __all__.append('SrcFormat')
2429
2430 ################################################################################
2431
2432 class SrcUploader(object):
2433     def __init__(self, *args, **kwargs):
2434         pass
2435
2436     def __repr__(self):
2437         return '<SrcUploader %s>' % self.uploader_id
2438
2439 __all__.append('SrcUploader')
2440
2441 ################################################################################
2442
2443 SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
2444                  ('SuiteID', 'suite_id'),
2445                  ('Version', 'version'),
2446                  ('Origin', 'origin'),
2447                  ('Label', 'label'),
2448                  ('Description', 'description'),
2449                  ('Untouchable', 'untouchable'),
2450                  ('Announce', 'announce'),
2451                  ('Codename', 'codename'),
2452                  ('OverrideCodename', 'overridecodename'),
2453                  ('ValidTime', 'validtime'),
2454                  ('Priority', 'priority'),
2455                  ('NotAutomatic', 'notautomatic'),
2456                  ('CopyChanges', 'copychanges'),
2457                  ('CopyDotDak', 'copydotdak'),
2458                  ('CommentsDir', 'commentsdir'),
2459                  ('OverrideSuite', 'overridesuite'),
2460                  ('ChangelogBase', 'changelogbase')]
2461
2462
2463 DAILY_APT_CONF="""
2464 Dir
2465 {
2466    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
2467    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
2468    CacheDir "/srv/ftp-master.debian.org/database/";
2469 };
2470
2471 Default
2472 {
2473    Packages::Compress "bzip2 gzip";
2474    Sources::Compress "bzip2 gzip";
2475    Contents::Compress "gzip";
2476    DeLinkLimit 0;
2477    MaxContentsChange 25000;
2478    FileMode 0664;
2479 }
2480
2481 TreeDefault
2482 {
2483    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
2484 };
2485
2486 """
2487
2488 apt_trees={}
2489 apt_trees["di"]={}
2490 apt_trees["testing"]="""
2491 tree "dists/testing"
2492 {
2493    FakeDI "dists/unstable";
2494    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
2495    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
2496    Sections "main contrib non-free";
2497    Architectures "%(arch)s";
2498    BinOverride "override.squeeze.$(SECTION)";
2499    ExtraOverride "override.squeeze.extra.$(SECTION)";
2500    SrcOverride "override.squeeze.$(SECTION).src";
2501 };
2502 """
2503 apt_trees["di"]["testing"]="""
2504 tree "dists/testing/main"
2505 {
2506    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
2507    Sections "debian-installer";
2508    Architectures "%(arch)s";
2509    BinOverride "override.squeeze.main.$(SECTION)";
2510    SrcOverride "override.squeeze.main.src";
2511    BinCacheDB "packages-debian-installer-$(ARCH).db";
2512    Packages::Extensions ".udeb";
2513    %(contentsline)s
2514 };
2515
2516 tree "dists/testing/non-free"
2517 {
2518    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
2519    Sections "debian-installer";
2520    Architectures "%(arch)s";
2521    BinOverride "override.squeeze.main.$(SECTION)";
2522    SrcOverride "override.squeeze.main.src";
2523    BinCacheDB "packages-debian-installer-$(ARCH).db";
2524    Packages::Extensions ".udeb";
2525    %(contentsline)s
2526 };
2527 """
2528
2529 apt_trees["unstable"]="""
2530 tree "dists/unstable"
2531 {
2532    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
2533    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
2534    Sections "main contrib non-free";
2535    Architectures "%(arch)s";
2536    BinOverride "override.sid.$(SECTION)";
2537    ExtraOverride "override.sid.extra.$(SECTION)";
2538    SrcOverride "override.sid.$(SECTION).src";
2539 };
2540 """
2541 apt_trees["di"]["unstable"]="""
2542 tree "dists/unstable/main"
2543 {
2544    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
2545    Sections "debian-installer";
2546    Architectures "%(arch)s";
2547    BinOverride "override.sid.main.$(SECTION)";
2548    SrcOverride "override.sid.main.src";
2549    BinCacheDB "packages-debian-installer-$(ARCH).db";
2550    Packages::Extensions ".udeb";
2551    %(contentsline)s
2552 };
2553
2554 tree "dists/unstable/non-free"
2555 {
2556    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
2557    Sections "debian-installer";
2558    Architectures "%(arch)s";
2559    BinOverride "override.sid.main.$(SECTION)";
2560    SrcOverride "override.sid.main.src";
2561    BinCacheDB "packages-debian-installer-$(ARCH).db";
2562    Packages::Extensions ".udeb";
2563    %(contentsline)s
2564 };
2565 """
2566
2567 apt_trees["experimental"]="""
2568 tree "dists/experimental"
2569 {
2570    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
2571    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
2572    Sections "main contrib non-free";
2573    Architectures "%(arch)s";
2574    BinOverride "override.sid.$(SECTION)";
2575    SrcOverride "override.sid.$(SECTION).src";
2576 };
2577 """
2578 apt_trees["di"]["experimental"]="""
2579 tree "dists/experimental/main"
2580 {
2581    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
2582    Sections "debian-installer";
2583    Architectures "%(arch)s";
2584    BinOverride "override.sid.main.$(SECTION)";
2585    SrcOverride "override.sid.main.src";
2586    BinCacheDB "packages-debian-installer-$(ARCH).db";
2587    Packages::Extensions ".udeb";
2588    %(contentsline)s
2589 };
2590
2591 tree "dists/experimental/non-free"
2592 {
2593    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
2594    Sections "debian-installer";
2595    Architectures "%(arch)s";
2596    BinOverride "override.sid.main.$(SECTION)";
2597    SrcOverride "override.sid.main.src";
2598    BinCacheDB "packages-debian-installer-$(ARCH).db";
2599    Packages::Extensions ".udeb";
2600    %(contentsline)s
2601 };
2602 """
2603
2604 apt_trees["testing-proposed-updates"]="""
2605 tree "dists/testing-proposed-updates"
2606 {
2607    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
2608    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
2609    Sections "main contrib non-free";
2610    Architectures "%(arch)s";
2611    BinOverride "override.squeeze.$(SECTION)";
2612    ExtraOverride "override.squeeze.extra.$(SECTION)";
2613    SrcOverride "override.squeeze.$(SECTION).src";
2614    Contents " ";
2615 };
2616 """
2617 apt_trees["di"]["testing-proposed-updates"]="""
2618 tree "dists/testing-proposed-updates/main"
2619 {
2620    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
2621    Sections "debian-installer";
2622    Architectures "%(arch)s";
2623    BinOverride "override.squeeze.main.$(SECTION)";
2624    SrcOverride "override.squeeze.main.src";
2625    BinCacheDB "packages-debian-installer-$(ARCH).db";
2626    Packages::Extensions ".udeb";
2627    Contents " ";
2628 };
2629 """
2630
2631 apt_trees["proposed-updates"]="""
2632 tree "dists/proposed-updates"
2633 {
2634    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
2635    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
2636    Sections "main contrib non-free";
2637    Architectures "%(arch)s";
2638    BinOverride "override.lenny.$(SECTION)";
2639    ExtraOverride "override.lenny.extra.$(SECTION)";
2640    SrcOverride "override.lenny.$(SECTION).src";
2641    Contents " ";
2642 };
2643 """
2644 apt_trees["di"]["proposed-updates"]="""
2645 tree "dists/proposed-updates/main"
2646 {
2647    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
2648    Sections "debian-installer";
2649    Architectures "%(arch)s";
2650    BinOverride "override.lenny.main.$(SECTION)";
2651    SrcOverride "override.lenny.main.src";
2652    BinCacheDB "packages-debian-installer-$(ARCH).db";
2653    Packages::Extensions ".udeb";
2654    Contents " ";
2655 };
2656 """
2657
2658 class Suite(object):
2659     def __init__(self, *args, **kwargs):
2660         pass
2661
2662     def __repr__(self):
2663         return '<Suite %s>' % self.suite_name
2664
2665     def __eq__(self, val):
2666         if isinstance(val, str):
2667             return (self.suite_name == val)
2668         # This signals to use the normal comparison operator
2669         return NotImplemented
2670
2671     def __ne__(self, val):
2672         if isinstance(val, str):
2673             return (self.suite_name != val)
2674         # This signals to use the normal comparison operator
2675         return NotImplemented
2676
2677     def details(self):
2678         ret = []
2679         for disp, field in SUITE_FIELDS:
2680             val = getattr(self, field, None)
2681             if val is not None:
2682                 ret.append("%s: %s" % (disp, val))
2683
2684         return "\n".join(ret)
2685
2686     def generate_packages_sources(self, arch):
2687         """
2688         Generate Packages/Sources files with apt-ftparchive for the given suite/arch
2689
2690         @type suite: string
2691         @param suite: Suite name
2692
2693         @type arch: string
2694         @param arch: Architecture name
2695         """
2696
2697         cnf = Config()
2698         try:
2699             # Write apt.conf
2700             (ac_fd, ac_name) = mkstemp(dir=cnf["Dir::TempPath"])
2701             os.write(ac_fd, DAILY_APT_CONF)
2702             # here we want to generate the tree entries
2703             os.write(ac_fd, apt_trees[self.suite_name] % {'arch': arch})
2704             # this special casing needs to go away, but this whole thing may just want an
2705             # aptconfig class anyways
2706             if arch != 'source':
2707                 if arch == 'hurd-i386' and self.suite_name == 'experimental':
2708                     pass
2709                 else:
2710                     if arch == "amd64":
2711                         os.write(ac_fd, apt_trees["di"][self.suite_name] %
2712                                  {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
2713                     else:
2714                         os.write(ac_fd, apt_trees["di"][self.suite_name] % {'arch': arch, 'contentsline': ''})
2715             os.close(ac_fd)
2716
2717             # Run apt-ftparchive generate
2718             # We might want to add a -q or -qq here
2719             os.environ['GZIP'] = '--rsyncable'
2720             os.chdir(cnf["Dir::TempPath"])
2721             (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
2722             sn="a-f %s,%s: " % (self.suite_name, arch)
2723             print sn + output.replace('\n', '\n%s' % (sn))
2724
2725         # Clean up any left behind files
2726         finally:
2727             if ac_fd:
2728                 try:
2729                     os.close(ac_fd)
2730                 except OSError:
2731                     pass
2732
2733             if ac_name:
2734                 try:
2735                     os.unlink(ac_name)
2736                 except OSError:
2737                     pass
2738
2739
2740
2741
2742 __all__.append('Suite')
2743
2744 @session_wrapper
2745 def get_suite_architecture(suite, architecture, session=None):
2746     """
2747     Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
2748     doesn't exist
2749
2750     @type suite: str
2751     @param suite: Suite name to search for
2752
2753     @type architecture: str
2754     @param architecture: Architecture name to search for
2755
2756     @type session: Session
2757     @param session: Optional SQL session object (a temporary one will be
2758     generated if not supplied)
2759
2760     @rtype: SuiteArchitecture
2761     @return: the SuiteArchitecture object or None
2762     """
2763
2764     q = session.query(SuiteArchitecture)
2765     q = q.join(Architecture).filter_by(arch_string=architecture)
2766     q = q.join(Suite).filter_by(suite_name=suite)
2767
2768     try:
2769         return q.one()
2770     except NoResultFound:
2771         return None
2772
2773 __all__.append('get_suite_architecture')
2774
2775 @session_wrapper
2776 def get_suite(suite, session=None):
2777     """
2778     Returns Suite object for given C{suite name}.
2779
2780     @type suite: string
2781     @param suite: The name of the suite
2782
2783     @type session: Session
2784     @param session: Optional SQLA session object (a temporary one will be
2785     generated if not supplied)
2786
2787     @rtype: Suite
2788     @return: Suite object for the requested suite name (None if not present)
2789     """
2790
2791     q = session.query(Suite).filter_by(suite_name=suite)
2792
2793     try:
2794         return q.one()
2795     except NoResultFound:
2796         return None
2797
2798 __all__.append('get_suite')
2799
2800 ################################################################################
2801
2802 class SuiteArchitecture(object):
2803     def __init__(self, *args, **kwargs):
2804         pass
2805
2806     def __repr__(self):
2807         return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
2808
2809 __all__.append('SuiteArchitecture')
2810
2811 @session_wrapper
2812 def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
2813     """
2814     Returns list of Architecture objects for given C{suite} name
2815
2816     @type suite: str
2817     @param suite: Suite name to search for
2818
2819     @type skipsrc: boolean
2820     @param skipsrc: Whether to skip returning the 'source' architecture entry
2821     (Default False)
2822
2823     @type skipall: boolean
2824     @param skipall: Whether to skip returning the 'all' architecture entry
2825     (Default False)
2826
2827     @type session: Session
2828     @param session: Optional SQL session object (a temporary one will be
2829     generated if not supplied)
2830
2831     @rtype: list
2832     @return: list of Architecture objects for the given name (may be empty)
2833     """
2834
2835     q = session.query(Architecture)
2836     q = q.join(SuiteArchitecture)
2837     q = q.join(Suite).filter_by(suite_name=suite)
2838
2839     if skipsrc:
2840         q = q.filter(Architecture.arch_string != 'source')
2841
2842     if skipall:
2843         q = q.filter(Architecture.arch_string != 'all')
2844
2845     q = q.order_by('arch_string')
2846
2847     return q.all()
2848
2849 __all__.append('get_suite_architectures')
2850
2851 ################################################################################
2852
2853 class SuiteSrcFormat(object):
2854     def __init__(self, *args, **kwargs):
2855         pass
2856
2857     def __repr__(self):
2858         return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
2859
2860 __all__.append('SuiteSrcFormat')
2861
2862 @session_wrapper
2863 def get_suite_src_formats(suite, session=None):
2864     """
2865     Returns list of allowed SrcFormat for C{suite}.
2866
2867     @type suite: str
2868     @param suite: Suite name to search for
2869
2870     @type session: Session
2871     @param session: Optional SQL session object (a temporary one will be
2872     generated if not supplied)
2873
2874     @rtype: list
2875     @return: the list of allowed source formats for I{suite}
2876     """
2877
2878     q = session.query(SrcFormat)
2879     q = q.join(SuiteSrcFormat)
2880     q = q.join(Suite).filter_by(suite_name=suite)
2881     q = q.order_by('format_name')
2882
2883     return q.all()
2884
2885 __all__.append('get_suite_src_formats')
2886
2887 ################################################################################
2888
2889 class Uid(object):
2890     def __init__(self, *args, **kwargs):
2891         pass
2892
2893     def __eq__(self, val):
2894         if isinstance(val, str):
2895             return (self.uid == val)
2896         # This signals to use the normal comparison operator
2897         return NotImplemented
2898
2899     def __ne__(self, val):
2900         if isinstance(val, str):
2901             return (self.uid != val)
2902         # This signals to use the normal comparison operator
2903         return NotImplemented
2904
2905     def __repr__(self):
2906         return '<Uid %s (%s)>' % (self.uid, self.name)
2907
2908 __all__.append('Uid')
2909
2910 @session_wrapper
2911 def add_database_user(uidname, session=None):
2912     """
2913     Adds a database user
2914
2915     @type uidname: string
2916     @param uidname: The uid of the user to add
2917
2918     @type session: SQLAlchemy
2919     @param session: Optional SQL session object (a temporary one will be
2920     generated if not supplied).  If not passed, a commit will be performed at
2921     the end of the function, otherwise the caller is responsible for commiting.
2922
2923     @rtype: Uid
2924     @return: the uid object for the given uidname
2925     """
2926
2927     session.execute("CREATE USER :uid", {'uid': uidname})
2928     session.commit_or_flush()
2929
2930 __all__.append('add_database_user')
2931
2932 @session_wrapper
2933 def get_or_set_uid(uidname, session=None):
2934     """
2935     Returns uid object for given uidname.
2936
2937     If no matching uidname is found, a row is inserted.
2938
2939     @type uidname: string
2940     @param uidname: The uid to add
2941
2942     @type session: SQLAlchemy
2943     @param session: Optional SQL session object (a temporary one will be
2944     generated if not supplied).  If not passed, a commit will be performed at
2945     the end of the function, otherwise the caller is responsible for commiting.
2946
2947     @rtype: Uid
2948     @return: the uid object for the given uidname
2949     """
2950
2951     q = session.query(Uid).filter_by(uid=uidname)
2952
2953     try:
2954         ret = q.one()
2955     except NoResultFound:
2956         uid = Uid()
2957         uid.uid = uidname
2958         session.add(uid)
2959         session.commit_or_flush()
2960         ret = uid
2961
2962     return ret
2963
2964 __all__.append('get_or_set_uid')
2965
2966 @session_wrapper
2967 def get_uid_from_fingerprint(fpr, session=None):
2968     q = session.query(Uid)
2969     q = q.join(Fingerprint).filter_by(fingerprint=fpr)
2970
2971     try:
2972         return q.one()
2973     except NoResultFound:
2974         return None
2975
2976 __all__.append('get_uid_from_fingerprint')
2977
2978 ################################################################################
2979
2980 class UploadBlock(object):
2981     def __init__(self, *args, **kwargs):
2982         pass
2983
2984     def __repr__(self):
2985         return '<UploadBlock %s (%s)>' % (self.source, self.upload_block_id)
2986
2987 __all__.append('UploadBlock')
2988
2989 ################################################################################
2990
2991 class DBConn(object):
2992     """
2993     database module init.
2994     """
2995     __shared_state = {}
2996
2997     def __init__(self, *args, **kwargs):
2998         self.__dict__ = self.__shared_state
2999
3000         if not getattr(self, 'initialised', False):
3001             self.initialised = True
3002             self.debug = kwargs.has_key('debug')
3003             self.__createconn()
3004
3005     def __setuptables(self):
3006         tables = (
3007             'architecture',
3008             'archive',
3009             'bin_associations',
3010             'binaries',
3011             'binary_acl',
3012             'binary_acl_map',
3013             'bin_contents',
3014             'build_queue',
3015             'build_queue_files',
3016             'component',
3017             'config',
3018             'changes_pending_binaries',
3019             'changes_pending_files',
3020             'changes_pending_files_map',
3021             'changes_pending_source',
3022             'changes_pending_source_files',
3023             'changes_pool_files',
3024             'deb_contents',
3025             'dsc_files',
3026             'files',
3027             'fingerprint',
3028             'keyrings',
3029             'changes',
3030             'keyring_acl_map',
3031             'location',
3032             'maintainer',
3033             'new_comments',
3034             'override',
3035             'override_type',
3036             'pending_bin_contents',
3037             'policy_queue',
3038             'priority',
3039             'section',
3040             'source',
3041             'source_acl',
3042             'src_associations',
3043             'src_format',
3044             'src_uploaders',
3045             'suite',
3046             'suite_architectures',
3047             'suite_src_formats',
3048             'suite_build_queue_copy',
3049             'udeb_contents',
3050             'uid',
3051             'upload_blocks',
3052         )
3053
3054         for table_name in tables:
3055             table = Table(table_name, self.db_meta, autoload=True)
3056             setattr(self, 'tbl_%s' % table_name, table)
3057
3058     def __setupmappers(self):
3059         mapper(Architecture, self.tbl_architecture,
3060                properties = dict(arch_id = self.tbl_architecture.c.id))
3061
3062         mapper(Archive, self.tbl_archive,
3063                properties = dict(archive_id = self.tbl_archive.c.id,
3064                                  archive_name = self.tbl_archive.c.name))
3065
3066         mapper(BinAssociation, self.tbl_bin_associations,
3067                properties = dict(ba_id = self.tbl_bin_associations.c.id,
3068                                  suite_id = self.tbl_bin_associations.c.suite,
3069                                  suite = relation(Suite),
3070                                  binary_id = self.tbl_bin_associations.c.bin,
3071                                  binary = relation(DBBinary)))
3072
3073         mapper(PendingBinContents, self.tbl_pending_bin_contents,
3074                properties = dict(contents_id =self.tbl_pending_bin_contents.c.id,
3075                                  filename = self.tbl_pending_bin_contents.c.filename,
3076                                  package = self.tbl_pending_bin_contents.c.package,
3077                                  version = self.tbl_pending_bin_contents.c.version,
3078                                  arch = self.tbl_pending_bin_contents.c.arch,
3079                                  otype = self.tbl_pending_bin_contents.c.type))
3080
3081         mapper(DebContents, self.tbl_deb_contents,
3082                properties = dict(binary_id=self.tbl_deb_contents.c.binary_id,
3083                                  package=self.tbl_deb_contents.c.package,
3084                                  suite=self.tbl_deb_contents.c.suite,
3085                                  arch=self.tbl_deb_contents.c.arch,
3086                                  section=self.tbl_deb_contents.c.section,
3087                                  filename=self.tbl_deb_contents.c.filename))
3088
3089         mapper(UdebContents, self.tbl_udeb_contents,
3090                properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id,
3091                                  package=self.tbl_udeb_contents.c.package,
3092                                  suite=self.tbl_udeb_contents.c.suite,
3093                                  arch=self.tbl_udeb_contents.c.arch,
3094                                  section=self.tbl_udeb_contents.c.section,
3095                                  filename=self.tbl_udeb_contents.c.filename))
3096
3097         mapper(BuildQueue, self.tbl_build_queue,
3098                properties = dict(queue_id = self.tbl_build_queue.c.id))
3099
3100         mapper(BuildQueueFile, self.tbl_build_queue_files,
3101                properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'),
3102                                  poolfile = relation(PoolFile, backref='buildqueueinstances')))
3103
3104         mapper(DBBinary, self.tbl_binaries,
3105                properties = dict(binary_id = self.tbl_binaries.c.id,
3106                                  package = self.tbl_binaries.c.package,
3107                                  version = self.tbl_binaries.c.version,
3108                                  maintainer_id = self.tbl_binaries.c.maintainer,
3109                                  maintainer = relation(Maintainer),
3110                                  source_id = self.tbl_binaries.c.source,
3111                                  source = relation(DBSource),
3112                                  arch_id = self.tbl_binaries.c.architecture,
3113                                  architecture = relation(Architecture),
3114                                  poolfile_id = self.tbl_binaries.c.file,
3115                                  poolfile = relation(PoolFile),
3116                                  binarytype = self.tbl_binaries.c.type,
3117                                  fingerprint_id = self.tbl_binaries.c.sig_fpr,
3118                                  fingerprint = relation(Fingerprint),
3119                                  install_date = self.tbl_binaries.c.install_date,
3120                                  binassociations = relation(BinAssociation,
3121                                                             primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
3122
3123         mapper(BinaryACL, self.tbl_binary_acl,
3124                properties = dict(binary_acl_id = self.tbl_binary_acl.c.id))
3125
3126         mapper(BinaryACLMap, self.tbl_binary_acl_map,
3127                properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id,
3128                                  fingerprint = relation(Fingerprint, backref="binary_acl_map"),
3129                                  architecture = relation(Architecture)))
3130
3131         mapper(Component, self.tbl_component,
3132                properties = dict(component_id = self.tbl_component.c.id,
3133                                  component_name = self.tbl_component.c.name))
3134
3135         mapper(DBConfig, self.tbl_config,
3136                properties = dict(config_id = self.tbl_config.c.id))
3137
3138         mapper(DSCFile, self.tbl_dsc_files,
3139                properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
3140                                  source_id = self.tbl_dsc_files.c.source,
3141                                  source = relation(DBSource),
3142                                  poolfile_id = self.tbl_dsc_files.c.file,
3143                                  poolfile = relation(PoolFile)))
3144
3145         mapper(PoolFile, self.tbl_files,
3146                properties = dict(file_id = self.tbl_files.c.id,
3147                                  filesize = self.tbl_files.c.size,
3148                                  location_id = self.tbl_files.c.location,
3149                                  location = relation(Location)))
3150
3151         mapper(Fingerprint, self.tbl_fingerprint,
3152                properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
3153                                  uid_id = self.tbl_fingerprint.c.uid,
3154                                  uid = relation(Uid),
3155                                  keyring_id = self.tbl_fingerprint.c.keyring,
3156                                  keyring = relation(Keyring),
3157                                  source_acl = relation(SourceACL),
3158                                  binary_acl = relation(BinaryACL)))
3159
3160         mapper(Keyring, self.tbl_keyrings,
3161                properties = dict(keyring_name = self.tbl_keyrings.c.name,
3162                                  keyring_id = self.tbl_keyrings.c.id))
3163
3164         mapper(DBChange, self.tbl_changes,
3165                properties = dict(change_id = self.tbl_changes.c.id,
3166                                  poolfiles = relation(PoolFile,
3167                                                       secondary=self.tbl_changes_pool_files,
3168                                                       backref="changeslinks"),
3169                                  seen = self.tbl_changes.c.seen,
3170                                  source = self.tbl_changes.c.source,
3171                                  binaries = self.tbl_changes.c.binaries,
3172                                  architecture = self.tbl_changes.c.architecture,
3173                                  distribution = self.tbl_changes.c.distribution,
3174                                  urgency = self.tbl_changes.c.urgency,
3175                                  maintainer = self.tbl_changes.c.maintainer,
3176                                  changedby = self.tbl_changes.c.changedby,
3177                                  date = self.tbl_changes.c.date,
3178                                  version = self.tbl_changes.c.version,
3179                                  files = relation(ChangePendingFile,
3180                                                   secondary=self.tbl_changes_pending_files_map,
3181                                                   backref="changesfile"),
3182                                  in_queue_id = self.tbl_changes.c.in_queue,
3183                                  in_queue = relation(PolicyQueue,
3184                                                      primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)),
3185                                  approved_for_id = self.tbl_changes.c.approved_for))
3186
3187         mapper(ChangePendingBinary, self.tbl_changes_pending_binaries,
3188                properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id))
3189
3190         mapper(ChangePendingFile, self.tbl_changes_pending_files,
3191                properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id,
3192                                  filename = self.tbl_changes_pending_files.c.filename,
3193                                  size = self.tbl_changes_pending_files.c.size,
3194                                  md5sum = self.tbl_changes_pending_files.c.md5sum,
3195                                  sha1sum = self.tbl_changes_pending_files.c.sha1sum,
3196                                  sha256sum = self.tbl_changes_pending_files.c.sha256sum))
3197
3198         mapper(ChangePendingSource, self.tbl_changes_pending_source,
3199                properties = dict(change_pending_source_id = self.tbl_changes_pending_source.c.id,
3200                                  change = relation(DBChange),
3201                                  maintainer = relation(Maintainer,
3202                                                        primaryjoin=(self.tbl_changes_pending_source.c.maintainer_id==self.tbl_maintainer.c.id)),
3203                                  changedby = relation(Maintainer,
3204                                                       primaryjoin=(self.tbl_changes_pending_source.c.changedby_id==self.tbl_maintainer.c.id)),
3205                                  fingerprint = relation(Fingerprint),
3206                                  source_files = relation(ChangePendingFile,
3207                                                          secondary=self.tbl_changes_pending_source_files,
3208                                                          backref="pending_sources")))
3209
3210
3211         mapper(KeyringACLMap, self.tbl_keyring_acl_map,
3212                properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id,
3213                                  keyring = relation(Keyring, backref="keyring_acl_map"),
3214                                  architecture = relation(Architecture)))
3215
3216         mapper(Location, self.tbl_location,
3217                properties = dict(location_id = self.tbl_location.c.id,
3218                                  component_id = self.tbl_location.c.component,
3219                                  component = relation(Component),
3220                                  archive_id = self.tbl_location.c.archive,
3221                                  archive = relation(Archive),
3222                                  archive_type = self.tbl_location.c.type))
3223
3224         mapper(Maintainer, self.tbl_maintainer,
3225                properties = dict(maintainer_id = self.tbl_maintainer.c.id))
3226
3227         mapper(NewComment, self.tbl_new_comments,
3228                properties = dict(comment_id = self.tbl_new_comments.c.id))
3229
3230         mapper(Override, self.tbl_override,
3231                properties = dict(suite_id = self.tbl_override.c.suite,
3232                                  suite = relation(Suite),
3233                                  package = self.tbl_override.c.package,
3234                                  component_id = self.tbl_override.c.component,
3235                                  component = relation(Component),
3236                                  priority_id = self.tbl_override.c.priority,
3237                                  priority = relation(Priority),
3238                                  section_id = self.tbl_override.c.section,
3239                                  section = relation(Section),
3240                                  overridetype_id = self.tbl_override.c.type,
3241                                  overridetype = relation(OverrideType)))
3242
3243         mapper(OverrideType, self.tbl_override_type,
3244                properties = dict(overridetype = self.tbl_override_type.c.type,
3245                                  overridetype_id = self.tbl_override_type.c.id))
3246
3247         mapper(PolicyQueue, self.tbl_policy_queue,
3248                properties = dict(policy_queue_id = self.tbl_policy_queue.c.id))
3249
3250         mapper(Priority, self.tbl_priority,
3251                properties = dict(priority_id = self.tbl_priority.c.id))
3252
3253         mapper(Section, self.tbl_section,
3254                properties = dict(section_id = self.tbl_section.c.id,
3255                                  section=self.tbl_section.c.section))
3256
3257         mapper(DBSource, self.tbl_source,
3258                properties = dict(source_id = self.tbl_source.c.id,
3259                                  version = self.tbl_source.c.version,
3260                                  maintainer_id = self.tbl_source.c.maintainer,
3261                                  maintainer = relation(Maintainer,
3262                                                        primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
3263                                  poolfile_id = self.tbl_source.c.file,
3264                                  poolfile = relation(PoolFile),
3265                                  fingerprint_id = self.tbl_source.c.sig_fpr,
3266                                  fingerprint = relation(Fingerprint),
3267                                  changedby_id = self.tbl_source.c.changedby,
3268                                  changedby = relation(Maintainer,
3269                                                       primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
3270                                  srcfiles = relation(DSCFile,
3271                                                      primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
3272                                  srcassociations = relation(SrcAssociation,
3273                                                             primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)),
3274                                  srcuploaders = relation(SrcUploader)))
3275
3276         mapper(SourceACL, self.tbl_source_acl,
3277                properties = dict(source_acl_id = self.tbl_source_acl.c.id))
3278
3279         mapper(SrcAssociation, self.tbl_src_associations,
3280                properties = dict(sa_id = self.tbl_src_associations.c.id,
3281                                  suite_id = self.tbl_src_associations.c.suite,
3282                                  suite = relation(Suite),
3283                                  source_id = self.tbl_src_associations.c.source,
3284                                  source = relation(DBSource)))
3285
3286         mapper(SrcFormat, self.tbl_src_format,
3287                properties = dict(src_format_id = self.tbl_src_format.c.id,
3288                                  format_name = self.tbl_src_format.c.format_name))
3289
3290         mapper(SrcUploader, self.tbl_src_uploaders,
3291                properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
3292                                  source_id = self.tbl_src_uploaders.c.source,
3293                                  source = relation(DBSource,
3294                                                    primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
3295                                  maintainer_id = self.tbl_src_uploaders.c.maintainer,
3296                                  maintainer = relation(Maintainer,
3297                                                        primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
3298
3299         mapper(Suite, self.tbl_suite,
3300                properties = dict(suite_id = self.tbl_suite.c.id,
3301                                  policy_queue = relation(PolicyQueue),
3302                                  copy_queues = relation(BuildQueue, secondary=self.tbl_suite_build_queue_copy)))
3303
3304         mapper(SuiteArchitecture, self.tbl_suite_architectures,
3305                properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
3306                                  suite = relation(Suite, backref='suitearchitectures'),
3307                                  arch_id = self.tbl_suite_architectures.c.architecture,
3308                                  architecture = relation(Architecture)))
3309
3310         mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
3311                properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
3312                                  suite = relation(Suite, backref='suitesrcformats'),
3313                                  src_format_id = self.tbl_suite_src_formats.c.src_format,
3314                                  src_format = relation(SrcFormat)))
3315
3316         mapper(Uid, self.tbl_uid,
3317                properties = dict(uid_id = self.tbl_uid.c.id,
3318                                  fingerprint = relation(Fingerprint)))
3319
3320         mapper(UploadBlock, self.tbl_upload_blocks,
3321                properties = dict(upload_block_id = self.tbl_upload_blocks.c.id,
3322                                  fingerprint = relation(Fingerprint, backref="uploadblocks"),
3323                                  uid = relation(Uid, backref="uploadblocks")))
3324
3325     ## Connection functions
3326     def __createconn(self):
3327         from config import Config
3328         cnf = Config()
3329         if cnf["DB::Host"]:
3330             # TCP/IP
3331             connstr = "postgres://%s" % cnf["DB::Host"]
3332             if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
3333                 connstr += ":%s" % cnf["DB::Port"]
3334             connstr += "/%s" % cnf["DB::Name"]
3335         else:
3336             # Unix Socket
3337             connstr = "postgres:///%s" % cnf["DB::Name"]
3338             if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
3339                 connstr += "?port=%s" % cnf["DB::Port"]
3340
3341         self.db_pg   = create_engine(connstr, echo=self.debug)
3342         self.db_meta = MetaData()
3343         self.db_meta.bind = self.db_pg
3344         self.db_smaker = sessionmaker(bind=self.db_pg,
3345                                       autoflush=True,
3346                                       autocommit=False)
3347
3348         self.__setuptables()
3349         self.__setupmappers()
3350
3351     def session(self):
3352         return self.db_smaker()
3353
3354 __all__.append('DBConn')
3355
3356