daklib/archive.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to manipulate the archive
  18
  19 This module provides classes to manipulate the archive.
  20 """
  21
  22 from daklib.dbconn import *
  23 import daklib.checks as checks
  24 from daklib.config import Config
  25 import daklib.upload as upload
  26 import daklib.utils as utils
  27 from daklib.fstransactions import FilesystemTransaction
  28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
  29
  30 import apt_pkg
  31 from datetime import datetime
  32 import os
  33 import shutil
  34 import subprocess
  35 from sqlalchemy.orm.exc import NoResultFound
  36 import tempfile
  37 import traceback
  38
  39 class ArchiveException(Exception):
  40     pass
  41
  42 class HashMismatchException(ArchiveException):
  43     pass
  44
  45 class ArchiveTransaction(object):
  46     """manipulate the archive in a transaction
  47     """
  48     def __init__(self):
  49         self.fs = FilesystemTransaction()
  50         self.session = DBConn().session()
  51
  52     def get_file(self, hashed_file, source_name):
  53         """Look for file C{hashed_file} in database
  54
  55         @type  hashed_file: L{daklib.upload.HashedFile}
  56         @param hashed_file: file to look for in the database
  57
  58         @raise KeyError: file was not found in the database
  59         @raise HashMismatchException: hash mismatch
  60
  61         @rtype:  L{daklib.dbconn.PoolFile}
  62         @return: database entry for the file
  63         """
  64         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  65         try:
  66             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
  67             if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
  68                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
  69             return poolfile
  70         except NoResultFound:
  71             raise KeyError('{0} not found in database.'.format(poolname))
  72
  73     def _install_file(self, directory, hashed_file, archive, component, source_name):
  74         """Install a file
  75
  76         Will not give an error when the file is already present.
  77
  78         @rtype:  L{daklib.dbconn.PoolFile}
  79         @return: batabase object for the new file
  80         """
  81         session = self.session
  82
  83         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  84         try:
  85             poolfile = self.get_file(hashed_file, source_name)
  86         except KeyError:
  87             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
  88             poolfile.md5sum = hashed_file.md5sum
  89             poolfile.sha1sum = hashed_file.sha1sum
  90             poolfile.sha256sum = hashed_file.sha256sum
  91             session.add(poolfile)
  92             session.flush()
  93
  94         try:
  95             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
  96         except NoResultFound:
  97             archive_file = ArchiveFile(archive, component, poolfile)
  98             session.add(archive_file)
  99             session.flush()
 100
 101             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
 102             hashed_file_path = os.path.join(directory, hashed_file.filename)
 103             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 104
 105         return poolfile
 106
 107     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
 108         """Install a binary package
 109
 110         @type  directory: str
 111         @param directory: directory the binary package is located in
 112
 113         @type  binary: L{daklib.upload.Binary}
 114         @param binary: binary package to install
 115
 116         @type  suite: L{daklib.dbconn.Suite}
 117         @param suite: target suite
 118
 119         @type  component: L{daklib.dbconn.Component}
 120         @param component: target component
 121
 122         @type  allow_tainted: bool
 123         @param allow_tainted: allow to copy additional files from tainted archives
 124
 125         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 126         @param fingerprint: optional fingerprint
 127
 128         @type  source_suites: list of L{daklib.dbconn.Suite} or C{True}
 129         @param source_suites: suites to copy the source from if they are not
 130                               in C{suite} or C{True} to allow copying from any
 131                               suite.
 132                               This can also be a SQLAlchemy (sub)query object.
 133
 134         @type  extra_source_archives: list of L{daklib.dbconn.Archive}
 135         @param extra_source_archives: extra archives to copy Built-Using sources from
 136
 137         @rtype:  L{daklib.dbconn.DBBinary}
 138         @return: databse object for the new package
 139         """
 140         session = self.session
 141         control = binary.control
 142         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 143         architecture = get_architecture(control['Architecture'], session)
 144
 145         (source_name, source_version) = binary.source
 146         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
 147         source = source_query.filter(DBSource.suites.contains(suite)).first()
 148         if source is None:
 149             if source_suites != True:
 150                 source_query = source_query.join(DBSource.suites) \
 151                     .filter(Suite.suite_id == source_suites.c.id)
 152             source = source_query.first()
 153             if source is None:
 154                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
 155             self.copy_source(source, suite, component)
 156
 157         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
 158
 159         unique = dict(
 160             package=control['Package'],
 161             version=control['Version'],
 162             architecture=architecture,
 163             )
 164         rest = dict(
 165             source=source,
 166             maintainer=maintainer,
 167             poolfile=db_file,
 168             binarytype=binary.type,
 169             fingerprint=fingerprint,
 170             )
 171
 172         try:
 173             db_binary = session.query(DBBinary).filter_by(**unique).one()
 174             for key, value in rest.iteritems():
 175                 if getattr(db_binary, key) != value:
 176                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
 177         except NoResultFound:
 178             db_binary = DBBinary(**unique)
 179             for key, value in rest.iteritems():
 180                 setattr(db_binary, key, value)
 181             session.add(db_binary)
 182             session.flush()
 183             import_metadata_into_db(db_binary, session)
 184
 185             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
 186
 187         if suite not in db_binary.suites:
 188             db_binary.suites.append(suite)
 189
 190         session.flush()
 191
 192         return db_binary
 193
 194     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
 195         """ensure source exists in the given archive
 196
 197         This is intended to be used to check that Built-Using sources exist.
 198
 199         @type  filename: str
 200         @param filename: filename to use in error messages
 201
 202         @type  source: L{daklib.dbconn.DBSource}
 203         @param source: source to look for
 204
 205         @type  archive: L{daklib.dbconn.Archive}
 206         @param archive: archive to look in
 207
 208         @type  extra_archives: list of L{daklib.dbconn.Archive}
 209         @param extra_archives: list of archives to copy the source package from
 210                                if it is not yet present in C{archive}
 211         """
 212         session = self.session
 213         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
 214         if db_file is not None:
 215             return True
 216
 217         # Try to copy file from one extra archive
 218         if extra_archives is None:
 219             extra_archives = []
 220         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
 221         if db_file is None:
 222             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
 223
 224         source_archive = db_file.archive
 225         for dsc_file in source.srcfiles:
 226             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
 227             # We were given an explicit list of archives so it is okay to copy from tainted archives.
 228             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 229
 230     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
 231         """Add Built-Using sources to C{db_binary.extra_sources}
 232         """
 233         session = self.session
 234         built_using = control.get('Built-Using', None)
 235
 236         if built_using is not None:
 237             for dep in apt_pkg.parse_depends(built_using):
 238                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
 239                 bu_source_name, bu_source_version, comp = dep[0]
 240                 assert comp == '=', 'Built-Using must contain strict dependencies'
 241
 242                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
 243                 if bu_source is None:
 244                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
 245
 246                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
 247
 248                 db_binary.extra_sources.append(bu_source)
 249
 250     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
 251         """Install a source package
 252
 253         @type  directory: str
 254         @param directory: directory the source package is located in
 255
 256         @type  source: L{daklib.upload.Source}
 257         @param source: source package to install
 258
 259         @type  suite: L{daklib.dbconn.Suite}
 260         @param suite: target suite
 261
 262         @type  component: L{daklib.dbconn.Component}
 263         @param component: target component
 264
 265         @type  changed_by: L{daklib.dbconn.Maintainer}
 266         @param changed_by: person who prepared this version of the package
 267
 268         @type  allow_tainted: bool
 269         @param allow_tainted: allow to copy additional files from tainted archives
 270
 271         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 272         @param fingerprint: optional fingerprint
 273
 274         @rtype:  L{daklib.dbconn.DBSource}
 275         @return: database object for the new source
 276         """
 277         session = self.session
 278         archive = suite.archive
 279         control = source.dsc
 280         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 281         source_name = control['Source']
 282
 283         ### Add source package to database
 284
 285         # We need to install the .dsc first as the DBSource object refers to it.
 286         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
 287
 288         unique = dict(
 289             source=source_name,
 290             version=control['Version'],
 291             )
 292         rest = dict(
 293             maintainer=maintainer,
 294             changedby=changed_by,
 295             #install_date=datetime.now().date(),
 296             poolfile=db_file_dsc,
 297             fingerprint=fingerprint,
 298             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
 299             )
 300
 301         created = False
 302         try:
 303             db_source = session.query(DBSource).filter_by(**unique).one()
 304             for key, value in rest.iteritems():
 305                 if getattr(db_source, key) != value:
 306                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
 307         except NoResultFound:
 308             created = True
 309             db_source = DBSource(**unique)
 310             for key, value in rest.iteritems():
 311                 setattr(db_source, key, value)
 312             # XXX: set as default in postgres?
 313             db_source.install_date = datetime.now().date()
 314             session.add(db_source)
 315             session.flush()
 316
 317             # Add .dsc file. Other files will be added later.
 318             db_dsc_file = DSCFile()
 319             db_dsc_file.source = db_source
 320             db_dsc_file.poolfile = db_file_dsc
 321             session.add(db_dsc_file)
 322             session.flush()
 323
 324         if suite in db_source.suites:
 325             return db_source
 326
 327         db_source.suites.append(suite)
 328
 329         if not created:
 330             return db_source
 331
 332         ### Now add remaining files and copy them to the archive.
 333
 334         for hashed_file in source.files.itervalues():
 335             hashed_file_path = os.path.join(directory, hashed_file.filename)
 336             if os.path.exists(hashed_file_path):
 337                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
 338                 session.add(db_file)
 339             else:
 340                 db_file = self.get_file(hashed_file, source_name)
 341                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
 342
 343             db_dsc_file = DSCFile()
 344             db_dsc_file.source = db_source
 345             db_dsc_file.poolfile = db_file
 346             session.add(db_dsc_file)
 347
 348         session.flush()
 349
 350         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
 351         import_metadata_into_db(db_source, session)
 352
 353         # Uploaders are the maintainer and co-maintainers from the Uploaders field
 354         db_source.uploaders.append(maintainer)
 355         if 'Uploaders' in control:
 356             def split_uploaders(field):
 357                 import re
 358                 for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
 359                     yield u.strip()
 360
 361             for u in split_uploaders(control['Uploaders']):
 362                 db_source.uploaders.append(get_or_set_maintainer(u, session))
 363         session.flush()
 364
 365         return db_source
 366
 367     def _copy_file(self, db_file, archive, component, allow_tainted=False):
 368         """Copy a file to the given archive and component
 369
 370         @type  db_file: L{daklib.dbconn.PoolFile}
 371         @param db_file: file to copy
 372
 373         @type  archive: L{daklib.dbconn.Archive}
 374         @param archive: target archive
 375
 376         @type  component: L{daklib.dbconn.Archive}
 377         @param component: target component
 378
 379         @type  allow_tainted: bool
 380         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 381         """
 382         session = self.session
 383
 384         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
 385             query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
 386             if not allow_tainted:
 387                 query = query.join(Archive).filter(Archive.tainted == False)
 388
 389             source_af = query.first()
 390             if source_af is None:
 391                 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
 392             target_af = ArchiveFile(archive, component, db_file)
 393             session.add(target_af)
 394             session.flush()
 395             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
 396
 397     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
 398         """Copy a binary package to the given suite and component
 399
 400         @type  db_binary: L{daklib.dbconn.DBBinary}
 401         @param db_binary: binary to copy
 402
 403         @type  suite: L{daklib.dbconn.Suite}
 404         @param suite: target suite
 405
 406         @type  component: L{daklib.dbconn.Component}
 407         @param component: target component
 408
 409         @type  allow_tainted: bool
 410         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 411
 412         @type  extra_archives: list of L{daklib.dbconn.Archive}
 413         @param extra_archives: extra archives to copy Built-Using sources from
 414         """
 415         session = self.session
 416         archive = suite.archive
 417         if archive.tainted:
 418             allow_tainted = True
 419
 420         filename = db_binary.poolfile.filename
 421
 422         # make sure source is present in target archive
 423         db_source = db_binary.source
 424         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
 425             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
 426
 427         # make sure built-using packages are present in target archive
 428         for db_source in db_binary.extra_sources:
 429             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 430
 431         # copy binary
 432         db_file = db_binary.poolfile
 433         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
 434         if suite not in db_binary.suites:
 435             db_binary.suites.append(suite)
 436         self.session.flush()
 437
 438     def copy_source(self, db_source, suite, component, allow_tainted=False):
 439         """Copy a source package to the given suite and component
 440
 441         @type  db_source: L{daklib.dbconn.DBSource}
 442         @param db_source: source to copy
 443
 444         @type  suite: L{daklib.dbconn.Suite}
 445         @param suite: target suite
 446
 447         @type  component: L{daklib.dbconn.Component}
 448         @param component: target component
 449
 450         @type  allow_tainted: bool
 451         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 452         """
 453         archive = suite.archive
 454         if archive.tainted:
 455             allow_tainted = True
 456         for db_dsc_file in db_source.srcfiles:
 457             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
 458         if suite not in db_source.suites:
 459             db_source.suites.append(suite)
 460         self.session.flush()
 461
 462     def remove_file(self, db_file, archive, component):
 463         """Remove a file from a given archive and component
 464
 465         @type  db_file: L{daklib.dbconn.PoolFile}
 466         @param db_file: file to remove
 467
 468         @type  archive: L{daklib.dbconn.Archive}
 469         @param archive: archive to remove the file from
 470
 471         @type  component: L{daklib.dbconn.Component}
 472         @param component: component to remove the file from
 473         """
 474         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
 475         self.fs.unlink(af.path)
 476         self.session.delete(af)
 477
 478     def remove_binary(self, binary, suite):
 479         """Remove a binary from a given suite and component
 480
 481         @type  binary: L{daklib.dbconn.DBBinary}
 482         @param binary: binary to remove
 483
 484         @type  suite: L{daklib.dbconn.Suite}
 485         @param suite: suite to remove the package from
 486         """
 487         binary.suites.remove(suite)
 488         self.session.flush()
 489
 490     def remove_source(self, source, suite):
 491         """Remove a source from a given suite and component
 492
 493         @type  source: L{daklib.dbconn.DBSource}
 494         @param source: source to remove
 495
 496         @type  suite: L{daklib.dbconn.Suite}
 497         @param suite: suite to remove the package from
 498
 499         @raise ArchiveException: source package is still referenced by other
 500                                  binaries in the suite
 501         """
 502         session = self.session
 503
 504         query = session.query(DBBinary).filter_by(source=source) \
 505             .filter(DBBinary.suites.contains(suite))
 506         if query.first() is not None:
 507             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
 508
 509         source.suites.remove(suite)
 510         session.flush()
 511
 512     def commit(self):
 513         """commit changes"""
 514         try:
 515             self.session.commit()
 516             self.fs.commit()
 517         finally:
 518             self.session.rollback()
 519             self.fs.rollback()
 520
 521     def rollback(self):
 522         """rollback changes"""
 523         self.session.rollback()
 524         self.fs.rollback()
 525
 526     def __enter__(self):
 527         return self
 528
 529     def __exit__(self, type, value, traceback):
 530         if type is None:
 531             self.commit()
 532         else:
 533             self.rollback()
 534         return None
 535
 536 class ArchiveUpload(object):
 537     """handle an upload
 538
 539     This class can be used in a with-statement::
 540
 541        with ArchiveUpload(...) as upload:
 542           ...
 543
 544     Doing so will automatically run any required cleanup and also rollback the
 545     transaction if it was not committed.
 546     """
 547     def __init__(self, directory, changes, keyrings):
 548         self.transaction = ArchiveTransaction()
 549         """transaction used to handle the upload
 550         @type: L{daklib.archive.ArchiveTransaction}
 551         """
 552
 553         self.session = self.transaction.session
 554         """database session"""
 555
 556         self.original_directory = directory
 557         self.original_changes = changes
 558
 559         self.changes = None
 560         """upload to process
 561         @type: L{daklib.upload.Changes}
 562         """
 563
 564         self.directory = None
 565         """directory with temporary copy of files. set by C{prepare}
 566         @type: str
 567         """
 568
 569         self.keyrings = keyrings
 570
 571         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
 572         """fingerprint of the key used to sign the upload
 573         @type: L{daklib.dbconn.Fingerprint}
 574         """
 575
 576         self.reject_reasons = []
 577         """reasons why the upload cannot by accepted
 578         @type: list of str
 579         """
 580
 581         self.warnings = []
 582         """warnings
 583         @note: Not used yet.
 584         @type: list of str
 585         """
 586
 587         self.final_suites = None
 588
 589         self.new = False
 590         """upload is NEW. set by C{check}
 591         @type: bool
 592         """
 593
 594         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
 595         self._new = self._new_queue.suite
 596
 597     def prepare(self):
 598         """prepare upload for further processing
 599
 600         This copies the files involved to a temporary directory.  If you use
 601         this method directly, you have to remove the directory given by the
 602         C{directory} attribute later on your own.
 603
 604         Instead of using the method directly, you can also use a with-statement::
 605
 606            with ArchiveUpload(...) as upload:
 607               ...
 608
 609         This will automatically handle any required cleanup.
 610         """
 611         assert self.directory is None
 612         assert self.original_changes.valid_signature
 613
 614         cnf = Config()
 615         session = self.transaction.session
 616
 617         self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
 618         with FilesystemTransaction() as fs:
 619             src = os.path.join(self.original_directory, self.original_changes.filename)
 620             dst = os.path.join(self.directory, self.original_changes.filename)
 621             fs.copy(src, dst)
 622
 623             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 624
 625             for f in self.changes.files.itervalues():
 626                 src = os.path.join(self.original_directory, f.filename)
 627                 dst = os.path.join(self.directory, f.filename)
 628                 if not os.path.exists(src):
 629                     continue
 630                 fs.copy(src, dst)
 631
 632             source = self.changes.source
 633             if source is not None:
 634                 for f in source.files.itervalues():
 635                     src = os.path.join(self.original_directory, f.filename)
 636                     dst = os.path.join(self.directory, f.filename)
 637                     if f.filename not in self.changes.files:
 638                         try:
 639                             db_file = self.transaction.get_file(f, source.dsc['Source'])
 640                             db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
 641                             fs.copy(db_archive_file.path, dst, symlink=True)
 642                         except KeyError:
 643                             # Ignore if get_file could not find it. Upload will
 644                             # probably be rejected later.
 645                             pass
 646
 647     def unpacked_source(self):
 648         """Path to unpacked source
 649
 650         Get path to the unpacked source. This method does unpack the source
 651         into a temporary directory under C{self.directory} if it has not
 652         been done so already.
 653
 654         @rtype:  str or C{None}
 655         @return: string giving the path to the unpacked source directory
 656                  or C{None} if no source was included in the upload.
 657         """
 658         assert self.directory is not None
 659
 660         source = self.changes.source
 661         if source is None:
 662             return None
 663         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
 664
 665         sourcedir = os.path.join(self.directory, 'source')
 666         if not os.path.exists(sourcedir):
 667             subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
 668         if not os.path.isdir(sourcedir):
 669             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
 670         return sourcedir
 671
 672     def _map_suite(self, suite_name):
 673         for rule in Config().value_list("SuiteMappings"):
 674             fields = rule.split()
 675             rtype = fields[0]
 676             if rtype == "map" or rtype == "silent-map":
 677                 (src, dst) = fields[1:3]
 678                 if src == suite_name:
 679                     suite_name = dst
 680                     if rtype != "silent-map":
 681                         self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
 682             elif rtype == "ignore":
 683                 ignored = fields[1]
 684                 if suite_name == ignored:
 685                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
 686                     suite_name = None
 687             elif rtype == "reject":
 688                 rejected = fields[1]
 689                 if suite_name == rejected:
 690                     self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
 691             ## XXX: propup-version and map-unreleased not yet implemented
 692         return suite_name
 693
 694     def _mapped_suites(self):
 695         """Get target suites after mappings
 696
 697         @rtype:  list of L{daklib.dbconn.Suite}
 698         @return: list giving the mapped target suites of this upload
 699         """
 700         session = self.session
 701
 702         suite_names = []
 703         for dist in self.changes.distributions:
 704             suite_name = self._map_suite(dist)
 705             if suite_name is not None:
 706                 suite_names.append(suite_name)
 707
 708         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
 709         return suites
 710
 711     def _check_new(self, suite):
 712         """Check if upload is NEW
 713
 714         An upload is NEW if it has binary or source packages that do not have
 715         an override in C{suite} OR if it references files ONLY in a tainted
 716         archive (eg. when it references files in NEW).
 717
 718         @rtype:  bool
 719         @return: C{True} if the upload is NEW, C{False} otherwise
 720         """
 721         session = self.session
 722
 723         # Check for missing overrides
 724         for b in self.changes.binaries:
 725             override = self._binary_override(suite, b)
 726             if override is None:
 727                 return True
 728
 729         if self.changes.source is not None:
 730             override = self._source_override(suite, self.changes.source)
 731             if override is None:
 732                 return True
 733
 734         # Check if we reference a file only in a tainted archive
 735         files = self.changes.files.values()
 736         if self.changes.source is not None:
 737             files.extend(self.changes.source.files.values())
 738         for f in files:
 739             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
 740             query_untainted = query.join(Archive).filter(Archive.tainted == False)
 741
 742             in_archive = (query.first() is not None)
 743             in_untainted_archive = (query_untainted.first() is not None)
 744
 745             if in_archive and not in_untainted_archive:
 746                 return True
 747
 748     def _final_suites(self):
 749         session = self.session
 750
 751         mapped_suites = self._mapped_suites()
 752         final_suites = set()
 753
 754         for suite in mapped_suites:
 755             overridesuite = suite
 756             if suite.overridesuite is not None:
 757                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 758             if self._check_new(overridesuite):
 759                 self.new = True
 760             final_suites.add(suite)
 761
 762         return final_suites
 763
 764     def _binary_override(self, suite, binary):
 765         """Get override entry for a binary
 766
 767         @type  suite: L{daklib.dbconn.Suite}
 768         @param suite: suite to get override for
 769
 770         @type  binary: L{daklib.upload.Binary}
 771         @param binary: binary to get override for
 772
 773         @rtype:  L{daklib.dbconn.Override} or C{None}
 774         @return: override for the given binary or C{None}
 775         """
 776         if suite.overridesuite is not None:
 777             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 778
 779         query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
 780                 .join(Component).filter(Component.component_name == binary.component) \
 781                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 782
 783         try:
 784             return query.one()
 785         except NoResultFound:
 786             return None
 787
 788     def _source_override(self, suite, source):
 789         """Get override entry for a source
 790
 791         @type  suite: L{daklib.dbconn.Suite}
 792         @param suite: suite to get override for
 793
 794         @type  source: L{daklib.upload.Source}
 795         @param source: source to get override for
 796
 797         @rtype:  L{daklib.dbconn.Override} or C{None}
 798         @return: override for the given source or C{None}
 799         """
 800         if suite.overridesuite is not None:
 801             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 802
 803         # XXX: component for source?
 804         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
 805                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 806
 807         try:
 808             return query.one()
 809         except NoResultFound:
 810             return None
 811
 812     def _binary_component(self, suite, binary, only_overrides=True):
 813         """get component for a binary
 814
 815         By default this will only look at overrides to get the right component;
 816         if C{only_overrides} is C{False} this method will also look at the
 817         Section field.
 818
 819         @type  suite: L{daklib.dbconn.Suite}
 820
 821         @type  binary: L{daklib.upload.Binary}
 822
 823         @type  only_overrides: bool
 824         @param only_overrides: only use overrides to get the right component
 825
 826         @rtype: L{daklib.dbconn.Component} or C{None}
 827         """
 828         override = self._binary_override(suite, binary)
 829         if override is not None:
 830             return override.component
 831         if only_overrides:
 832             return None
 833         return get_mapped_component(binary.component, self.session)
 834
 835     def check(self, force=False):
 836         """run checks against the upload
 837
 838         @type  force: bool
 839         @param force: ignore failing forcable checks
 840
 841         @rtype:  bool
 842         @return: C{True} if all checks passed, C{False} otherwise
 843         """
 844         # XXX: needs to be better structured.
 845         assert self.changes.valid_signature
 846
 847         try:
 848             for chk in (
 849                     checks.SignatureCheck,
 850                     checks.ChangesCheck,
 851                     checks.TransitionCheck,
 852                     checks.UploadBlockCheck,
 853                     checks.HashesCheck,
 854                     checks.SourceCheck,
 855                     checks.BinaryCheck,
 856                     checks.BinaryTimestampCheck,
 857                     checks.ACLCheck,
 858                     checks.SingleDistributionCheck,
 859                     checks.NoSourceOnlyCheck,
 860                     checks.LintianCheck,
 861                     ):
 862                 chk().check(self)
 863
 864             final_suites = self._final_suites()
 865             if len(final_suites) == 0:
 866                 self.reject_reasons.append('Ended with no suite to install to.')
 867                 return False
 868
 869             for chk in (
 870                     checks.SourceFormatCheck,
 871                     checks.SuiteArchitectureCheck,
 872                     checks.VersionCheck,
 873                     ):
 874                 for suite in final_suites:
 875                     chk().per_suite_check(self, suite)
 876
 877             if len(self.reject_reasons) != 0:
 878                 return False
 879
 880             self.final_suites = final_suites
 881             return True
 882         except checks.Reject as e:
 883             self.reject_reasons.append(unicode(e))
 884         except Exception as e:
 885             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
 886         return False
 887
 888     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
 889         """Install upload to the given suite
 890
 891         @type  suite: L{daklib.dbconn.Suite}
 892         @param suite: suite to install the package into. This is the real suite,
 893                       ie. after any redirection to NEW or a policy queue
 894
 895         @param source_component_func: function to get the L{daklib.dbconn.Component}
 896                                       for a L{daklib.upload.Source} object
 897
 898         @param binary_component_func: function to get the L{daklib.dbconn.Component}
 899                                       for a L{daklib.upload.Binary} object
 900
 901         @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
 902
 903         @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
 904
 905         @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
 906                  object for the install source or C{None} if no source was
 907                  included. The second is a list of L{daklib.dbconn.DBBinary}
 908                  objects for the installed binary packages.
 909         """
 910         # XXX: move this function to ArchiveTransaction?
 911
 912         control = self.changes.changes
 913         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 914
 915         if source_suites is None:
 916             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
 917
 918         source = self.changes.source
 919         if source is not None:
 920             component = source_component_func(source)
 921             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
 922         else:
 923             db_source = None
 924
 925         db_binaries = []
 926         for binary in self.changes.binaries:
 927             component = binary_component_func(binary)
 928             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
 929             db_binaries.append(db_binary)
 930
 931         if suite.copychanges:
 932             src = os.path.join(self.directory, self.changes.filename)
 933             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
 934             self.transaction.fs.copy(src, dst)
 935
 936         return (db_source, db_binaries)
 937
 938     def _install_changes(self):
 939         assert self.changes.valid_signature
 940         control = self.changes.changes
 941         session = self.transaction.session
 942         config = Config()
 943
 944         changelog_id = None
 945         # Only add changelog for sourceful uploads and binNMUs
 946         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
 947             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
 948             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
 949             assert changelog_id is not None
 950
 951         db_changes = DBChange()
 952         db_changes.changesname = self.changes.filename
 953         db_changes.source = control['Source']
 954         db_changes.binaries = control.get('Binary', None)
 955         db_changes.architecture = control['Architecture']
 956         db_changes.version = control['Version']
 957         db_changes.distribution = control['Distribution']
 958         db_changes.urgency = control['Urgency']
 959         db_changes.maintainer = control['Maintainer']
 960         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
 961         db_changes.date = control['Date']
 962         db_changes.fingerprint = self.fingerprint.fingerprint
 963         db_changes.changelog_id = changelog_id
 964         db_changes.closes = self.changes.closed_bugs
 965
 966         self.transaction.session.add(db_changes)
 967         self.transaction.session.flush()
 968
 969         return db_changes
 970
 971     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
 972         u = PolicyQueueUpload()
 973         u.policy_queue = policy_queue
 974         u.target_suite = target_suite
 975         u.changes = db_changes
 976         u.source = db_source
 977         u.binaries = db_binaries
 978         self.transaction.session.add(u)
 979         self.transaction.session.flush()
 980
 981         dst = os.path.join(policy_queue.path, self.changes.filename)
 982         self.transaction.fs.copy(self.changes.path, dst)
 983
 984         return u
 985
 986     def try_autobyhand(self):
 987         """Try AUTOBYHAND
 988
 989         Try to handle byhand packages automatically.
 990
 991         @rtype:  list of L{daklib.upload.HashedFile}
 992         @return: list of remaining byhand files
 993         """
 994         assert len(self.reject_reasons) == 0
 995         assert self.changes.valid_signature
 996         assert self.final_suites is not None
 997
 998         byhand = self.changes.byhand_files
 999         if len(byhand) == 0:
1000             return True
1001
1002         suites = list(self.final_suites)
1003         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1004         suite = suites[0]
1005
1006         cnf = Config()
1007         control = self.changes.changes
1008         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1009
1010         remaining = []
1011         for f in byhand:
1012             parts = f.filename.split('_', 2)
1013             if len(parts) != 3:
1014                 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1015                 remaining.append(f)
1016                 continue
1017
1018             package, version, archext = parts
1019             arch, ext = archext.split('.', 1)
1020
1021             rule = automatic_byhand_packages.get(package)
1022             if rule is None:
1023                 remaining.append(f)
1024                 continue
1025
1026             if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
1027                 remaining.append(f)
1028                 continue
1029
1030             script = rule['Script']
1031             retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1032             if retcode != 0:
1033                 print "W: error processing {0}.".format(f.filename)
1034                 remaining.append(f)
1035
1036         return len(remaining) == 0
1037
1038     def _install_byhand(self, policy_queue_upload, hashed_file):
1039         """install byhand file
1040
1041         @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1042
1043         @type  hashed_file: L{daklib.upload.HashedFile}
1044         """
1045         fs = self.transaction.fs
1046         session = self.transaction.session
1047         policy_queue = policy_queue_upload.policy_queue
1048
1049         byhand_file = PolicyQueueByhandFile()
1050         byhand_file.upload = policy_queue_upload
1051         byhand_file.filename = hashed_file.filename
1052         session.add(byhand_file)
1053         session.flush()
1054
1055         src = os.path.join(self.directory, hashed_file.filename)
1056         dst = os.path.join(policy_queue.path, hashed_file.filename)
1057         fs.copy(src, dst)
1058
1059         return byhand_file
1060
1061     def _do_bts_versiontracking(self):
1062         cnf = Config()
1063         fs = self.transaction.fs
1064
1065         btsdir = cnf.get('Dir::BTSVersionTrack')
1066         if btsdir is None or btsdir == '':
1067             return
1068
1069         base = os.path.join(btsdir, self.changes.filename[:-8])
1070
1071         # version history
1072         sourcedir = self.unpacked_source()
1073         if sourcedir is not None:
1074             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1075             versions = fs.create("{0}.versions".format(base), mode=0o644)
1076             for line in fh.readlines():
1077                 if re_changelog_versions.match(line):
1078                     versions.write(line)
1079             fh.close()
1080             versions.close()
1081
1082         # binary -> source mapping
1083         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1084         for binary in self.changes.binaries:
1085             control = binary.control
1086             source_package, source_version = binary.source
1087             line = " ".join([control['Package'], control['Version'], source_package, source_version])
1088             print >>debinfo, line
1089         debinfo.close()
1090
1091     def _policy_queue(self, suite):
1092         if suite.policy_queue is not None:
1093             return suite.policy_queue
1094         return None
1095
1096     def install(self):
1097         """install upload
1098
1099         Install upload to a suite or policy queue.  This method does B{not}
1100         handle uploads to NEW.
1101
1102         You need to have called the C{check} method before calling this method.
1103         """
1104         assert len(self.reject_reasons) == 0
1105         assert self.changes.valid_signature
1106         assert self.final_suites is not None
1107         assert not self.new
1108
1109         db_changes = self._install_changes()
1110
1111         for suite in self.final_suites:
1112             overridesuite = suite
1113             if suite.overridesuite is not None:
1114                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1115
1116             policy_queue = self._policy_queue(suite)
1117
1118             redirected_suite = suite
1119             if policy_queue is not None:
1120                 redirected_suite = policy_queue.suite
1121
1122             source_component_func = lambda source: self._source_override(overridesuite, source).component
1123             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1124
1125             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1126
1127             if policy_queue is not None:
1128                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1129
1130             # copy to build queues
1131             if policy_queue is None or policy_queue.send_to_build_queues:
1132                 for build_queue in suite.copy_queues:
1133                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1134
1135         self._do_bts_versiontracking()
1136
1137     def install_to_new(self):
1138         """install upload to NEW
1139
1140         Install upload to NEW.  This method does B{not} handle regular uploads
1141         to suites or policy queues.
1142
1143         You need to have called the C{check} method before calling this method.
1144         """
1145         # Uploads to NEW are special as we don't have overrides.
1146         assert len(self.reject_reasons) == 0
1147         assert self.changes.valid_signature
1148         assert self.final_suites is not None
1149
1150         source = self.changes.source
1151         binaries = self.changes.binaries
1152         byhand = self.changes.byhand_files
1153
1154         new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1155         if len(byhand) > 0:
1156             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1157         new_suite = new_queue.suite
1158
1159         # we need a suite to guess components
1160         suites = list(self.final_suites)
1161         assert len(suites) == 1, "NEW uploads must be to a single suite"
1162         suite = suites[0]
1163
1164         def binary_component_func(binary):
1165             return self._binary_component(suite, binary, only_overrides=False)
1166
1167         # guess source component
1168         # XXX: should be moved into an extra method
1169         binary_component_names = set()
1170         for binary in binaries:
1171             component = binary_component_func(binary)
1172             binary_component_names.add(component.component_name)
1173         source_component_name = None
1174         for c in self.session.query(Component).order_by(Component.component_id):
1175             guess = c.component_name
1176             if guess in binary_component_names:
1177                 source_component_name = guess
1178                 break
1179         if source_component_name is None:
1180             raise Exception('Could not guess source component.')
1181         source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1182         source_component_func = lambda source: source_component
1183
1184         db_changes = self._install_changes()
1185         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1186         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1187
1188         for f in byhand:
1189             self._install_byhand(policy_upload, f)
1190
1191         self._do_bts_versiontracking()
1192
1193     def commit(self):
1194         """commit changes"""
1195         self.transaction.commit()
1196
1197     def rollback(self):
1198         """rollback changes"""
1199         self.transaction.rollback()
1200
1201     def __enter__(self):
1202         self.prepare()
1203         return self
1204
1205     def __exit__(self, type, value, traceback):
1206         if self.directory is not None:
1207             shutil.rmtree(self.directory)
1208             self.directory = None
1209         self.changes = None
1210         self.transaction.rollback()
1211         return None