daklib/archive.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to manipulate the archive
  18
  19 This module provides classes to manipulate the archive.
  20 """
  21
  22 from daklib.dbconn import *
  23 import daklib.checks as checks
  24 from daklib.config import Config
  25 import daklib.upload as upload
  26 import daklib.utils as utils
  27 from daklib.fstransactions import FilesystemTransaction
  28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
  29
  30 import apt_pkg
  31 from datetime import datetime
  32 import os
  33 import shutil
  34 import subprocess
  35 from sqlalchemy.orm.exc import NoResultFound
  36 import tempfile
  37 import traceback
  38
  39 class ArchiveException(Exception):
  40     pass
  41
  42 class HashMismatchException(ArchiveException):
  43     pass
  44
  45 class ArchiveTransaction(object):
  46     """manipulate the archive in a transaction
  47     """
  48     def __init__(self):
  49         self.fs = FilesystemTransaction()
  50         self.session = DBConn().session()
  51
  52     def get_file(self, hashed_file, source_name):
  53         """Look for file C{hashed_file} in database
  54
  55         @type  hashed_file: L{daklib.upload.HashedFile}
  56         @param hashed_file: file to look for in the database
  57
  58         @raise KeyError: file was not found in the database
  59         @raise HashMismatchException: hash mismatch
  60
  61         @rtype:  L{daklib.dbconn.PoolFile}
  62         @return: database entry for the file
  63         """
  64         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  65         try:
  66             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
  67             if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
  68                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
  69             return poolfile
  70         except NoResultFound:
  71             raise KeyError('{0} not found in database.'.format(poolname))
  72
  73     def _install_file(self, directory, hashed_file, archive, component, source_name):
  74         """Install a file
  75
  76         Will not give an error when the file is already present.
  77
  78         @rtype:  L{daklib.dbconn.PoolFile}
  79         @return: batabase object for the new file
  80         """
  81         session = self.session
  82
  83         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  84         try:
  85             poolfile = self.get_file(hashed_file, source_name)
  86         except KeyError:
  87             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
  88             poolfile.md5sum = hashed_file.md5sum
  89             poolfile.sha1sum = hashed_file.sha1sum
  90             poolfile.sha256sum = hashed_file.sha256sum
  91             session.add(poolfile)
  92             session.flush()
  93
  94         try:
  95             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
  96         except NoResultFound:
  97             archive_file = ArchiveFile(archive, component, poolfile)
  98             session.add(archive_file)
  99             session.flush()
 100
 101             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
 102             hashed_file_path = os.path.join(directory, hashed_file.filename)
 103             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 104
 105         return poolfile
 106
 107     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
 108         """Install a binary package
 109
 110         @type  directory: str
 111         @param directory: directory the binary package is located in
 112
 113         @type  binary: L{daklib.upload.Binary}
 114         @param binary: binary package to install
 115
 116         @type  suite: L{daklib.dbconn.Suite}
 117         @param suite: target suite
 118
 119         @type  component: L{daklib.dbconn.Component}
 120         @param component: target component
 121
 122         @type  allow_tainted: bool
 123         @param allow_tainted: allow to copy additional files from tainted archives
 124
 125         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 126         @param fingerprint: optional fingerprint
 127
 128         @type  source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
 129         @param source_suites: suites to copy the source from if they are not
 130                               in C{suite} or C{True} to allow copying from any
 131                               suite.
 132
 133         @type  extra_source_archives: list of L{daklib.dbconn.Archive}
 134         @param extra_source_archives: extra archives to copy Built-Using sources from
 135
 136         @rtype:  L{daklib.dbconn.DBBinary}
 137         @return: databse object for the new package
 138         """
 139         session = self.session
 140         control = binary.control
 141         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 142         architecture = get_architecture(control['Architecture'], session)
 143
 144         (source_name, source_version) = binary.source
 145         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
 146         source = source_query.filter(DBSource.suites.contains(suite)).first()
 147         if source is None:
 148             if source_suites != True:
 149                 source_query = source_query.join(DBSource.suites) \
 150                     .filter(Suite.suite_id == source_suites.c.id)
 151             source = source_query.first()
 152             if source is None:
 153                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
 154             self.copy_source(source, suite, component)
 155
 156         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
 157
 158         unique = dict(
 159             package=control['Package'],
 160             version=control['Version'],
 161             architecture=architecture,
 162             )
 163         rest = dict(
 164             source=source,
 165             maintainer=maintainer,
 166             poolfile=db_file,
 167             binarytype=binary.type,
 168             fingerprint=fingerprint,
 169             )
 170
 171         try:
 172             db_binary = session.query(DBBinary).filter_by(**unique).one()
 173             for key, value in rest.iteritems():
 174                 if getattr(db_binary, key) != value:
 175                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
 176         except NoResultFound:
 177             db_binary = DBBinary(**unique)
 178             for key, value in rest.iteritems():
 179                 setattr(db_binary, key, value)
 180             session.add(db_binary)
 181             session.flush()
 182             import_metadata_into_db(db_binary, session)
 183
 184             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
 185
 186         if suite not in db_binary.suites:
 187             db_binary.suites.append(suite)
 188
 189         session.flush()
 190
 191         return db_binary
 192
 193     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
 194         """ensure source exists in the given archive
 195
 196         This is intended to be used to check that Built-Using sources exist.
 197
 198         @type  filename: str
 199         @param filename: filename to use in error messages
 200
 201         @type  source: L{daklib.dbconn.DBSource}
 202         @param source: source to look for
 203
 204         @type  archive: L{daklib.dbconn.Archive}
 205         @param archive: archive to look in
 206
 207         @type  extra_archives: list of L{daklib.dbconn.Archive}
 208         @param extra_archives: list of archives to copy the source package from
 209                                if it is not yet present in C{archive}
 210         """
 211         session = self.session
 212         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
 213         if db_file is not None:
 214             return True
 215
 216         # Try to copy file from one extra archive
 217         if extra_archives is None:
 218             extra_archives = []
 219         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
 220         if db_file is None:
 221             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
 222
 223         source_archive = db_file.archive
 224         for dsc_file in source.srcfiles:
 225             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
 226             # We were given an explicit list of archives so it is okay to copy from tainted archives.
 227             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 228
 229     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
 230         """Add Built-Using sources to C{db_binary.extra_sources}
 231         """
 232         session = self.session
 233         built_using = control.get('Built-Using', None)
 234
 235         if built_using is not None:
 236             for dep in apt_pkg.parse_depends(built_using):
 237                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
 238                 bu_source_name, bu_source_version, comp = dep[0]
 239                 assert comp == '=', 'Built-Using must contain strict dependencies'
 240
 241                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
 242                 if bu_source is None:
 243                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
 244
 245                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
 246
 247                 db_binary.extra_sources.append(bu_source)
 248
 249     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
 250         """Install a source package
 251
 252         @type  directory: str
 253         @param directory: directory the source package is located in
 254
 255         @type  source: L{daklib.upload.Source}
 256         @param source: source package to install
 257
 258         @type  suite: L{daklib.dbconn.Suite}
 259         @param suite: target suite
 260
 261         @type  component: L{daklib.dbconn.Component}
 262         @param component: target component
 263
 264         @type  changed_by: L{daklib.dbconn.Maintainer}
 265         @param changed_by: person who prepared this version of the package
 266
 267         @type  allow_tainted: bool
 268         @param allow_tainted: allow to copy additional files from tainted archives
 269
 270         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 271         @param fingerprint: optional fingerprint
 272
 273         @rtype:  L{daklib.dbconn.DBSource}
 274         @return: database object for the new source
 275         """
 276         session = self.session
 277         archive = suite.archive
 278         control = source.dsc
 279         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 280         source_name = control['Source']
 281
 282         ### Add source package to database
 283
 284         # We need to install the .dsc first as the DBSource object refers to it.
 285         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
 286
 287         unique = dict(
 288             source=source_name,
 289             version=control['Version'],
 290             )
 291         rest = dict(
 292             maintainer=maintainer,
 293             changedby=changed_by,
 294             #install_date=datetime.now().date(),
 295             poolfile=db_file_dsc,
 296             fingerprint=fingerprint,
 297             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
 298             )
 299
 300         created = False
 301         try:
 302             db_source = session.query(DBSource).filter_by(**unique).one()
 303             for key, value in rest.iteritems():
 304                 if getattr(db_source, key) != value:
 305                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
 306         except NoResultFound:
 307             created = True
 308             db_source = DBSource(**unique)
 309             for key, value in rest.iteritems():
 310                 setattr(db_source, key, value)
 311             # XXX: set as default in postgres?
 312             db_source.install_date = datetime.now().date()
 313             session.add(db_source)
 314             session.flush()
 315
 316             # Add .dsc file. Other files will be added later.
 317             db_dsc_file = DSCFile()
 318             db_dsc_file.source = db_source
 319             db_dsc_file.poolfile = db_file_dsc
 320             session.add(db_dsc_file)
 321             session.flush()
 322
 323         if suite in db_source.suites:
 324             return db_source
 325
 326         db_source.suites.append(suite)
 327
 328         if not created:
 329             return db_source
 330
 331         ### Now add remaining files and copy them to the archive.
 332
 333         for hashed_file in source.files.itervalues():
 334             hashed_file_path = os.path.join(directory, hashed_file.filename)
 335             if os.path.exists(hashed_file_path):
 336                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
 337                 session.add(db_file)
 338             else:
 339                 db_file = self.get_file(hashed_file, source_name)
 340                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
 341
 342             db_dsc_file = DSCFile()
 343             db_dsc_file.source = db_source
 344             db_dsc_file.poolfile = db_file
 345             session.add(db_dsc_file)
 346
 347         session.flush()
 348
 349         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
 350         import_metadata_into_db(db_source, session)
 351
 352         # Uploaders are the maintainer and co-maintainers from the Uploaders field
 353         db_source.uploaders.append(maintainer)
 354         if 'Uploaders' in control:
 355             def split_uploaders(field):
 356                 import re
 357                 for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
 358                     yield u.strip()
 359
 360             for u in split_uploaders(control['Uploaders']):
 361                 db_source.uploaders.append(get_or_set_maintainer(u, session))
 362         session.flush()
 363
 364         return db_source
 365
 366     def _copy_file(self, db_file, archive, component, allow_tainted=False):
 367         """Copy a file to the given archive and component
 368
 369         @type  db_file: L{daklib.dbconn.PoolFile}
 370         @param db_file: file to copy
 371
 372         @type  archive: L{daklib.dbconn.Archive}
 373         @param archive: target archive
 374
 375         @type  component: L{daklib.dbconn.Archive}
 376         @param component: target component
 377
 378         @type  allow_tainted: bool
 379         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 380         """
 381         session = self.session
 382
 383         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
 384             query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
 385             if not allow_tainted:
 386                 query = query.join(Archive).filter(Archive.tainted == False)
 387
 388             source_af = query.first()
 389             if source_af is None:
 390                 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
 391             target_af = ArchiveFile(archive, component, db_file)
 392             session.add(target_af)
 393             session.flush()
 394             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
 395
 396     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
 397         """Copy a binary package to the given suite and component
 398
 399         @type  db_binary: L{daklib.dbconn.DBBinary}
 400         @param db_binary: binary to copy
 401
 402         @type  suite: L{daklib.dbconn.Suite}
 403         @param suite: target suite
 404
 405         @type  component: L{daklib.dbconn.Component}
 406         @param component: target component
 407
 408         @type  allow_tainted: bool
 409         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 410
 411         @type  extra_archives: list of L{daklib.dbconn.Archive}
 412         @param extra_archives: extra archives to copy Built-Using sources from
 413         """
 414         session = self.session
 415         archive = suite.archive
 416         if archive.tainted:
 417             allow_tainted = True
 418
 419         filename = db_binary.poolfile.filename
 420
 421         # make sure source is present in target archive
 422         db_source = db_binary.source
 423         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
 424             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
 425
 426         # make sure built-using packages are present in target archive
 427         for db_source in db_binary.extra_sources:
 428             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 429
 430         # copy binary
 431         db_file = db_binary.poolfile
 432         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
 433         if suite not in db_binary.suites:
 434             db_binary.suites.append(suite)
 435         self.session.flush()
 436
 437     def copy_source(self, db_source, suite, component, allow_tainted=False):
 438         """Copy a source package to the given suite and component
 439
 440         @type  db_source: L{daklib.dbconn.DBSource}
 441         @param db_source: source to copy
 442
 443         @type  suite: L{daklib.dbconn.Suite}
 444         @param suite: target suite
 445
 446         @type  component: L{daklib.dbconn.Component}
 447         @param component: target component
 448
 449         @type  allow_tainted: bool
 450         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 451         """
 452         archive = suite.archive
 453         if archive.tainted:
 454             allow_tainted = True
 455         for db_dsc_file in db_source.srcfiles:
 456             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
 457         if suite not in db_source.suites:
 458             db_source.suites.append(suite)
 459         self.session.flush()
 460
 461     def remove_file(self, db_file, archive, component):
 462         """Remove a file from a given archive and component
 463
 464         @type  db_file: L{daklib.dbconn.PoolFile}
 465         @param db_file: file to remove
 466
 467         @type  archive: L{daklib.dbconn.Archive}
 468         @param archive: archive to remove the file from
 469
 470         @type  component: L{daklib.dbconn.Component}
 471         @param component: component to remove the file from
 472         """
 473         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
 474         self.fs.unlink(af.path)
 475         self.session.delete(af)
 476
 477     def remove_binary(self, binary, suite):
 478         """Remove a binary from a given suite and component
 479
 480         @type  binary: L{daklib.dbconn.DBBinary}
 481         @param binary: binary to remove
 482
 483         @type  suite: L{daklib.dbconn.Suite}
 484         @param suite: suite to remove the package from
 485         """
 486         binary.suites.remove(suite)
 487         self.session.flush()
 488
 489     def remove_source(self, source, suite):
 490         """Remove a source from a given suite and component
 491
 492         @type  source: L{daklib.dbconn.DBSource}
 493         @param source: source to remove
 494
 495         @type  suite: L{daklib.dbconn.Suite}
 496         @param suite: suite to remove the package from
 497
 498         @raise ArchiveException: source package is still referenced by other
 499                                  binaries in the suite
 500         """
 501         session = self.session
 502
 503         query = session.query(DBBinary).filter_by(source=source) \
 504             .filter(DBBinary.suites.contains(suite))
 505         if query.first() is not None:
 506             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
 507
 508         source.suites.remove(suite)
 509         session.flush()
 510
 511     def commit(self):
 512         """commit changes"""
 513         try:
 514             self.session.commit()
 515             self.fs.commit()
 516         finally:
 517             self.session.rollback()
 518             self.fs.rollback()
 519
 520     def rollback(self):
 521         """rollback changes"""
 522         self.session.rollback()
 523         self.fs.rollback()
 524
 525     def __enter__(self):
 526         return self
 527
 528     def __exit__(self, type, value, traceback):
 529         if type is None:
 530             self.commit()
 531         else:
 532             self.rollback()
 533         return None
 534
 535 class ArchiveUpload(object):
 536     """handle an upload
 537
 538     This class can be used in a with-statement::
 539
 540        with ArchiveUpload(...) as upload:
 541           ...
 542
 543     Doing so will automatically run any required cleanup and also rollback the
 544     transaction if it was not committed.
 545     """
 546     def __init__(self, directory, changes, keyrings):
 547         self.transaction = ArchiveTransaction()
 548         """transaction used to handle the upload
 549         @type: L{daklib.archive.ArchiveTransaction}
 550         """
 551
 552         self.session = self.transaction.session
 553         """database session"""
 554
 555         self.original_directory = directory
 556         self.original_changes = changes
 557
 558         self.changes = None
 559         """upload to process
 560         @type: L{daklib.upload.Changes}
 561         """
 562
 563         self.directory = None
 564         """directory with temporary copy of files. set by C{prepare}
 565         @type: str
 566         """
 567
 568         self.keyrings = keyrings
 569
 570         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
 571         """fingerprint of the key used to sign the upload
 572         @type: L{daklib.dbconn.Fingerprint}
 573         """
 574
 575         self.reject_reasons = []
 576         """reasons why the upload cannot by accepted
 577         @type: list of str
 578         """
 579
 580         self.warnings = []
 581         """warnings
 582         @note: Not used yet.
 583         @type: list of str
 584         """
 585
 586         self.final_suites = None
 587
 588         self.new = False
 589         """upload is NEW. set by C{check}
 590         @type: bool
 591         """
 592
 593         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
 594         self._new = self._new_queue.suite
 595
 596     def prepare(self):
 597         """prepare upload for further processing
 598
 599         This copies the files involved to a temporary directory.  If you use
 600         this method directly, you have to remove the directory given by the
 601         C{directory} attribute later on your own.
 602
 603         Instead of using the method directly, you can also use a with-statement::
 604
 605            with ArchiveUpload(...) as upload:
 606               ...
 607
 608         This will automatically handle any required cleanup.
 609         """
 610         assert self.directory is None
 611         assert self.original_changes.valid_signature
 612
 613         cnf = Config()
 614         session = self.transaction.session
 615
 616         self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
 617         with FilesystemTransaction() as fs:
 618             src = os.path.join(self.original_directory, self.original_changes.filename)
 619             dst = os.path.join(self.directory, self.original_changes.filename)
 620             fs.copy(src, dst)
 621
 622             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 623
 624             for f in self.changes.files.itervalues():
 625                 src = os.path.join(self.original_directory, f.filename)
 626                 dst = os.path.join(self.directory, f.filename)
 627                 if not os.path.exists(src):
 628                     continue
 629                 fs.copy(src, dst)
 630
 631             source = self.changes.source
 632             if source is not None:
 633                 for f in source.files.itervalues():
 634                     src = os.path.join(self.original_directory, f.filename)
 635                     dst = os.path.join(self.directory, f.filename)
 636                     if not os.path.exists(dst):
 637                         try:
 638                             db_file = self.transaction.get_file(f, source.dsc['Source'])
 639                             db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
 640                             fs.copy(db_archive_file.path, dst, symlink=True)
 641                         except KeyError:
 642                             # Ignore if get_file could not find it. Upload will
 643                             # probably be rejected later.
 644                             pass
 645
 646     def unpacked_source(self):
 647         """Path to unpacked source
 648
 649         Get path to the unpacked source. This method does unpack the source
 650         into a temporary directory under C{self.directory} if it has not
 651         been done so already.
 652
 653         @rtype:  str or C{None}
 654         @return: string giving the path to the unpacked source directory
 655                  or C{None} if no source was included in the upload.
 656         """
 657         assert self.directory is not None
 658
 659         source = self.changes.source
 660         if source is None:
 661             return None
 662         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
 663
 664         sourcedir = os.path.join(self.directory, 'source')
 665         if not os.path.exists(sourcedir):
 666             subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
 667         if not os.path.isdir(sourcedir):
 668             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
 669         return sourcedir
 670
 671     def _map_suite(self, suite_name):
 672         for rule in Config().value_list("SuiteMappings"):
 673             fields = rule.split()
 674             rtype = fields[0]
 675             if rtype == "map" or rtype == "silent-map":
 676                 (src, dst) = fields[1:3]
 677                 if src == suite_name:
 678                     suite_name = dst
 679                     if rtype != "silent-map":
 680                         self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
 681             elif rtype == "ignore":
 682                 ignored = fields[1]
 683                 if suite_name == ignored:
 684                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
 685                     suite_name = None
 686             elif rtype == "reject":
 687                 rejected = fields[1]
 688                 if suite_name == rejected:
 689                     self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
 690             ## XXX: propup-version and map-unreleased not yet implemented
 691         return suite_name
 692
 693     def _mapped_suites(self):
 694         """Get target suites after mappings
 695
 696         @rtype:  list of L{daklib.dbconn.Suite}
 697         @return: list giving the mapped target suites of this upload
 698         """
 699         session = self.session
 700
 701         suite_names = []
 702         for dist in self.changes.distributions:
 703             suite_name = self._map_suite(dist)
 704             if suite_name is not None:
 705                 suite_names.append(suite_name)
 706
 707         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
 708         return suites
 709
 710     def _check_new(self, suite):
 711         """Check if upload is NEW
 712
 713         An upload is NEW if it has binary or source packages that do not have
 714         an override in C{suite} OR if it references files ONLY in a tainted
 715         archive (eg. when it references files in NEW).
 716
 717         @rtype:  bool
 718         @return: C{True} if the upload is NEW, C{False} otherwise
 719         """
 720         session = self.session
 721
 722         # Check for missing overrides
 723         for b in self.changes.binaries:
 724             override = self._binary_override(suite, b)
 725             if override is None:
 726                 return True
 727
 728         if self.changes.source is not None:
 729             override = self._source_override(suite, self.changes.source)
 730             if override is None:
 731                 return True
 732
 733         # Check if we reference a file only in a tainted archive
 734         files = self.changes.files.values()
 735         if self.changes.source is not None:
 736             files.extend(self.changes.source.files.values())
 737         for f in files:
 738             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
 739             query_untainted = query.join(Archive).filter(Archive.tainted == False)
 740
 741             in_archive = (query.first() is not None)
 742             in_untainted_archive = (query_untainted.first() is not None)
 743
 744             if in_archive and not in_untainted_archive:
 745                 return True
 746
 747     def _final_suites(self):
 748         session = self.session
 749
 750         mapped_suites = self._mapped_suites()
 751         final_suites = set()
 752
 753         for suite in mapped_suites:
 754             overridesuite = suite
 755             if suite.overridesuite is not None:
 756                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 757             if self._check_new(overridesuite):
 758                 self.new = True
 759             final_suites.add(suite)
 760
 761         return final_suites
 762
 763     def _binary_override(self, suite, binary):
 764         """Get override entry for a binary
 765
 766         @type  suite: L{daklib.dbconn.Suite}
 767         @param suite: suite to get override for
 768
 769         @type  binary: L{daklib.upload.Binary}
 770         @param binary: binary to get override for
 771
 772         @rtype:  L{daklib.dbconn.Override} or C{None}
 773         @return: override for the given binary or C{None}
 774         """
 775         if suite.overridesuite is not None:
 776             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 777
 778         query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
 779                 .join(Component).filter(Component.component_name == binary.component) \
 780                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 781
 782         try:
 783             return query.one()
 784         except NoResultFound:
 785             return None
 786
 787     def _source_override(self, suite, source):
 788         """Get override entry for a source
 789
 790         @type  suite: L{daklib.dbconn.Suite}
 791         @param suite: suite to get override for
 792
 793         @type  source: L{daklib.upload.Source}
 794         @param source: source to get override for
 795
 796         @rtype:  L{daklib.dbconn.Override} or C{None}
 797         @return: override for the given source or C{None}
 798         """
 799         if suite.overridesuite is not None:
 800             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 801
 802         # XXX: component for source?
 803         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
 804                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 805
 806         try:
 807             return query.one()
 808         except NoResultFound:
 809             return None
 810
 811     def _binary_component(self, suite, binary, only_overrides=True):
 812         """get component for a binary
 813
 814         By default this will only look at overrides to get the right component;
 815         if C{only_overrides} is C{False} this method will also look at the
 816         Section field.
 817
 818         @type  suite: L{daklib.dbconn.Suite}
 819
 820         @type  binary: L{daklib.upload.Binary}
 821
 822         @type  only_overrides: bool
 823         @param only_overrides: only use overrides to get the right component
 824
 825         @rtype: L{daklib.dbconn.Component} or C{None}
 826         """
 827         override = self._binary_override(suite, binary)
 828         if override is not None:
 829             return override.component
 830         if only_overrides:
 831             return None
 832         return get_mapped_component(binary.component, self.session)
 833
 834     def check(self, force=False):
 835         """run checks against the upload
 836
 837         @type  force: bool
 838         @param force: ignore failing forcable checks
 839
 840         @rtype:  bool
 841         @return: C{True} if all checks passed, C{False} otherwise
 842         """
 843         # XXX: needs to be better structured.
 844         assert self.changes.valid_signature
 845
 846         try:
 847             for chk in (
 848                     checks.SignatureCheck,
 849                     checks.ChangesCheck,
 850                     checks.TransitionCheck,
 851                     checks.UploadBlockCheck,
 852                     checks.HashesCheck,
 853                     checks.SourceCheck,
 854                     checks.BinaryCheck,
 855                     checks.BinaryTimestampCheck,
 856                     checks.ACLCheck,
 857                     checks.SingleDistributionCheck,
 858                     checks.NoSourceOnlyCheck,
 859                     checks.LintianCheck,
 860                     ):
 861                 chk().check(self)
 862
 863             final_suites = self._final_suites()
 864             if len(final_suites) == 0:
 865                 self.reject_reasons.append('Ended with no suite to install to.')
 866                 return False
 867
 868             for chk in (
 869                     checks.SourceFormatCheck,
 870                     checks.SuiteArchitectureCheck,
 871                     checks.VersionCheck,
 872                     ):
 873                 for suite in final_suites:
 874                     chk().per_suite_check(self, suite)
 875
 876             if len(self.reject_reasons) != 0:
 877                 return False
 878
 879             self.final_suites = final_suites
 880             return True
 881         except checks.Reject as e:
 882             self.reject_reasons.append(unicode(e))
 883         except Exception as e:
 884             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
 885         return False
 886
 887     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
 888         """Install upload to the given suite
 889
 890         @type  suite: L{daklib.dbconn.Suite}
 891         @param suite: suite to install the package into. This is the real suite,
 892                       ie. after any redirection to NEW or a policy queue
 893
 894         @param source_component_func: function to get the L{daklib.dbconn.Component}
 895                                       for a L{daklib.upload.Source} object
 896
 897         @param binary_component_func: function to get the L{daklib.dbconn.Component}
 898                                       for a L{daklib.upload.Binary} object
 899
 900         @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
 901
 902         @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
 903
 904         @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
 905                  object for the install source or C{None} if no source was
 906                  included. The second is a list of L{daklib.dbconn.DBBinary}
 907                  objects for the installed binary packages.
 908         """
 909         # XXX: move this function to ArchiveTransaction?
 910
 911         control = self.changes.changes
 912         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 913
 914         if source_suites is None:
 915             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()
 916
 917         source = self.changes.source
 918         if source is not None:
 919             component = source_component_func(source)
 920             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
 921         else:
 922             db_source = None
 923
 924         db_binaries = []
 925         for binary in self.changes.binaries:
 926             component = binary_component_func(binary)
 927             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
 928             db_binaries.append(db_binary)
 929
 930         if suite.copychanges:
 931             src = os.path.join(self.directory, self.changes.filename)
 932             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
 933             self.transaction.fs.copy(src, dst)
 934
 935         return (db_source, db_binaries)
 936
 937     def _install_changes(self):
 938         assert self.changes.valid_signature
 939         control = self.changes.changes
 940         session = self.transaction.session
 941         config = Config()
 942
 943         changelog_id = None
 944         # Only add changelog for sourceful uploads and binNMUs
 945         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
 946             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
 947             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
 948             assert changelog_id is not None
 949
 950         db_changes = DBChange()
 951         db_changes.changesname = self.changes.filename
 952         db_changes.source = control['Source']
 953         db_changes.binaries = control.get('Binary', None)
 954         db_changes.architecture = control['Architecture']
 955         db_changes.version = control['Version']
 956         db_changes.distribution = control['Distribution']
 957         db_changes.urgency = control['Urgency']
 958         db_changes.maintainer = control['Maintainer']
 959         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
 960         db_changes.date = control['Date']
 961         db_changes.fingerprint = self.fingerprint.fingerprint
 962         db_changes.changelog_id = changelog_id
 963         db_changes.closes = self.changes.closed_bugs
 964
 965         self.transaction.session.add(db_changes)
 966         self.transaction.session.flush()
 967
 968         return db_changes
 969
 970     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
 971         u = PolicyQueueUpload()
 972         u.policy_queue = policy_queue
 973         u.target_suite = target_suite
 974         u.changes = db_changes
 975         u.source = db_source
 976         u.binaries = db_binaries
 977         self.transaction.session.add(u)
 978         self.transaction.session.flush()
 979
 980         dst = os.path.join(policy_queue.path, self.changes.filename)
 981         self.transaction.fs.copy(self.changes.path, dst)
 982
 983         return u
 984
 985     def try_autobyhand(self):
 986         """Try AUTOBYHAND
 987
 988         Try to handle byhand packages automatically.
 989
 990         @rtype:  list of L{daklib.upload.HashedFile}
 991         @return: list of remaining byhand files
 992         """
 993         assert len(self.reject_reasons) == 0
 994         assert self.changes.valid_signature
 995         assert self.final_suites is not None
 996
 997         byhand = self.changes.byhand_files
 998         if len(byhand) == 0:
 999             return True
1000
1001         suites = list(self.final_suites)
1002         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1003         suite = suites[0]
1004
1005         cnf = Config()
1006         control = self.changes.changes
1007         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1008
1009         remaining = []
1010         for f in byhand:
1011             parts = f.filename.split('_', 2)
1012             if len(parts) != 3:
1013                 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1014                 remaining.append(f)
1015                 continue
1016
1017             package, version, archext = parts
1018             arch, ext = archext.split('.', 1)
1019
1020             rule = automatic_byhand_packages.get(package)
1021             if rule is None:
1022                 remaining.append(f)
1023                 continue
1024
1025             if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
1026                 remaining.append(f)
1027                 continue
1028
1029             script = rule['Script']
1030             retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1031             if retcode != 0:
1032                 print "W: error processing {0}.".format(f.filename)
1033                 remaining.append(f)
1034
1035         return len(remaining) == 0
1036
1037     def _install_byhand(self, policy_queue_upload, hashed_file):
1038         """install byhand file
1039
1040         @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1041
1042         @type  hashed_file: L{daklib.upload.HashedFile}
1043         """
1044         fs = self.transaction.fs
1045         session = self.transaction.session
1046         policy_queue = policy_queue_upload.policy_queue
1047
1048         byhand_file = PolicyQueueByhandFile()
1049         byhand_file.upload = policy_queue_upload
1050         byhand_file.filename = hashed_file.filename
1051         session.add(byhand_file)
1052         session.flush()
1053
1054         src = os.path.join(self.directory, hashed_file.filename)
1055         dst = os.path.join(policy_queue.path, hashed_file.filename)
1056         fs.copy(src, dst)
1057
1058         return byhand_file
1059
1060     def _do_bts_versiontracking(self):
1061         cnf = Config()
1062         fs = self.transaction.fs
1063
1064         btsdir = cnf.get('Dir::BTSVersionTrack')
1065         if btsdir is None or btsdir == '':
1066             return
1067
1068         base = os.path.join(btsdir, self.changes.filename[:-8])
1069
1070         # version history
1071         sourcedir = self.unpacked_source()
1072         if sourcedir is not None:
1073             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1074             versions = fs.create("{0}.versions".format(base), mode=0o644)
1075             for line in fh.readlines():
1076                 if re_changelog_versions.match(line):
1077                     versions.write(line)
1078             fh.close()
1079             versions.close()
1080
1081         # binary -> source mapping
1082         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1083         for binary in self.changes.binaries:
1084             control = binary.control
1085             source_package, source_version = binary.source
1086             line = " ".join([control['Package'], control['Version'], source_package, source_version])
1087             print >>debinfo, line
1088         debinfo.close()
1089
1090     def _policy_queue(self, suite):
1091         if suite.policy_queue is not None:
1092             return suite.policy_queue
1093         return None
1094
1095     def install(self):
1096         """install upload
1097
1098         Install upload to a suite or policy queue.  This method does B{not}
1099         handle uploads to NEW.
1100
1101         You need to have called the C{check} method before calling this method.
1102         """
1103         assert len(self.reject_reasons) == 0
1104         assert self.changes.valid_signature
1105         assert self.final_suites is not None
1106         assert not self.new
1107
1108         db_changes = self._install_changes()
1109
1110         for suite in self.final_suites:
1111             overridesuite = suite
1112             if suite.overridesuite is not None:
1113                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1114
1115             policy_queue = self._policy_queue(suite)
1116
1117             redirected_suite = suite
1118             if policy_queue is not None:
1119                 redirected_suite = policy_queue.suite
1120
1121             source_component_func = lambda source: self._source_override(overridesuite, source).component
1122             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1123
1124             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1125
1126             if policy_queue is not None:
1127                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1128
1129             # copy to build queues
1130             if policy_queue is None or policy_queue.send_to_build_queues:
1131                 source_suites = self.session.query(Suite).filter_by(suite_id=suite.suite_id).subquery()
1132                 for build_queue in suite.copy_queues:
1133                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1134
1135         self._do_bts_versiontracking()
1136
1137     def install_to_new(self):
1138         """install upload to NEW
1139
1140         Install upload to NEW.  This method does B{not} handle regular uploads
1141         to suites or policy queues.
1142
1143         You need to have called the C{check} method before calling this method.
1144         """
1145         # Uploads to NEW are special as we don't have overrides.
1146         assert len(self.reject_reasons) == 0
1147         assert self.changes.valid_signature
1148         assert self.final_suites is not None
1149
1150         source = self.changes.source
1151         binaries = self.changes.binaries
1152         byhand = self.changes.byhand_files
1153
1154         new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1155         if len(byhand) > 0:
1156             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1157         new_suite = new_queue.suite
1158
1159         # we need a suite to guess components
1160         suites = list(self.final_suites)
1161         assert len(suites) == 1, "NEW uploads must be to a single suite"
1162         suite = suites[0]
1163
1164         def binary_component_func(binary):
1165             return self._binary_component(suite, binary, only_overrides=False)
1166
1167         # guess source component
1168         # XXX: should be moved into an extra method
1169         binary_component_names = set()
1170         for binary in binaries:
1171             component = binary_component_func(binary)
1172             binary_component_names.add(component.component_name)
1173         source_component_name = None
1174         for c in self.session.query(Component).order_by(Component.component_id):
1175             guess = c.component_name
1176             if guess in binary_component_names:
1177                 source_component_name = guess
1178                 break
1179         if source_component_name is None:
1180             raise Exception('Could not guess source component.')
1181         source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1182         source_component_func = lambda source: source_component
1183
1184         db_changes = self._install_changes()
1185         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1186         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1187
1188         for f in byhand:
1189             self._install_byhand(policy_upload, f)
1190
1191         self._do_bts_versiontracking()
1192
1193     def commit(self):
1194         """commit changes"""
1195         self.transaction.commit()
1196
1197     def rollback(self):
1198         """rollback changes"""
1199         self.transaction.rollback()
1200
1201     def __enter__(self):
1202         self.prepare()
1203         return self
1204
1205     def __exit__(self, type, value, traceback):
1206         if self.directory is not None:
1207             shutil.rmtree(self.directory)
1208             self.directory = None
1209         self.changes = None
1210         self.transaction.rollback()
1211         return None