daklib/archive.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to manipulate the archive
  18
  19 This module provides classes to manipulate the archive.
  20 """
  21
  22 from daklib.dbconn import *
  23 import daklib.checks as checks
  24 from daklib.config import Config
  25 import daklib.upload as upload
  26 import daklib.utils as utils
  27 from daklib.fstransactions import FilesystemTransaction
  28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
  29
  30 import apt_pkg
  31 from datetime import datetime
  32 import os
  33 import shutil
  34 import subprocess
  35 from sqlalchemy.orm.exc import NoResultFound
  36 import sqlalchemy.exc
  37 import tempfile
  38 import traceback
  39
  40 class ArchiveException(Exception):
  41     pass
  42
  43 class HashMismatchException(ArchiveException):
  44     pass
  45
  46 class ArchiveTransaction(object):
  47     """manipulate the archive in a transaction
  48     """
  49     def __init__(self):
  50         self.fs = FilesystemTransaction()
  51         self.session = DBConn().session()
  52
  53     def get_file(self, hashed_file, source_name, check_hashes=True):
  54         """Look for file C{hashed_file} in database
  55
  56         @type  hashed_file: L{daklib.upload.HashedFile}
  57         @param hashed_file: file to look for in the database
  58
  59         @type  source_name: str
  60         @param source_name: source package name
  61
  62         @type  check_hashes: bool
  63         @param check_hashes: check size and hashes match
  64
  65         @raise KeyError: file was not found in the database
  66         @raise HashMismatchException: hash mismatch
  67
  68         @rtype:  L{daklib.dbconn.PoolFile}
  69         @return: database entry for the file
  70         """
  71         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  72         try:
  73             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
  74             if check_hashes and (poolfile.filesize != hashed_file.size
  75                                  or poolfile.md5sum != hashed_file.md5sum
  76                                  or poolfile.sha1sum != hashed_file.sha1sum
  77                                  or poolfile.sha256sum != hashed_file.sha256sum):
  78                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
  79             return poolfile
  80         except NoResultFound:
  81             raise KeyError('{0} not found in database.'.format(poolname))
  82
  83     def _install_file(self, directory, hashed_file, archive, component, source_name):
  84         """Install a file
  85
  86         Will not give an error when the file is already present.
  87
  88         @rtype:  L{daklib.dbconn.PoolFile}
  89         @return: database object for the new file
  90         """
  91         session = self.session
  92
  93         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  94         try:
  95             poolfile = self.get_file(hashed_file, source_name)
  96         except KeyError:
  97             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
  98             poolfile.md5sum = hashed_file.md5sum
  99             poolfile.sha1sum = hashed_file.sha1sum
 100             poolfile.sha256sum = hashed_file.sha256sum
 101             session.add(poolfile)
 102             session.flush()
 103
 104         try:
 105             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
 106         except NoResultFound:
 107             archive_file = ArchiveFile(archive, component, poolfile)
 108             session.add(archive_file)
 109             session.flush()
 110
 111             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
 112             hashed_file_path = os.path.join(directory, hashed_file.filename)
 113             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 114
 115         return poolfile
 116
 117     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
 118         """Install a binary package
 119
 120         @type  directory: str
 121         @param directory: directory the binary package is located in
 122
 123         @type  binary: L{daklib.upload.Binary}
 124         @param binary: binary package to install
 125
 126         @type  suite: L{daklib.dbconn.Suite}
 127         @param suite: target suite
 128
 129         @type  component: L{daklib.dbconn.Component}
 130         @param component: target component
 131
 132         @type  allow_tainted: bool
 133         @param allow_tainted: allow to copy additional files from tainted archives
 134
 135         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 136         @param fingerprint: optional fingerprint
 137
 138         @type  source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
 139         @param source_suites: suites to copy the source from if they are not
 140                               in C{suite} or C{True} to allow copying from any
 141                               suite.
 142
 143         @type  extra_source_archives: list of L{daklib.dbconn.Archive}
 144         @param extra_source_archives: extra archives to copy Built-Using sources from
 145
 146         @rtype:  L{daklib.dbconn.DBBinary}
 147         @return: databse object for the new package
 148         """
 149         session = self.session
 150         control = binary.control
 151         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 152         architecture = get_architecture(control['Architecture'], session)
 153
 154         (source_name, source_version) = binary.source
 155         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
 156         source = source_query.filter(DBSource.suites.contains(suite)).first()
 157         if source is None:
 158             if source_suites != True:
 159                 source_query = source_query.join(DBSource.suites) \
 160                     .filter(Suite.suite_id == source_suites.c.id)
 161             source = source_query.first()
 162             if source is None:
 163                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
 164             self.copy_source(source, suite, component)
 165
 166         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
 167
 168         unique = dict(
 169             package=control['Package'],
 170             version=control['Version'],
 171             architecture=architecture,
 172             )
 173         rest = dict(
 174             source=source,
 175             maintainer=maintainer,
 176             poolfile=db_file,
 177             binarytype=binary.type,
 178             fingerprint=fingerprint,
 179             )
 180
 181         try:
 182             db_binary = session.query(DBBinary).filter_by(**unique).one()
 183             for key, value in rest.iteritems():
 184                 if getattr(db_binary, key) != value:
 185                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
 186         except NoResultFound:
 187             db_binary = DBBinary(**unique)
 188             for key, value in rest.iteritems():
 189                 setattr(db_binary, key, value)
 190             session.add(db_binary)
 191             session.flush()
 192             import_metadata_into_db(db_binary, session)
 193
 194             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
 195
 196         if suite not in db_binary.suites:
 197             db_binary.suites.append(suite)
 198
 199         session.flush()
 200
 201         return db_binary
 202
 203     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
 204         """ensure source exists in the given archive
 205
 206         This is intended to be used to check that Built-Using sources exist.
 207
 208         @type  filename: str
 209         @param filename: filename to use in error messages
 210
 211         @type  source: L{daklib.dbconn.DBSource}
 212         @param source: source to look for
 213
 214         @type  archive: L{daklib.dbconn.Archive}
 215         @param archive: archive to look in
 216
 217         @type  extra_archives: list of L{daklib.dbconn.Archive}
 218         @param extra_archives: list of archives to copy the source package from
 219                                if it is not yet present in C{archive}
 220         """
 221         session = self.session
 222         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
 223         if db_file is not None:
 224             return True
 225
 226         # Try to copy file from one extra archive
 227         if extra_archives is None:
 228             extra_archives = []
 229         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
 230         if db_file is None:
 231             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
 232
 233         source_archive = db_file.archive
 234         for dsc_file in source.srcfiles:
 235             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
 236             # We were given an explicit list of archives so it is okay to copy from tainted archives.
 237             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 238
 239     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
 240         """Add Built-Using sources to C{db_binary.extra_sources}
 241         """
 242         session = self.session
 243         built_using = control.get('Built-Using', None)
 244
 245         if built_using is not None:
 246             for dep in apt_pkg.parse_depends(built_using):
 247                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
 248                 bu_source_name, bu_source_version, comp = dep[0]
 249                 assert comp == '=', 'Built-Using must contain strict dependencies'
 250
 251                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
 252                 if bu_source is None:
 253                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
 254
 255                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
 256
 257                 db_binary.extra_sources.append(bu_source)
 258
 259     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
 260         """Install a source package
 261
 262         @type  directory: str
 263         @param directory: directory the source package is located in
 264
 265         @type  source: L{daklib.upload.Source}
 266         @param source: source package to install
 267
 268         @type  suite: L{daklib.dbconn.Suite}
 269         @param suite: target suite
 270
 271         @type  component: L{daklib.dbconn.Component}
 272         @param component: target component
 273
 274         @type  changed_by: L{daklib.dbconn.Maintainer}
 275         @param changed_by: person who prepared this version of the package
 276
 277         @type  allow_tainted: bool
 278         @param allow_tainted: allow to copy additional files from tainted archives
 279
 280         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 281         @param fingerprint: optional fingerprint
 282
 283         @rtype:  L{daklib.dbconn.DBSource}
 284         @return: database object for the new source
 285         """
 286         session = self.session
 287         archive = suite.archive
 288         control = source.dsc
 289         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 290         source_name = control['Source']
 291
 292         ### Add source package to database
 293
 294         # We need to install the .dsc first as the DBSource object refers to it.
 295         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
 296
 297         unique = dict(
 298             source=source_name,
 299             version=control['Version'],
 300             )
 301         rest = dict(
 302             maintainer=maintainer,
 303             changedby=changed_by,
 304             #install_date=datetime.now().date(),
 305             poolfile=db_file_dsc,
 306             fingerprint=fingerprint,
 307             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
 308             )
 309
 310         created = False
 311         try:
 312             db_source = session.query(DBSource).filter_by(**unique).one()
 313             for key, value in rest.iteritems():
 314                 if getattr(db_source, key) != value:
 315                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
 316         except NoResultFound:
 317             created = True
 318             db_source = DBSource(**unique)
 319             for key, value in rest.iteritems():
 320                 setattr(db_source, key, value)
 321             # XXX: set as default in postgres?
 322             db_source.install_date = datetime.now().date()
 323             session.add(db_source)
 324             session.flush()
 325
 326             # Add .dsc file. Other files will be added later.
 327             db_dsc_file = DSCFile()
 328             db_dsc_file.source = db_source
 329             db_dsc_file.poolfile = db_file_dsc
 330             session.add(db_dsc_file)
 331             session.flush()
 332
 333         if suite in db_source.suites:
 334             return db_source
 335
 336         db_source.suites.append(suite)
 337
 338         if not created:
 339             for f in db_source.srcfiles:
 340                 self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
 341             return db_source
 342
 343         ### Now add remaining files and copy them to the archive.
 344
 345         for hashed_file in source.files.itervalues():
 346             hashed_file_path = os.path.join(directory, hashed_file.filename)
 347             if os.path.exists(hashed_file_path):
 348                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
 349                 session.add(db_file)
 350             else:
 351                 db_file = self.get_file(hashed_file, source_name)
 352                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
 353
 354             db_dsc_file = DSCFile()
 355             db_dsc_file.source = db_source
 356             db_dsc_file.poolfile = db_file
 357             session.add(db_dsc_file)
 358
 359         session.flush()
 360
 361         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
 362         import_metadata_into_db(db_source, session)
 363
 364         # Uploaders are the maintainer and co-maintainers from the Uploaders field
 365         db_source.uploaders.append(maintainer)
 366         if 'Uploaders' in control:
 367             from daklib.textutils import split_uploaders
 368             for u in split_uploaders(control['Uploaders']):
 369                 db_source.uploaders.append(get_or_set_maintainer(u, session))
 370         session.flush()
 371
 372         return db_source
 373
 374     def _copy_file(self, db_file, archive, component, allow_tainted=False):
 375         """Copy a file to the given archive and component
 376
 377         @type  db_file: L{daklib.dbconn.PoolFile}
 378         @param db_file: file to copy
 379
 380         @type  archive: L{daklib.dbconn.Archive}
 381         @param archive: target archive
 382
 383         @type  component: L{daklib.dbconn.Archive}
 384         @param component: target component
 385
 386         @type  allow_tainted: bool
 387         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 388         """
 389         session = self.session
 390
 391         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
 392             query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
 393             if not allow_tainted:
 394                 query = query.join(Archive).filter(Archive.tainted == False)
 395
 396             source_af = query.first()
 397             if source_af is None:
 398                 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
 399             target_af = ArchiveFile(archive, component, db_file)
 400             session.add(target_af)
 401             session.flush()
 402             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
 403
 404     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
 405         """Copy a binary package to the given suite and component
 406
 407         @type  db_binary: L{daklib.dbconn.DBBinary}
 408         @param db_binary: binary to copy
 409
 410         @type  suite: L{daklib.dbconn.Suite}
 411         @param suite: target suite
 412
 413         @type  component: L{daklib.dbconn.Component}
 414         @param component: target component
 415
 416         @type  allow_tainted: bool
 417         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 418
 419         @type  extra_archives: list of L{daklib.dbconn.Archive}
 420         @param extra_archives: extra archives to copy Built-Using sources from
 421         """
 422         session = self.session
 423         archive = suite.archive
 424         if archive.tainted:
 425             allow_tainted = True
 426
 427         filename = db_binary.poolfile.filename
 428
 429         # make sure source is present in target archive
 430         db_source = db_binary.source
 431         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
 432             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
 433
 434         # make sure built-using packages are present in target archive
 435         for db_source in db_binary.extra_sources:
 436             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 437
 438         # copy binary
 439         db_file = db_binary.poolfile
 440         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
 441         if suite not in db_binary.suites:
 442             db_binary.suites.append(suite)
 443         self.session.flush()
 444
 445     def copy_source(self, db_source, suite, component, allow_tainted=False):
 446         """Copy a source package to the given suite and component
 447
 448         @type  db_source: L{daklib.dbconn.DBSource}
 449         @param db_source: source to copy
 450
 451         @type  suite: L{daklib.dbconn.Suite}
 452         @param suite: target suite
 453
 454         @type  component: L{daklib.dbconn.Component}
 455         @param component: target component
 456
 457         @type  allow_tainted: bool
 458         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 459         """
 460         archive = suite.archive
 461         if archive.tainted:
 462             allow_tainted = True
 463         for db_dsc_file in db_source.srcfiles:
 464             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
 465         if suite not in db_source.suites:
 466             db_source.suites.append(suite)
 467         self.session.flush()
 468
 469     def remove_file(self, db_file, archive, component):
 470         """Remove a file from a given archive and component
 471
 472         @type  db_file: L{daklib.dbconn.PoolFile}
 473         @param db_file: file to remove
 474
 475         @type  archive: L{daklib.dbconn.Archive}
 476         @param archive: archive to remove the file from
 477
 478         @type  component: L{daklib.dbconn.Component}
 479         @param component: component to remove the file from
 480         """
 481         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
 482         self.fs.unlink(af.path)
 483         self.session.delete(af)
 484
 485     def remove_binary(self, binary, suite):
 486         """Remove a binary from a given suite and component
 487
 488         @type  binary: L{daklib.dbconn.DBBinary}
 489         @param binary: binary to remove
 490
 491         @type  suite: L{daklib.dbconn.Suite}
 492         @param suite: suite to remove the package from
 493         """
 494         binary.suites.remove(suite)
 495         self.session.flush()
 496
 497     def remove_source(self, source, suite):
 498         """Remove a source from a given suite and component
 499
 500         @type  source: L{daklib.dbconn.DBSource}
 501         @param source: source to remove
 502
 503         @type  suite: L{daklib.dbconn.Suite}
 504         @param suite: suite to remove the package from
 505
 506         @raise ArchiveException: source package is still referenced by other
 507                                  binaries in the suite
 508         """
 509         session = self.session
 510
 511         query = session.query(DBBinary).filter_by(source=source) \
 512             .filter(DBBinary.suites.contains(suite))
 513         if query.first() is not None:
 514             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
 515
 516         source.suites.remove(suite)
 517         session.flush()
 518
 519     def commit(self):
 520         """commit changes"""
 521         try:
 522             self.session.commit()
 523             self.fs.commit()
 524         finally:
 525             self.session.rollback()
 526             self.fs.rollback()
 527
 528     def rollback(self):
 529         """rollback changes"""
 530         self.session.rollback()
 531         self.fs.rollback()
 532
 533     def __enter__(self):
 534         return self
 535
 536     def __exit__(self, type, value, traceback):
 537         if type is None:
 538             self.commit()
 539         else:
 540             self.rollback()
 541         return None
 542
 543 class ArchiveUpload(object):
 544     """handle an upload
 545
 546     This class can be used in a with-statement::
 547
 548        with ArchiveUpload(...) as upload:
 549           ...
 550
 551     Doing so will automatically run any required cleanup and also rollback the
 552     transaction if it was not committed.
 553     """
 554     def __init__(self, directory, changes, keyrings):
 555         self.transaction = ArchiveTransaction()
 556         """transaction used to handle the upload
 557         @type: L{daklib.archive.ArchiveTransaction}
 558         """
 559
 560         self.session = self.transaction.session
 561         """database session"""
 562
 563         self.original_directory = directory
 564         self.original_changes = changes
 565
 566         self.changes = None
 567         """upload to process
 568         @type: L{daklib.upload.Changes}
 569         """
 570
 571         self.directory = None
 572         """directory with temporary copy of files. set by C{prepare}
 573         @type: str
 574         """
 575
 576         self.keyrings = keyrings
 577
 578         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
 579         """fingerprint of the key used to sign the upload
 580         @type: L{daklib.dbconn.Fingerprint}
 581         """
 582
 583         self.reject_reasons = []
 584         """reasons why the upload cannot by accepted
 585         @type: list of str
 586         """
 587
 588         self.warnings = []
 589         """warnings
 590         @note: Not used yet.
 591         @type: list of str
 592         """
 593
 594         self.final_suites = None
 595
 596         self.new = False
 597         """upload is NEW. set by C{check}
 598         @type: bool
 599         """
 600
 601         self._checked = False
 602         """checks passes. set by C{check}
 603         @type: bool
 604         """
 605
 606         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
 607         self._new = self._new_queue.suite
 608
 609     def warn(self, message):
 610         """add a warning message
 611
 612         Adds a warning message that can later be seen in C{self.warnings}
 613
 614         @type  message: string
 615         @param message: warning message
 616         """
 617         self.warnings.append(message)
 618
 619     def prepare(self):
 620         """prepare upload for further processing
 621
 622         This copies the files involved to a temporary directory.  If you use
 623         this method directly, you have to remove the directory given by the
 624         C{directory} attribute later on your own.
 625
 626         Instead of using the method directly, you can also use a with-statement::
 627
 628            with ArchiveUpload(...) as upload:
 629               ...
 630
 631         This will automatically handle any required cleanup.
 632         """
 633         assert self.directory is None
 634         assert self.original_changes.valid_signature
 635
 636         cnf = Config()
 637         session = self.transaction.session
 638
 639         self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
 640                                             mode=0o2750, group=cnf.unprivgroup)
 641         with FilesystemTransaction() as fs:
 642             src = os.path.join(self.original_directory, self.original_changes.filename)
 643             dst = os.path.join(self.directory, self.original_changes.filename)
 644             fs.copy(src, dst, mode=0o640)
 645
 646             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 647
 648             for f in self.changes.files.itervalues():
 649                 src = os.path.join(self.original_directory, f.filename)
 650                 dst = os.path.join(self.directory, f.filename)
 651                 if not os.path.exists(src):
 652                     continue
 653                 fs.copy(src, dst, mode=0o640)
 654
 655             source = None
 656             try:
 657                 source = self.changes.source
 658             except Exception:
 659                 # Do not raise an exception here if the .dsc is invalid.
 660                 pass
 661
 662             if source is not None:
 663                 for f in source.files.itervalues():
 664                     src = os.path.join(self.original_directory, f.filename)
 665                     dst = os.path.join(self.directory, f.filename)
 666                     if not os.path.exists(dst):
 667                         try:
 668                             db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
 669                             db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
 670                             fs.copy(db_archive_file.path, dst, symlink=True)
 671                         except KeyError:
 672                             # Ignore if get_file could not find it. Upload will
 673                             # probably be rejected later.
 674                             pass
 675
 676     def unpacked_source(self):
 677         """Path to unpacked source
 678
 679         Get path to the unpacked source. This method does unpack the source
 680         into a temporary directory under C{self.directory} if it has not
 681         been done so already.
 682
 683         @rtype:  str or C{None}
 684         @return: string giving the path to the unpacked source directory
 685                  or C{None} if no source was included in the upload.
 686         """
 687         assert self.directory is not None
 688
 689         source = self.changes.source
 690         if source is None:
 691             return None
 692         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
 693
 694         sourcedir = os.path.join(self.directory, 'source')
 695         if not os.path.exists(sourcedir):
 696             devnull = open('/dev/null', 'w')
 697             subprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
 698         if not os.path.isdir(sourcedir):
 699             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
 700         return sourcedir
 701
 702     def _map_suite(self, suite_name):
 703         for rule in Config().value_list("SuiteMappings"):
 704             fields = rule.split()
 705             rtype = fields[0]
 706             if rtype == "map" or rtype == "silent-map":
 707                 (src, dst) = fields[1:3]
 708                 if src == suite_name:
 709                     suite_name = dst
 710                     if rtype != "silent-map":
 711                         self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
 712             elif rtype == "ignore":
 713                 ignored = fields[1]
 714                 if suite_name == ignored:
 715                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
 716                     suite_name = None
 717             elif rtype == "reject":
 718                 rejected = fields[1]
 719                 if suite_name == rejected:
 720                     self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
 721             ## XXX: propup-version and map-unreleased not yet implemented
 722         return suite_name
 723
 724     def _mapped_suites(self):
 725         """Get target suites after mappings
 726
 727         @rtype:  list of L{daklib.dbconn.Suite}
 728         @return: list giving the mapped target suites of this upload
 729         """
 730         session = self.session
 731
 732         suite_names = []
 733         for dist in self.changes.distributions:
 734             suite_name = self._map_suite(dist)
 735             if suite_name is not None:
 736                 suite_names.append(suite_name)
 737
 738         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
 739         return suites
 740
 741     def _check_new(self, suite):
 742         """Check if upload is NEW
 743
 744         An upload is NEW if it has binary or source packages that do not have
 745         an override in C{suite} OR if it references files ONLY in a tainted
 746         archive (eg. when it references files in NEW).
 747
 748         @rtype:  bool
 749         @return: C{True} if the upload is NEW, C{False} otherwise
 750         """
 751         session = self.session
 752         new = False
 753
 754         # Check for missing overrides
 755         for b in self.changes.binaries:
 756             override = self._binary_override(suite, b)
 757             if override is None:
 758                 self.warnings.append('binary:{0} is NEW.'.format(b.control['Package']))
 759                 new = True
 760
 761         if self.changes.source is not None:
 762             override = self._source_override(suite, self.changes.source)
 763             if override is None:
 764                 self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
 765                 new = True
 766
 767         # Check if we reference a file only in a tainted archive
 768         files = self.changes.files.values()
 769         if self.changes.source is not None:
 770             files.extend(self.changes.source.files.values())
 771         for f in files:
 772             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
 773             query_untainted = query.join(Archive).filter(Archive.tainted == False)
 774
 775             in_archive = (query.first() is not None)
 776             in_untainted_archive = (query_untainted.first() is not None)
 777
 778             if in_archive and not in_untainted_archive:
 779                 self.warnings.append('{0} is only available in NEW.'.format(f.filename))
 780                 new = True
 781
 782         return new
 783
 784     def _final_suites(self):
 785         session = self.session
 786
 787         mapped_suites = self._mapped_suites()
 788         final_suites = set()
 789
 790         for suite in mapped_suites:
 791             overridesuite = suite
 792             if suite.overridesuite is not None:
 793                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 794             if self._check_new(overridesuite):
 795                 self.new = True
 796             final_suites.add(suite)
 797
 798         return final_suites
 799
 800     def _binary_override(self, suite, binary):
 801         """Get override entry for a binary
 802
 803         @type  suite: L{daklib.dbconn.Suite}
 804         @param suite: suite to get override for
 805
 806         @type  binary: L{daklib.upload.Binary}
 807         @param binary: binary to get override for
 808
 809         @rtype:  L{daklib.dbconn.Override} or C{None}
 810         @return: override for the given binary or C{None}
 811         """
 812         if suite.overridesuite is not None:
 813             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 814
 815         mapped_component = get_mapped_component(binary.component)
 816         if mapped_component is None:
 817             return None
 818
 819         query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
 820                 .join(Component).filter(Component.component_name == mapped_component.component_name) \
 821                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 822
 823         try:
 824             return query.one()
 825         except NoResultFound:
 826             return None
 827
 828     def _source_override(self, suite, source):
 829         """Get override entry for a source
 830
 831         @type  suite: L{daklib.dbconn.Suite}
 832         @param suite: suite to get override for
 833
 834         @type  source: L{daklib.upload.Source}
 835         @param source: source to get override for
 836
 837         @rtype:  L{daklib.dbconn.Override} or C{None}
 838         @return: override for the given source or C{None}
 839         """
 840         if suite.overridesuite is not None:
 841             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 842
 843         # XXX: component for source?
 844         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
 845                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 846
 847         try:
 848             return query.one()
 849         except NoResultFound:
 850             return None
 851
 852     def _binary_component(self, suite, binary, only_overrides=True):
 853         """get component for a binary
 854
 855         By default this will only look at overrides to get the right component;
 856         if C{only_overrides} is C{False} this method will also look at the
 857         Section field.
 858
 859         @type  suite: L{daklib.dbconn.Suite}
 860
 861         @type  binary: L{daklib.upload.Binary}
 862
 863         @type  only_overrides: bool
 864         @param only_overrides: only use overrides to get the right component
 865
 866         @rtype: L{daklib.dbconn.Component} or C{None}
 867         """
 868         override = self._binary_override(suite, binary)
 869         if override is not None:
 870             return override.component
 871         if only_overrides:
 872             return None
 873         return get_mapped_component(binary.component, self.session)
 874
 875     def check(self, force=False):
 876         """run checks against the upload
 877
 878         @type  force: bool
 879         @param force: ignore failing forcable checks
 880
 881         @rtype:  bool
 882         @return: C{True} if all checks passed, C{False} otherwise
 883         """
 884         # XXX: needs to be better structured.
 885         assert self.changes.valid_signature
 886
 887         try:
 888             # Validate signatures and hashes before we do any real work:
 889             for chk in (
 890                     checks.SignatureAndHashesCheck,
 891                     checks.ChangesCheck,
 892                     checks.ExternalHashesCheck,
 893                     checks.SourceCheck,
 894                     checks.BinaryCheck,
 895                     checks.BinaryTimestampCheck,
 896                     checks.SingleDistributionCheck,
 897                     ):
 898                 chk().check(self)
 899
 900             final_suites = self._final_suites()
 901             if len(final_suites) == 0:
 902                 self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
 903                 return False
 904
 905             self.final_suites = final_suites
 906
 907             for chk in (
 908                     checks.TransitionCheck,
 909                     checks.ACLCheck,
 910                     checks.NoSourceOnlyCheck,
 911                     checks.LintianCheck,
 912                     ):
 913                 chk().check(self)
 914
 915             for chk in (
 916                     checks.ACLCheck,
 917                     checks.SourceFormatCheck,
 918                     checks.SuiteArchitectureCheck,
 919                     checks.VersionCheck,
 920                     ):
 921                 for suite in final_suites:
 922                     chk().per_suite_check(self, suite)
 923
 924             if len(self.reject_reasons) != 0:
 925                 return False
 926
 927             self._checked = True
 928             return True
 929         except checks.Reject as e:
 930             self.reject_reasons.append(unicode(e))
 931         except Exception as e:
 932             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
 933         return False
 934
 935     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
 936         """Install upload to the given suite
 937
 938         @type  suite: L{daklib.dbconn.Suite}
 939         @param suite: suite to install the package into. This is the real suite,
 940                       ie. after any redirection to NEW or a policy queue
 941
 942         @param source_component_func: function to get the L{daklib.dbconn.Component}
 943                                       for a L{daklib.upload.Source} object
 944
 945         @param binary_component_func: function to get the L{daklib.dbconn.Component}
 946                                       for a L{daklib.upload.Binary} object
 947
 948         @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
 949
 950         @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
 951
 952         @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
 953                  object for the install source or C{None} if no source was
 954                  included. The second is a list of L{daklib.dbconn.DBBinary}
 955                  objects for the installed binary packages.
 956         """
 957         # XXX: move this function to ArchiveTransaction?
 958
 959         control = self.changes.changes
 960         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 961
 962         if source_suites is None:
 963             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()
 964
 965         source = self.changes.source
 966         if source is not None:
 967             component = source_component_func(source)
 968             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
 969         else:
 970             db_source = None
 971
 972         db_binaries = []
 973         for binary in self.changes.binaries:
 974             component = binary_component_func(binary)
 975             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
 976             db_binaries.append(db_binary)
 977
 978         if suite.copychanges:
 979             src = os.path.join(self.directory, self.changes.filename)
 980             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
 981             self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
 982
 983         return (db_source, db_binaries)
 984
 985     def _install_changes(self):
 986         assert self.changes.valid_signature
 987         control = self.changes.changes
 988         session = self.transaction.session
 989         config = Config()
 990
 991         changelog_id = None
 992         # Only add changelog for sourceful uploads and binNMUs
 993         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
 994             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
 995             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
 996             assert changelog_id is not None
 997
 998         db_changes = DBChange()
 999         db_changes.changesname = self.changes.filename
1000         db_changes.source = control['Source']
1001         db_changes.binaries = control.get('Binary', None)
1002         db_changes.architecture = control['Architecture']
1003         db_changes.version = control['Version']
1004         db_changes.distribution = control['Distribution']
1005         db_changes.urgency = control['Urgency']
1006         db_changes.maintainer = control['Maintainer']
1007         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
1008         db_changes.date = control['Date']
1009         db_changes.fingerprint = self.fingerprint.fingerprint
1010         db_changes.changelog_id = changelog_id
1011         db_changes.closes = self.changes.closed_bugs
1012
1013         try:
1014             self.transaction.session.add(db_changes)
1015             self.transaction.session.flush()
1016         except sqlalchemy.exc.IntegrityError:
1017             raise ArchiveException('{0} is already known.'.format(self.changes.filename))
1018
1019         return db_changes
1020
1021     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
1022         u = PolicyQueueUpload()
1023         u.policy_queue = policy_queue
1024         u.target_suite = target_suite
1025         u.changes = db_changes
1026         u.source = db_source
1027         u.binaries = db_binaries
1028         self.transaction.session.add(u)
1029         self.transaction.session.flush()
1030
1031         dst = os.path.join(policy_queue.path, self.changes.filename)
1032         self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)
1033
1034         return u
1035
1036     def try_autobyhand(self):
1037         """Try AUTOBYHAND
1038
1039         Try to handle byhand packages automatically.
1040
1041         @rtype:  list of L{daklib.upload.HashedFile}
1042         @return: list of remaining byhand files
1043         """
1044         assert len(self.reject_reasons) == 0
1045         assert self.changes.valid_signature
1046         assert self.final_suites is not None
1047         assert self._checked
1048
1049         byhand = self.changes.byhand_files
1050         if len(byhand) == 0:
1051             return True
1052
1053         suites = list(self.final_suites)
1054         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1055         suite = suites[0]
1056
1057         cnf = Config()
1058         control = self.changes.changes
1059         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1060
1061         remaining = []
1062         for f in byhand:
1063             parts = f.filename.split('_', 2)
1064             if len(parts) != 3:
1065                 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1066                 remaining.append(f)
1067                 continue
1068
1069             package, version, archext = parts
1070             arch, ext = archext.split('.', 1)
1071
1072             try:
1073                 rule = automatic_byhand_packages.subtree(package)
1074             except KeyError:
1075                 remaining.append(f)
1076                 continue
1077
1078             if rule['Source'] != self.changes.source_name or rule['Section'] != f.section or rule['Extension'] != ext:
1079                 remaining.append(f)
1080                 continue
1081
1082             script = rule['Script']
1083             retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1084             if retcode != 0:
1085                 print "W: error processing {0}.".format(f.filename)
1086                 remaining.append(f)
1087
1088         return len(remaining) == 0
1089
1090     def _install_byhand(self, policy_queue_upload, hashed_file):
1091         """install byhand file
1092
1093         @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1094
1095         @type  hashed_file: L{daklib.upload.HashedFile}
1096         """
1097         fs = self.transaction.fs
1098         session = self.transaction.session
1099         policy_queue = policy_queue_upload.policy_queue
1100
1101         byhand_file = PolicyQueueByhandFile()
1102         byhand_file.upload = policy_queue_upload
1103         byhand_file.filename = hashed_file.filename
1104         session.add(byhand_file)
1105         session.flush()
1106
1107         src = os.path.join(self.directory, hashed_file.filename)
1108         dst = os.path.join(policy_queue.path, hashed_file.filename)
1109         fs.copy(src, dst, mode=policy_queue.change_perms)
1110
1111         return byhand_file
1112
1113     def _do_bts_versiontracking(self):
1114         cnf = Config()
1115         fs = self.transaction.fs
1116
1117         btsdir = cnf.get('Dir::BTSVersionTrack')
1118         if btsdir is None or btsdir == '':
1119             return
1120
1121         base = os.path.join(btsdir, self.changes.filename[:-8])
1122
1123         # version history
1124         sourcedir = self.unpacked_source()
1125         if sourcedir is not None:
1126             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1127             versions = fs.create("{0}.versions".format(base), mode=0o644)
1128             for line in fh.readlines():
1129                 if re_changelog_versions.match(line):
1130                     versions.write(line)
1131             fh.close()
1132             versions.close()
1133
1134         # binary -> source mapping
1135         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1136         for binary in self.changes.binaries:
1137             control = binary.control
1138             source_package, source_version = binary.source
1139             line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
1140             print >>debinfo, line
1141         debinfo.close()
1142
1143     def _policy_queue(self, suite):
1144         if suite.policy_queue is not None:
1145             return suite.policy_queue
1146         return None
1147
1148     def install(self):
1149         """install upload
1150
1151         Install upload to a suite or policy queue.  This method does B{not}
1152         handle uploads to NEW.
1153
1154         You need to have called the C{check} method before calling this method.
1155         """
1156         assert len(self.reject_reasons) == 0
1157         assert self.changes.valid_signature
1158         assert self.final_suites is not None
1159         assert self._checked
1160         assert not self.new
1161
1162         db_changes = self._install_changes()
1163
1164         for suite in self.final_suites:
1165             overridesuite = suite
1166             if suite.overridesuite is not None:
1167                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1168
1169             policy_queue = self._policy_queue(suite)
1170
1171             redirected_suite = suite
1172             if policy_queue is not None:
1173                 redirected_suite = policy_queue.suite
1174
1175             # source can be in the suite we install to or any suite we enhance
1176             source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
1177             for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
1178                     .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
1179                     .filter(VersionCheck.check == 'Enhances'):
1180                 source_suite_ids.add(enhanced_suite_id)
1181
1182             source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()
1183
1184             source_component_func = lambda source: self._source_override(overridesuite, source).component
1185             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1186
1187             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1188
1189             if policy_queue is not None:
1190                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1191
1192             # copy to build queues
1193             if policy_queue is None or policy_queue.send_to_build_queues:
1194                 for build_queue in suite.copy_queues:
1195                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1196
1197         self._do_bts_versiontracking()
1198
1199     def install_to_new(self):
1200         """install upload to NEW
1201
1202         Install upload to NEW.  This method does B{not} handle regular uploads
1203         to suites or policy queues.
1204
1205         You need to have called the C{check} method before calling this method.
1206         """
1207         # Uploads to NEW are special as we don't have overrides.
1208         assert len(self.reject_reasons) == 0
1209         assert self.changes.valid_signature
1210         assert self.final_suites is not None
1211
1212         source = self.changes.source
1213         binaries = self.changes.binaries
1214         byhand = self.changes.byhand_files
1215
1216         # we need a suite to guess components
1217         suites = list(self.final_suites)
1218         assert len(suites) == 1, "NEW uploads must be to a single suite"
1219         suite = suites[0]
1220
1221         # decide which NEW queue to use
1222         if suite.new_queue is None:
1223             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1224         else:
1225             new_queue = suite.new_queue
1226         if len(byhand) > 0:
1227             # There is only one global BYHAND queue
1228             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1229         new_suite = new_queue.suite
1230
1231
1232         def binary_component_func(binary):
1233             return self._binary_component(suite, binary, only_overrides=False)
1234
1235         # guess source component
1236         # XXX: should be moved into an extra method
1237         binary_component_names = set()
1238         for binary in binaries:
1239             component = binary_component_func(binary)
1240             binary_component_names.add(component.component_name)
1241         source_component_name = None
1242         for c in self.session.query(Component).order_by(Component.component_id):
1243             guess = c.component_name
1244             if guess in binary_component_names:
1245                 source_component_name = guess
1246                 break
1247         if source_component_name is None:
1248             source_component = self.session.query(Component).order_by(Component.component_id).first()
1249         else:
1250             source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1251         source_component_func = lambda source: source_component
1252
1253         db_changes = self._install_changes()
1254         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1255         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1256
1257         for f in byhand:
1258             self._install_byhand(policy_upload, f)
1259
1260         self._do_bts_versiontracking()
1261
1262     def commit(self):
1263         """commit changes"""
1264         self.transaction.commit()
1265
1266     def rollback(self):
1267         """rollback changes"""
1268         self.transaction.rollback()
1269
1270     def __enter__(self):
1271         self.prepare()
1272         return self
1273
1274     def __exit__(self, type, value, traceback):
1275         if self.directory is not None:
1276             shutil.rmtree(self.directory)
1277             self.directory = None
1278         self.changes = None
1279         self.transaction.rollback()
1280         return None