daklib/archive.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to manipulate the archive
  18
  19 This module provides classes to manipulate the archive.
  20 """
  21
  22 from daklib.dbconn import *
  23 import daklib.checks as checks
  24 from daklib.config import Config
  25 import daklib.upload as upload
  26 import daklib.utils as utils
  27 from daklib.fstransactions import FilesystemTransaction
  28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
  29
  30 import apt_pkg
  31 from datetime import datetime
  32 import os
  33 import shutil
  34 import subprocess
  35 from sqlalchemy.orm.exc import NoResultFound
  36 import sqlalchemy.exc
  37 import tempfile
  38 import traceback
  39
  40 class ArchiveException(Exception):
  41     pass
  42
  43 class HashMismatchException(ArchiveException):
  44     pass
  45
  46 class ArchiveTransaction(object):
  47     """manipulate the archive in a transaction
  48     """
  49     def __init__(self):
  50         self.fs = FilesystemTransaction()
  51         self.session = DBConn().session()
  52
  53     def get_file(self, hashed_file, source_name, check_hashes=True):
  54         """Look for file C{hashed_file} in database
  55
  56         @type  hashed_file: L{daklib.upload.HashedFile}
  57         @param hashed_file: file to look for in the database
  58
  59         @type  source_name: str
  60         @param source_name: source package name
  61
  62         @type  check_hashes: bool
  63         @param check_hashes: check size and hashes match
  64
  65         @raise KeyError: file was not found in the database
  66         @raise HashMismatchException: hash mismatch
  67
  68         @rtype:  L{daklib.dbconn.PoolFile}
  69         @return: database entry for the file
  70         """
  71         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  72         try:
  73             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
  74             if check_hashes and (poolfile.filesize != hashed_file.size
  75                                  or poolfile.md5sum != hashed_file.md5sum
  76                                  or poolfile.sha1sum != hashed_file.sha1sum
  77                                  or poolfile.sha256sum != hashed_file.sha256sum):
  78                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
  79             return poolfile
  80         except NoResultFound:
  81             raise KeyError('{0} not found in database.'.format(poolname))
  82
  83     def _install_file(self, directory, hashed_file, archive, component, source_name):
  84         """Install a file
  85
  86         Will not give an error when the file is already present.
  87
  88         @rtype:  L{daklib.dbconn.PoolFile}
  89         @return: database object for the new file
  90         """
  91         session = self.session
  92
  93         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  94         try:
  95             poolfile = self.get_file(hashed_file, source_name)
  96         except KeyError:
  97             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
  98             poolfile.md5sum = hashed_file.md5sum
  99             poolfile.sha1sum = hashed_file.sha1sum
 100             poolfile.sha256sum = hashed_file.sha256sum
 101             session.add(poolfile)
 102             session.flush()
 103
 104         try:
 105             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
 106         except NoResultFound:
 107             archive_file = ArchiveFile(archive, component, poolfile)
 108             session.add(archive_file)
 109             session.flush()
 110
 111             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
 112             hashed_file_path = os.path.join(directory, hashed_file.filename)
 113             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 114
 115         return poolfile
 116
 117     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
 118         """Install a binary package
 119
 120         @type  directory: str
 121         @param directory: directory the binary package is located in
 122
 123         @type  binary: L{daklib.upload.Binary}
 124         @param binary: binary package to install
 125
 126         @type  suite: L{daklib.dbconn.Suite}
 127         @param suite: target suite
 128
 129         @type  component: L{daklib.dbconn.Component}
 130         @param component: target component
 131
 132         @type  allow_tainted: bool
 133         @param allow_tainted: allow to copy additional files from tainted archives
 134
 135         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 136         @param fingerprint: optional fingerprint
 137
 138         @type  source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
 139         @param source_suites: suites to copy the source from if they are not
 140                               in C{suite} or C{True} to allow copying from any
 141                               suite.
 142
 143         @type  extra_source_archives: list of L{daklib.dbconn.Archive}
 144         @param extra_source_archives: extra archives to copy Built-Using sources from
 145
 146         @rtype:  L{daklib.dbconn.DBBinary}
 147         @return: databse object for the new package
 148         """
 149         session = self.session
 150         control = binary.control
 151         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 152         architecture = get_architecture(control['Architecture'], session)
 153
 154         (source_name, source_version) = binary.source
 155         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
 156         source = source_query.filter(DBSource.suites.contains(suite)).first()
 157         if source is None:
 158             if source_suites != True:
 159                 source_query = source_query.join(DBSource.suites) \
 160                     .filter(Suite.suite_id == source_suites.c.id)
 161             source = source_query.first()
 162             if source is None:
 163                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
 164             self.copy_source(source, suite, component)
 165
 166         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
 167
 168         unique = dict(
 169             package=control['Package'],
 170             version=control['Version'],
 171             architecture=architecture,
 172             )
 173         rest = dict(
 174             source=source,
 175             maintainer=maintainer,
 176             poolfile=db_file,
 177             binarytype=binary.type,
 178             fingerprint=fingerprint,
 179             )
 180
 181         try:
 182             db_binary = session.query(DBBinary).filter_by(**unique).one()
 183             for key, value in rest.iteritems():
 184                 if getattr(db_binary, key) != value:
 185                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
 186         except NoResultFound:
 187             db_binary = DBBinary(**unique)
 188             for key, value in rest.iteritems():
 189                 setattr(db_binary, key, value)
 190             session.add(db_binary)
 191             session.flush()
 192             import_metadata_into_db(db_binary, session)
 193
 194             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
 195
 196         if suite not in db_binary.suites:
 197             db_binary.suites.append(suite)
 198
 199         session.flush()
 200
 201         return db_binary
 202
 203     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
 204         """ensure source exists in the given archive
 205
 206         This is intended to be used to check that Built-Using sources exist.
 207
 208         @type  filename: str
 209         @param filename: filename to use in error messages
 210
 211         @type  source: L{daklib.dbconn.DBSource}
 212         @param source: source to look for
 213
 214         @type  archive: L{daklib.dbconn.Archive}
 215         @param archive: archive to look in
 216
 217         @type  extra_archives: list of L{daklib.dbconn.Archive}
 218         @param extra_archives: list of archives to copy the source package from
 219                                if it is not yet present in C{archive}
 220         """
 221         session = self.session
 222         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
 223         if db_file is not None:
 224             return True
 225
 226         # Try to copy file from one extra archive
 227         if extra_archives is None:
 228             extra_archives = []
 229         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
 230         if db_file is None:
 231             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
 232
 233         source_archive = db_file.archive
 234         for dsc_file in source.srcfiles:
 235             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
 236             # We were given an explicit list of archives so it is okay to copy from tainted archives.
 237             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 238
 239     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
 240         """Add Built-Using sources to C{db_binary.extra_sources}
 241         """
 242         session = self.session
 243         built_using = control.get('Built-Using', None)
 244
 245         if built_using is not None:
 246             for dep in apt_pkg.parse_depends(built_using):
 247                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
 248                 bu_source_name, bu_source_version, comp = dep[0]
 249                 assert comp == '=', 'Built-Using must contain strict dependencies'
 250
 251                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
 252                 if bu_source is None:
 253                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
 254
 255                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
 256
 257                 db_binary.extra_sources.append(bu_source)
 258
 259     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
 260         """Install a source package
 261
 262         @type  directory: str
 263         @param directory: directory the source package is located in
 264
 265         @type  source: L{daklib.upload.Source}
 266         @param source: source package to install
 267
 268         @type  suite: L{daklib.dbconn.Suite}
 269         @param suite: target suite
 270
 271         @type  component: L{daklib.dbconn.Component}
 272         @param component: target component
 273
 274         @type  changed_by: L{daklib.dbconn.Maintainer}
 275         @param changed_by: person who prepared this version of the package
 276
 277         @type  allow_tainted: bool
 278         @param allow_tainted: allow to copy additional files from tainted archives
 279
 280         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 281         @param fingerprint: optional fingerprint
 282
 283         @rtype:  L{daklib.dbconn.DBSource}
 284         @return: database object for the new source
 285         """
 286         session = self.session
 287         archive = suite.archive
 288         control = source.dsc
 289         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 290         source_name = control['Source']
 291
 292         ### Add source package to database
 293
 294         # We need to install the .dsc first as the DBSource object refers to it.
 295         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
 296
 297         unique = dict(
 298             source=source_name,
 299             version=control['Version'],
 300             )
 301         rest = dict(
 302             maintainer=maintainer,
 303             changedby=changed_by,
 304             #install_date=datetime.now().date(),
 305             poolfile=db_file_dsc,
 306             fingerprint=fingerprint,
 307             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
 308             )
 309
 310         created = False
 311         try:
 312             db_source = session.query(DBSource).filter_by(**unique).one()
 313             for key, value in rest.iteritems():
 314                 if getattr(db_source, key) != value:
 315                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
 316         except NoResultFound:
 317             created = True
 318             db_source = DBSource(**unique)
 319             for key, value in rest.iteritems():
 320                 setattr(db_source, key, value)
 321             # XXX: set as default in postgres?
 322             db_source.install_date = datetime.now().date()
 323             session.add(db_source)
 324             session.flush()
 325
 326             # Add .dsc file. Other files will be added later.
 327             db_dsc_file = DSCFile()
 328             db_dsc_file.source = db_source
 329             db_dsc_file.poolfile = db_file_dsc
 330             session.add(db_dsc_file)
 331             session.flush()
 332
 333         if suite in db_source.suites:
 334             return db_source
 335
 336         db_source.suites.append(suite)
 337
 338         if not created:
 339             for f in db_source.srcfiles:
 340                 self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
 341             return db_source
 342
 343         ### Now add remaining files and copy them to the archive.
 344
 345         for hashed_file in source.files.itervalues():
 346             hashed_file_path = os.path.join(directory, hashed_file.filename)
 347             if os.path.exists(hashed_file_path):
 348                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
 349                 session.add(db_file)
 350             else:
 351                 db_file = self.get_file(hashed_file, source_name)
 352                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
 353
 354             db_dsc_file = DSCFile()
 355             db_dsc_file.source = db_source
 356             db_dsc_file.poolfile = db_file
 357             session.add(db_dsc_file)
 358
 359         session.flush()
 360
 361         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
 362         import_metadata_into_db(db_source, session)
 363
 364         # Uploaders are the maintainer and co-maintainers from the Uploaders field
 365         db_source.uploaders.append(maintainer)
 366         if 'Uploaders' in control:
 367             from daklib.textutils import split_uploaders
 368             for u in split_uploaders(control['Uploaders']):
 369                 db_source.uploaders.append(get_or_set_maintainer(u, session))
 370         session.flush()
 371
 372         return db_source
 373
 374     def _copy_file(self, db_file, archive, component, allow_tainted=False):
 375         """Copy a file to the given archive and component
 376
 377         @type  db_file: L{daklib.dbconn.PoolFile}
 378         @param db_file: file to copy
 379
 380         @type  archive: L{daklib.dbconn.Archive}
 381         @param archive: target archive
 382
 383         @type  component: L{daklib.dbconn.Archive}
 384         @param component: target component
 385
 386         @type  allow_tainted: bool
 387         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 388         """
 389         session = self.session
 390
 391         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
 392             query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
 393             if not allow_tainted:
 394                 query = query.join(Archive).filter(Archive.tainted == False)
 395
 396             source_af = query.first()
 397             if source_af is None:
 398                 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
 399             target_af = ArchiveFile(archive, component, db_file)
 400             session.add(target_af)
 401             session.flush()
 402             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
 403
 404     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
 405         """Copy a binary package to the given suite and component
 406
 407         @type  db_binary: L{daklib.dbconn.DBBinary}
 408         @param db_binary: binary to copy
 409
 410         @type  suite: L{daklib.dbconn.Suite}
 411         @param suite: target suite
 412
 413         @type  component: L{daklib.dbconn.Component}
 414         @param component: target component
 415
 416         @type  allow_tainted: bool
 417         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 418
 419         @type  extra_archives: list of L{daklib.dbconn.Archive}
 420         @param extra_archives: extra archives to copy Built-Using sources from
 421         """
 422         session = self.session
 423         archive = suite.archive
 424         if archive.tainted:
 425             allow_tainted = True
 426
 427         filename = db_binary.poolfile.filename
 428
 429         # make sure source is present in target archive
 430         db_source = db_binary.source
 431         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
 432             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
 433
 434         # make sure built-using packages are present in target archive
 435         for db_source in db_binary.extra_sources:
 436             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 437
 438         # copy binary
 439         db_file = db_binary.poolfile
 440         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
 441         if suite not in db_binary.suites:
 442             db_binary.suites.append(suite)
 443         self.session.flush()
 444
 445     def copy_source(self, db_source, suite, component, allow_tainted=False):
 446         """Copy a source package to the given suite and component
 447
 448         @type  db_source: L{daklib.dbconn.DBSource}
 449         @param db_source: source to copy
 450
 451         @type  suite: L{daklib.dbconn.Suite}
 452         @param suite: target suite
 453
 454         @type  component: L{daklib.dbconn.Component}
 455         @param component: target component
 456
 457         @type  allow_tainted: bool
 458         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 459         """
 460         archive = suite.archive
 461         if archive.tainted:
 462             allow_tainted = True
 463         for db_dsc_file in db_source.srcfiles:
 464             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
 465         if suite not in db_source.suites:
 466             db_source.suites.append(suite)
 467         self.session.flush()
 468
 469     def remove_file(self, db_file, archive, component):
 470         """Remove a file from a given archive and component
 471
 472         @type  db_file: L{daklib.dbconn.PoolFile}
 473         @param db_file: file to remove
 474
 475         @type  archive: L{daklib.dbconn.Archive}
 476         @param archive: archive to remove the file from
 477
 478         @type  component: L{daklib.dbconn.Component}
 479         @param component: component to remove the file from
 480         """
 481         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
 482         self.fs.unlink(af.path)
 483         self.session.delete(af)
 484
 485     def remove_binary(self, binary, suite):
 486         """Remove a binary from a given suite and component
 487
 488         @type  binary: L{daklib.dbconn.DBBinary}
 489         @param binary: binary to remove
 490
 491         @type  suite: L{daklib.dbconn.Suite}
 492         @param suite: suite to remove the package from
 493         """
 494         binary.suites.remove(suite)
 495         self.session.flush()
 496
 497     def remove_source(self, source, suite):
 498         """Remove a source from a given suite and component
 499
 500         @type  source: L{daklib.dbconn.DBSource}
 501         @param source: source to remove
 502
 503         @type  suite: L{daklib.dbconn.Suite}
 504         @param suite: suite to remove the package from
 505
 506         @raise ArchiveException: source package is still referenced by other
 507                                  binaries in the suite
 508         """
 509         session = self.session
 510
 511         query = session.query(DBBinary).filter_by(source=source) \
 512             .filter(DBBinary.suites.contains(suite))
 513         if query.first() is not None:
 514             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
 515
 516         source.suites.remove(suite)
 517         session.flush()
 518
 519     def commit(self):
 520         """commit changes"""
 521         try:
 522             self.session.commit()
 523             self.fs.commit()
 524         finally:
 525             self.session.rollback()
 526             self.fs.rollback()
 527
 528     def rollback(self):
 529         """rollback changes"""
 530         self.session.rollback()
 531         self.fs.rollback()
 532
 533     def __enter__(self):
 534         return self
 535
 536     def __exit__(self, type, value, traceback):
 537         if type is None:
 538             self.commit()
 539         else:
 540             self.rollback()
 541         return None
 542
 543 class ArchiveUpload(object):
 544     """handle an upload
 545
 546     This class can be used in a with-statement::
 547
 548        with ArchiveUpload(...) as upload:
 549           ...
 550
 551     Doing so will automatically run any required cleanup and also rollback the
 552     transaction if it was not committed.
 553     """
 554     def __init__(self, directory, changes, keyrings):
 555         self.transaction = ArchiveTransaction()
 556         """transaction used to handle the upload
 557         @type: L{daklib.archive.ArchiveTransaction}
 558         """
 559
 560         self.session = self.transaction.session
 561         """database session"""
 562
 563         self.original_directory = directory
 564         self.original_changes = changes
 565
 566         self.changes = None
 567         """upload to process
 568         @type: L{daklib.upload.Changes}
 569         """
 570
 571         self.directory = None
 572         """directory with temporary copy of files. set by C{prepare}
 573         @type: str
 574         """
 575
 576         self.keyrings = keyrings
 577
 578         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
 579         """fingerprint of the key used to sign the upload
 580         @type: L{daklib.dbconn.Fingerprint}
 581         """
 582
 583         self.reject_reasons = []
 584         """reasons why the upload cannot by accepted
 585         @type: list of str
 586         """
 587
 588         self.warnings = []
 589         """warnings
 590         @note: Not used yet.
 591         @type: list of str
 592         """
 593
 594         self.final_suites = None
 595
 596         self.new = False
 597         """upload is NEW. set by C{check}
 598         @type: bool
 599         """
 600
 601         self._checked = False
 602         """checks passes. set by C{check}
 603         @type: bool
 604         """
 605
 606         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
 607         self._new = self._new_queue.suite
 608
 609     def warn(self, message):
 610         """add a warning message
 611
 612         Adds a warning message that can later be seen in C{self.warnings}
 613
 614         @type  message: string
 615         @param message: warning message
 616         """
 617         self.warnings.append(message)
 618
 619     def prepare(self):
 620         """prepare upload for further processing
 621
 622         This copies the files involved to a temporary directory.  If you use
 623         this method directly, you have to remove the directory given by the
 624         C{directory} attribute later on your own.
 625
 626         Instead of using the method directly, you can also use a with-statement::
 627
 628            with ArchiveUpload(...) as upload:
 629               ...
 630
 631         This will automatically handle any required cleanup.
 632         """
 633         assert self.directory is None
 634         assert self.original_changes.valid_signature
 635
 636         cnf = Config()
 637         session = self.transaction.session
 638
 639         group = cnf.get('Dinstall::UnprivGroup') or None
 640         self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
 641                                             mode=0o2750, group=group)
 642         with FilesystemTransaction() as fs:
 643             src = os.path.join(self.original_directory, self.original_changes.filename)
 644             dst = os.path.join(self.directory, self.original_changes.filename)
 645             fs.copy(src, dst, mode=0o640)
 646
 647             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 648
 649             for f in self.changes.files.itervalues():
 650                 src = os.path.join(self.original_directory, f.filename)
 651                 dst = os.path.join(self.directory, f.filename)
 652                 if not os.path.exists(src):
 653                     continue
 654                 fs.copy(src, dst, mode=0o640)
 655
 656             source = None
 657             try:
 658                 source = self.changes.source
 659             except Exception:
 660                 # Do not raise an exception here if the .dsc is invalid.
 661                 pass
 662
 663             if source is not None:
 664                 for f in source.files.itervalues():
 665                     src = os.path.join(self.original_directory, f.filename)
 666                     dst = os.path.join(self.directory, f.filename)
 667                     if not os.path.exists(dst):
 668                         try:
 669                             db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
 670                             db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
 671                             fs.copy(db_archive_file.path, dst, mode=0o640)
 672                         except KeyError:
 673                             # Ignore if get_file could not find it. Upload will
 674                             # probably be rejected later.
 675                             pass
 676
 677     def unpacked_source(self):
 678         """Path to unpacked source
 679
 680         Get path to the unpacked source. This method does unpack the source
 681         into a temporary directory under C{self.directory} if it has not
 682         been done so already.
 683
 684         @rtype:  str or C{None}
 685         @return: string giving the path to the unpacked source directory
 686                  or C{None} if no source was included in the upload.
 687         """
 688         assert self.directory is not None
 689
 690         source = self.changes.source
 691         if source is None:
 692             return None
 693         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
 694
 695         sourcedir = os.path.join(self.directory, 'source')
 696         if not os.path.exists(sourcedir):
 697             devnull = open('/dev/null', 'w')
 698             subprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
 699         if not os.path.isdir(sourcedir):
 700             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
 701         return sourcedir
 702
 703     def _map_suite(self, suite_name):
 704         for rule in Config().value_list("SuiteMappings"):
 705             fields = rule.split()
 706             rtype = fields[0]
 707             if rtype == "map" or rtype == "silent-map":
 708                 (src, dst) = fields[1:3]
 709                 if src == suite_name:
 710                     suite_name = dst
 711                     if rtype != "silent-map":
 712                         self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
 713             elif rtype == "ignore":
 714                 ignored = fields[1]
 715                 if suite_name == ignored:
 716                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
 717                     suite_name = None
 718             elif rtype == "reject":
 719                 rejected = fields[1]
 720                 if suite_name == rejected:
 721                     raise checks.Reject('Uploads to {0} are not accepted.'.format(rejected))
 722             ## XXX: propup-version and map-unreleased not yet implemented
 723         return suite_name
 724
 725     def _mapped_suites(self):
 726         """Get target suites after mappings
 727
 728         @rtype:  list of L{daklib.dbconn.Suite}
 729         @return: list giving the mapped target suites of this upload
 730         """
 731         session = self.session
 732
 733         suite_names = []
 734         for dist in self.changes.distributions:
 735             suite_name = self._map_suite(dist)
 736             if suite_name is not None:
 737                 suite_names.append(suite_name)
 738
 739         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
 740         return suites
 741
 742     def _check_new(self, suite):
 743         """Check if upload is NEW
 744
 745         An upload is NEW if it has binary or source packages that do not have
 746         an override in C{suite} OR if it references files ONLY in a tainted
 747         archive (eg. when it references files in NEW).
 748
 749         @rtype:  bool
 750         @return: C{True} if the upload is NEW, C{False} otherwise
 751         """
 752         session = self.session
 753         new = False
 754
 755         # Check for missing overrides
 756         for b in self.changes.binaries:
 757             override = self._binary_override(suite, b)
 758             if override is None:
 759                 self.warnings.append('binary:{0} is NEW.'.format(b.control['Package']))
 760                 new = True
 761
 762         if self.changes.source is not None:
 763             override = self._source_override(suite, self.changes.source)
 764             if override is None:
 765                 self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
 766                 new = True
 767
 768         # Check if we reference a file only in a tainted archive
 769         files = self.changes.files.values()
 770         if self.changes.source is not None:
 771             files.extend(self.changes.source.files.values())
 772         for f in files:
 773             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
 774             query_untainted = query.join(Archive).filter(Archive.tainted == False)
 775
 776             in_archive = (query.first() is not None)
 777             in_untainted_archive = (query_untainted.first() is not None)
 778
 779             if in_archive and not in_untainted_archive:
 780                 self.warnings.append('{0} is only available in NEW.'.format(f.filename))
 781                 new = True
 782
 783         return new
 784
 785     def _final_suites(self):
 786         session = self.session
 787
 788         mapped_suites = self._mapped_suites()
 789         final_suites = set()
 790
 791         for suite in mapped_suites:
 792             overridesuite = suite
 793             if suite.overridesuite is not None:
 794                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 795             if self._check_new(overridesuite):
 796                 self.new = True
 797             final_suites.add(suite)
 798
 799         return final_suites
 800
 801     def _binary_override(self, suite, binary):
 802         """Get override entry for a binary
 803
 804         @type  suite: L{daklib.dbconn.Suite}
 805         @param suite: suite to get override for
 806
 807         @type  binary: L{daklib.upload.Binary}
 808         @param binary: binary to get override for
 809
 810         @rtype:  L{daklib.dbconn.Override} or C{None}
 811         @return: override for the given binary or C{None}
 812         """
 813         if suite.overridesuite is not None:
 814             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 815
 816         mapped_component = get_mapped_component(binary.component)
 817         if mapped_component is None:
 818             return None
 819
 820         query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
 821                 .join(Component).filter(Component.component_name == mapped_component.component_name) \
 822                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 823
 824         try:
 825             return query.one()
 826         except NoResultFound:
 827             return None
 828
 829     def _source_override(self, suite, source):
 830         """Get override entry for a source
 831
 832         @type  suite: L{daklib.dbconn.Suite}
 833         @param suite: suite to get override for
 834
 835         @type  source: L{daklib.upload.Source}
 836         @param source: source to get override for
 837
 838         @rtype:  L{daklib.dbconn.Override} or C{None}
 839         @return: override for the given source or C{None}
 840         """
 841         if suite.overridesuite is not None:
 842             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 843
 844         # XXX: component for source?
 845         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
 846                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 847
 848         try:
 849             return query.one()
 850         except NoResultFound:
 851             return None
 852
 853     def _binary_component(self, suite, binary, only_overrides=True):
 854         """get component for a binary
 855
 856         By default this will only look at overrides to get the right component;
 857         if C{only_overrides} is C{False} this method will also look at the
 858         Section field.
 859
 860         @type  suite: L{daklib.dbconn.Suite}
 861
 862         @type  binary: L{daklib.upload.Binary}
 863
 864         @type  only_overrides: bool
 865         @param only_overrides: only use overrides to get the right component
 866
 867         @rtype: L{daklib.dbconn.Component} or C{None}
 868         """
 869         override = self._binary_override(suite, binary)
 870         if override is not None:
 871             return override.component
 872         if only_overrides:
 873             return None
 874         return get_mapped_component(binary.component, self.session)
 875
 876     def check(self, force=False):
 877         """run checks against the upload
 878
 879         @type  force: bool
 880         @param force: ignore failing forcable checks
 881
 882         @rtype:  bool
 883         @return: C{True} if all checks passed, C{False} otherwise
 884         """
 885         # XXX: needs to be better structured.
 886         assert self.changes.valid_signature
 887
 888         try:
 889             # Validate signatures and hashes before we do any real work:
 890             for chk in (
 891                     checks.SignatureAndHashesCheck,
 892                     checks.ChangesCheck,
 893                     checks.ExternalHashesCheck,
 894                     checks.SourceCheck,
 895                     checks.BinaryCheck,
 896                     checks.BinaryTimestampCheck,
 897                     checks.SingleDistributionCheck,
 898                     ):
 899                 chk().check(self)
 900
 901             final_suites = self._final_suites()
 902             if len(final_suites) == 0:
 903                 self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
 904                 return False
 905
 906             self.final_suites = final_suites
 907
 908             for chk in (
 909                     checks.TransitionCheck,
 910                     checks.ACLCheck,
 911                     checks.NoSourceOnlyCheck,
 912                     checks.LintianCheck,
 913                     ):
 914                 chk().check(self)
 915
 916             for chk in (
 917                     checks.ACLCheck,
 918                     checks.SourceFormatCheck,
 919                     checks.SuiteArchitectureCheck,
 920                     checks.VersionCheck,
 921                     ):
 922                 for suite in final_suites:
 923                     chk().per_suite_check(self, suite)
 924
 925             if len(self.reject_reasons) != 0:
 926                 return False
 927
 928             self._checked = True
 929             return True
 930         except checks.Reject as e:
 931             self.reject_reasons.append(unicode(e))
 932         except Exception as e:
 933             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
 934         return False
 935
 936     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
 937         """Install upload to the given suite
 938
 939         @type  suite: L{daklib.dbconn.Suite}
 940         @param suite: suite to install the package into. This is the real suite,
 941                       ie. after any redirection to NEW or a policy queue
 942
 943         @param source_component_func: function to get the L{daklib.dbconn.Component}
 944                                       for a L{daklib.upload.Source} object
 945
 946         @param binary_component_func: function to get the L{daklib.dbconn.Component}
 947                                       for a L{daklib.upload.Binary} object
 948
 949         @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
 950
 951         @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
 952
 953         @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
 954                  object for the install source or C{None} if no source was
 955                  included. The second is a list of L{daklib.dbconn.DBBinary}
 956                  objects for the installed binary packages.
 957         """
 958         # XXX: move this function to ArchiveTransaction?
 959
 960         control = self.changes.changes
 961         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 962
 963         if source_suites is None:
 964             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()
 965
 966         source = self.changes.source
 967         if source is not None:
 968             component = source_component_func(source)
 969             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
 970         else:
 971             db_source = None
 972
 973         db_binaries = []
 974         for binary in self.changes.binaries:
 975             component = binary_component_func(binary)
 976             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
 977             db_binaries.append(db_binary)
 978
 979         if suite.copychanges:
 980             src = os.path.join(self.directory, self.changes.filename)
 981             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
 982             self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
 983
 984         return (db_source, db_binaries)
 985
 986     def _install_changes(self):
 987         assert self.changes.valid_signature
 988         control = self.changes.changes
 989         session = self.transaction.session
 990         config = Config()
 991
 992         changelog_id = None
 993         # Only add changelog for sourceful uploads and binNMUs
 994         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
 995             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
 996             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
 997             assert changelog_id is not None
 998
 999         db_changes = DBChange()
1000         db_changes.changesname = self.changes.filename
1001         db_changes.source = control['Source']
1002         db_changes.binaries = control.get('Binary', None)
1003         db_changes.architecture = control['Architecture']
1004         db_changes.version = control['Version']
1005         db_changes.distribution = control['Distribution']
1006         db_changes.urgency = control['Urgency']
1007         db_changes.maintainer = control['Maintainer']
1008         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
1009         db_changes.date = control['Date']
1010         db_changes.fingerprint = self.fingerprint.fingerprint
1011         db_changes.changelog_id = changelog_id
1012         db_changes.closes = self.changes.closed_bugs
1013
1014         try:
1015             self.transaction.session.add(db_changes)
1016             self.transaction.session.flush()
1017         except sqlalchemy.exc.IntegrityError:
1018             raise ArchiveException('{0} is already known.'.format(self.changes.filename))
1019
1020         return db_changes
1021
1022     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
1023         u = PolicyQueueUpload()
1024         u.policy_queue = policy_queue
1025         u.target_suite = target_suite
1026         u.changes = db_changes
1027         u.source = db_source
1028         u.binaries = db_binaries
1029         self.transaction.session.add(u)
1030         self.transaction.session.flush()
1031
1032         dst = os.path.join(policy_queue.path, self.changes.filename)
1033         self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)
1034
1035         return u
1036
1037     def try_autobyhand(self):
1038         """Try AUTOBYHAND
1039
1040         Try to handle byhand packages automatically.
1041
1042         @rtype:  list of L{daklib.upload.HashedFile}
1043         @return: list of remaining byhand files
1044         """
1045         assert len(self.reject_reasons) == 0
1046         assert self.changes.valid_signature
1047         assert self.final_suites is not None
1048         assert self._checked
1049
1050         byhand = self.changes.byhand_files
1051         if len(byhand) == 0:
1052             return True
1053
1054         suites = list(self.final_suites)
1055         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1056         suite = suites[0]
1057
1058         cnf = Config()
1059         control = self.changes.changes
1060         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1061
1062         remaining = []
1063         for f in byhand:
1064             parts = f.filename.split('_', 2)
1065             if len(parts) != 3:
1066                 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1067                 remaining.append(f)
1068                 continue
1069
1070             package, version, archext = parts
1071             arch, ext = archext.split('.', 1)
1072
1073             try:
1074                 rule = automatic_byhand_packages.subtree(package)
1075             except KeyError:
1076                 remaining.append(f)
1077                 continue
1078
1079             if rule['Source'] != self.changes.source_name \
1080                     or rule['Section'] != f.section \
1081                     or ('Extension' in rule and rule['Extension'] != ext):
1082                 remaining.append(f)
1083                 continue
1084
1085             script = rule['Script']
1086             retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1087             if retcode != 0:
1088                 print "W: error processing {0}.".format(f.filename)
1089                 remaining.append(f)
1090
1091         return len(remaining) == 0
1092
1093     def _install_byhand(self, policy_queue_upload, hashed_file):
1094         """install byhand file
1095
1096         @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1097
1098         @type  hashed_file: L{daklib.upload.HashedFile}
1099         """
1100         fs = self.transaction.fs
1101         session = self.transaction.session
1102         policy_queue = policy_queue_upload.policy_queue
1103
1104         byhand_file = PolicyQueueByhandFile()
1105         byhand_file.upload = policy_queue_upload
1106         byhand_file.filename = hashed_file.filename
1107         session.add(byhand_file)
1108         session.flush()
1109
1110         src = os.path.join(self.directory, hashed_file.filename)
1111         dst = os.path.join(policy_queue.path, hashed_file.filename)
1112         fs.copy(src, dst, mode=policy_queue.change_perms)
1113
1114         return byhand_file
1115
1116     def _do_bts_versiontracking(self):
1117         cnf = Config()
1118         fs = self.transaction.fs
1119
1120         btsdir = cnf.get('Dir::BTSVersionTrack')
1121         if btsdir is None or btsdir == '':
1122             return
1123
1124         base = os.path.join(btsdir, self.changes.filename[:-8])
1125
1126         # version history
1127         sourcedir = self.unpacked_source()
1128         if sourcedir is not None:
1129             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1130             versions = fs.create("{0}.versions".format(base), mode=0o644)
1131             for line in fh.readlines():
1132                 if re_changelog_versions.match(line):
1133                     versions.write(line)
1134             fh.close()
1135             versions.close()
1136
1137         # binary -> source mapping
1138         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1139         for binary in self.changes.binaries:
1140             control = binary.control
1141             source_package, source_version = binary.source
1142             line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
1143             print >>debinfo, line
1144         debinfo.close()
1145
1146     def _policy_queue(self, suite):
1147         if suite.policy_queue is not None:
1148             return suite.policy_queue
1149         return None
1150
1151     def install(self):
1152         """install upload
1153
1154         Install upload to a suite or policy queue.  This method does B{not}
1155         handle uploads to NEW.
1156
1157         You need to have called the C{check} method before calling this method.
1158         """
1159         assert len(self.reject_reasons) == 0
1160         assert self.changes.valid_signature
1161         assert self.final_suites is not None
1162         assert self._checked
1163         assert not self.new
1164
1165         db_changes = self._install_changes()
1166
1167         for suite in self.final_suites:
1168             overridesuite = suite
1169             if suite.overridesuite is not None:
1170                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1171
1172             policy_queue = self._policy_queue(suite)
1173
1174             redirected_suite = suite
1175             if policy_queue is not None:
1176                 redirected_suite = policy_queue.suite
1177
1178             # source can be in the suite we install to or any suite we enhance
1179             source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
1180             for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
1181                     .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
1182                     .filter(VersionCheck.check == 'Enhances'):
1183                 source_suite_ids.add(enhanced_suite_id)
1184
1185             source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()
1186
1187             source_component_func = lambda source: self._source_override(overridesuite, source).component
1188             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1189
1190             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1191
1192             if policy_queue is not None:
1193                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1194
1195             # copy to build queues
1196             if policy_queue is None or policy_queue.send_to_build_queues:
1197                 for build_queue in suite.copy_queues:
1198                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1199
1200         self._do_bts_versiontracking()
1201
1202     def install_to_new(self):
1203         """install upload to NEW
1204
1205         Install upload to NEW.  This method does B{not} handle regular uploads
1206         to suites or policy queues.
1207
1208         You need to have called the C{check} method before calling this method.
1209         """
1210         # Uploads to NEW are special as we don't have overrides.
1211         assert len(self.reject_reasons) == 0
1212         assert self.changes.valid_signature
1213         assert self.final_suites is not None
1214
1215         source = self.changes.source
1216         binaries = self.changes.binaries
1217         byhand = self.changes.byhand_files
1218
1219         # we need a suite to guess components
1220         suites = list(self.final_suites)
1221         assert len(suites) == 1, "NEW uploads must be to a single suite"
1222         suite = suites[0]
1223
1224         # decide which NEW queue to use
1225         if suite.new_queue is None:
1226             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1227         else:
1228             new_queue = suite.new_queue
1229         if len(byhand) > 0:
1230             # There is only one global BYHAND queue
1231             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1232         new_suite = new_queue.suite
1233
1234
1235         def binary_component_func(binary):
1236             return self._binary_component(suite, binary, only_overrides=False)
1237
1238         # guess source component
1239         # XXX: should be moved into an extra method
1240         binary_component_names = set()
1241         for binary in binaries:
1242             component = binary_component_func(binary)
1243             binary_component_names.add(component.component_name)
1244         source_component_name = None
1245         for c in self.session.query(Component).order_by(Component.component_id):
1246             guess = c.component_name
1247             if guess in binary_component_names:
1248                 source_component_name = guess
1249                 break
1250         if source_component_name is None:
1251             source_component = self.session.query(Component).order_by(Component.component_id).first()
1252         else:
1253             source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1254         source_component_func = lambda source: source_component
1255
1256         db_changes = self._install_changes()
1257         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1258         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1259
1260         for f in byhand:
1261             self._install_byhand(policy_upload, f)
1262
1263         self._do_bts_versiontracking()
1264
1265     def commit(self):
1266         """commit changes"""
1267         self.transaction.commit()
1268
1269     def rollback(self):
1270         """rollback changes"""
1271         self.transaction.rollback()
1272
1273     def __enter__(self):
1274         self.prepare()
1275         return self
1276
1277     def __exit__(self, type, value, traceback):
1278         if self.directory is not None:
1279             shutil.rmtree(self.directory)
1280             self.directory = None
1281         self.changes = None
1282         self.transaction.rollback()
1283         return None