daklib/archive.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to manipulate the archive
  18
  19 This module provides classes to manipulate the archive.
  20 """
  21
  22 from daklib.dbconn import *
  23 import daklib.checks as checks
  24 from daklib.config import Config
  25 import daklib.upload as upload
  26 import daklib.utils as utils
  27 from daklib.fstransactions import FilesystemTransaction
  28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
  29 import daklib.daksubprocess
  30
  31 import apt_pkg
  32 from datetime import datetime
  33 import os
  34 import shutil
  35 from sqlalchemy.orm.exc import NoResultFound
  36 import sqlalchemy.exc
  37 import tempfile
  38 import traceback
  39
  40 class ArchiveException(Exception):
  41     pass
  42
  43 class HashMismatchException(ArchiveException):
  44     pass
  45
  46 class ArchiveTransaction(object):
  47     """manipulate the archive in a transaction
  48     """
  49     def __init__(self):
  50         self.fs = FilesystemTransaction()
  51         self.session = DBConn().session()
  52
  53     def get_file(self, hashed_file, source_name, check_hashes=True):
  54         """Look for file C{hashed_file} in database
  55
  56         @type  hashed_file: L{daklib.upload.HashedFile}
  57         @param hashed_file: file to look for in the database
  58
  59         @type  source_name: str
  60         @param source_name: source package name
  61
  62         @type  check_hashes: bool
  63         @param check_hashes: check size and hashes match
  64
  65         @raise KeyError: file was not found in the database
  66         @raise HashMismatchException: hash mismatch
  67
  68         @rtype:  L{daklib.dbconn.PoolFile}
  69         @return: database entry for the file
  70         """
  71         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  72         try:
  73             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
  74             if check_hashes and (poolfile.filesize != hashed_file.size
  75                                  or poolfile.md5sum != hashed_file.md5sum
  76                                  or poolfile.sha1sum != hashed_file.sha1sum
  77                                  or poolfile.sha256sum != hashed_file.sha256sum):
  78                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
  79             return poolfile
  80         except NoResultFound:
  81             raise KeyError('{0} not found in database.'.format(poolname))
  82
  83     def _install_file(self, directory, hashed_file, archive, component, source_name):
  84         """Install a file
  85
  86         Will not give an error when the file is already present.
  87
  88         @rtype:  L{daklib.dbconn.PoolFile}
  89         @return: database object for the new file
  90         """
  91         session = self.session
  92
  93         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  94         try:
  95             poolfile = self.get_file(hashed_file, source_name)
  96         except KeyError:
  97             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
  98             poolfile.md5sum = hashed_file.md5sum
  99             poolfile.sha1sum = hashed_file.sha1sum
 100             poolfile.sha256sum = hashed_file.sha256sum
 101             session.add(poolfile)
 102             session.flush()
 103
 104         try:
 105             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
 106         except NoResultFound:
 107             archive_file = ArchiveFile(archive, component, poolfile)
 108             session.add(archive_file)
 109             session.flush()
 110
 111             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
 112             hashed_file_path = os.path.join(directory, hashed_file.filename)
 113             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 114
 115         return poolfile
 116
 117     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
 118         """Install a binary package
 119
 120         @type  directory: str
 121         @param directory: directory the binary package is located in
 122
 123         @type  binary: L{daklib.upload.Binary}
 124         @param binary: binary package to install
 125
 126         @type  suite: L{daklib.dbconn.Suite}
 127         @param suite: target suite
 128
 129         @type  component: L{daklib.dbconn.Component}
 130         @param component: target component
 131
 132         @type  allow_tainted: bool
 133         @param allow_tainted: allow to copy additional files from tainted archives
 134
 135         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 136         @param fingerprint: optional fingerprint
 137
 138         @type  source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
 139         @param source_suites: suites to copy the source from if they are not
 140                               in C{suite} or C{True} to allow copying from any
 141                               suite.
 142
 143         @type  extra_source_archives: list of L{daklib.dbconn.Archive}
 144         @param extra_source_archives: extra archives to copy Built-Using sources from
 145
 146         @rtype:  L{daklib.dbconn.DBBinary}
 147         @return: databse object for the new package
 148         """
 149         session = self.session
 150         control = binary.control
 151         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 152         architecture = get_architecture(control['Architecture'], session)
 153
 154         (source_name, source_version) = binary.source
 155         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
 156         source = source_query.filter(DBSource.suites.contains(suite)).first()
 157         if source is None:
 158             if source_suites != True:
 159                 source_query = source_query.join(DBSource.suites) \
 160                     .filter(Suite.suite_id == source_suites.c.id)
 161             source = source_query.first()
 162             if source is None:
 163                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
 164             self.copy_source(source, suite, component)
 165
 166         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
 167
 168         unique = dict(
 169             package=control['Package'],
 170             version=control['Version'],
 171             architecture=architecture,
 172             )
 173         rest = dict(
 174             source=source,
 175             maintainer=maintainer,
 176             poolfile=db_file,
 177             binarytype=binary.type,
 178             fingerprint=fingerprint,
 179             )
 180
 181         try:
 182             db_binary = session.query(DBBinary).filter_by(**unique).one()
 183             for key, value in rest.iteritems():
 184                 if getattr(db_binary, key) != value:
 185                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
 186         except NoResultFound:
 187             db_binary = DBBinary(**unique)
 188             for key, value in rest.iteritems():
 189                 setattr(db_binary, key, value)
 190             session.add(db_binary)
 191             session.flush()
 192             import_metadata_into_db(db_binary, session)
 193
 194             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
 195
 196         if suite not in db_binary.suites:
 197             db_binary.suites.append(suite)
 198
 199         session.flush()
 200
 201         return db_binary
 202
 203     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
 204         """ensure source exists in the given archive
 205
 206         This is intended to be used to check that Built-Using sources exist.
 207
 208         @type  filename: str
 209         @param filename: filename to use in error messages
 210
 211         @type  source: L{daklib.dbconn.DBSource}
 212         @param source: source to look for
 213
 214         @type  archive: L{daklib.dbconn.Archive}
 215         @param archive: archive to look in
 216
 217         @type  extra_archives: list of L{daklib.dbconn.Archive}
 218         @param extra_archives: list of archives to copy the source package from
 219                                if it is not yet present in C{archive}
 220         """
 221         session = self.session
 222         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
 223         if db_file is not None:
 224             return True
 225
 226         # Try to copy file from one extra archive
 227         if extra_archives is None:
 228             extra_archives = []
 229         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
 230         if db_file is None:
 231             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
 232
 233         source_archive = db_file.archive
 234         for dsc_file in source.srcfiles:
 235             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
 236             # We were given an explicit list of archives so it is okay to copy from tainted archives.
 237             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 238
 239     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
 240         """Add Built-Using sources to C{db_binary.extra_sources}
 241         """
 242         session = self.session
 243         built_using = control.get('Built-Using', None)
 244
 245         if built_using is not None:
 246             for dep in apt_pkg.parse_depends(built_using):
 247                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
 248                 bu_source_name, bu_source_version, comp = dep[0]
 249                 assert comp == '=', 'Built-Using must contain strict dependencies'
 250
 251                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
 252                 if bu_source is None:
 253                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
 254
 255                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
 256
 257                 db_binary.extra_sources.append(bu_source)
 258
 259     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
 260         """Install a source package
 261
 262         @type  directory: str
 263         @param directory: directory the source package is located in
 264
 265         @type  source: L{daklib.upload.Source}
 266         @param source: source package to install
 267
 268         @type  suite: L{daklib.dbconn.Suite}
 269         @param suite: target suite
 270
 271         @type  component: L{daklib.dbconn.Component}
 272         @param component: target component
 273
 274         @type  changed_by: L{daklib.dbconn.Maintainer}
 275         @param changed_by: person who prepared this version of the package
 276
 277         @type  allow_tainted: bool
 278         @param allow_tainted: allow to copy additional files from tainted archives
 279
 280         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 281         @param fingerprint: optional fingerprint
 282
 283         @rtype:  L{daklib.dbconn.DBSource}
 284         @return: database object for the new source
 285         """
 286         session = self.session
 287         archive = suite.archive
 288         control = source.dsc
 289         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 290         source_name = control['Source']
 291
 292         ### Add source package to database
 293
 294         # We need to install the .dsc first as the DBSource object refers to it.
 295         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
 296
 297         unique = dict(
 298             source=source_name,
 299             version=control['Version'],
 300             )
 301         rest = dict(
 302             maintainer=maintainer,
 303             changedby=changed_by,
 304             #install_date=datetime.now().date(),
 305             poolfile=db_file_dsc,
 306             fingerprint=fingerprint,
 307             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
 308             )
 309
 310         created = False
 311         try:
 312             db_source = session.query(DBSource).filter_by(**unique).one()
 313             for key, value in rest.iteritems():
 314                 if getattr(db_source, key) != value:
 315                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
 316         except NoResultFound:
 317             created = True
 318             db_source = DBSource(**unique)
 319             for key, value in rest.iteritems():
 320                 setattr(db_source, key, value)
 321             # XXX: set as default in postgres?
 322             db_source.install_date = datetime.now().date()
 323             session.add(db_source)
 324             session.flush()
 325
 326             # Add .dsc file. Other files will be added later.
 327             db_dsc_file = DSCFile()
 328             db_dsc_file.source = db_source
 329             db_dsc_file.poolfile = db_file_dsc
 330             session.add(db_dsc_file)
 331             session.flush()
 332
 333         if suite in db_source.suites:
 334             return db_source
 335
 336         db_source.suites.append(suite)
 337
 338         if not created:
 339             for f in db_source.srcfiles:
 340                 self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
 341             return db_source
 342
 343         ### Now add remaining files and copy them to the archive.
 344
 345         for hashed_file in source.files.itervalues():
 346             hashed_file_path = os.path.join(directory, hashed_file.filename)
 347             if os.path.exists(hashed_file_path):
 348                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
 349                 session.add(db_file)
 350             else:
 351                 db_file = self.get_file(hashed_file, source_name)
 352                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
 353
 354             db_dsc_file = DSCFile()
 355             db_dsc_file.source = db_source
 356             db_dsc_file.poolfile = db_file
 357             session.add(db_dsc_file)
 358
 359         session.flush()
 360
 361         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
 362         import_metadata_into_db(db_source, session)
 363
 364         # Uploaders are the maintainer and co-maintainers from the Uploaders field
 365         db_source.uploaders.append(maintainer)
 366         if 'Uploaders' in control:
 367             from daklib.textutils import split_uploaders
 368             for u in split_uploaders(control['Uploaders']):
 369                 db_source.uploaders.append(get_or_set_maintainer(u, session))
 370         session.flush()
 371
 372         return db_source
 373
 374     def _copy_file(self, db_file, archive, component, allow_tainted=False):
 375         """Copy a file to the given archive and component
 376
 377         @type  db_file: L{daklib.dbconn.PoolFile}
 378         @param db_file: file to copy
 379
 380         @type  archive: L{daklib.dbconn.Archive}
 381         @param archive: target archive
 382
 383         @type  component: L{daklib.dbconn.Archive}
 384         @param component: target component
 385
 386         @type  allow_tainted: bool
 387         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 388         """
 389         session = self.session
 390
 391         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
 392             query = session.query(ArchiveFile).filter_by(file=db_file)
 393             if not allow_tainted:
 394                 query = query.join(Archive).filter(Archive.tainted == False)
 395
 396             source_af = query.first()
 397             if source_af is None:
 398                 raise ArchiveException('cp: Could not find {0} in any archive.'.format(db_file.filename))
 399             target_af = ArchiveFile(archive, component, db_file)
 400             session.add(target_af)
 401             session.flush()
 402             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
 403
 404     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
 405         """Copy a binary package to the given suite and component
 406
 407         @type  db_binary: L{daklib.dbconn.DBBinary}
 408         @param db_binary: binary to copy
 409
 410         @type  suite: L{daklib.dbconn.Suite}
 411         @param suite: target suite
 412
 413         @type  component: L{daklib.dbconn.Component}
 414         @param component: target component
 415
 416         @type  allow_tainted: bool
 417         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 418
 419         @type  extra_archives: list of L{daklib.dbconn.Archive}
 420         @param extra_archives: extra archives to copy Built-Using sources from
 421         """
 422         session = self.session
 423         archive = suite.archive
 424         if archive.tainted:
 425             allow_tainted = True
 426
 427         filename = db_binary.poolfile.filename
 428
 429         # make sure source is present in target archive
 430         db_source = db_binary.source
 431         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
 432             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
 433
 434         # make sure built-using packages are present in target archive
 435         for db_source in db_binary.extra_sources:
 436             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 437
 438         # copy binary
 439         db_file = db_binary.poolfile
 440         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
 441         if suite not in db_binary.suites:
 442             db_binary.suites.append(suite)
 443         self.session.flush()
 444
 445     def copy_source(self, db_source, suite, component, allow_tainted=False):
 446         """Copy a source package to the given suite and component
 447
 448         @type  db_source: L{daklib.dbconn.DBSource}
 449         @param db_source: source to copy
 450
 451         @type  suite: L{daklib.dbconn.Suite}
 452         @param suite: target suite
 453
 454         @type  component: L{daklib.dbconn.Component}
 455         @param component: target component
 456
 457         @type  allow_tainted: bool
 458         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 459         """
 460         archive = suite.archive
 461         if archive.tainted:
 462             allow_tainted = True
 463         for db_dsc_file in db_source.srcfiles:
 464             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
 465         if suite not in db_source.suites:
 466             db_source.suites.append(suite)
 467         self.session.flush()
 468
 469     def remove_file(self, db_file, archive, component):
 470         """Remove a file from a given archive and component
 471
 472         @type  db_file: L{daklib.dbconn.PoolFile}
 473         @param db_file: file to remove
 474
 475         @type  archive: L{daklib.dbconn.Archive}
 476         @param archive: archive to remove the file from
 477
 478         @type  component: L{daklib.dbconn.Component}
 479         @param component: component to remove the file from
 480         """
 481         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
 482         self.fs.unlink(af.path)
 483         self.session.delete(af)
 484
 485     def remove_binary(self, binary, suite):
 486         """Remove a binary from a given suite and component
 487
 488         @type  binary: L{daklib.dbconn.DBBinary}
 489         @param binary: binary to remove
 490
 491         @type  suite: L{daklib.dbconn.Suite}
 492         @param suite: suite to remove the package from
 493         """
 494         binary.suites.remove(suite)
 495         self.session.flush()
 496
 497     def remove_source(self, source, suite):
 498         """Remove a source from a given suite and component
 499
 500         @type  source: L{daklib.dbconn.DBSource}
 501         @param source: source to remove
 502
 503         @type  suite: L{daklib.dbconn.Suite}
 504         @param suite: suite to remove the package from
 505
 506         @raise ArchiveException: source package is still referenced by other
 507                                  binaries in the suite
 508         """
 509         session = self.session
 510
 511         query = session.query(DBBinary).filter_by(source=source) \
 512             .filter(DBBinary.suites.contains(suite))
 513         if query.first() is not None:
 514             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
 515
 516         source.suites.remove(suite)
 517         session.flush()
 518
 519     def commit(self):
 520         """commit changes"""
 521         try:
 522             self.session.commit()
 523             self.fs.commit()
 524         finally:
 525             self.session.rollback()
 526             self.fs.rollback()
 527
 528     def rollback(self):
 529         """rollback changes"""
 530         self.session.rollback()
 531         self.fs.rollback()
 532
 533     def __enter__(self):
 534         return self
 535
 536     def __exit__(self, type, value, traceback):
 537         if type is None:
 538             self.commit()
 539         else:
 540             self.rollback()
 541         return None
 542
 543 class ArchiveUpload(object):
 544     """handle an upload
 545
 546     This class can be used in a with-statement::
 547
 548        with ArchiveUpload(...) as upload:
 549           ...
 550
 551     Doing so will automatically run any required cleanup and also rollback the
 552     transaction if it was not committed.
 553     """
 554     def __init__(self, directory, changes, keyrings):
 555         self.transaction = ArchiveTransaction()
 556         """transaction used to handle the upload
 557         @type: L{daklib.archive.ArchiveTransaction}
 558         """
 559
 560         self.session = self.transaction.session
 561         """database session"""
 562
 563         self.original_directory = directory
 564         self.original_changes = changes
 565
 566         self.changes = None
 567         """upload to process
 568         @type: L{daklib.upload.Changes}
 569         """
 570
 571         self.directory = None
 572         """directory with temporary copy of files. set by C{prepare}
 573         @type: str
 574         """
 575
 576         self.keyrings = keyrings
 577
 578         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
 579         """fingerprint of the key used to sign the upload
 580         @type: L{daklib.dbconn.Fingerprint}
 581         """
 582
 583         self.reject_reasons = []
 584         """reasons why the upload cannot by accepted
 585         @type: list of str
 586         """
 587
 588         self.warnings = []
 589         """warnings
 590         @note: Not used yet.
 591         @type: list of str
 592         """
 593
 594         self.final_suites = None
 595
 596         self.new = False
 597         """upload is NEW. set by C{check}
 598         @type: bool
 599         """
 600
 601         self._checked = False
 602         """checks passes. set by C{check}
 603         @type: bool
 604         """
 605
 606         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
 607         self._new = self._new_queue.suite
 608
 609     def warn(self, message):
 610         """add a warning message
 611
 612         Adds a warning message that can later be seen in C{self.warnings}
 613
 614         @type  message: string
 615         @param message: warning message
 616         """
 617         self.warnings.append(message)
 618
 619     def prepare(self):
 620         """prepare upload for further processing
 621
 622         This copies the files involved to a temporary directory.  If you use
 623         this method directly, you have to remove the directory given by the
 624         C{directory} attribute later on your own.
 625
 626         Instead of using the method directly, you can also use a with-statement::
 627
 628            with ArchiveUpload(...) as upload:
 629               ...
 630
 631         This will automatically handle any required cleanup.
 632         """
 633         assert self.directory is None
 634         assert self.original_changes.valid_signature
 635
 636         cnf = Config()
 637         session = self.transaction.session
 638
 639         group = cnf.get('Dinstall::UnprivGroup') or None
 640         self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
 641                                             mode=0o2750, group=group)
 642         with FilesystemTransaction() as fs:
 643             src = os.path.join(self.original_directory, self.original_changes.filename)
 644             dst = os.path.join(self.directory, self.original_changes.filename)
 645             fs.copy(src, dst, mode=0o640)
 646
 647             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 648
 649             for f in self.changes.files.itervalues():
 650                 src = os.path.join(self.original_directory, f.filename)
 651                 dst = os.path.join(self.directory, f.filename)
 652                 if not os.path.exists(src):
 653                     continue
 654                 fs.copy(src, dst, mode=0o640)
 655
 656             source = None
 657             try:
 658                 source = self.changes.source
 659             except Exception:
 660                 # Do not raise an exception here if the .dsc is invalid.
 661                 pass
 662
 663             if source is not None:
 664                 for f in source.files.itervalues():
 665                     src = os.path.join(self.original_directory, f.filename)
 666                     dst = os.path.join(self.directory, f.filename)
 667                     if not os.path.exists(dst):
 668                         try:
 669                             db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
 670                             db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
 671                             fs.copy(db_archive_file.path, dst, mode=0o640)
 672                         except KeyError:
 673                             # Ignore if get_file could not find it. Upload will
 674                             # probably be rejected later.
 675                             pass
 676
 677     def unpacked_source(self):
 678         """Path to unpacked source
 679
 680         Get path to the unpacked source. This method does unpack the source
 681         into a temporary directory under C{self.directory} if it has not
 682         been done so already.
 683
 684         @rtype:  str or C{None}
 685         @return: string giving the path to the unpacked source directory
 686                  or C{None} if no source was included in the upload.
 687         """
 688         assert self.directory is not None
 689
 690         source = self.changes.source
 691         if source is None:
 692             return None
 693         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
 694
 695         sourcedir = os.path.join(self.directory, 'source')
 696         if not os.path.exists(sourcedir):
 697             devnull = open('/dev/null', 'w')
 698             daklib.daksubprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
 699         if not os.path.isdir(sourcedir):
 700             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
 701         return sourcedir
 702
 703     def _map_suite(self, suite_name):
 704         for rule in Config().value_list("SuiteMappings"):
 705             fields = rule.split()
 706             rtype = fields[0]
 707             if rtype == "map" or rtype == "silent-map":
 708                 (src, dst) = fields[1:3]
 709                 if src == suite_name:
 710                     suite_name = dst
 711                     if rtype != "silent-map":
 712                         self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
 713             elif rtype == "ignore":
 714                 ignored = fields[1]
 715                 if suite_name == ignored:
 716                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
 717                     suite_name = None
 718             elif rtype == "reject":
 719                 rejected = fields[1]
 720                 if suite_name == rejected:
 721                     raise checks.Reject('Uploads to {0} are not accepted.'.format(rejected))
 722             ## XXX: propup-version and map-unreleased not yet implemented
 723         return suite_name
 724
 725     def _mapped_suites(self):
 726         """Get target suites after mappings
 727
 728         @rtype:  list of L{daklib.dbconn.Suite}
 729         @return: list giving the mapped target suites of this upload
 730         """
 731         session = self.session
 732
 733         suite_names = []
 734         for dist in self.changes.distributions:
 735             suite_name = self._map_suite(dist)
 736             if suite_name is not None:
 737                 suite_names.append(suite_name)
 738
 739         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
 740         return suites
 741
 742     def _check_new_binary_overrides(self, suite):
 743         new = False
 744
 745         binaries = self.changes.binaries
 746         source = self.changes.source
 747         if source is not None and not source.package_list.fallback:
 748             packages = source.package_list.packages_for_suite(suite)
 749             binaries = [ entry for entry in packages ]
 750
 751         for b in binaries:
 752             override = self._binary_override(suite, b)
 753             if override is None:
 754                 self.warnings.append('binary:{0} is NEW.'.format(b.name))
 755                 new = True
 756
 757         return new
 758
 759     def _check_new(self, suite):
 760         """Check if upload is NEW
 761
 762         An upload is NEW if it has binary or source packages that do not have
 763         an override in C{suite} OR if it references files ONLY in a tainted
 764         archive (eg. when it references files in NEW).
 765
 766         @rtype:  bool
 767         @return: C{True} if the upload is NEW, C{False} otherwise
 768         """
 769         session = self.session
 770         new = False
 771
 772         # Check for missing overrides
 773         if self._check_new_binary_overrides(suite):
 774             new = True
 775         if self.changes.source is not None:
 776             override = self._source_override(suite, self.changes.source)
 777             if override is None:
 778                 self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
 779                 new = True
 780
 781         # Check if we reference a file only in a tainted archive
 782         files = self.changes.files.values()
 783         if self.changes.source is not None:
 784             files.extend(self.changes.source.files.values())
 785         for f in files:
 786             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
 787             query_untainted = query.join(Archive).filter(Archive.tainted == False)
 788
 789             in_archive = (query.first() is not None)
 790             in_untainted_archive = (query_untainted.first() is not None)
 791
 792             if in_archive and not in_untainted_archive:
 793                 self.warnings.append('{0} is only available in NEW.'.format(f.filename))
 794                 new = True
 795
 796         return new
 797
 798     def _final_suites(self):
 799         session = self.session
 800
 801         mapped_suites = self._mapped_suites()
 802         final_suites = set()
 803
 804         for suite in mapped_suites:
 805             overridesuite = suite
 806             if suite.overridesuite is not None:
 807                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 808             if self._check_new(overridesuite):
 809                 self.new = True
 810             final_suites.add(suite)
 811
 812         return final_suites
 813
 814     def _binary_override(self, suite, binary):
 815         """Get override entry for a binary
 816
 817         @type  suite: L{daklib.dbconn.Suite}
 818         @param suite: suite to get override for
 819
 820         @type  binary: L{daklib.upload.Binary} or L{daklib.packagelist.PackageListEntry}
 821         @param binary: binary to get override for
 822
 823         @rtype:  L{daklib.dbconn.Override} or C{None}
 824         @return: override for the given binary or C{None}
 825         """
 826         if suite.overridesuite is not None:
 827             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 828
 829         mapped_component = get_mapped_component(binary.component)
 830         if mapped_component is None:
 831             return None
 832
 833         query = self.session.query(Override).filter_by(suite=suite, package=binary.name) \
 834                 .join(Component).filter(Component.component_name == mapped_component.component_name) \
 835                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 836
 837         try:
 838             return query.one()
 839         except NoResultFound:
 840             return None
 841
 842     def _source_override(self, suite, source):
 843         """Get override entry for a source
 844
 845         @type  suite: L{daklib.dbconn.Suite}
 846         @param suite: suite to get override for
 847
 848         @type  source: L{daklib.upload.Source}
 849         @param source: source to get override for
 850
 851         @rtype:  L{daklib.dbconn.Override} or C{None}
 852         @return: override for the given source or C{None}
 853         """
 854         if suite.overridesuite is not None:
 855             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 856
 857         # XXX: component for source?
 858         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
 859                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 860
 861         try:
 862             return query.one()
 863         except NoResultFound:
 864             return None
 865
 866     def _binary_component(self, suite, binary, only_overrides=True):
 867         """get component for a binary
 868
 869         By default this will only look at overrides to get the right component;
 870         if C{only_overrides} is C{False} this method will also look at the
 871         Section field.
 872
 873         @type  suite: L{daklib.dbconn.Suite}
 874
 875         @type  binary: L{daklib.upload.Binary}
 876
 877         @type  only_overrides: bool
 878         @param only_overrides: only use overrides to get the right component
 879
 880         @rtype: L{daklib.dbconn.Component} or C{None}
 881         """
 882         override = self._binary_override(suite, binary)
 883         if override is not None:
 884             return override.component
 885         if only_overrides:
 886             return None
 887         return get_mapped_component(binary.component, self.session)
 888
 889     def check(self, force=False):
 890         """run checks against the upload
 891
 892         @type  force: bool
 893         @param force: ignore failing forcable checks
 894
 895         @rtype:  bool
 896         @return: C{True} if all checks passed, C{False} otherwise
 897         """
 898         # XXX: needs to be better structured.
 899         assert self.changes.valid_signature
 900
 901         try:
 902             # Validate signatures and hashes before we do any real work:
 903             for chk in (
 904                     checks.SignatureAndHashesCheck,
 905                     checks.ChangesCheck,
 906                     checks.ExternalHashesCheck,
 907                     checks.SourceCheck,
 908                     checks.BinaryCheck,
 909                     checks.BinaryTimestampCheck,
 910                     checks.SingleDistributionCheck,
 911                     ):
 912                 chk().check(self)
 913
 914             final_suites = self._final_suites()
 915             if len(final_suites) == 0:
 916                 self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
 917                 return False
 918
 919             self.final_suites = final_suites
 920
 921             for chk in (
 922                     checks.TransitionCheck,
 923                     checks.ACLCheck,
 924                     checks.NoSourceOnlyCheck,
 925                     checks.LintianCheck,
 926                     ):
 927                 chk().check(self)
 928
 929             for chk in (
 930                     checks.ACLCheck,
 931                     checks.SourceFormatCheck,
 932                     checks.SuiteArchitectureCheck,
 933                     checks.VersionCheck,
 934                     ):
 935                 for suite in final_suites:
 936                     chk().per_suite_check(self, suite)
 937
 938             if len(self.reject_reasons) != 0:
 939                 return False
 940
 941             self._checked = True
 942             return True
 943         except checks.Reject as e:
 944             self.reject_reasons.append(unicode(e))
 945         except Exception as e:
 946             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
 947         return False
 948
 949     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
 950         """Install upload to the given suite
 951
 952         @type  suite: L{daklib.dbconn.Suite}
 953         @param suite: suite to install the package into. This is the real suite,
 954                       ie. after any redirection to NEW or a policy queue
 955
 956         @param source_component_func: function to get the L{daklib.dbconn.Component}
 957                                       for a L{daklib.upload.Source} object
 958
 959         @param binary_component_func: function to get the L{daklib.dbconn.Component}
 960                                       for a L{daklib.upload.Binary} object
 961
 962         @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
 963
 964         @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
 965
 966         @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
 967                  object for the install source or C{None} if no source was
 968                  included. The second is a list of L{daklib.dbconn.DBBinary}
 969                  objects for the installed binary packages.
 970         """
 971         # XXX: move this function to ArchiveTransaction?
 972
 973         control = self.changes.changes
 974         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 975
 976         if source_suites is None:
 977             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()
 978
 979         source = self.changes.source
 980         if source is not None:
 981             component = source_component_func(source)
 982             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
 983         else:
 984             db_source = None
 985
 986         db_binaries = []
 987         for binary in self.changes.binaries:
 988             component = binary_component_func(binary)
 989             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
 990             db_binaries.append(db_binary)
 991
 992         if suite.copychanges:
 993             src = os.path.join(self.directory, self.changes.filename)
 994             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
 995             self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
 996
 997         return (db_source, db_binaries)
 998
 999     def _install_changes(self):
1000         assert self.changes.valid_signature
1001         control = self.changes.changes
1002         session = self.transaction.session
1003         config = Config()
1004
1005         changelog_id = None
1006         # Only add changelog for sourceful uploads and binNMUs
1007         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
1008             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
1009             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
1010             assert changelog_id is not None
1011
1012         db_changes = DBChange()
1013         db_changes.changesname = self.changes.filename
1014         db_changes.source = control['Source']
1015         db_changes.binaries = control.get('Binary', None)
1016         db_changes.architecture = control['Architecture']
1017         db_changes.version = control['Version']
1018         db_changes.distribution = control['Distribution']
1019         db_changes.urgency = control['Urgency']
1020         db_changes.maintainer = control['Maintainer']
1021         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
1022         db_changes.date = control['Date']
1023         db_changes.fingerprint = self.fingerprint.fingerprint
1024         db_changes.changelog_id = changelog_id
1025         db_changes.closes = self.changes.closed_bugs
1026
1027         try:
1028             self.transaction.session.add(db_changes)
1029             self.transaction.session.flush()
1030         except sqlalchemy.exc.IntegrityError:
1031             raise ArchiveException('{0} is already known.'.format(self.changes.filename))
1032
1033         return db_changes
1034
1035     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
1036         u = PolicyQueueUpload()
1037         u.policy_queue = policy_queue
1038         u.target_suite = target_suite
1039         u.changes = db_changes
1040         u.source = db_source
1041         u.binaries = db_binaries
1042         self.transaction.session.add(u)
1043         self.transaction.session.flush()
1044
1045         dst = os.path.join(policy_queue.path, self.changes.filename)
1046         self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)
1047
1048         return u
1049
1050     def try_autobyhand(self):
1051         """Try AUTOBYHAND
1052
1053         Try to handle byhand packages automatically.
1054
1055         @rtype:  list of L{daklib.upload.HashedFile}
1056         @return: list of remaining byhand files
1057         """
1058         assert len(self.reject_reasons) == 0
1059         assert self.changes.valid_signature
1060         assert self.final_suites is not None
1061         assert self._checked
1062
1063         byhand = self.changes.byhand_files
1064         if len(byhand) == 0:
1065             return True
1066
1067         suites = list(self.final_suites)
1068         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1069         suite = suites[0]
1070
1071         cnf = Config()
1072         control = self.changes.changes
1073         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1074
1075         remaining = []
1076         for f in byhand:
1077             if '_' in f.filename:
1078                 parts = f.filename.split('_', 2)
1079                 if len(parts) != 3:
1080                     print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1081                     remaining.append(f)
1082                     continue
1083
1084                 package, version, archext = parts
1085                 arch, ext = archext.split('.', 1)
1086             else:
1087                 parts = f.filename.split('.')
1088                 if len(parts) < 2:
1089                     print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1090                     remaining.append(f)
1091                     continue
1092
1093                 package = parts[0]
1094                 version = '0'
1095                 arch = 'all'
1096                 ext = parts[-1]
1097
1098             try:
1099                 rule = automatic_byhand_packages.subtree(package)
1100             except KeyError:
1101                 remaining.append(f)
1102                 continue
1103
1104             if rule['Source'] != self.changes.source_name \
1105                     or rule['Section'] != f.section \
1106                     or ('Extension' in rule and rule['Extension'] != ext):
1107                 remaining.append(f)
1108                 continue
1109
1110             script = rule['Script']
1111             retcode = daklib.daksubprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1112             if retcode != 0:
1113                 print "W: error processing {0}.".format(f.filename)
1114                 remaining.append(f)
1115
1116         return len(remaining) == 0
1117
1118     def _install_byhand(self, policy_queue_upload, hashed_file):
1119         """install byhand file
1120
1121         @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1122
1123         @type  hashed_file: L{daklib.upload.HashedFile}
1124         """
1125         fs = self.transaction.fs
1126         session = self.transaction.session
1127         policy_queue = policy_queue_upload.policy_queue
1128
1129         byhand_file = PolicyQueueByhandFile()
1130         byhand_file.upload = policy_queue_upload
1131         byhand_file.filename = hashed_file.filename
1132         session.add(byhand_file)
1133         session.flush()
1134
1135         src = os.path.join(self.directory, hashed_file.filename)
1136         dst = os.path.join(policy_queue.path, hashed_file.filename)
1137         fs.copy(src, dst, mode=policy_queue.change_perms)
1138
1139         return byhand_file
1140
1141     def _do_bts_versiontracking(self):
1142         cnf = Config()
1143         fs = self.transaction.fs
1144
1145         btsdir = cnf.get('Dir::BTSVersionTrack')
1146         if btsdir is None or btsdir == '':
1147             return
1148
1149         base = os.path.join(btsdir, self.changes.filename[:-8])
1150
1151         # version history
1152         sourcedir = self.unpacked_source()
1153         if sourcedir is not None:
1154             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1155             versions = fs.create("{0}.versions".format(base), mode=0o644)
1156             for line in fh.readlines():
1157                 if re_changelog_versions.match(line):
1158                     versions.write(line)
1159             fh.close()
1160             versions.close()
1161
1162         # binary -> source mapping
1163         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1164         for binary in self.changes.binaries:
1165             control = binary.control
1166             source_package, source_version = binary.source
1167             line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
1168             print >>debinfo, line
1169         debinfo.close()
1170
1171     def _policy_queue(self, suite):
1172         if suite.policy_queue is not None:
1173             return suite.policy_queue
1174         return None
1175
1176     def install(self):
1177         """install upload
1178
1179         Install upload to a suite or policy queue.  This method does B{not}
1180         handle uploads to NEW.
1181
1182         You need to have called the C{check} method before calling this method.
1183         """
1184         assert len(self.reject_reasons) == 0
1185         assert self.changes.valid_signature
1186         assert self.final_suites is not None
1187         assert self._checked
1188         assert not self.new
1189
1190         db_changes = self._install_changes()
1191
1192         for suite in self.final_suites:
1193             overridesuite = suite
1194             if suite.overridesuite is not None:
1195                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1196
1197             policy_queue = self._policy_queue(suite)
1198
1199             redirected_suite = suite
1200             if policy_queue is not None:
1201                 redirected_suite = policy_queue.suite
1202
1203             # source can be in the suite we install to or any suite we enhance
1204             source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
1205             for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
1206                     .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
1207                     .filter(VersionCheck.check == 'Enhances'):
1208                 source_suite_ids.add(enhanced_suite_id)
1209
1210             source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()
1211
1212             source_component_func = lambda source: self._source_override(overridesuite, source).component
1213             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1214
1215             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1216
1217             if policy_queue is not None:
1218                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1219
1220             # copy to build queues
1221             if policy_queue is None or policy_queue.send_to_build_queues:
1222                 for build_queue in suite.copy_queues:
1223                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1224
1225         self._do_bts_versiontracking()
1226
1227     def install_to_new(self):
1228         """install upload to NEW
1229
1230         Install upload to NEW.  This method does B{not} handle regular uploads
1231         to suites or policy queues.
1232
1233         You need to have called the C{check} method before calling this method.
1234         """
1235         # Uploads to NEW are special as we don't have overrides.
1236         assert len(self.reject_reasons) == 0
1237         assert self.changes.valid_signature
1238         assert self.final_suites is not None
1239
1240         source = self.changes.source
1241         binaries = self.changes.binaries
1242         byhand = self.changes.byhand_files
1243
1244         # we need a suite to guess components
1245         suites = list(self.final_suites)
1246         assert len(suites) == 1, "NEW uploads must be to a single suite"
1247         suite = suites[0]
1248
1249         # decide which NEW queue to use
1250         if suite.new_queue is None:
1251             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1252         else:
1253             new_queue = suite.new_queue
1254         if len(byhand) > 0:
1255             # There is only one global BYHAND queue
1256             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1257         new_suite = new_queue.suite
1258
1259
1260         def binary_component_func(binary):
1261             return self._binary_component(suite, binary, only_overrides=False)
1262
1263         # guess source component
1264         # XXX: should be moved into an extra method
1265         binary_component_names = set()
1266         for binary in binaries:
1267             component = binary_component_func(binary)
1268             binary_component_names.add(component.component_name)
1269         source_component_name = None
1270         for c in self.session.query(Component).order_by(Component.component_id):
1271             guess = c.component_name
1272             if guess in binary_component_names:
1273                 source_component_name = guess
1274                 break
1275         if source_component_name is None:
1276             source_component = self.session.query(Component).order_by(Component.component_id).first()
1277         else:
1278             source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1279         source_component_func = lambda source: source_component
1280
1281         db_changes = self._install_changes()
1282         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1283         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1284
1285         for f in byhand:
1286             self._install_byhand(policy_upload, f)
1287
1288         self._do_bts_versiontracking()
1289
1290     def commit(self):
1291         """commit changes"""
1292         self.transaction.commit()
1293
1294     def rollback(self):
1295         """rollback changes"""
1296         self.transaction.rollback()
1297
1298     def __enter__(self):
1299         self.prepare()
1300         return self
1301
1302     def __exit__(self, type, value, traceback):
1303         if self.directory is not None:
1304             shutil.rmtree(self.directory)
1305             self.directory = None
1306         self.changes = None
1307         self.transaction.rollback()
1308         return None