daklib/archive.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to manipulate the archive
  18
  19 This module provides classes to manipulate the archive.
  20 """
  21
  22 from daklib.dbconn import *
  23 import daklib.checks as checks
  24 from daklib.config import Config
  25 import daklib.upload as upload
  26 import daklib.utils as utils
  27 from daklib.fstransactions import FilesystemTransaction
  28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
  29
  30 import apt_pkg
  31 from datetime import datetime
  32 import os
  33 import shutil
  34 import subprocess
  35 from sqlalchemy.orm.exc import NoResultFound
  36 import tempfile
  37 import traceback
  38
  39 class ArchiveException(Exception):
  40     pass
  41
  42 class HashMismatchException(ArchiveException):
  43     pass
  44
  45 class ArchiveTransaction(object):
  46     """manipulate the archive in a transaction
  47     """
  48     def __init__(self):
  49         self.fs = FilesystemTransaction()
  50         self.session = DBConn().session()
  51
  52     def get_file(self, hashed_file, source_name, check_hashes=True):
  53         """Look for file C{hashed_file} in database
  54
  55         @type  hashed_file: L{daklib.upload.HashedFile}
  56         @param hashed_file: file to look for in the database
  57
  58         @type  source_name: str
  59         @param source_name: source package name
  60
  61         @type  check_hashes: bool
  62         @param check_hashes: check size and hashes match
  63
  64         @raise KeyError: file was not found in the database
  65         @raise HashMismatchException: hash mismatch
  66
  67         @rtype:  L{daklib.dbconn.PoolFile}
  68         @return: database entry for the file
  69         """
  70         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  71         try:
  72             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
  73             if check_hashes and (poolfile.filesize != hashed_file.size
  74                                  or poolfile.md5sum != hashed_file.md5sum
  75                                  or poolfile.sha1sum != hashed_file.sha1sum
  76                                  or poolfile.sha256sum != hashed_file.sha256sum):
  77                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
  78             return poolfile
  79         except NoResultFound:
  80             raise KeyError('{0} not found in database.'.format(poolname))
  81
  82     def _install_file(self, directory, hashed_file, archive, component, source_name):
  83         """Install a file
  84
  85         Will not give an error when the file is already present.
  86
  87         @rtype:  L{daklib.dbconn.PoolFile}
  88         @return: batabase object for the new file
  89         """
  90         session = self.session
  91
  92         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  93         try:
  94             poolfile = self.get_file(hashed_file, source_name)
  95         except KeyError:
  96             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
  97             poolfile.md5sum = hashed_file.md5sum
  98             poolfile.sha1sum = hashed_file.sha1sum
  99             poolfile.sha256sum = hashed_file.sha256sum
 100             session.add(poolfile)
 101             session.flush()
 102
 103         try:
 104             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
 105         except NoResultFound:
 106             archive_file = ArchiveFile(archive, component, poolfile)
 107             session.add(archive_file)
 108             session.flush()
 109
 110             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
 111             hashed_file_path = os.path.join(directory, hashed_file.filename)
 112             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 113
 114         return poolfile
 115
 116     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
 117         """Install a binary package
 118
 119         @type  directory: str
 120         @param directory: directory the binary package is located in
 121
 122         @type  binary: L{daklib.upload.Binary}
 123         @param binary: binary package to install
 124
 125         @type  suite: L{daklib.dbconn.Suite}
 126         @param suite: target suite
 127
 128         @type  component: L{daklib.dbconn.Component}
 129         @param component: target component
 130
 131         @type  allow_tainted: bool
 132         @param allow_tainted: allow to copy additional files from tainted archives
 133
 134         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 135         @param fingerprint: optional fingerprint
 136
 137         @type  source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
 138         @param source_suites: suites to copy the source from if they are not
 139                               in C{suite} or C{True} to allow copying from any
 140                               suite.
 141
 142         @type  extra_source_archives: list of L{daklib.dbconn.Archive}
 143         @param extra_source_archives: extra archives to copy Built-Using sources from
 144
 145         @rtype:  L{daklib.dbconn.DBBinary}
 146         @return: databse object for the new package
 147         """
 148         session = self.session
 149         control = binary.control
 150         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 151         architecture = get_architecture(control['Architecture'], session)
 152
 153         (source_name, source_version) = binary.source
 154         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
 155         source = source_query.filter(DBSource.suites.contains(suite)).first()
 156         if source is None:
 157             if source_suites != True:
 158                 source_query = source_query.join(DBSource.suites) \
 159                     .filter(Suite.suite_id == source_suites.c.id)
 160             source = source_query.first()
 161             if source is None:
 162                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
 163             self.copy_source(source, suite, component)
 164
 165         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
 166
 167         unique = dict(
 168             package=control['Package'],
 169             version=control['Version'],
 170             architecture=architecture,
 171             )
 172         rest = dict(
 173             source=source,
 174             maintainer=maintainer,
 175             poolfile=db_file,
 176             binarytype=binary.type,
 177             fingerprint=fingerprint,
 178             )
 179
 180         try:
 181             db_binary = session.query(DBBinary).filter_by(**unique).one()
 182             for key, value in rest.iteritems():
 183                 if getattr(db_binary, key) != value:
 184                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
 185         except NoResultFound:
 186             db_binary = DBBinary(**unique)
 187             for key, value in rest.iteritems():
 188                 setattr(db_binary, key, value)
 189             session.add(db_binary)
 190             session.flush()
 191             import_metadata_into_db(db_binary, session)
 192
 193             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
 194
 195         if suite not in db_binary.suites:
 196             db_binary.suites.append(suite)
 197
 198         session.flush()
 199
 200         return db_binary
 201
 202     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
 203         """ensure source exists in the given archive
 204
 205         This is intended to be used to check that Built-Using sources exist.
 206
 207         @type  filename: str
 208         @param filename: filename to use in error messages
 209
 210         @type  source: L{daklib.dbconn.DBSource}
 211         @param source: source to look for
 212
 213         @type  archive: L{daklib.dbconn.Archive}
 214         @param archive: archive to look in
 215
 216         @type  extra_archives: list of L{daklib.dbconn.Archive}
 217         @param extra_archives: list of archives to copy the source package from
 218                                if it is not yet present in C{archive}
 219         """
 220         session = self.session
 221         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
 222         if db_file is not None:
 223             return True
 224
 225         # Try to copy file from one extra archive
 226         if extra_archives is None:
 227             extra_archives = []
 228         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
 229         if db_file is None:
 230             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
 231
 232         source_archive = db_file.archive
 233         for dsc_file in source.srcfiles:
 234             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
 235             # We were given an explicit list of archives so it is okay to copy from tainted archives.
 236             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 237
 238     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
 239         """Add Built-Using sources to C{db_binary.extra_sources}
 240         """
 241         session = self.session
 242         built_using = control.get('Built-Using', None)
 243
 244         if built_using is not None:
 245             for dep in apt_pkg.parse_depends(built_using):
 246                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
 247                 bu_source_name, bu_source_version, comp = dep[0]
 248                 assert comp == '=', 'Built-Using must contain strict dependencies'
 249
 250                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
 251                 if bu_source is None:
 252                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
 253
 254                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
 255
 256                 db_binary.extra_sources.append(bu_source)
 257
 258     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
 259         """Install a source package
 260
 261         @type  directory: str
 262         @param directory: directory the source package is located in
 263
 264         @type  source: L{daklib.upload.Source}
 265         @param source: source package to install
 266
 267         @type  suite: L{daklib.dbconn.Suite}
 268         @param suite: target suite
 269
 270         @type  component: L{daklib.dbconn.Component}
 271         @param component: target component
 272
 273         @type  changed_by: L{daklib.dbconn.Maintainer}
 274         @param changed_by: person who prepared this version of the package
 275
 276         @type  allow_tainted: bool
 277         @param allow_tainted: allow to copy additional files from tainted archives
 278
 279         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 280         @param fingerprint: optional fingerprint
 281
 282         @rtype:  L{daklib.dbconn.DBSource}
 283         @return: database object for the new source
 284         """
 285         session = self.session
 286         archive = suite.archive
 287         control = source.dsc
 288         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 289         source_name = control['Source']
 290
 291         ### Add source package to database
 292
 293         # We need to install the .dsc first as the DBSource object refers to it.
 294         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
 295
 296         unique = dict(
 297             source=source_name,
 298             version=control['Version'],
 299             )
 300         rest = dict(
 301             maintainer=maintainer,
 302             changedby=changed_by,
 303             #install_date=datetime.now().date(),
 304             poolfile=db_file_dsc,
 305             fingerprint=fingerprint,
 306             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
 307             )
 308
 309         created = False
 310         try:
 311             db_source = session.query(DBSource).filter_by(**unique).one()
 312             for key, value in rest.iteritems():
 313                 if getattr(db_source, key) != value:
 314                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
 315         except NoResultFound:
 316             created = True
 317             db_source = DBSource(**unique)
 318             for key, value in rest.iteritems():
 319                 setattr(db_source, key, value)
 320             # XXX: set as default in postgres?
 321             db_source.install_date = datetime.now().date()
 322             session.add(db_source)
 323             session.flush()
 324
 325             # Add .dsc file. Other files will be added later.
 326             db_dsc_file = DSCFile()
 327             db_dsc_file.source = db_source
 328             db_dsc_file.poolfile = db_file_dsc
 329             session.add(db_dsc_file)
 330             session.flush()
 331
 332         if suite in db_source.suites:
 333             return db_source
 334
 335         db_source.suites.append(suite)
 336
 337         if not created:
 338             for f in db_source.srcfiles:
 339                 self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
 340             return db_source
 341
 342         ### Now add remaining files and copy them to the archive.
 343
 344         for hashed_file in source.files.itervalues():
 345             hashed_file_path = os.path.join(directory, hashed_file.filename)
 346             if os.path.exists(hashed_file_path):
 347                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
 348                 session.add(db_file)
 349             else:
 350                 db_file = self.get_file(hashed_file, source_name)
 351                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
 352
 353             db_dsc_file = DSCFile()
 354             db_dsc_file.source = db_source
 355             db_dsc_file.poolfile = db_file
 356             session.add(db_dsc_file)
 357
 358         session.flush()
 359
 360         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
 361         import_metadata_into_db(db_source, session)
 362
 363         # Uploaders are the maintainer and co-maintainers from the Uploaders field
 364         db_source.uploaders.append(maintainer)
 365         if 'Uploaders' in control:
 366             from daklib.textutils import split_uploaders
 367             for u in split_uploaders(control['Uploaders']):
 368                 db_source.uploaders.append(get_or_set_maintainer(u, session))
 369         session.flush()
 370
 371         return db_source
 372
 373     def _copy_file(self, db_file, archive, component, allow_tainted=False):
 374         """Copy a file to the given archive and component
 375
 376         @type  db_file: L{daklib.dbconn.PoolFile}
 377         @param db_file: file to copy
 378
 379         @type  archive: L{daklib.dbconn.Archive}
 380         @param archive: target archive
 381
 382         @type  component: L{daklib.dbconn.Archive}
 383         @param component: target component
 384
 385         @type  allow_tainted: bool
 386         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 387         """
 388         session = self.session
 389
 390         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
 391             query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
 392             if not allow_tainted:
 393                 query = query.join(Archive).filter(Archive.tainted == False)
 394
 395             source_af = query.first()
 396             if source_af is None:
 397                 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
 398             target_af = ArchiveFile(archive, component, db_file)
 399             session.add(target_af)
 400             session.flush()
 401             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
 402
 403     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
 404         """Copy a binary package to the given suite and component
 405
 406         @type  db_binary: L{daklib.dbconn.DBBinary}
 407         @param db_binary: binary to copy
 408
 409         @type  suite: L{daklib.dbconn.Suite}
 410         @param suite: target suite
 411
 412         @type  component: L{daklib.dbconn.Component}
 413         @param component: target component
 414
 415         @type  allow_tainted: bool
 416         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 417
 418         @type  extra_archives: list of L{daklib.dbconn.Archive}
 419         @param extra_archives: extra archives to copy Built-Using sources from
 420         """
 421         session = self.session
 422         archive = suite.archive
 423         if archive.tainted:
 424             allow_tainted = True
 425
 426         filename = db_binary.poolfile.filename
 427
 428         # make sure source is present in target archive
 429         db_source = db_binary.source
 430         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
 431             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
 432
 433         # make sure built-using packages are present in target archive
 434         for db_source in db_binary.extra_sources:
 435             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 436
 437         # copy binary
 438         db_file = db_binary.poolfile
 439         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
 440         if suite not in db_binary.suites:
 441             db_binary.suites.append(suite)
 442         self.session.flush()
 443
 444     def copy_source(self, db_source, suite, component, allow_tainted=False):
 445         """Copy a source package to the given suite and component
 446
 447         @type  db_source: L{daklib.dbconn.DBSource}
 448         @param db_source: source to copy
 449
 450         @type  suite: L{daklib.dbconn.Suite}
 451         @param suite: target suite
 452
 453         @type  component: L{daklib.dbconn.Component}
 454         @param component: target component
 455
 456         @type  allow_tainted: bool
 457         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 458         """
 459         archive = suite.archive
 460         if archive.tainted:
 461             allow_tainted = True
 462         for db_dsc_file in db_source.srcfiles:
 463             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
 464         if suite not in db_source.suites:
 465             db_source.suites.append(suite)
 466         self.session.flush()
 467
 468     def remove_file(self, db_file, archive, component):
 469         """Remove a file from a given archive and component
 470
 471         @type  db_file: L{daklib.dbconn.PoolFile}
 472         @param db_file: file to remove
 473
 474         @type  archive: L{daklib.dbconn.Archive}
 475         @param archive: archive to remove the file from
 476
 477         @type  component: L{daklib.dbconn.Component}
 478         @param component: component to remove the file from
 479         """
 480         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
 481         self.fs.unlink(af.path)
 482         self.session.delete(af)
 483
 484     def remove_binary(self, binary, suite):
 485         """Remove a binary from a given suite and component
 486
 487         @type  binary: L{daklib.dbconn.DBBinary}
 488         @param binary: binary to remove
 489
 490         @type  suite: L{daklib.dbconn.Suite}
 491         @param suite: suite to remove the package from
 492         """
 493         binary.suites.remove(suite)
 494         self.session.flush()
 495
 496     def remove_source(self, source, suite):
 497         """Remove a source from a given suite and component
 498
 499         @type  source: L{daklib.dbconn.DBSource}
 500         @param source: source to remove
 501
 502         @type  suite: L{daklib.dbconn.Suite}
 503         @param suite: suite to remove the package from
 504
 505         @raise ArchiveException: source package is still referenced by other
 506                                  binaries in the suite
 507         """
 508         session = self.session
 509
 510         query = session.query(DBBinary).filter_by(source=source) \
 511             .filter(DBBinary.suites.contains(suite))
 512         if query.first() is not None:
 513             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
 514
 515         source.suites.remove(suite)
 516         session.flush()
 517
 518     def commit(self):
 519         """commit changes"""
 520         try:
 521             self.session.commit()
 522             self.fs.commit()
 523         finally:
 524             self.session.rollback()
 525             self.fs.rollback()
 526
 527     def rollback(self):
 528         """rollback changes"""
 529         self.session.rollback()
 530         self.fs.rollback()
 531
 532     def __enter__(self):
 533         return self
 534
 535     def __exit__(self, type, value, traceback):
 536         if type is None:
 537             self.commit()
 538         else:
 539             self.rollback()
 540         return None
 541
 542 class ArchiveUpload(object):
 543     """handle an upload
 544
 545     This class can be used in a with-statement::
 546
 547        with ArchiveUpload(...) as upload:
 548           ...
 549
 550     Doing so will automatically run any required cleanup and also rollback the
 551     transaction if it was not committed.
 552     """
 553     def __init__(self, directory, changes, keyrings):
 554         self.transaction = ArchiveTransaction()
 555         """transaction used to handle the upload
 556         @type: L{daklib.archive.ArchiveTransaction}
 557         """
 558
 559         self.session = self.transaction.session
 560         """database session"""
 561
 562         self.original_directory = directory
 563         self.original_changes = changes
 564
 565         self.changes = None
 566         """upload to process
 567         @type: L{daklib.upload.Changes}
 568         """
 569
 570         self.directory = None
 571         """directory with temporary copy of files. set by C{prepare}
 572         @type: str
 573         """
 574
 575         self.keyrings = keyrings
 576
 577         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
 578         """fingerprint of the key used to sign the upload
 579         @type: L{daklib.dbconn.Fingerprint}
 580         """
 581
 582         self.reject_reasons = []
 583         """reasons why the upload cannot by accepted
 584         @type: list of str
 585         """
 586
 587         self.warnings = []
 588         """warnings
 589         @note: Not used yet.
 590         @type: list of str
 591         """
 592
 593         self.final_suites = None
 594
 595         self.new = False
 596         """upload is NEW. set by C{check}
 597         @type: bool
 598         """
 599
 600         self._checked = False
 601         """checks passes. set by C{check}
 602         @type: bool
 603         """
 604
 605         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
 606         self._new = self._new_queue.suite
 607
 608     def warn(self, message):
 609         """add a warning message
 610
 611         Adds a warning message that can later be seen in C{self.warnings}
 612
 613         @type  message: string
 614         @param message: warning message
 615         """
 616         self.warnings.append(message)
 617
 618     def prepare(self):
 619         """prepare upload for further processing
 620
 621         This copies the files involved to a temporary directory.  If you use
 622         this method directly, you have to remove the directory given by the
 623         C{directory} attribute later on your own.
 624
 625         Instead of using the method directly, you can also use a with-statement::
 626
 627            with ArchiveUpload(...) as upload:
 628               ...
 629
 630         This will automatically handle any required cleanup.
 631         """
 632         assert self.directory is None
 633         assert self.original_changes.valid_signature
 634
 635         cnf = Config()
 636         session = self.transaction.session
 637
 638         self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
 639                                             mode=0o2750, group=cnf.unprivgroup)
 640         with FilesystemTransaction() as fs:
 641             src = os.path.join(self.original_directory, self.original_changes.filename)
 642             dst = os.path.join(self.directory, self.original_changes.filename)
 643             fs.copy(src, dst, mode=0o640)
 644
 645             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 646
 647             for f in self.changes.files.itervalues():
 648                 src = os.path.join(self.original_directory, f.filename)
 649                 dst = os.path.join(self.directory, f.filename)
 650                 if not os.path.exists(src):
 651                     continue
 652                 fs.copy(src, dst, mode=0o640)
 653
 654             source = self.changes.source
 655             if source is not None:
 656                 for f in source.files.itervalues():
 657                     src = os.path.join(self.original_directory, f.filename)
 658                     dst = os.path.join(self.directory, f.filename)
 659                     if not os.path.exists(dst):
 660                         try:
 661                             db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
 662                             db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
 663                             fs.copy(db_archive_file.path, dst, symlink=True)
 664                         except KeyError:
 665                             # Ignore if get_file could not find it. Upload will
 666                             # probably be rejected later.
 667                             pass
 668
 669     def unpacked_source(self):
 670         """Path to unpacked source
 671
 672         Get path to the unpacked source. This method does unpack the source
 673         into a temporary directory under C{self.directory} if it has not
 674         been done so already.
 675
 676         @rtype:  str or C{None}
 677         @return: string giving the path to the unpacked source directory
 678                  or C{None} if no source was included in the upload.
 679         """
 680         assert self.directory is not None
 681
 682         source = self.changes.source
 683         if source is None:
 684             return None
 685         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
 686
 687         sourcedir = os.path.join(self.directory, 'source')
 688         if not os.path.exists(sourcedir):
 689             devnull = open('/dev/null', 'w')
 690             subprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
 691         if not os.path.isdir(sourcedir):
 692             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
 693         return sourcedir
 694
 695     def _map_suite(self, suite_name):
 696         for rule in Config().value_list("SuiteMappings"):
 697             fields = rule.split()
 698             rtype = fields[0]
 699             if rtype == "map" or rtype == "silent-map":
 700                 (src, dst) = fields[1:3]
 701                 if src == suite_name:
 702                     suite_name = dst
 703                     if rtype != "silent-map":
 704                         self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
 705             elif rtype == "ignore":
 706                 ignored = fields[1]
 707                 if suite_name == ignored:
 708                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
 709                     suite_name = None
 710             elif rtype == "reject":
 711                 rejected = fields[1]
 712                 if suite_name == rejected:
 713                     self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
 714             ## XXX: propup-version and map-unreleased not yet implemented
 715         return suite_name
 716
 717     def _mapped_suites(self):
 718         """Get target suites after mappings
 719
 720         @rtype:  list of L{daklib.dbconn.Suite}
 721         @return: list giving the mapped target suites of this upload
 722         """
 723         session = self.session
 724
 725         suite_names = []
 726         for dist in self.changes.distributions:
 727             suite_name = self._map_suite(dist)
 728             if suite_name is not None:
 729                 suite_names.append(suite_name)
 730
 731         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
 732         return suites
 733
 734     def _check_new(self, suite):
 735         """Check if upload is NEW
 736
 737         An upload is NEW if it has binary or source packages that do not have
 738         an override in C{suite} OR if it references files ONLY in a tainted
 739         archive (eg. when it references files in NEW).
 740
 741         @rtype:  bool
 742         @return: C{True} if the upload is NEW, C{False} otherwise
 743         """
 744         session = self.session
 745         new = False
 746
 747         # Check for missing overrides
 748         for b in self.changes.binaries:
 749             override = self._binary_override(suite, b)
 750             if override is None:
 751                 self.warnings.append('binary:{0} is NEW.'.format(b.control['Package']))
 752                 new = True
 753
 754         if self.changes.source is not None:
 755             override = self._source_override(suite, self.changes.source)
 756             if override is None:
 757                 self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
 758                 new = True
 759
 760         # Check if we reference a file only in a tainted archive
 761         files = self.changes.files.values()
 762         if self.changes.source is not None:
 763             files.extend(self.changes.source.files.values())
 764         for f in files:
 765             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
 766             query_untainted = query.join(Archive).filter(Archive.tainted == False)
 767
 768             in_archive = (query.first() is not None)
 769             in_untainted_archive = (query_untainted.first() is not None)
 770
 771             if in_archive and not in_untainted_archive:
 772                 self.warnings.append('{0} is only available in NEW.'.format(f.filename))
 773                 new = True
 774
 775         return new
 776
 777     def _final_suites(self):
 778         session = self.session
 779
 780         mapped_suites = self._mapped_suites()
 781         final_suites = set()
 782
 783         for suite in mapped_suites:
 784             overridesuite = suite
 785             if suite.overridesuite is not None:
 786                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 787             if self._check_new(overridesuite):
 788                 self.new = True
 789             final_suites.add(suite)
 790
 791         return final_suites
 792
 793     def _binary_override(self, suite, binary):
 794         """Get override entry for a binary
 795
 796         @type  suite: L{daklib.dbconn.Suite}
 797         @param suite: suite to get override for
 798
 799         @type  binary: L{daklib.upload.Binary}
 800         @param binary: binary to get override for
 801
 802         @rtype:  L{daklib.dbconn.Override} or C{None}
 803         @return: override for the given binary or C{None}
 804         """
 805         if suite.overridesuite is not None:
 806             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 807
 808         mapped_component = get_mapped_component(binary.component)
 809         if mapped_component is None:
 810             return None
 811
 812         query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
 813                 .join(Component).filter(Component.component_name == mapped_component.component_name) \
 814                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 815
 816         try:
 817             return query.one()
 818         except NoResultFound:
 819             return None
 820
 821     def _source_override(self, suite, source):
 822         """Get override entry for a source
 823
 824         @type  suite: L{daklib.dbconn.Suite}
 825         @param suite: suite to get override for
 826
 827         @type  source: L{daklib.upload.Source}
 828         @param source: source to get override for
 829
 830         @rtype:  L{daklib.dbconn.Override} or C{None}
 831         @return: override for the given source or C{None}
 832         """
 833         if suite.overridesuite is not None:
 834             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 835
 836         # XXX: component for source?
 837         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
 838                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 839
 840         try:
 841             return query.one()
 842         except NoResultFound:
 843             return None
 844
 845     def _binary_component(self, suite, binary, only_overrides=True):
 846         """get component for a binary
 847
 848         By default this will only look at overrides to get the right component;
 849         if C{only_overrides} is C{False} this method will also look at the
 850         Section field.
 851
 852         @type  suite: L{daklib.dbconn.Suite}
 853
 854         @type  binary: L{daklib.upload.Binary}
 855
 856         @type  only_overrides: bool
 857         @param only_overrides: only use overrides to get the right component
 858
 859         @rtype: L{daklib.dbconn.Component} or C{None}
 860         """
 861         override = self._binary_override(suite, binary)
 862         if override is not None:
 863             return override.component
 864         if only_overrides:
 865             return None
 866         return get_mapped_component(binary.component, self.session)
 867
 868     def check(self, force=False):
 869         """run checks against the upload
 870
 871         @type  force: bool
 872         @param force: ignore failing forcable checks
 873
 874         @rtype:  bool
 875         @return: C{True} if all checks passed, C{False} otherwise
 876         """
 877         # XXX: needs to be better structured.
 878         assert self.changes.valid_signature
 879
 880         try:
 881             # Validate signatures and hashes before we do any real work:
 882             for chk in (
 883                     checks.SignatureCheck,
 884                     checks.ChangesCheck,
 885                     checks.HashesCheck,
 886                     checks.ExternalHashesCheck,
 887                     checks.SourceCheck,
 888                     checks.BinaryCheck,
 889                     checks.BinaryTimestampCheck,
 890                     checks.SingleDistributionCheck,
 891                     ):
 892                 chk().check(self)
 893
 894             final_suites = self._final_suites()
 895             if len(final_suites) == 0:
 896                 self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
 897                 return False
 898
 899             self.final_suites = final_suites
 900
 901             for chk in (
 902                     checks.TransitionCheck,
 903                     checks.ACLCheck,
 904                     checks.NoSourceOnlyCheck,
 905                     checks.LintianCheck,
 906                     ):
 907                 chk().check(self)
 908
 909             for chk in (
 910                     checks.ACLCheck,
 911                     checks.SourceFormatCheck,
 912                     checks.SuiteArchitectureCheck,
 913                     checks.VersionCheck,
 914                     ):
 915                 for suite in final_suites:
 916                     chk().per_suite_check(self, suite)
 917
 918             if len(self.reject_reasons) != 0:
 919                 return False
 920
 921             self._checked = True
 922             return True
 923         except checks.Reject as e:
 924             self.reject_reasons.append(unicode(e))
 925         except Exception as e:
 926             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
 927         return False
 928
 929     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
 930         """Install upload to the given suite
 931
 932         @type  suite: L{daklib.dbconn.Suite}
 933         @param suite: suite to install the package into. This is the real suite,
 934                       ie. after any redirection to NEW or a policy queue
 935
 936         @param source_component_func: function to get the L{daklib.dbconn.Component}
 937                                       for a L{daklib.upload.Source} object
 938
 939         @param binary_component_func: function to get the L{daklib.dbconn.Component}
 940                                       for a L{daklib.upload.Binary} object
 941
 942         @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
 943
 944         @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
 945
 946         @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
 947                  object for the install source or C{None} if no source was
 948                  included. The second is a list of L{daklib.dbconn.DBBinary}
 949                  objects for the installed binary packages.
 950         """
 951         # XXX: move this function to ArchiveTransaction?
 952
 953         control = self.changes.changes
 954         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 955
 956         if source_suites is None:
 957             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()
 958
 959         source = self.changes.source
 960         if source is not None:
 961             component = source_component_func(source)
 962             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
 963         else:
 964             db_source = None
 965
 966         db_binaries = []
 967         for binary in self.changes.binaries:
 968             component = binary_component_func(binary)
 969             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
 970             db_binaries.append(db_binary)
 971
 972         if suite.copychanges:
 973             src = os.path.join(self.directory, self.changes.filename)
 974             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
 975             self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
 976
 977         return (db_source, db_binaries)
 978
 979     def _install_changes(self):
 980         assert self.changes.valid_signature
 981         control = self.changes.changes
 982         session = self.transaction.session
 983         config = Config()
 984
 985         changelog_id = None
 986         # Only add changelog for sourceful uploads and binNMUs
 987         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
 988             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
 989             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
 990             assert changelog_id is not None
 991
 992         db_changes = DBChange()
 993         db_changes.changesname = self.changes.filename
 994         db_changes.source = control['Source']
 995         db_changes.binaries = control.get('Binary', None)
 996         db_changes.architecture = control['Architecture']
 997         db_changes.version = control['Version']
 998         db_changes.distribution = control['Distribution']
 999         db_changes.urgency = control['Urgency']
1000         db_changes.maintainer = control['Maintainer']
1001         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
1002         db_changes.date = control['Date']
1003         db_changes.fingerprint = self.fingerprint.fingerprint
1004         db_changes.changelog_id = changelog_id
1005         db_changes.closes = self.changes.closed_bugs
1006
1007         self.transaction.session.add(db_changes)
1008         self.transaction.session.flush()
1009
1010         return db_changes
1011
1012     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
1013         u = PolicyQueueUpload()
1014         u.policy_queue = policy_queue
1015         u.target_suite = target_suite
1016         u.changes = db_changes
1017         u.source = db_source
1018         u.binaries = db_binaries
1019         self.transaction.session.add(u)
1020         self.transaction.session.flush()
1021
1022         dst = os.path.join(policy_queue.path, self.changes.filename)
1023         self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)
1024
1025         return u
1026
1027     def try_autobyhand(self):
1028         """Try AUTOBYHAND
1029
1030         Try to handle byhand packages automatically.
1031
1032         @rtype:  list of L{daklib.upload.HashedFile}
1033         @return: list of remaining byhand files
1034         """
1035         assert len(self.reject_reasons) == 0
1036         assert self.changes.valid_signature
1037         assert self.final_suites is not None
1038         assert self._checked
1039
1040         byhand = self.changes.byhand_files
1041         if len(byhand) == 0:
1042             return True
1043
1044         suites = list(self.final_suites)
1045         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1046         suite = suites[0]
1047
1048         cnf = Config()
1049         control = self.changes.changes
1050         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1051
1052         remaining = []
1053         for f in byhand:
1054             parts = f.filename.split('_', 2)
1055             if len(parts) != 3:
1056                 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1057                 remaining.append(f)
1058                 continue
1059
1060             package, version, archext = parts
1061             arch, ext = archext.split('.', 1)
1062
1063             try:
1064                 rule = automatic_byhand_packages.subtree(package)
1065             except KeyError:
1066                 remaining.append(f)
1067                 continue
1068
1069             if rule['Source'] != self.changes.source_name or rule['Section'] != f.section or rule['Extension'] != ext:
1070                 remaining.append(f)
1071                 continue
1072
1073             script = rule['Script']
1074             retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1075             if retcode != 0:
1076                 print "W: error processing {0}.".format(f.filename)
1077                 remaining.append(f)
1078
1079         return len(remaining) == 0
1080
1081     def _install_byhand(self, policy_queue_upload, hashed_file):
1082         """install byhand file
1083
1084         @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1085
1086         @type  hashed_file: L{daklib.upload.HashedFile}
1087         """
1088         fs = self.transaction.fs
1089         session = self.transaction.session
1090         policy_queue = policy_queue_upload.policy_queue
1091
1092         byhand_file = PolicyQueueByhandFile()
1093         byhand_file.upload = policy_queue_upload
1094         byhand_file.filename = hashed_file.filename
1095         session.add(byhand_file)
1096         session.flush()
1097
1098         src = os.path.join(self.directory, hashed_file.filename)
1099         dst = os.path.join(policy_queue.path, hashed_file.filename)
1100         fs.copy(src, dst, mode=policy_queue.change_perms)
1101
1102         return byhand_file
1103
1104     def _do_bts_versiontracking(self):
1105         cnf = Config()
1106         fs = self.transaction.fs
1107
1108         btsdir = cnf.get('Dir::BTSVersionTrack')
1109         if btsdir is None or btsdir == '':
1110             return
1111
1112         base = os.path.join(btsdir, self.changes.filename[:-8])
1113
1114         # version history
1115         sourcedir = self.unpacked_source()
1116         if sourcedir is not None:
1117             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1118             versions = fs.create("{0}.versions".format(base), mode=0o644)
1119             for line in fh.readlines():
1120                 if re_changelog_versions.match(line):
1121                     versions.write(line)
1122             fh.close()
1123             versions.close()
1124
1125         # binary -> source mapping
1126         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1127         for binary in self.changes.binaries:
1128             control = binary.control
1129             source_package, source_version = binary.source
1130             line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
1131             print >>debinfo, line
1132         debinfo.close()
1133
1134     def _policy_queue(self, suite):
1135         if suite.policy_queue is not None:
1136             return suite.policy_queue
1137         return None
1138
1139     def install(self):
1140         """install upload
1141
1142         Install upload to a suite or policy queue.  This method does B{not}
1143         handle uploads to NEW.
1144
1145         You need to have called the C{check} method before calling this method.
1146         """
1147         assert len(self.reject_reasons) == 0
1148         assert self.changes.valid_signature
1149         assert self.final_suites is not None
1150         assert self._checked
1151         assert not self.new
1152
1153         db_changes = self._install_changes()
1154
1155         for suite in self.final_suites:
1156             overridesuite = suite
1157             if suite.overridesuite is not None:
1158                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1159
1160             policy_queue = self._policy_queue(suite)
1161
1162             redirected_suite = suite
1163             if policy_queue is not None:
1164                 redirected_suite = policy_queue.suite
1165
1166             # source can be in the suite we install to or any suite we enhance
1167             source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
1168             for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
1169                     .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
1170                     .filter(VersionCheck.check == 'Enhances'):
1171                 source_suite_ids.add(enhanced_suite_id)
1172
1173             source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()
1174
1175             source_component_func = lambda source: self._source_override(overridesuite, source).component
1176             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1177
1178             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1179
1180             if policy_queue is not None:
1181                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1182
1183             # copy to build queues
1184             if policy_queue is None or policy_queue.send_to_build_queues:
1185                 for build_queue in suite.copy_queues:
1186                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1187
1188         self._do_bts_versiontracking()
1189
1190     def install_to_new(self):
1191         """install upload to NEW
1192
1193         Install upload to NEW.  This method does B{not} handle regular uploads
1194         to suites or policy queues.
1195
1196         You need to have called the C{check} method before calling this method.
1197         """
1198         # Uploads to NEW are special as we don't have overrides.
1199         assert len(self.reject_reasons) == 0
1200         assert self.changes.valid_signature
1201         assert self.final_suites is not None
1202
1203         source = self.changes.source
1204         binaries = self.changes.binaries
1205         byhand = self.changes.byhand_files
1206
1207         # we need a suite to guess components
1208         suites = list(self.final_suites)
1209         assert len(suites) == 1, "NEW uploads must be to a single suite"
1210         suite = suites[0]
1211
1212         # decide which NEW queue to use
1213         if suite.new_queue is None:
1214             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1215         else:
1216             new_queue = suite.new_queue
1217         if len(byhand) > 0:
1218             # There is only one global BYHAND queue
1219             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1220         new_suite = new_queue.suite
1221
1222
1223         def binary_component_func(binary):
1224             return self._binary_component(suite, binary, only_overrides=False)
1225
1226         # guess source component
1227         # XXX: should be moved into an extra method
1228         binary_component_names = set()
1229         for binary in binaries:
1230             component = binary_component_func(binary)
1231             binary_component_names.add(component.component_name)
1232         source_component_name = None
1233         for c in self.session.query(Component).order_by(Component.component_id):
1234             guess = c.component_name
1235             if guess in binary_component_names:
1236                 source_component_name = guess
1237                 break
1238         if source_component_name is None:
1239             source_component = self.session.query(Component).order_by(Component.component_id).first()
1240         else:
1241             source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1242         source_component_func = lambda source: source_component
1243
1244         db_changes = self._install_changes()
1245         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1246         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1247
1248         for f in byhand:
1249             self._install_byhand(policy_upload, f)
1250
1251         self._do_bts_versiontracking()
1252
1253     def commit(self):
1254         """commit changes"""
1255         self.transaction.commit()
1256
1257     def rollback(self):
1258         """rollback changes"""
1259         self.transaction.rollback()
1260
1261     def __enter__(self):
1262         self.prepare()
1263         return self
1264
1265     def __exit__(self, type, value, traceback):
1266         if self.directory is not None:
1267             shutil.rmtree(self.directory)
1268             self.directory = None
1269         self.changes = None
1270         self.transaction.rollback()
1271         return None