daklib/archive.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to manipulate the archive
  18
  19 This module provides classes to manipulate the archive.
  20 """
  21
  22 from .dbconn import *
  23 import daklib.checks as checks
  24 from daklib.config import Config
  25 import daklib.upload as upload
  26 import daklib.utils as utils
  27 from .fstransactions import FilesystemTransaction
  28 from .regexes import re_changelog_versions, re_bin_only_nmu
  29
  30 import apt_pkg
  31 from datetime import datetime
  32 import os
  33 import shutil
  34 import subprocess
  35 from sqlalchemy.orm.exc import NoResultFound
  36 import tempfile
  37 import traceback
  38
  39 class ArchiveException(Exception):
  40     pass
  41
  42 class HashMismatchException(ArchiveException):
  43     pass
  44
  45 class ArchiveTransaction(object):
  46     """manipulate the archive in a transaction
  47     """
  48     def __init__(self):
  49         self.fs = FilesystemTransaction()
  50         self.session = DBConn().session()
  51
  52     def get_file(self, hashed_file, source_name):
  53         """Look for file `hashed_file` in database
  54
  55         Args:
  56            hashed_file (daklib.upload.HashedFile): file to look for in the database
  57
  58         Raises:
  59            KeyError: file was not found in the database
  60            HashMismatchException: hash mismatch
  61
  62         Returns:
  63            `daklib.dbconn.PoolFile` object for the database
  64         """
  65         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  66         try:
  67             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
  68             if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
  69                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
  70             return poolfile
  71         except NoResultFound:
  72             raise KeyError('{0} not found in database.'.format(poolname))
  73
  74     def _install_file(self, directory, hashed_file, archive, component, source_name):
  75         """Install a file
  76
  77         Will not give an error when the file is already present.
  78
  79         Returns:
  80            `daklib.dbconn.PoolFile` object for the new file
  81         """
  82         session = self.session
  83
  84         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  85         try:
  86             poolfile = self.get_file(hashed_file, source_name)
  87         except KeyError:
  88             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
  89             poolfile.md5sum = hashed_file.md5sum
  90             poolfile.sha1sum = hashed_file.sha1sum
  91             poolfile.sha256sum = hashed_file.sha256sum
  92             session.add(poolfile)
  93             session.flush()
  94
  95         try:
  96             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
  97         except NoResultFound:
  98             archive_file = ArchiveFile(archive, component, poolfile)
  99             session.add(archive_file)
 100             session.flush()
 101
 102             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
 103             hashed_file_path = os.path.join(directory, hashed_file.filename)
 104             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 105
 106         return poolfile
 107
 108     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
 109         """Install a binary package
 110
 111         Args:
 112            directory (str): directory the binary package is located in
 113            binary (daklib.upload.Binary): binary package to install
 114            suite (daklib.dbconn.Suite): target suite
 115            component (daklib.dbconn.Component): target component
 116
 117         Kwargs:
 118            allow_tainted (bool): allow to copy additional files from tainted archives
 119            fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
 120            source_suites (list of daklib.dbconn.Suite or True): suites to copy
 121               the source from if they are not in `suite` or True to allow
 122               copying from any suite.
 123               This can also be a SQLAlchemy (sub)query object.
 124            extra_source_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
 125
 126         Returns:
 127            `daklib.dbconn.DBBinary` object for the new package
 128         """
 129         session = self.session
 130         control = binary.control
 131         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 132         architecture = get_architecture(control['Architecture'], session)
 133
 134         (source_name, source_version) = binary.source
 135         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
 136         source = source_query.filter(DBSource.suites.contains(suite)).first()
 137         if source is None:
 138             if source_suites != True:
 139                 source_query = source_query.join(DBSource.suites) \
 140                     .filter(Suite.suite_id == source_suites.c.id)
 141             source = source_query.first()
 142             if source is None:
 143                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
 144             self.copy_source(source, suite, component)
 145
 146         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
 147
 148         unique = dict(
 149             package=control['Package'],
 150             version=control['Version'],
 151             architecture=architecture,
 152             )
 153         rest = dict(
 154             source=source,
 155             maintainer=maintainer,
 156             poolfile=db_file,
 157             binarytype=binary.type,
 158             fingerprint=fingerprint,
 159             )
 160
 161         try:
 162             db_binary = session.query(DBBinary).filter_by(**unique).one()
 163             for key, value in rest.iteritems():
 164                 if getattr(db_binary, key) != value:
 165                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
 166         except NoResultFound:
 167             db_binary = DBBinary(**unique)
 168             for key, value in rest.iteritems():
 169                 setattr(db_binary, key, value)
 170             session.add(db_binary)
 171             session.flush()
 172             import_metadata_into_db(db_binary, session)
 173
 174             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
 175
 176         if suite not in db_binary.suites:
 177             db_binary.suites.append(suite)
 178
 179         session.flush()
 180
 181         return db_binary
 182
 183     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
 184         """ensure source exists in the given archive
 185
 186         This is intended to be used to check that Built-Using sources exist.
 187
 188         Args:
 189            filename (str): filename to use in error messages
 190            source (daklib.dbconn.DBSource): source to look for
 191            archive (daklib.dbconn.Archive): archive to look in
 192
 193         Kwargs:
 194            extra_archives (list of daklib.dbconn.Archive): list of archives to copy
 195                the source package from if it is not yet present in `archive`
 196         """
 197         session = self.session
 198         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
 199         if db_file is not None:
 200             return True
 201
 202         # Try to copy file from one extra archive
 203         if extra_archives is None:
 204             extra_archives = []
 205         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
 206         if db_file is None:
 207             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
 208
 209         source_archive = db_file.archive
 210         for dsc_file in source.srcfiles:
 211             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
 212             # We were given an explicit list of archives so it is okay to copy from tainted archives.
 213             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 214
 215     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
 216         """Add Built-Using sources to `db_binary.extra_sources`
 217         """
 218         session = self.session
 219         built_using = control.get('Built-Using', None)
 220
 221         if built_using is not None:
 222             for dep in apt_pkg.parse_depends(built_using):
 223                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
 224                 bu_source_name, bu_source_version, comp = dep[0]
 225                 assert comp == '=', 'Built-Using must contain strict dependencies'
 226
 227                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
 228                 if bu_source is None:
 229                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
 230
 231                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
 232
 233                 db_binary.extra_sources.append(bu_source)
 234
 235     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
 236         """Install a source package
 237
 238         Args:
 239            directory (str): directory the source package is located in
 240            source (daklib.upload.Source): source package to install
 241            suite (daklib.dbconn.Suite): target suite
 242            component (daklib.dbconn.Component): target component
 243            changed_by (daklib.dbconn.Maintainer): person who prepared this version of the package
 244
 245         Kwargs:
 246            allow_tainted (bool): allow to copy additional files from tainted archives
 247            fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
 248
 249         Returns:
 250            `daklib.dbconn.DBSource` object for the new source
 251         """
 252         session = self.session
 253         archive = suite.archive
 254         control = source.dsc
 255         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 256         source_name = control['Source']
 257
 258         ### Add source package to database
 259
 260         # We need to install the .dsc first as the DBSource object refers to it.
 261         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
 262
 263         unique = dict(
 264             source=source_name,
 265             version=control['Version'],
 266             )
 267         rest = dict(
 268             maintainer=maintainer,
 269             changedby=changed_by,
 270             #install_date=datetime.now().date(),
 271             poolfile=db_file_dsc,
 272             fingerprint=fingerprint,
 273             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
 274             )
 275
 276         created = False
 277         try:
 278             db_source = session.query(DBSource).filter_by(**unique).one()
 279             for key, value in rest.iteritems():
 280                 if getattr(db_source, key) != value:
 281                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
 282         except NoResultFound:
 283             created = True
 284             db_source = DBSource(**unique)
 285             for key, value in rest.iteritems():
 286                 setattr(db_source, key, value)
 287             # XXX: set as default in postgres?
 288             db_source.install_date = datetime.now().date()
 289             session.add(db_source)
 290             session.flush()
 291
 292             # Add .dsc file. Other files will be added later.
 293             db_dsc_file = DSCFile()
 294             db_dsc_file.source = db_source
 295             db_dsc_file.poolfile = db_file_dsc
 296             session.add(db_dsc_file)
 297             session.flush()
 298
 299         if suite in db_source.suites:
 300             return db_source
 301
 302         db_source.suites.append(suite)
 303
 304         if not created:
 305             return db_source
 306
 307         ### Now add remaining files and copy them to the archive.
 308
 309         for hashed_file in source.files.itervalues():
 310             hashed_file_path = os.path.join(directory, hashed_file.filename)
 311             if os.path.exists(hashed_file_path):
 312                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
 313                 session.add(db_file)
 314             else:
 315                 db_file = self.get_file(hashed_file, source_name)
 316                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
 317
 318             db_dsc_file = DSCFile()
 319             db_dsc_file.source = db_source
 320             db_dsc_file.poolfile = db_file
 321             session.add(db_dsc_file)
 322
 323         session.flush()
 324
 325         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
 326         import_metadata_into_db(db_source, session)
 327
 328         # Uploaders are the maintainer and co-maintainers from the Uploaders field
 329         db_source.uploaders.append(maintainer)
 330         if 'Uploaders' in control:
 331             def split_uploaders(field):
 332                 import re
 333                 for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
 334                     yield u.strip()
 335
 336             for u in split_uploaders(control['Uploaders']):
 337                 db_source.uploaders.append(get_or_set_maintainer(u, session))
 338         session.flush()
 339
 340         return db_source
 341
 342     def _copy_file(self, db_file, archive, component, allow_tainted=False):
 343         """Copy a file to the given archive and component
 344
 345         Args:
 346            db_file (daklib.dbconn.PoolFile): file to copy
 347            archive (daklib.dbconn.Archive): target archive
 348            component (daklib.dbconn.Component): target component
 349
 350         Kwargs:
 351            allow_tainted (bool): allow to copy from tainted archives (such as NEW)
 352         """
 353         session = self.session
 354
 355         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
 356             query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
 357             if not allow_tainted:
 358                 query = query.join(Archive).filter(Archive.tainted == False)
 359
 360             source_af = query.first()
 361             if source_af is None:
 362                 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
 363             target_af = ArchiveFile(archive, component, db_file)
 364             session.add(target_af)
 365             session.flush()
 366             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
 367
 368     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
 369         """Copy a binary package to the given suite and component
 370
 371         Args:
 372            db_binary (daklib.dbconn.DBBinary): binary to copy
 373            suite (daklib.dbconn.Suite): target suite
 374            component (daklib.dbconn.Component): target component
 375
 376         Kwargs:
 377            allow_tainted (bool): allow to copy from tainted archives (such as NEW)
 378            extra_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
 379         """
 380         session = self.session
 381         archive = suite.archive
 382         if archive.tainted:
 383             allow_tainted = True
 384
 385         filename = db_binary.poolfile.filename
 386
 387         # make sure source is present in target archive
 388         db_source = db_binary.source
 389         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
 390             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
 391
 392         # make sure built-using packages are present in target archive
 393         for db_source in db_binary.extra_sources:
 394             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 395
 396         # copy binary
 397         db_file = db_binary.poolfile
 398         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
 399         if suite not in db_binary.suites:
 400             db_binary.suites.append(suite)
 401         self.session.flush()
 402
 403     def copy_source(self, db_source, suite, component, allow_tainted=False):
 404         """Copy a source package to the given suite and component
 405
 406         Args:
 407            db_source (daklib.dbconn.DBSource): source to copy
 408            suite (daklib.dbconn.Suite): target suite
 409            component (daklib.dbconn.Component): target component
 410
 411         Kwargs:
 412            allow_tainted (bool): allow to copy from tainted archives (such as NEW)
 413         """
 414         archive = suite.archive
 415         if archive.tainted:
 416             allow_tainted = True
 417         for db_dsc_file in db_source.srcfiles:
 418             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
 419         if suite not in db_source.suites:
 420             db_source.suites.append(suite)
 421         self.session.flush()
 422
 423     def remove_file(self, db_file, archive, component):
 424         """Remove a file from a given archive and component
 425
 426         Args:
 427            db_file (daklib.dbconn.PoolFile): file to remove
 428            archive (daklib.dbconn.Archive): archive to remove the file from
 429            component (daklib.dbconn.Component): component to remove the file from
 430         """
 431         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
 432         self.fs.unlink(af.path)
 433         self.session.delete(af)
 434
 435     def remove_binary(self, binary, suite):
 436         """Remove a binary from a given suite and component
 437
 438         Args:
 439            binary (daklib.dbconn.DBBinary): binary to remove
 440            suite (daklib.dbconn.Suite): suite to remove the package from
 441         """
 442         binary.suites.remove(suite)
 443         self.session.flush()
 444
 445     def remove_source(self, source, suite):
 446         """Remove a source from a given suite and component
 447
 448         Raises:
 449            ArchiveException: source package is still referenced by other
 450                              binaries in the suite
 451
 452         Args:
 453            binary (daklib.dbconn.DBSource): source to remove
 454            suite (daklib.dbconn.Suite): suite to remove the package from
 455         """
 456         session = self.session
 457
 458         query = session.query(DBBinary).filter_by(source=source) \
 459             .filter(DBBinary.suites.contains(suite))
 460         if query.first() is not None:
 461             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
 462
 463         source.suites.remove(suite)
 464         session.flush()
 465
 466     def commit(self):
 467         """commit changes"""
 468         try:
 469             self.session.commit()
 470             self.fs.commit()
 471         finally:
 472             self.session.rollback()
 473             self.fs.rollback()
 474
 475     def rollback(self):
 476         """rollback changes"""
 477         self.session.rollback()
 478         self.fs.rollback()
 479
 480     def __enter__(self):
 481         return self
 482
 483     def __exit__(self, type, value, traceback):
 484         if type is None:
 485             self.commit()
 486         else:
 487             self.rollback()
 488         return None
 489
 490 class ArchiveUpload(object):
 491     """handle an upload
 492
 493     This class can be used in a with-statement:
 494
 495        with ArchiveUpload(...) as upload:
 496           ...
 497
 498     Doing so will automatically run any required cleanup and also rollback the
 499     transaction if it was not committed.
 500
 501     Attributes:
 502        changes (daklib.upload.Changes): upload to process
 503        directory (str): directory with temporary copy of files. set by `prepare`
 504        fingerprint (daklib.dbconn.Fingerprint): fingerprint used to sign the upload
 505        new (bool): upload is NEW. set by `check`
 506        reject_reasons (list of str): reasons why the upload cannot be accepted
 507        session: database session
 508        transaction (daklib.archive.ArchiveTransaction): transaction used to handle the upload
 509        warnings (list of str): warnings (NOT USED YET)
 510     """
 511     def __init__(self, directory, changes, keyrings):
 512         self.transaction = ArchiveTransaction()
 513         self.session = self.transaction.session
 514
 515         self.original_directory = directory
 516         self.original_changes = changes
 517         self.changes = None
 518         self.directory = None
 519         self.keyrings = keyrings
 520
 521         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
 522
 523         self.reject_reasons = []
 524         self.warnings = []
 525         self.final_suites = None
 526         self.new = False
 527
 528         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
 529         self._new = self._new_queue.suite
 530
 531     def prepare(self):
 532         """prepare upload for further processing
 533
 534         This copies the files involved to a temporary directory.  If you use
 535         this method directly, you have to remove the directory given by the
 536         `directory` attribute later on your own.
 537
 538         Instead of using the method directly, you can also use a with-statement:
 539
 540            with ArchiveUpload(...) as upload:
 541               ...
 542
 543         This will automatically handle any required cleanup.
 544         """
 545         assert self.directory is None
 546         assert self.original_changes.valid_signature
 547
 548         cnf = Config()
 549         session = self.transaction.session
 550
 551         self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
 552         with FilesystemTransaction() as fs:
 553             src = os.path.join(self.original_directory, self.original_changes.filename)
 554             dst = os.path.join(self.directory, self.original_changes.filename)
 555             fs.copy(src, dst)
 556
 557             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 558
 559             for f in self.changes.files.itervalues():
 560                 src = os.path.join(self.original_directory, f.filename)
 561                 dst = os.path.join(self.directory, f.filename)
 562                 fs.copy(src, dst)
 563
 564             source = self.changes.source
 565             if source is not None:
 566                 for f in source.files.itervalues():
 567                     src = os.path.join(self.original_directory, f.filename)
 568                     dst = os.path.join(self.directory, f.filename)
 569                     if f.filename not in self.changes.files:
 570                         db_file = self.transaction.get_file(f, source.dsc['Source'])
 571                         db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
 572                         fs.copy(db_archive_file.path, dst, symlink=True)
 573
 574     def unpacked_source(self):
 575         """Path to unpacked source
 576
 577         Get path to the unpacked source. This method does unpack the source
 578         into a temporary directory under `self.directory` if it has not
 579         been done so already.
 580
 581         Returns:
 582            String giving the path to the unpacked source directory
 583            or None if no source was included in the upload.
 584         """
 585         assert self.directory is not None
 586
 587         source = self.changes.source
 588         if source is None:
 589             return None
 590         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
 591
 592         sourcedir = os.path.join(self.directory, 'source')
 593         if not os.path.exists(sourcedir):
 594             subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
 595         if not os.path.isdir(sourcedir):
 596             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
 597         return sourcedir
 598
 599     def _map_suite(self, suite_name):
 600         for rule in Config().value_list("SuiteMappings"):
 601             fields = rule.split()
 602             rtype = fields[0]
 603             if rtype == "map" or rtype == "silent-map":
 604                 (src, dst) = fields[1:3]
 605                 if src == suite_name:
 606                     suite_name = dst
 607                     if rtype != "silent-map":
 608                         self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
 609             elif rtype == "ignore":
 610                 ignored = fields[1]
 611                 if suite_name == ignored:
 612                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
 613                     suite_name = None
 614             elif rtype == "reject":
 615                 rejected = fields[1]
 616                 if suite_name == rejected:
 617                     self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
 618             ## XXX: propup-version and map-unreleased not yet implemented
 619         return suite_name
 620
 621     def _mapped_suites(self):
 622         """Get target suites after mappings
 623
 624         Returns:
 625            list of daklib.dbconn.Suite giving the mapped target suites of this upload
 626         """
 627         session = self.session
 628
 629         suite_names = []
 630         for dist in self.changes.distributions:
 631             suite_name = self._map_suite(dist)
 632             if suite_name is not None:
 633                 suite_names.append(suite_name)
 634
 635         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
 636         return suites
 637
 638     def _mapped_component(self, component_name):
 639         """get component after mappings
 640
 641         Evaluate component mappings from ComponentMappings in dak.conf for the
 642         given component name.
 643
 644         NOTE: ansgar wants to get rid of this. It's currently only used for
 645         the security archive
 646
 647         Args:
 648            component_name (str): component name
 649
 650         Returns:
 651            `daklib.dbconn.Component` object
 652         """
 653         cnf = Config()
 654         for m in cnf.value_list("ComponentMappings"):
 655             (src, dst) = m.split()
 656             if component_name == src:
 657                 component_name = dst
 658         component = self.session.query(Component).filter_by(component_name=component_name).one()
 659         return component
 660
 661     def _check_new(self, suite):
 662         """Check if upload is NEW
 663
 664         An upload is NEW if it has binary or source packages that do not have
 665         an override in `suite` OR if it references files ONLY in a tainted
 666         archive (eg. when it references files in NEW).
 667
 668         Returns:
 669            True if the upload is NEW, False otherwise
 670         """
 671         session = self.session
 672
 673         # Check for missing overrides
 674         for b in self.changes.binaries:
 675             override = self._binary_override(suite, b)
 676             if override is None:
 677                 return True
 678
 679         if self.changes.source is not None:
 680             override = self._source_override(suite, self.changes.source)
 681             if override is None:
 682                 return True
 683
 684         # Check if we reference a file only in a tainted archive
 685         files = self.changes.files.values()
 686         if self.changes.source is not None:
 687             files.extend(self.changes.source.files.values())
 688         for f in files:
 689             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
 690             query_untainted = query.join(Archive).filter(Archive.tainted == False)
 691
 692             in_archive = (query.first() is not None)
 693             in_untainted_archive = (query_untainted.first() is not None)
 694
 695             if in_archive and not in_untainted_archive:
 696                 return True
 697
 698     def _final_suites(self):
 699         session = self.session
 700
 701         mapped_suites = self._mapped_suites()
 702         final_suites = set()
 703
 704         for suite in mapped_suites:
 705             overridesuite = suite
 706             if suite.overridesuite is not None:
 707                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 708             if self._check_new(overridesuite):
 709                 self.new = True
 710             final_suites.add(suite)
 711
 712         return final_suites
 713
 714     def _binary_override(self, suite, binary):
 715         """Get override entry for a binary
 716
 717         Args:
 718            suite (daklib.dbconn.Suite)
 719            binary (daklib.upload.Binary)
 720
 721         Returns:
 722            daklib.dbconn.Override or None
 723         """
 724         if suite.overridesuite is not None:
 725             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 726
 727         query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
 728                 .join(Component).filter(Component.component_name == binary.component) \
 729                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 730
 731         try:
 732             return query.one()
 733         except NoResultFound:
 734             return None
 735
 736     def _source_override(self, suite, source):
 737         """Get override entry for a source
 738
 739         Args:
 740            suite (daklib.dbconn.Suite)
 741            source (daklib.upload.Source)
 742
 743         Returns:
 744            daklib.dbconn.Override or None
 745         """
 746         if suite.overridesuite is not None:
 747             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 748
 749         # XXX: component for source?
 750         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
 751                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 752
 753         try:
 754             return query.one()
 755         except NoResultFound:
 756             return None
 757
 758     def _binary_component(self, suite, binary, only_overrides=True):
 759         """get component for a binary
 760
 761         By default this will only look at overrides to get the right component;
 762         if `only_overrides` is False this method will also look at the Section field.
 763
 764         Args:
 765            suite (daklib.dbconn.Suite)
 766            binary (daklib.upload.Binary)
 767
 768         Kwargs:
 769            only_overrides (bool): only use overrides to get the right component.
 770                defaults to True.
 771
 772         Returns:
 773            `daklib.dbconn.Component` object or None
 774         """
 775         override = self._binary_override(suite, binary)
 776         if override is not None:
 777             return override.component
 778         if only_overrides:
 779             return None
 780         return self._mapped_component(binary.component)
 781
 782     def check(self, force=False):
 783         """run checks against the upload
 784
 785         Args:
 786            force (bool): ignore failing forcable checks
 787
 788         Returns:
 789            True if all checks passed, False otherwise
 790         """
 791         # XXX: needs to be better structured.
 792         assert self.changes.valid_signature
 793
 794         try:
 795             for chk in (
 796                     checks.SignatureCheck,
 797                     checks.ChangesCheck,
 798                     checks.TransitionCheck,
 799                     checks.UploadBlockCheck,
 800                     checks.HashesCheck,
 801                     checks.SourceCheck,
 802                     checks.BinaryCheck,
 803                     checks.BinaryTimestampCheck,
 804                     checks.ACLCheck,
 805                     checks.SingleDistributionCheck,
 806                     checks.NoSourceOnlyCheck,
 807                     checks.LintianCheck,
 808                     ):
 809                 chk().check(self)
 810
 811             final_suites = self._final_suites()
 812             if len(final_suites) == 0:
 813                 self.reject_reasons.append('Ended with no suite to install to.')
 814                 return False
 815
 816             for chk in (
 817                     checks.SourceFormatCheck,
 818                     checks.SuiteArchitectureCheck,
 819                     checks.VersionCheck,
 820                     ):
 821                 for suite in final_suites:
 822                     chk().per_suite_check(self, suite)
 823
 824             if len(self.reject_reasons) != 0:
 825                 return False
 826
 827             self.final_suites = final_suites
 828             return True
 829         except checks.Reject as e:
 830             self.reject_reasons.append(unicode(e))
 831         except Exception as e:
 832             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
 833         return False
 834
 835     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
 836         """Install upload to the given suite
 837
 838         Args:
 839            suite (daklib.dbconn.Suite): suite to install the package into.
 840               This is the real suite, ie. after any redirection to NEW or a policy queue
 841            source_component_func: function to get the `daklib.dbconn.Component`
 842               for a `daklib.upload.Source` object
 843            binary_component_func: function to get the `daklib.dbconn.Component`
 844               for a `daklib.upload.Binary` object
 845
 846         Kwargs:
 847            source_suites: see `daklib.archive.ArchiveTransaction.install_binary`
 848            extra_source_archives: see `daklib.archive.ArchiveTransaction.install_binary`
 849
 850         Returns:
 851            tuple with two elements. The first is a `daklib.dbconn.DBSource`
 852            object for the install source or None if no source was included.
 853            The second is a list of `daklib.dbconn.DBBinary` objects for the
 854            installed binary packages.
 855         """
 856         # XXX: move this function to ArchiveTransaction?
 857
 858         control = self.changes.changes
 859         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 860
 861         if source_suites is None:
 862             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
 863
 864         source = self.changes.source
 865         if source is not None:
 866             component = source_component_func(source)
 867             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
 868         else:
 869             db_source = None
 870
 871         db_binaries = []
 872         for binary in self.changes.binaries:
 873             component = binary_component_func(binary)
 874             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
 875             db_binaries.append(db_binary)
 876
 877         if suite.copychanges:
 878             src = os.path.join(self.directory, self.changes.filename)
 879             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
 880             self.transaction.fs.copy(src, dst)
 881
 882         return (db_source, db_binaries)
 883
 884     def _install_changes(self):
 885         assert self.changes.valid_signature
 886         control = self.changes.changes
 887         session = self.transaction.session
 888         config = Config()
 889
 890         changelog_id = None
 891         # Only add changelog for sourceful uploads and binNMUs
 892         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
 893             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
 894             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
 895             assert changelog_id is not None
 896
 897         db_changes = DBChange()
 898         db_changes.changesname = self.changes.filename
 899         db_changes.source = control['Source']
 900         db_changes.binaries = control.get('Binary', None)
 901         db_changes.architecture = control['Architecture']
 902         db_changes.version = control['Version']
 903         db_changes.distribution = control['Distribution']
 904         db_changes.urgency = control['Urgency']
 905         db_changes.maintainer = control['Maintainer']
 906         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
 907         db_changes.date = control['Date']
 908         db_changes.fingerprint = self.fingerprint.fingerprint
 909         db_changes.changelog_id = changelog_id
 910         db_changes.closes = self.changes.closed_bugs
 911
 912         self.transaction.session.add(db_changes)
 913         self.transaction.session.flush()
 914
 915         return db_changes
 916
 917     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
 918         u = PolicyQueueUpload()
 919         u.policy_queue = policy_queue
 920         u.target_suite = target_suite
 921         u.changes = db_changes
 922         u.source = db_source
 923         u.binaries = db_binaries
 924         self.transaction.session.add(u)
 925         self.transaction.session.flush()
 926
 927         dst = os.path.join(policy_queue.path, self.changes.filename)
 928         self.transaction.fs.copy(self.changes.path, dst)
 929
 930         return u
 931
 932     def try_autobyhand(self):
 933         """Try AUTOBYHAND
 934
 935         Try to handle byhand packages automatically.
 936
 937         Returns:
 938            list of `daklib.upload.hashed_file` for the remaining byhand packages
 939         """
 940         assert len(self.reject_reasons) == 0
 941         assert self.changes.valid_signature
 942         assert self.final_suites is not None
 943
 944         byhand = self.changes.byhand_files
 945         if len(byhand) == 0:
 946             return True
 947
 948         suites = list(self.final_suites)
 949         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
 950         suite = suites[0]
 951
 952         cnf = Config()
 953         control = self.changes.changes
 954         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
 955
 956         remaining = []
 957         for f in byhand:
 958             parts = f.filename.split('_', 2)
 959             if len(parts) != 3:
 960                 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
 961                 remaining.append(f)
 962                 continue
 963
 964             package, version, archext = parts
 965             arch, ext = archext.split('.', 1)
 966
 967             rule = automatic_byhand_packages.get(package)
 968             if rule is None:
 969                 remaining.append(f)
 970                 continue
 971
 972             if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
 973                 remaining.append(f)
 974                 continue
 975
 976             script = rule['Script']
 977             retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
 978             if retcode != 0:
 979                 print "W: error processing {0}.".format(f.filename)
 980                 remaining.append(f)
 981
 982         return len(remaining) == 0
 983
 984     def _install_byhand(self, policy_queue_upload, hashed_file):
 985         """
 986         Args:
 987            policy_queue_upload (daklib.dbconn.PolicyQueueUpload): XXX
 988            hashed_file (daklib.upload.HashedFile): XXX
 989         """
 990         fs = self.transaction.fs
 991         session = self.transaction.session
 992         policy_queue = policy_queue_upload.policy_queue
 993
 994         byhand_file = PolicyQueueByhandFile()
 995         byhand_file.upload = policy_queue_upload
 996         byhand_file.filename = hashed_file.filename
 997         session.add(byhand_file)
 998         session.flush()
 999
1000         src = os.path.join(self.directory, hashed_file.filename)
1001         dst = os.path.join(policy_queue.path, hashed_file.filename)
1002         fs.copy(src, dst)
1003
1004         return byhand_file
1005
1006     def _do_bts_versiontracking(self):
1007         cnf = Config()
1008         fs = self.transaction.fs
1009
1010         btsdir = cnf.get('Dir::BTSVersionTrack')
1011         if btsdir is None or btsdir == '':
1012             return
1013
1014         base = os.path.join(btsdir, self.changes.filename[:-8])
1015
1016         # version history
1017         sourcedir = self.unpacked_source()
1018         if sourcedir is not None:
1019             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1020             versions = fs.create("{0}.versions".format(base), mode=0o644)
1021             for line in fh.readlines():
1022                 if re_changelog_versions.match(line):
1023                     versions.write(line)
1024             fh.close()
1025             versions.close()
1026
1027         # binary -> source mapping
1028         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1029         for binary in self.changes.binaries:
1030             control = binary.control
1031             source_package, source_version = binary.source
1032             line = " ".join([control['Package'], control['Version'], source_package, source_version])
1033             print >>debinfo, line
1034         debinfo.close()
1035
1036     def _policy_queue(self, suite):
1037         if suite.policy_queue is not None:
1038             return suite.policy_queue
1039         return None
1040
1041     def install(self):
1042         """install upload
1043
1044         Install upload to a suite or policy queue.  This method does *not*
1045         handle uploads to NEW.
1046
1047         You need to have called the `check` method before calling this method.
1048         """
1049         assert len(self.reject_reasons) == 0
1050         assert self.changes.valid_signature
1051         assert self.final_suites is not None
1052         assert not self.new
1053
1054         db_changes = self._install_changes()
1055
1056         for suite in self.final_suites:
1057             overridesuite = suite
1058             if suite.overridesuite is not None:
1059                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1060
1061             policy_queue = self._policy_queue(suite)
1062
1063             redirected_suite = suite
1064             if policy_queue is not None:
1065                 redirected_suite = policy_queue.suite
1066
1067             source_component_func = lambda source: self._source_override(overridesuite, source).component
1068             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1069
1070             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1071
1072             if policy_queue is not None:
1073                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1074
1075             # copy to build queues
1076             if policy_queue is None or policy_queue.send_to_build_queues:
1077                 for build_queue in suite.copy_queues:
1078                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1079
1080         self._do_bts_versiontracking()
1081
1082     def install_to_new(self):
1083         """install upload to NEW
1084
1085         Install upload to NEW.  This method does *not* handle regular uploads
1086         to suites or policy queues.
1087
1088         You need to have called the `check` method before calling this method.
1089         """
1090         # Uploads to NEW are special as we don't have overrides.
1091         assert len(self.reject_reasons) == 0
1092         assert self.changes.valid_signature
1093         assert self.final_suites is not None
1094
1095         source = self.changes.source
1096         binaries = self.changes.binaries
1097         byhand = self.changes.byhand_files
1098
1099         new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1100         if len(byhand) > 0:
1101             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1102         new_suite = new_queue.suite
1103
1104         # we need a suite to guess components
1105         suites = list(self.final_suites)
1106         assert len(suites) == 1, "NEW uploads must be to a single suite"
1107         suite = suites[0]
1108
1109         def binary_component_func(binary):
1110             return self._binary_component(suite, binary, only_overrides=False)
1111
1112         # guess source component
1113         # XXX: should be moved into an extra method
1114         binary_component_names = set()
1115         for binary in binaries:
1116             component = binary_component_func(binary)
1117             binary_component_names.add(component.component_name)
1118         source_component_name = None
1119         for c in self.session.query(Component).order_by(Component.component_id):
1120             guess = c.component_name
1121             if guess in binary_component_names:
1122                 source_component_name = guess
1123                 break
1124         if source_component_name is None:
1125             raise Exception('Could not guess source component.')
1126         source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1127         source_component_func = lambda source: source_component
1128
1129         db_changes = self._install_changes()
1130         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1131         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1132
1133         for f in byhand:
1134             self._install_byhand(policy_upload, f)
1135
1136         self._do_bts_versiontracking()
1137
1138     def commit(self):
1139         """commit changes"""
1140         self.transaction.commit()
1141
1142     def rollback(self):
1143         """rollback changes"""
1144         self.transaction.rollback()
1145
1146     def __enter__(self):
1147         self.prepare()
1148         return self
1149
1150     def __exit__(self, type, value, traceback):
1151         if self.directory is not None:
1152             shutil.rmtree(self.directory)
1153             self.directory = None
1154         self.changes = None
1155         self.transaction.rollback()
1156         return None