daklib/archive.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to manipulate the archive
  18
  19 This module provides classes to manipulate the archive.
  20 """
  21
  22 from .dbconn import *
  23 import daklib.checks as checks
  24 from daklib.config import Config
  25 import daklib.upload as upload
  26 import daklib.utils as utils
  27 from .fstransactions import FilesystemTransaction
  28 from .regexes import re_changelog_versions, re_bin_only_nmu
  29
  30 import apt_pkg
  31 from datetime import datetime
  32 import os
  33 import shutil
  34 import subprocess
  35 from sqlalchemy.orm.exc import NoResultFound
  36 import tempfile
  37 import traceback
  38
  39 class ArchiveException(Exception):
  40     pass
  41
  42 class HashMismatchException(ArchiveException):
  43     pass
  44
  45 class ArchiveTransaction(object):
  46     """manipulate the archive in a transaction
  47     """
  48     def __init__(self):
  49         self.fs = FilesystemTransaction()
  50         self.session = DBConn().session()
  51
  52     def get_file(self, hashed_file, source_name):
  53         """Look for file `hashed_file` in database
  54
  55         Args:
  56            hashed_file (daklib.upload.HashedFile): file to look for in the database
  57
  58         Raises:
  59            KeyError: file was not found in the database
  60            HashMismatchException: hash mismatch
  61
  62         Returns:
  63            `daklib.dbconn.PoolFile` object for the database
  64         """
  65         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  66         try:
  67             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
  68             if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
  69                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
  70             return poolfile
  71         except NoResultFound:
  72             raise KeyError('{0} not found in database.'.format(poolname))
  73
  74     def _install_file(self, directory, hashed_file, archive, component, source_name):
  75         """Install a file
  76
  77         Will not give an error when the file is already present.
  78
  79         Returns:
  80            `daklib.dbconn.PoolFile` object for the new file
  81         """
  82         session = self.session
  83
  84         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  85         try:
  86             poolfile = self.get_file(hashed_file, source_name)
  87         except KeyError:
  88             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
  89             poolfile.md5sum = hashed_file.md5sum
  90             poolfile.sha1sum = hashed_file.sha1sum
  91             poolfile.sha256sum = hashed_file.sha256sum
  92             session.add(poolfile)
  93             session.flush()
  94
  95         try:
  96             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
  97         except NoResultFound:
  98             archive_file = ArchiveFile(archive, component, poolfile)
  99             session.add(archive_file)
 100             session.flush()
 101
 102             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
 103             hashed_file_path = os.path.join(directory, hashed_file.filename)
 104             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 105
 106         return poolfile
 107
 108     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
 109         """Install a binary package
 110
 111         Args:
 112            directory (str): directory the binary package is located in
 113            binary (daklib.upload.Binary): binary package to install
 114            suite (daklib.dbconn.Suite): target suite
 115            component (daklib.dbconn.Component): target component
 116
 117         Kwargs:
 118            allow_tainted (bool): allow to copy additional files from tainted archives
 119            fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
 120            source_suites (list of daklib.dbconn.Suite or True): suites to copy
 121               the source from if they are not in `suite` or True to allow
 122               copying from any suite.
 123               This can also be a SQLAlchemy (sub)query object.
 124            extra_source_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
 125
 126         Returns:
 127            `daklib.dbconn.DBBinary` object for the new package
 128         """
 129         session = self.session
 130         control = binary.control
 131         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 132         architecture = get_architecture(control['Architecture'], session)
 133
 134         (source_name, source_version) = binary.source
 135         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
 136         source = source_query.filter(DBSource.suites.contains(suite)).first()
 137         if source is None:
 138             if source_suites != True:
 139                 source_query = source_query.join(DBSource.suites) \
 140                     .filter(Suite.suite_id == source_suites.c.id)
 141             source = source_query.first()
 142             if source is None:
 143                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
 144             self.copy_source(source, suite, component)
 145
 146         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
 147
 148         unique = dict(
 149             package=control['Package'],
 150             version=control['Version'],
 151             architecture=architecture,
 152             )
 153         rest = dict(
 154             source=source,
 155             maintainer=maintainer,
 156             poolfile=db_file,
 157             binarytype=binary.type,
 158             fingerprint=fingerprint,
 159             )
 160
 161         try:
 162             db_binary = session.query(DBBinary).filter_by(**unique).one()
 163             for key, value in rest.iteritems():
 164                 if getattr(db_binary, key) != value:
 165                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
 166         except NoResultFound:
 167             db_binary = DBBinary(**unique)
 168             for key, value in rest.iteritems():
 169                 setattr(db_binary, key, value)
 170             session.add(db_binary)
 171             session.flush()
 172             import_metadata_into_db(db_binary, session)
 173
 174             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
 175
 176         if suite not in db_binary.suites:
 177             db_binary.suites.append(suite)
 178
 179         session.flush()
 180
 181         return db_binary
 182
 183     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
 184         """ensure source exists in the given archive
 185
 186         This is intended to be used to check that Built-Using sources exist.
 187
 188         Args:
 189            filename (str): filename to use in error messages
 190            source (daklib.dbconn.DBSource): source to look for
 191            archive (daklib.dbconn.Archive): archive to look in
 192
 193         Kwargs:
 194            extra_archives (list of daklib.dbconn.Archive): list of archives to copy
 195                the source package from if it is not yet present in `archive`
 196         """
 197         session = self.session
 198         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
 199         if db_file is not None:
 200             return True
 201
 202         # Try to copy file from one extra archive
 203         if extra_archives is None:
 204             extra_archives = []
 205         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
 206         if db_file is None:
 207             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
 208
 209         source_archive = db_file.archive
 210         for dsc_file in source.srcfiles:
 211             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
 212             # We were given an explicit list of archives so it is okay to copy from tainted archives.
 213             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 214
 215     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
 216         """Add Built-Using sources to `db_binary.extra_sources`
 217         """
 218         session = self.session
 219         built_using = control.get('Built-Using', None)
 220
 221         if built_using is not None:
 222             for dep in apt_pkg.parse_depends(built_using):
 223                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
 224                 bu_source_name, bu_source_version, comp = dep[0]
 225                 assert comp == '=', 'Built-Using must contain strict dependencies'
 226
 227                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
 228                 if bu_source is None:
 229                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
 230
 231                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
 232
 233                 db_binary.extra_sources.append(bu_source)
 234
 235     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
 236         """Install a source package
 237
 238         Args:
 239            directory (str): directory the source package is located in
 240            source (daklib.upload.Source): source package to install
 241            suite (daklib.dbconn.Suite): target suite
 242            component (daklib.dbconn.Component): target component
 243            changed_by (daklib.dbconn.Maintainer): person who prepared this version of the package
 244
 245         Kwargs:
 246            allow_tainted (bool): allow to copy additional files from tainted archives
 247            fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
 248
 249         Returns:
 250            `daklib.dbconn.DBSource` object for the new source
 251         """
 252         session = self.session
 253         archive = suite.archive
 254         control = source.dsc
 255         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 256         source_name = control['Source']
 257
 258         ### Add source package to database
 259
 260         # We need to install the .dsc first as the DBSource object refers to it.
 261         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
 262
 263         unique = dict(
 264             source=source_name,
 265             version=control['Version'],
 266             )
 267         rest = dict(
 268             maintainer=maintainer,
 269             changedby=changed_by,
 270             #install_date=datetime.now().date(),
 271             poolfile=db_file_dsc,
 272             fingerprint=fingerprint,
 273             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
 274             )
 275
 276         created = False
 277         try:
 278             db_source = session.query(DBSource).filter_by(**unique).one()
 279             for key, value in rest.iteritems():
 280                 if getattr(db_source, key) != value:
 281                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
 282         except NoResultFound:
 283             created = True
 284             db_source = DBSource(**unique)
 285             for key, value in rest.iteritems():
 286                 setattr(db_source, key, value)
 287             # XXX: set as default in postgres?
 288             db_source.install_date = datetime.now().date()
 289             session.add(db_source)
 290             session.flush()
 291
 292             # Add .dsc file. Other files will be added later.
 293             db_dsc_file = DSCFile()
 294             db_dsc_file.source = db_source
 295             db_dsc_file.poolfile = db_file_dsc
 296             session.add(db_dsc_file)
 297             session.flush()
 298
 299         if suite in db_source.suites:
 300             return db_source
 301
 302         db_source.suites.append(suite)
 303
 304         if not created:
 305             return db_source
 306
 307         ### Now add remaining files and copy them to the archive.
 308
 309         for hashed_file in source.files.itervalues():
 310             hashed_file_path = os.path.join(directory, hashed_file.filename)
 311             if os.path.exists(hashed_file_path):
 312                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
 313                 session.add(db_file)
 314             else:
 315                 db_file = self.get_file(hashed_file, source_name)
 316                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
 317
 318             db_dsc_file = DSCFile()
 319             db_dsc_file.source = db_source
 320             db_dsc_file.poolfile = db_file
 321             session.add(db_dsc_file)
 322
 323         session.flush()
 324
 325         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
 326         import_metadata_into_db(db_source, session)
 327
 328         # Uploaders are the maintainer and co-maintainers from the Uploaders field
 329         db_source.uploaders.append(maintainer)
 330         if 'Uploaders' in control:
 331             def split_uploaders(field):
 332                 import re
 333                 for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
 334                     yield u.strip()
 335
 336             for u in split_uploaders(control['Uploaders']):
 337                 db_source.uploaders.append(get_or_set_maintainer(u, session))
 338         session.flush()
 339
 340         return db_source
 341
 342     def _copy_file(self, db_file, archive, component, allow_tainted=False):
 343         """Copy a file to the given archive and component
 344
 345         Args:
 346            db_file (daklib.dbconn.PoolFile): file to copy
 347            archive (daklib.dbconn.Archive): target archive
 348            component (daklib.dbconn.Component): target component
 349
 350         Kwargs:
 351            allow_tainted (bool): allow to copy from tainted archives (such as NEW)
 352         """
 353         session = self.session
 354
 355         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
 356             query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
 357             if not allow_tainted:
 358                 query = query.join(Archive).filter(Archive.tainted == False)
 359
 360             source_af = query.first()
 361             if source_af is None:
 362                 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
 363             target_af = ArchiveFile(archive, component, db_file)
 364             session.add(target_af)
 365             session.flush()
 366             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
 367
 368     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
 369         """Copy a binary package to the given suite and component
 370
 371         Args:
 372            db_binary (daklib.dbconn.DBBinary): binary to copy
 373            suite (daklib.dbconn.Suite): target suite
 374            component (daklib.dbconn.Component): target component
 375
 376         Kwargs:
 377            allow_tainted (bool): allow to copy from tainted archives (such as NEW)
 378            extra_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
 379         """
 380         session = self.session
 381         archive = suite.archive
 382         if archive.tainted:
 383             allow_tainted = True
 384
 385         filename = db_binary.poolfile.filename
 386
 387         # make sure source is present in target archive
 388         db_source = db_binary.source
 389         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
 390             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
 391
 392         # make sure built-using packages are present in target archive
 393         for db_source in db_binary.extra_sources:
 394             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 395
 396         # copy binary
 397         db_file = db_binary.poolfile
 398         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
 399         if suite not in db_binary.suites:
 400             db_binary.suites.append(suite)
 401         self.session.flush()
 402
 403     def copy_source(self, db_source, suite, component, allow_tainted=False):
 404         """Copy a source package to the given suite and component
 405
 406         Args:
 407            db_source (daklib.dbconn.DBSource): source to copy
 408            suite (daklib.dbconn.Suite): target suite
 409            component (daklib.dbconn.Component): target component
 410
 411         Kwargs:
 412            allow_tainted (bool): allow to copy from tainted archives (such as NEW)
 413         """
 414         archive = suite.archive
 415         if archive.tainted:
 416             allow_tainted = True
 417         for db_dsc_file in db_source.srcfiles:
 418             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
 419         if suite not in db_source.suites:
 420             db_source.suites.append(suite)
 421         self.session.flush()
 422
 423     def remove_file(self, db_file, archive, component):
 424         """Remove a file from a given archive and component
 425
 426         Args:
 427            db_file (daklib.dbconn.PoolFile): file to remove
 428            archive (daklib.dbconn.Archive): archive to remove the file from
 429            component (daklib.dbconn.Component): component to remove the file from
 430         """
 431         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
 432         self.fs.unlink(af.path)
 433         self.session.delete(af)
 434
 435     def remove_binary(self, binary, suite):
 436         """Remove a binary from a given suite and component
 437
 438         Args:
 439            binary (daklib.dbconn.DBBinary): binary to remove
 440            suite (daklib.dbconn.Suite): suite to remove the package from
 441         """
 442         binary.suites.remove(suite)
 443         self.session.flush()
 444
 445     def remove_source(self, source, suite):
 446         """Remove a source from a given suite and component
 447
 448         Raises:
 449            ArchiveException: source package is still referenced by other
 450                              binaries in the suite
 451
 452         Args:
 453            binary (daklib.dbconn.DBSource): source to remove
 454            suite (daklib.dbconn.Suite): suite to remove the package from
 455         """
 456         session = self.session
 457
 458         query = session.query(DBBinary).filter_by(source=source) \
 459             .filter(DBBinary.suites.contains(suite))
 460         if query.first() is not None:
 461             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
 462
 463         source.suites.remove(suite)
 464         session.flush()
 465
 466     def commit(self):
 467         """commit changes"""
 468         try:
 469             self.session.commit()
 470             self.fs.commit()
 471         finally:
 472             self.session.rollback()
 473             self.fs.rollback()
 474
 475     def rollback(self):
 476         """rollback changes"""
 477         self.session.rollback()
 478         self.fs.rollback()
 479
 480     def __enter__(self):
 481         return self
 482
 483     def __exit__(self, type, value, traceback):
 484         if type is None:
 485             self.commit()
 486         else:
 487             self.rollback()
 488         return None
 489
 490 class ArchiveUpload(object):
 491     """handle an upload
 492
 493     This class can be used in a with-statement:
 494
 495        with ArchiveUpload(...) as upload:
 496           ...
 497
 498     Doing so will automatically run any required cleanup and also rollback the
 499     transaction if it was not committed.
 500
 501     Attributes:
 502        changes (daklib.upload.Changes): upload to process
 503        directory (str): directory with temporary copy of files. set by `prepare`
 504        fingerprint (daklib.dbconn.Fingerprint): fingerprint used to sign the upload
 505        new (bool): upload is NEW. set by `check`
 506        reject_reasons (list of str): reasons why the upload cannot be accepted
 507        session: database session
 508        transaction (daklib.archive.ArchiveTransaction): transaction used to handle the upload
 509        warnings (list of str): warnings (NOT USED YET)
 510     """
 511     def __init__(self, directory, changes, keyrings):
 512         self.transaction = ArchiveTransaction()
 513         self.session = self.transaction.session
 514
 515         self.original_directory = directory
 516         self.original_changes = changes
 517         self.changes = None
 518         self.directory = None
 519         self.keyrings = keyrings
 520
 521         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
 522
 523         self.reject_reasons = []
 524         self.warnings = []
 525         self.final_suites = None
 526         self.new = False
 527
 528         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
 529         self._new = self._new_queue.suite
 530
 531     def prepare(self):
 532         """prepare upload for further processing
 533
 534         This copies the files involved to a temporary directory.  If you use
 535         this method directly, you have to remove the directory given by the
 536         `directory` attribute later on your own.
 537
 538         Instead of using the method directly, you can also use a with-statement:
 539
 540            with ArchiveUpload(...) as upload:
 541               ...
 542
 543         This will automatically handle any required cleanup.
 544         """
 545         assert self.directory is None
 546         assert self.original_changes.valid_signature
 547
 548         cnf = Config()
 549         session = self.transaction.session
 550
 551         self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
 552         with FilesystemTransaction() as fs:
 553             src = os.path.join(self.original_directory, self.original_changes.filename)
 554             dst = os.path.join(self.directory, self.original_changes.filename)
 555             fs.copy(src, dst)
 556
 557             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 558
 559             for f in self.changes.files.itervalues():
 560                 src = os.path.join(self.original_directory, f.filename)
 561                 dst = os.path.join(self.directory, f.filename)
 562                 fs.copy(src, dst)
 563
 564             source = self.changes.source
 565             if source is not None:
 566                 for f in source.files.itervalues():
 567                     src = os.path.join(self.original_directory, f.filename)
 568                     dst = os.path.join(self.directory, f.filename)
 569                     if f.filename not in self.changes.files:
 570                         db_file = self.transaction.get_file(f, source.dsc['Source'])
 571                         db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
 572                         fs.copy(db_archive_file.path, dst, symlink=True)
 573
 574     def unpacked_source(self):
 575         """Path to unpacked source
 576
 577         Get path to the unpacked source. This method does unpack the source
 578         into a temporary directory under `self.directory` if it has not
 579         been done so already.
 580
 581         Returns:
 582            String giving the path to the unpacked source directory
 583            or None if no source was included in the upload.
 584         """
 585         assert self.directory is not None
 586
 587         source = self.changes.source
 588         if source is None:
 589             return None
 590         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
 591
 592         sourcedir = os.path.join(self.directory, 'source')
 593         if not os.path.exists(sourcedir):
 594             subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
 595         if not os.path.isdir(sourcedir):
 596             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
 597         return sourcedir
 598
 599     def _map_suite(self, suite_name):
 600         for rule in Config().value_list("SuiteMappings"):
 601             fields = rule.split()
 602             rtype = fields[0]
 603             if rtype == "map" or rtype == "silent-map":
 604                 (src, dst) = fields[1:3]
 605                 if src == suite_name:
 606                     suite_name = dst
 607                     if rtype != "silent-map":
 608                         self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
 609             elif rtype == "ignore":
 610                 ignored = fields[1]
 611                 if suite_name == ignored:
 612                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
 613                     suite_name = None
 614             elif rtype == "reject":
 615                 rejected = fields[1]
 616                 if suite_name == rejected:
 617                     self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
 618             ## XXX: propup-version and map-unreleased not yet implemented
 619         return suite_name
 620
 621     def _mapped_suites(self):
 622         """Get target suites after mappings
 623
 624         Returns:
 625            list of daklib.dbconn.Suite giving the mapped target suites of this upload
 626         """
 627         session = self.session
 628
 629         suite_names = []
 630         for dist in self.changes.distributions:
 631             suite_name = self._map_suite(dist)
 632             if suite_name is not None:
 633                 suite_names.append(suite_name)
 634
 635         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
 636         return suites
 637
 638     def _mapped_component(self, component_name):
 639         """get component after mappings
 640
 641         Evaluate component mappings from ComponentMappings in dak.conf for the
 642         given component name.
 643
 644         NOTE: ansgar wants to get rid of this. It's currently only used for
 645         the security archive
 646
 647         Args:
 648            component_name (str): component name
 649
 650         Returns:
 651            `daklib.dbconn.Component` object
 652         """
 653         cnf = Config()
 654         for m in cnf.value_list("ComponentMappings"):
 655             (src, dst) = m.split()
 656             if component_name == src:
 657                 component_name = dst
 658         component = self.session.query(Component).filter_by(component_name=component_name).one()
 659         return component
 660
 661     def _check_new(self, suite):
 662         """Check if upload is NEW
 663
 664         An upload is NEW if it has binary or source packages that do not have
 665         an override in `suite` OR if it references files ONLY in a tainted
 666         archive (eg. when it references files in NEW).
 667
 668         Returns:
 669            True if the upload is NEW, False otherwise
 670         """
 671         session = self.session
 672
 673         # Check for missing overrides
 674         for b in self.changes.binaries:
 675             override = self._binary_override(suite, b)
 676             if override is None:
 677                 return True
 678
 679         if self.changes.source is not None:
 680             override = self._source_override(suite, self.changes.source)
 681             if override is None:
 682                 return True
 683
 684         # Check if we reference a file only in a tainted archive
 685         files = self.changes.files.values()
 686         if self.changes.source is not None:
 687             files.extend(self.changes.source.files.values())
 688         for f in files:
 689             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
 690             query_untainted = query.join(Archive).filter(Archive.tainted == False)
 691
 692             in_archive = (query.first() is not None)
 693             in_untainted_archive = (query_untainted.first() is not None)
 694
 695             if in_archive and not in_untainted_archive:
 696                 return True
 697
 698     def _final_suites(self):
 699         session = self.session
 700
 701         mapped_suites = self._mapped_suites()
 702         final_suites = set()
 703
 704         for suite in mapped_suites:
 705             overridesuite = suite
 706             if suite.overridesuite is not None:
 707                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 708             if self._check_new(overridesuite):
 709                 self.new = True
 710             final_suites.add(suite)
 711
 712         return final_suites
 713
 714     def _binary_override(self, suite, binary):
 715         """Get override entry for a binary
 716
 717         Args:
 718            suite (daklib.dbconn.Suite)
 719            binary (daklib.upload.Binary)
 720
 721         Returns:
 722            daklib.dbconn.Override or None
 723         """
 724         if suite.overridesuite is not None:
 725             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 726
 727         query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
 728                 .join(Component).filter(Component.component_name == binary.component) \
 729                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 730
 731         try:
 732             return query.one()
 733         except NoResultFound:
 734             return None
 735
 736     def _source_override(self, suite, source):
 737         """Get override entry for a source
 738
 739         Args:
 740            suite (daklib.dbconn.Suite)
 741            source (daklib.upload.Source)
 742
 743         Returns:
 744            daklib.dbconn.Override or None
 745         """
 746         if suite.overridesuite is not None:
 747             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 748
 749         # XXX: component for source?
 750         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
 751                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 752
 753         try:
 754             return query.one()
 755         except NoResultFound:
 756             return None
 757
 758     def _binary_component(self, suite, binary, only_overrides=True):
 759         """get component for a binary
 760
 761         By default this will only look at overrides to get the right component;
 762         if `only_overrides` is False this method will also look at the Section field.
 763
 764         Args:
 765            suite (daklib.dbconn.Suite)
 766            binary (daklib.upload.Binary)
 767
 768         Kwargs:
 769            only_overrides (bool): only use overrides to get the right component.
 770                defaults to True.
 771
 772         Returns:
 773            `daklib.dbconn.Component` object or None
 774         """
 775         override = self._binary_override(suite, binary)
 776         if override is not None:
 777             return override.component
 778         if only_overrides:
 779             return None
 780         return self._mapped_component(binary.component)
 781
 782     def check(self, force=False):
 783         """run checks against the upload
 784
 785         Args:
 786            force (bool): ignore failing forcable checks
 787
 788         Returns:
 789            True if all checks passed, False otherwise
 790         """
 791         # XXX: needs to be better structured.
 792         assert self.changes.valid_signature
 793
 794         try:
 795             for chk in (
 796                     checks.SignatureCheck,
 797                     checks.ChangesCheck,
 798                     checks.HashesCheck,
 799                     checks.SourceCheck,
 800                     checks.BinaryCheck,
 801                     checks.BinaryTimestampCheck,
 802                     checks.ACLCheck,
 803                     checks.SingleDistributionCheck,
 804                     checks.NoSourceOnlyCheck,
 805                     checks.LintianCheck,
 806                     ):
 807                 chk().check(self)
 808
 809             final_suites = self._final_suites()
 810             if len(final_suites) == 0:
 811                 self.reject_reasons.append('Ended with no suite to install to.')
 812                 return False
 813
 814             for chk in (
 815                     checks.SourceFormatCheck,
 816                     checks.SuiteArchitectureCheck,
 817                     checks.VersionCheck,
 818                     ):
 819                 for suite in final_suites:
 820                     chk().per_suite_check(self, suite)
 821
 822             if len(self.reject_reasons) != 0:
 823                 return False
 824
 825             self.final_suites = final_suites
 826             return True
 827         except checks.Reject as e:
 828             self.reject_reasons.append(unicode(e))
 829         except Exception as e:
 830             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
 831         return False
 832
 833     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
 834         """Install upload to the given suite
 835
 836         Args:
 837            suite (daklib.dbconn.Suite): suite to install the package into.
 838               This is the real suite, ie. after any redirection to NEW or a policy queue
 839            source_component_func: function to get the `daklib.dbconn.Component`
 840               for a `daklib.upload.Source` object
 841            binary_component_func: function to get the `daklib.dbconn.Component`
 842               for a `daklib.upload.Binary` object
 843
 844         Kwargs:
 845            source_suites: see `daklib.archive.ArchiveTransaction.install_binary`
 846            extra_source_archives: see `daklib.archive.ArchiveTransaction.install_binary`
 847
 848         Returns:
 849            tuple with two elements. The first is a `daklib.dbconn.DBSource`
 850            object for the install source or None if no source was included.
 851            The second is a list of `daklib.dbconn.DBBinary` objects for the
 852            installed binary packages.
 853         """
 854         # XXX: move this function to ArchiveTransaction?
 855
 856         control = self.changes.changes
 857         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
 858
 859         if source_suites is None:
 860             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
 861
 862         source = self.changes.source
 863         if source is not None:
 864             component = source_component_func(source)
 865             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
 866         else:
 867             db_source = None
 868
 869         db_binaries = []
 870         for binary in self.changes.binaries:
 871             component = binary_component_func(binary)
 872             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
 873             db_binaries.append(db_binary)
 874
 875         if suite.copychanges:
 876             src = os.path.join(self.directory, self.changes.filename)
 877             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
 878             self.transaction.fs.copy(src, dst)
 879
 880         return (db_source, db_binaries)
 881
 882     def _install_changes(self):
 883         assert self.changes.valid_signature
 884         control = self.changes.changes
 885         session = self.transaction.session
 886         config = Config()
 887
 888         changelog_id = None
 889         # Only add changelog for sourceful uploads and binNMUs
 890         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
 891             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
 892             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
 893             assert changelog_id is not None
 894
 895         db_changes = DBChange()
 896         db_changes.changesname = self.changes.filename
 897         db_changes.source = control['Source']
 898         db_changes.binaries = control.get('Binary', None)
 899         db_changes.architecture = control['Architecture']
 900         db_changes.version = control['Version']
 901         db_changes.distribution = control['Distribution']
 902         db_changes.urgency = control['Urgency']
 903         db_changes.maintainer = control['Maintainer']
 904         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
 905         db_changes.date = control['Date']
 906         db_changes.fingerprint = self.fingerprint.fingerprint
 907         db_changes.changelog_id = changelog_id
 908         db_changes.closes = self.changes.closed_bugs
 909
 910         self.transaction.session.add(db_changes)
 911         self.transaction.session.flush()
 912
 913         return db_changes
 914
 915     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
 916         u = PolicyQueueUpload()
 917         u.policy_queue = policy_queue
 918         u.target_suite = target_suite
 919         u.changes = db_changes
 920         u.source = db_source
 921         u.binaries = db_binaries
 922         self.transaction.session.add(u)
 923         self.transaction.session.flush()
 924
 925         dst = os.path.join(policy_queue.path, self.changes.filename)
 926         self.transaction.fs.copy(self.changes.path, dst)
 927
 928         return u
 929
 930     def try_autobyhand(self):
 931         """Try AUTOBYHAND
 932
 933         Try to handle byhand packages automatically.
 934
 935         Returns:
 936            list of `daklib.upload.hashed_file` for the remaining byhand packages
 937         """
 938         assert len(self.reject_reasons) == 0
 939         assert self.changes.valid_signature
 940         assert self.final_suites is not None
 941
 942         byhand = self.changes.byhand_files
 943         if len(byhand) == 0:
 944             return True
 945
 946         suites = list(self.final_suites)
 947         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
 948         suite = suites[0]
 949
 950         cnf = Config()
 951         control = self.changes.changes
 952         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
 953
 954         remaining = []
 955         for f in byhand:
 956             parts = f.filename.split('_', 2)
 957             if len(parts) != 3:
 958                 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
 959                 remaining.append(f)
 960                 continue
 961
 962             package, version, archext = parts
 963             arch, ext = archext.split('.', 1)
 964
 965             rule = automatic_byhand_packages.get(package)
 966             if rule is None:
 967                 remaining.append(f)
 968                 continue
 969
 970             if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
 971                 remaining.append(f)
 972                 continue
 973
 974             script = rule['Script']
 975             retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
 976             if retcode != 0:
 977                 print "W: error processing {0}.".format(f.filename)
 978                 remaining.append(f)
 979
 980         return len(remaining) == 0
 981
 982     def _install_byhand(self, policy_queue_upload, hashed_file):
 983         """
 984         Args:
 985            policy_queue_upload (daklib.dbconn.PolicyQueueUpload): XXX
 986            hashed_file (daklib.upload.HashedFile): XXX
 987         """
 988         fs = self.transaction.fs
 989         session = self.transaction.session
 990         policy_queue = policy_queue_upload.policy_queue
 991
 992         byhand_file = PolicyQueueByhandFile()
 993         byhand_file.upload = policy_queue_upload
 994         byhand_file.filename = hashed_file.filename
 995         session.add(byhand_file)
 996         session.flush()
 997
 998         src = os.path.join(self.directory, hashed_file.filename)
 999         dst = os.path.join(policy_queue.path, hashed_file.filename)
1000         fs.copy(src, dst)
1001
1002         return byhand_file
1003
1004     def _do_bts_versiontracking(self):
1005         cnf = Config()
1006         fs = self.transaction.fs
1007
1008         btsdir = cnf.get('Dir::BTSVersionTrack')
1009         if btsdir is None or btsdir == '':
1010             return
1011
1012         base = os.path.join(btsdir, self.changes.filename[:-8])
1013
1014         # version history
1015         sourcedir = self.unpacked_source()
1016         if sourcedir is not None:
1017             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1018             versions = fs.create("{0}.versions".format(base), mode=0o644)
1019             for line in fh.readlines():
1020                 if re_changelog_versions.match(line):
1021                     versions.write(line)
1022             fh.close()
1023             versions.close()
1024
1025         # binary -> source mapping
1026         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1027         for binary in self.changes.binaries:
1028             control = binary.control
1029             source_package, source_version = binary.source
1030             line = " ".join([control['Package'], control['Version'], source_package, source_version])
1031             print >>debinfo, line
1032         debinfo.close()
1033
1034     def install(self):
1035         """install upload
1036
1037         Install upload to a suite or policy queue.  This method does *not*
1038         handle uploads to NEW.
1039
1040         You need to have called the `check` method before calling this method.
1041         """
1042         assert len(self.reject_reasons) == 0
1043         assert self.changes.valid_signature
1044         assert self.final_suites is not None
1045         assert not self.new
1046
1047         db_changes = self._install_changes()
1048
1049         for suite in self.final_suites:
1050             overridesuite = suite
1051             if suite.overridesuite is not None:
1052                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1053
1054             redirected_suite = suite
1055             if suite.policy_queue is not None:
1056                 redirected_suite = suite.policy_queue.suite
1057
1058             source_component_func = lambda source: self._source_override(overridesuite, source).component
1059             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1060
1061             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1062
1063             if suite.policy_queue is not None:
1064                 self._install_policy(suite.policy_queue, suite, db_changes, db_source, db_binaries)
1065
1066             # copy to build queues
1067             if suite.policy_queue is None or suite.policy_queue.send_to_build_queues:
1068                 for build_queue in suite.copy_queues:
1069                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1070
1071         self._do_bts_versiontracking()
1072
1073     def install_to_new(self):
1074         """install upload to NEW
1075
1076         Install upload to NEW.  This method does *not* handle regular uploads
1077         to suites or policy queues.
1078
1079         You need to have called the `check` method before calling this method.
1080         """
1081         # Uploads to NEW are special as we don't have overrides.
1082         assert len(self.reject_reasons) == 0
1083         assert self.changes.valid_signature
1084         assert self.final_suites is not None
1085
1086         source = self.changes.source
1087         binaries = self.changes.binaries
1088         byhand = self.changes.byhand_files
1089
1090         new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1091         if len(byhand) > 0:
1092             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1093         new_suite = new_queue.suite
1094
1095         # we need a suite to guess components
1096         suites = list(self.final_suites)
1097         assert len(suites) == 1, "NEW uploads must be to a single suite"
1098         suite = suites[0]
1099
1100         def binary_component_func(binary):
1101             return self._binary_component(suite, binary, only_overrides=False)
1102
1103         # guess source component
1104         # XXX: should be moved into an extra method
1105         binary_component_names = set()
1106         for binary in binaries:
1107             component = binary_component_func(binary)
1108             binary_component_names.add(component.component_name)
1109         source_component_name = None
1110         for c in self.session.query(Component).order_by(Component.component_id):
1111             guess = c.component_name
1112             if guess in binary_component_names:
1113                 source_component_name = guess
1114                 break
1115         if source_component_name is None:
1116             raise Exception('Could not guess source component.')
1117         source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1118         source_component_func = lambda source: source_component
1119
1120         db_changes = self._install_changes()
1121         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1122         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1123
1124         for f in byhand:
1125             self._install_byhand(policy_upload, f)
1126
1127         self._do_bts_versiontracking()
1128
1129     def commit(self):
1130         """commit changes"""
1131         self.transaction.commit()
1132
1133     def rollback(self):
1134         """rollback changes"""
1135         self.transaction.rollback()
1136
1137     def __enter__(self):
1138         self.prepare()
1139         return self
1140
1141     def __exit__(self, type, value, traceback):
1142         if self.directory is not None:
1143             shutil.rmtree(self.directory)
1144             self.directory = None
1145         self.changes = None
1146         self.transaction.rollback()
1147         return None