daklib/policy.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to process policy queue uploads"""
  18
  19 from .config import Config
  20 from .dbconn import BinaryMetadata, Component, MetadataKey, Override, OverrideType, Suite, get_mapped_component, get_mapped_component_name
  21 from .fstransactions import FilesystemTransaction
  22 from .regexes import re_file_changes, re_file_safe
  23 from .packagelist import PackageList
  24 import daklib.utils as utils
  25
  26 import errno
  27 import os
  28 import shutil
  29 import tempfile
  30
  31 class UploadCopy(object):
  32     """export a policy queue upload
  33
  34     This class can be used in a with-statement::
  35
  36        with UploadCopy(...) as copy:
  37           ...
  38
  39     Doing so will provide a temporary copy of the upload in the directory
  40     given by the C{directory} attribute.  The copy will be removed on leaving
  41     the with-block.
  42     """
  43     def __init__(self, upload, group=None):
  44         """initializer
  45
  46         @type  upload: L{daklib.dbconn.PolicyQueueUpload}
  47         @param upload: upload to handle
  48         """
  49
  50         self.directory = None
  51         self.upload = upload
  52         self.group = group
  53
  54     def export(self, directory, mode=None, symlink=True, ignore_existing=False):
  55         """export a copy of the upload
  56
  57         @type  directory: str
  58         @param directory: directory to export to
  59
  60         @type  mode: int
  61         @param mode: permissions to use for the copied files
  62
  63         @type  symlink: bool
  64         @param symlink: use symlinks instead of copying the files
  65
  66         @type  ignore_existing: bool
  67         @param ignore_existing: ignore already existing files
  68         """
  69         with FilesystemTransaction() as fs:
  70             source = self.upload.source
  71             queue = self.upload.policy_queue
  72
  73             if source is not None:
  74                 for dsc_file in source.srcfiles:
  75                     f = dsc_file.poolfile
  76                     dst = os.path.join(directory, os.path.basename(f.filename))
  77                     if not os.path.exists(dst) or not ignore_existing:
  78                         fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
  79
  80             for binary in self.upload.binaries:
  81                 f = binary.poolfile
  82                 dst = os.path.join(directory, os.path.basename(f.filename))
  83                 if not os.path.exists(dst) or not ignore_existing:
  84                     fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
  85
  86             # copy byhand files
  87             for byhand in self.upload.byhand:
  88                 src = os.path.join(queue.path, byhand.filename)
  89                 dst = os.path.join(directory, byhand.filename)
  90                 if os.path.exists(src) and (not os.path.exists(dst) or not ignore_existing):
  91                     fs.copy(src, dst, mode=mode, symlink=symlink)
  92
  93             # copy .changes
  94             src = os.path.join(queue.path, self.upload.changes.changesname)
  95             dst = os.path.join(directory, self.upload.changes.changesname)
  96             if not os.path.exists(dst) or not ignore_existing:
  97                 fs.copy(src, dst, mode=mode, symlink=symlink)
  98
  99     def __enter__(self):
 100         assert self.directory is None
 101
 102         mode = 0o0700
 103         symlink = True
 104         if self.group is not None:
 105             mode = 0o2750
 106             symlink = False
 107
 108         cnf = Config()
 109         self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
 110                                             mode=mode,
 111                                             group=self.group)
 112         self.export(self.directory, symlink=symlink)
 113         return self
 114
 115     def __exit__(self, *args):
 116         if self.directory is not None:
 117             shutil.rmtree(self.directory)
 118             self.directory = None
 119         return None
 120
 121 class PolicyQueueUploadHandler(object):
 122     """process uploads to policy queues
 123
 124     This class allows to accept or reject uploads and to get a list of missing
 125     overrides (for NEW processing).
 126     """
 127     def __init__(self, upload, session):
 128         """initializer
 129
 130         @type  upload: L{daklib.dbconn.PolicyQueueUpload}
 131         @param upload: upload to process
 132
 133         @param session: database session
 134         """
 135         self.upload = upload
 136         self.session = session
 137
 138     @property
 139     def _overridesuite(self):
 140         overridesuite = self.upload.target_suite
 141         if overridesuite.overridesuite is not None:
 142             overridesuite = self.session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
 143         return overridesuite
 144
 145     def _source_override(self, component_name):
 146         package = self.upload.source.source
 147         suite = self._overridesuite
 148         component = get_mapped_component(component_name, self.session)
 149         query = self.session.query(Override).filter_by(package=package, suite=suite) \
 150             .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
 151             .filter(Override.component == component)
 152         return query.first()
 153
 154     def _binary_override(self, name, binarytype, component_name):
 155         suite = self._overridesuite
 156         component = get_mapped_component(component_name, self.session)
 157         query = self.session.query(Override).filter_by(package=name, suite=suite) \
 158             .join(OverrideType).filter(OverrideType.overridetype == binarytype) \
 159             .filter(Override.component == component)
 160         return query.first()
 161
 162     @property
 163     def _changes_prefix(self):
 164         changesname = self.upload.changes.changesname
 165         assert changesname.endswith('.changes')
 166         assert re_file_changes.match(changesname)
 167         return changesname[0:-8]
 168
 169     def accept(self):
 170         """mark upload as accepted"""
 171         assert len(self.missing_overrides()) == 0
 172
 173         fn1 = 'ACCEPT.{0}'.format(self._changes_prefix)
 174         fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1)
 175         try:
 176             fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
 177             os.write(fh, 'OK\n')
 178             os.close(fh)
 179         except OSError as e:
 180             if e.errno == errno.EEXIST:
 181                 pass
 182             else:
 183                 raise
 184
 185     def reject(self, reason):
 186         """mark upload as rejected
 187
 188         @type  reason: str
 189         @param reason: reason for the rejection
 190         """
 191         cnf = Config()
 192
 193         fn1 = 'REJECT.{0}'.format(self._changes_prefix)
 194         assert re_file_safe.match(fn1)
 195
 196         fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1)
 197         try:
 198             fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
 199             os.write(fh, 'NOTOK\n')
 200             os.write(fh, 'From: {0} <{1}>\n\n'.format(utils.whoami(), cnf['Dinstall::MyAdminAddress']))
 201             os.write(fh, reason)
 202             os.close(fh)
 203         except OSError as e:
 204             if e.errno == errno.EEXIST:
 205                 pass
 206             else:
 207                 raise
 208
 209     def get_action(self):
 210         """get current action
 211
 212         @rtype:  str
 213         @return: string giving the current action, one of 'ACCEPT', 'ACCEPTED', 'REJECT'
 214         """
 215         changes_prefix = self._changes_prefix
 216
 217         for action in ('ACCEPT', 'ACCEPTED', 'REJECT'):
 218             fn1 = '{0}.{1}'.format(action, changes_prefix)
 219             fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1)
 220             if os.path.exists(fn):
 221                 return action
 222
 223         return None
 224
 225     def missing_overrides(self, hints=None):
 226         """get missing override entries for the upload
 227
 228         @type  hints: list of dict
 229         @param hints: suggested hints for new overrides in the same format as
 230                       the return value
 231
 232         @return: list of dicts with the following keys:
 233
 234                  - package: package name
 235                  - priority: default priority (from upload)
 236                  - section: default section (from upload)
 237                  - component: default component (from upload)
 238                  - type: type of required override ('dsc', 'deb' or 'udeb')
 239
 240                  All values are strings.
 241         """
 242         # TODO: use Package-List field
 243         missing = []
 244         components = set()
 245
 246         source = self.upload.source
 247
 248         if hints is None:
 249             hints = []
 250         hints_map = dict([ ((o['type'], o['package']), o) for o in hints ])
 251
 252         def check_override(name, type, priority, section, included):
 253             component = 'main'
 254             if section.find('/') != -1:
 255                 component = section.split('/', 1)[0]
 256             override = self._binary_override(name, type, component)
 257             if override is None and not any(o['package'] == name and o['type'] == type for o in missing):
 258                 hint = hints_map.get((type, name))
 259                 if hint is not None:
 260                     missing.append(hint)
 261                     component = hint['component']
 262                 else:
 263                     missing.append(dict(
 264                             package = name,
 265                             priority = priority,
 266                             section = section,
 267                             component = component,
 268                             type = type,
 269                             included = included
 270                             ))
 271             components.add(component)
 272
 273         for binary in self.upload.binaries:
 274             binary_proxy = binary.proxy
 275             priority = binary_proxy['Priority']
 276             section = binary_proxy['Section']
 277             check_override(binary.package, binary.binarytype, priority, section, included=True)
 278
 279         if source is not None:
 280             source_proxy = source.proxy
 281             package_list = PackageList(source_proxy)
 282             if not package_list.fallback:
 283                 packages = package_list.packages_for_suite(self.upload.target_suite)
 284                 for p in packages:
 285                     check_override(p.name, p.type, p.priority, p.section, included=False)
 286
 287             # see daklib.archive.source_component_from_package_list
 288             # which we cannot use here as we might not have a Package-List
 289             # field for old packages
 290             mapped_components = [ get_mapped_component_name(c) for c in components ]
 291             query = self.session.query(Component).order_by(Component.ordering) \
 292                     .filter(Component.component_name.in_(mapped_components))
 293             source_component = query.first().component_name
 294
 295             override = self._source_override(source_component)
 296             if override is None:
 297                 hint = hints_map.get(('dsc', source.source))
 298                 if hint is not None:
 299                     missing.append(hint)
 300                 else:
 301                     section = 'misc'
 302                     if source_component != 'main':
 303                         section = "{0}/{1}".format(source_component, section)
 304                     missing.append(dict(
 305                             package = source.source,
 306                             priority = 'extra',
 307                             section = section,
 308                             component = source_component,
 309                             type = 'dsc',
 310                             included = True,
 311                             ))
 312
 313         return missing