X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fpolicy.py;h=aa5f12f0ac141e10e04552c11449ff842618cea2;hb=391f5ec09a119131dc846b796ca791f4cecc69e4;hp=c989f1f269969672b46ef5bee97feaecc1ed404c;hpb=b740656a6cca0f69bb1a094fc0c8bcb930c56bf2;p=dak.git diff --git a/daklib/policy.py b/daklib/policy.py index c989f1f2..aa5f12f0 100644 --- a/daklib/policy.py +++ b/daklib/policy.py @@ -17,9 +17,11 @@ """module to process policy queue uploads""" from .config import Config -from .dbconn import BinaryMetadata, Component, MetadataKey, Override, OverrideType +from .dbconn import BinaryMetadata, Component, MetadataKey, Override, OverrideType, Suite, get_mapped_component, get_mapped_component_name from .fstransactions import FilesystemTransaction from .regexes import re_file_changes, re_file_safe +from .packagelist import PackageList +import daklib.utils as utils import errno import os @@ -29,31 +31,40 @@ import tempfile class UploadCopy(object): """export a policy queue upload - This class can be used in a with-statements: + This class can be used in a with-statement:: with UploadCopy(...) as copy: ... Doing so will provide a temporary copy of the upload in the directory - given by the `directory` attribute. The copy will be removed on leaving + given by the C{directory} attribute. The copy will be removed on leaving the with-block. - - Args: - upload (daklib.dbconn.PolicyQueueUpload) """ - def __init__(self, upload): + def __init__(self, upload, group=None): + """initializer + + @type upload: L{daklib.dbconn.PolicyQueueUpload} + @param upload: upload to handle + """ + self.directory = None self.upload = upload + self.group = group - def export(self, directory, mode=None, symlink=True): + def export(self, directory, mode=None, symlink=True, ignore_existing=False): """export a copy of the upload - Args: - directory (str) + @type directory: str + @param directory: directory to export to + + @type mode: int + @param mode: permissions to use for the copied files - Kwargs: - mode (int): permissions to use for the copied files - symlink (bool): use symlinks instead of copying the files (default: True) + @type symlink: bool + @param symlink: use symlinks instead of copying the files + + @type ignore_existing: bool + @param ignore_existing: ignore already existing files """ with FilesystemTransaction() as fs: source = self.upload.source @@ -63,29 +74,42 @@ class UploadCopy(object): for dsc_file in source.srcfiles: f = dsc_file.poolfile dst = os.path.join(directory, os.path.basename(f.filename)) - fs.copy(f.fullpath, dst, mode=mode, symlink=symlink) + if not os.path.exists(dst) or not ignore_existing: + fs.copy(f.fullpath, dst, mode=mode, symlink=symlink) + for binary in self.upload.binaries: f = binary.poolfile dst = os.path.join(directory, os.path.basename(f.filename)) - fs.copy(f.fullpath, dst, mode=mode, symlink=symlink) + if not os.path.exists(dst) or not ignore_existing: + fs.copy(f.fullpath, dst, mode=mode, symlink=symlink) # copy byhand files for byhand in self.upload.byhand: src = os.path.join(queue.path, byhand.filename) dst = os.path.join(directory, byhand.filename) - fs.copy(src, dst, mode=mode, symlink=symlink) + if os.path.exists(src) and (not os.path.exists(dst) or not ignore_existing): + fs.copy(src, dst, mode=mode, symlink=symlink) # copy .changes src = os.path.join(queue.path, self.upload.changes.changesname) dst = os.path.join(directory, self.upload.changes.changesname) - fs.copy(src, dst, mode=mode, symlink=symlink) + if not os.path.exists(dst) or not ignore_existing: + fs.copy(src, dst, mode=mode, symlink=symlink) def __enter__(self): assert self.directory is None + mode = 0o0700 + symlink = True + if self.group is not None: + mode = 0o2750 + symlink = False + cnf = Config() - self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath')) - self.export(self.directory, symlink=True) + self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'), + mode=mode, + group=self.group) + self.export(self.directory, symlink=symlink) return self def __exit__(self, *args): @@ -103,9 +127,10 @@ class PolicyQueueUploadHandler(object): def __init__(self, upload, session): """initializer - Args: - upload (daklib.dbconn.PolicyQueueUpload): upload to process - session: database session + @type upload: L{daklib.dbconn.PolicyQueueUpload} + @param upload: upload to process + + @param session: database session """ self.upload = upload self.session = session @@ -120,29 +145,20 @@ class PolicyQueueUploadHandler(object): def _source_override(self, component_name): package = self.upload.source.source suite = self._overridesuite + component = get_mapped_component(component_name, self.session) query = self.session.query(Override).filter_by(package=package, suite=suite) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \ - .join(Component).filter(Component.component_name == component_name) + .filter(Override.component == component) return query.first() - def _binary_override(self, binary, component_name): - package = binary.package + def _binary_override(self, name, binarytype, component_name): suite = self._overridesuite - overridetype = binary.binarytype - query = self.session.query(Override).filter_by(package=package, suite=suite) \ - .join(OverrideType).filter(OverrideType.overridetype == overridetype) \ - .join(Component).filter(Component.component_name == component_name) + component = get_mapped_component(component_name, self.session) + query = self.session.query(Override).filter_by(package=name, suite=suite) \ + .join(OverrideType).filter(OverrideType.overridetype == binarytype) \ + .filter(Override.component == component) return query.first() - def _binary_metadata(self, binary, key): - metadata_key = self.session.query(MetadataKey).filter_by(key=key).first() - if metadata_key is None: - return None - metadata = self.session.query(BinaryMetadata).filter_by(binary=binary, key=metadata_key).first() - if metadata is None: - return None - return metadata.value - @property def _changes_prefix(self): changesname = self.upload.changes.changesname @@ -169,16 +185,19 @@ class PolicyQueueUploadHandler(object): def reject(self, reason): """mark upload as rejected - Args: - reason (str): reason for the rejection + @type reason: str + @param reason: reason for the rejection """ + cnf = Config() + fn1 = 'REJECT.{0}'.format(self._changes_prefix) assert re_file_safe.match(fn1) fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1) try: - fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) os.write(fh, 'NOTOK\n') + os.write(fh, 'From: {0} <{1}>\n\n'.format(utils.whoami(), cnf['Dinstall::MyAdminAddress'])) os.write(fh, reason) os.close(fh) except OSError as e: @@ -190,8 +209,8 @@ class PolicyQueueUploadHandler(object): def get_action(self): """get current action - Returns: - string giving the current action, one of 'ACCEPT', 'ACCEPTED', 'REJECT' + @rtype: str + @return: string giving the current action, one of 'ACCEPT', 'ACCEPTED', 'REJECT' """ changes_prefix = self._changes_prefix @@ -206,57 +225,73 @@ class PolicyQueueUploadHandler(object): def missing_overrides(self, hints=None): """get missing override entries for the upload - Kwargs: - hints (list of dict): suggested hints for new overrides in the same - format as the return value - - Returns: - list of dicts with the following keys: - package: package name - priority: default priority (from upload) - section: default section (from upload) - component: default component (from upload) - type: type of required override ('dsc', 'deb' or 'udeb') - All values are strings. + @type hints: list of dict + @param hints: suggested hints for new overrides in the same format as + the return value + + @return: list of dicts with the following keys: + + - package: package name + - priority: default priority (from upload) + - section: default section (from upload) + - component: default component (from upload) + - type: type of required override ('dsc', 'deb' or 'udeb') + + All values are strings. """ # TODO: use Package-List field missing = [] components = set() + source = self.upload.source + if hints is None: hints = [] hints_map = dict([ ((o['type'], o['package']), o) for o in hints ]) - for binary in self.upload.binaries: - priority = self._binary_metadata(binary, 'Priority') - section = self._binary_metadata(binary, 'Section') + def check_override(name, type, priority, section, included): component = 'main' if section.find('/') != -1: component = section.split('/', 1)[0] - override = self._binary_override(binary, component) - if override is None: - hint = hints_map.get((binary.binarytype, binary.package)) + override = self._binary_override(name, type, component) + if override is None and not any(o['package'] == name and o['type'] == type for o in missing): + hint = hints_map.get((type, name)) if hint is not None: missing.append(hint) component = hint['component'] else: missing.append(dict( - package = binary.package, + package = name, priority = priority, section = section, component = component, - type = binary.binarytype, + type = type, + included = included )) components.add(component) - source_component = '(unknown)' - for component in ('main', 'contrib', 'non-free'): - if component in components: - source_component = component - break + for binary in self.upload.binaries: + binary_proxy = binary.proxy + priority = binary_proxy['Priority'] + section = binary_proxy['Section'] + check_override(binary.package, binary.binarytype, priority, section, included=True) - source = self.upload.source if source is not None: + source_proxy = source.proxy + package_list = PackageList(source_proxy) + if not package_list.fallback: + packages = package_list.packages_for_suite(self.upload.target_suite) + for p in packages: + check_override(p.name, p.type, p.priority, p.section, included=False) + + # see daklib.archive.source_component_from_package_list + # which we cannot use here as we might not have a Package-List + # field for old packages + mapped_components = [ get_mapped_component_name(c) for c in components ] + query = self.session.query(Component).order_by(Component.ordering) \ + .filter(Component.component_name.in_(mapped_components)) + source_component = query.first().component_name + override = self._source_override(source_component) if override is None: hint = hints_map.get(('dsc', source.source)) @@ -264,14 +299,15 @@ class PolicyQueueUploadHandler(object): missing.append(hint) else: section = 'misc' - if component != 'main': - section = "{0}/{1}".format(component, section) + if source_component != 'main': + section = "{0}/{1}".format(source_component, section) missing.append(dict( package = source.source, priority = 'extra', section = section, component = source_component, type = 'dsc', + included = True, )) return missing