X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Farchive.py;h=6577d3591f479e960a0ae23b213e1f5c75ccfef3;hb=6b05407646140b4ce2b16463bb96bf471dd5e646;hp=e4804433be13afdb6cc5d18ac74a4ab5a40e477a;hpb=4db39fc2fdceac5ddae30507a65f24b3fe694137;p=dak.git diff --git a/daklib/archive.py b/daklib/archive.py index e4804433..6577d359 100644 --- a/daklib/archive.py +++ b/daklib/archive.py @@ -26,13 +26,14 @@ import daklib.upload as upload import daklib.utils as utils from daklib.fstransactions import FilesystemTransaction from daklib.regexes import re_changelog_versions, re_bin_only_nmu +import daklib.daksubprocess import apt_pkg from datetime import datetime import os import shutil -import subprocess from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.orm import object_session import sqlalchemy.exc import tempfile import traceback @@ -109,7 +110,7 @@ class ArchiveTransaction(object): session.flush() path = os.path.join(archive.path, 'pool', component.component_name, poolname) - hashed_file_path = os.path.join(directory, hashed_file.filename) + hashed_file_path = os.path.join(directory, hashed_file.input_filename) self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode) return poolfile @@ -175,6 +176,10 @@ class ArchiveTransaction(object): maintainer=maintainer, poolfile=db_file, binarytype=binary.type, + ) + # Other attributes that are ignored for purposes of equality with + # an existing source + rest2 = dict( fingerprint=fingerprint, ) @@ -187,6 +192,8 @@ class ArchiveTransaction(object): db_binary = DBBinary(**unique) for key, value in rest.iteritems(): setattr(db_binary, key, value) + for key, value in rest2.iteritems(): + setattr(db_binary, key, value) session.add(db_binary) session.flush() import_metadata_into_db(db_binary, session) @@ -240,51 +247,18 @@ class ArchiveTransaction(object): """Add Built-Using sources to C{db_binary.extra_sources} """ session = self.session - built_using = control.get('Built-Using', None) - - if built_using is not None: - for dep in apt_pkg.parse_depends(built_using): - assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field' - bu_source_name, bu_source_version, comp = dep[0] - assert comp == '=', 'Built-Using must contain strict dependencies' - - bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first() - if bu_source is None: - raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version)) - - self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives) - - db_binary.extra_sources.append(bu_source) - - def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None): - """Install a source package - - @type directory: str - @param directory: directory the source package is located in - - @type source: L{daklib.upload.Source} - @param source: source package to install - @type suite: L{daklib.dbconn.Suite} - @param suite: target suite + for bu_source_name, bu_source_version in daklib.utils.parse_built_using(control): + bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first() + if bu_source is None: + raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version)) - @type component: L{daklib.dbconn.Component} - @param component: target component + self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives) - @type changed_by: L{daklib.dbconn.Maintainer} - @param changed_by: person who prepared this version of the package - - @type allow_tainted: bool - @param allow_tainted: allow to copy additional files from tainted archives - - @type fingerprint: L{daklib.dbconn.Fingerprint} - @param fingerprint: optional fingerprint + db_binary.extra_sources.append(bu_source) - @rtype: L{daklib.dbconn.DBSource} - @return: database object for the new source - """ + def install_source_to_archive(self, directory, source, archive, component, changed_by, allow_tainted=False, fingerprint=None): session = self.session - archive = suite.archive control = source.dsc maintainer = get_or_set_maintainer(control['Maintainer'], session) source_name = control['Source'] @@ -300,12 +274,15 @@ class ArchiveTransaction(object): ) rest = dict( maintainer=maintainer, - changedby=changed_by, - #install_date=datetime.now().date(), poolfile=db_file_dsc, - fingerprint=fingerprint, dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'), ) + # Other attributes that are ignored for purposes of equality with + # an existing source + rest2 = dict( + changedby=changed_by, + fingerprint=fingerprint, + ) created = False try: @@ -318,8 +295,8 @@ class ArchiveTransaction(object): db_source = DBSource(**unique) for key, value in rest.iteritems(): setattr(db_source, key, value) - # XXX: set as default in postgres? - db_source.install_date = datetime.now().date() + for key, value in rest2.iteritems(): + setattr(db_source, key, value) session.add(db_source) session.flush() @@ -330,11 +307,6 @@ class ArchiveTransaction(object): session.add(db_dsc_file) session.flush() - if suite in db_source.suites: - return db_source - - db_source.suites.append(suite) - if not created: for f in db_source.srcfiles: self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted) @@ -343,7 +315,7 @@ class ArchiveTransaction(object): ### Now add remaining files and copy them to the archive. for hashed_file in source.files.itervalues(): - hashed_file_path = os.path.join(directory, hashed_file.filename) + hashed_file_path = os.path.join(directory, hashed_file.input_filename) if os.path.exists(hashed_file_path): db_file = self._install_file(directory, hashed_file, archive, component, source_name) session.add(db_file) @@ -371,6 +343,42 @@ class ArchiveTransaction(object): return db_source + def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None): + """Install a source package + + @type directory: str + @param directory: directory the source package is located in + + @type source: L{daklib.upload.Source} + @param source: source package to install + + @type suite: L{daklib.dbconn.Suite} + @param suite: target suite + + @type component: L{daklib.dbconn.Component} + @param component: target component + + @type changed_by: L{daklib.dbconn.Maintainer} + @param changed_by: person who prepared this version of the package + + @type allow_tainted: bool + @param allow_tainted: allow to copy additional files from tainted archives + + @type fingerprint: L{daklib.dbconn.Fingerprint} + @param fingerprint: optional fingerprint + + @rtype: L{daklib.dbconn.DBSource} + @return: database object for the new source + """ + db_source = self.install_source_to_archive(directory, source, suite.archive, component, changed_by, allow_tainted, fingerprint) + + if suite in db_source.suites: + return db_source + db_source.suites.append(suite) + self.session.flush() + + return db_source + def _copy_file(self, db_file, archive, component, allow_tainted=False): """Copy a file to the given archive and component @@ -388,7 +396,7 @@ class ArchiveTransaction(object): """ session = self.session - if session.query(ArchiveFile).filter_by(archive=archive, file=db_file).first() is None: + if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None: query = session.query(ArchiveFile).filter_by(file=db_file) if not allow_tainted: query = query.join(Archive).filter(Archive.tainted == False) @@ -530,6 +538,9 @@ class ArchiveTransaction(object): self.session.rollback() self.fs.rollback() + def flush(self): + self.session.flush() + def __enter__(self): return self @@ -540,6 +551,34 @@ class ArchiveTransaction(object): self.rollback() return None +def source_component_from_package_list(package_list, suite): + """Get component for a source package + + This function will look at the Package-List field to determine the + component the source package belongs to. This is the first component + the source package provides binaries for (first with respect to the + ordering of components). + + It the source package has no Package-List field, None is returned. + + @type package_list: L{daklib.packagelist.PackageList} + @param package_list: package list of the source to get the override for + + @type suite: L{daklib.dbconn.Suite} + @param suite: suite to consider for binaries produced + + @rtype: L{daklib.dbconn.Component} or C{None} + @return: component for the given source or C{None} + """ + if package_list.fallback: + return None + session = object_session(suite) + packages = package_list.packages_for_suite(suite) + components = set(p.component for p in packages) + query = session.query(Component).order_by(Component.ordering) \ + .filter(Component.component_name.in_(components)) + return query.first() + class ArchiveUpload(object): """handle an upload @@ -695,7 +734,7 @@ class ArchiveUpload(object): sourcedir = os.path.join(self.directory, 'source') if not os.path.exists(sourcedir): devnull = open('/dev/null', 'w') - subprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull) + daklib.daksubprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull) if not os.path.isdir(sourcedir): raise Exception("{0} is not a directory after extracting source package".format(sourcedir)) return sourcedir @@ -739,12 +778,39 @@ class ArchiveUpload(object): suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names)) return suites - def _check_new(self, suite): + def _check_new_binary_overrides(self, suite, overridesuite): + new = False + source = self.changes.source + + if source is not None and not source.package_list.fallback: + packages = source.package_list.packages_for_suite(suite) + binaries = [ entry for entry in packages ] + for b in binaries: + override = self._binary_override(overridesuite, b) + if override is None: + self.warnings.append('binary:{0} is NEW.'.format(b.name)) + new = True + else: + binaries = self.changes.binaries + for b in binaries: + if utils.is_in_debug_section(b.control) and suite.debug_suite is not None: + continue + override = self._binary_override(overridesuite, b) + if override is None: + self.warnings.append('binary:{0} is NEW.'.format(b.name)) + new = True + + return new + + def _check_new(self, suite, overridesuite): """Check if upload is NEW An upload is NEW if it has binary or source packages that do not have - an override in C{suite} OR if it references files ONLY in a tainted - archive (eg. when it references files in NEW). + an override in C{overridesuite} OR if it references files ONLY in a + tainted archive (eg. when it references files in NEW). + + Debug packages (*-dbgsym in Section: debug) are not considered as NEW + if C{suite} has a seperate debug suite. @rtype: bool @return: C{True} if the upload is NEW, C{False} otherwise @@ -753,14 +819,10 @@ class ArchiveUpload(object): new = False # Check for missing overrides - for b in self.changes.binaries: - override = self._binary_override(suite, b) - if override is None: - self.warnings.append('binary:{0} is NEW.'.format(b.control['Package'])) - new = True - + if self._check_new_binary_overrides(suite, overridesuite): + new = True if self.changes.source is not None: - override = self._source_override(suite, self.changes.source) + override = self._source_override(overridesuite, self.changes.source) if override is None: self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source'])) new = True @@ -792,7 +854,7 @@ class ArchiveUpload(object): overridesuite = suite if suite.overridesuite is not None: overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one() - if self._check_new(overridesuite): + if self._check_new(suite, overridesuite): self.new = True final_suites.add(suite) @@ -804,7 +866,7 @@ class ArchiveUpload(object): @type suite: L{daklib.dbconn.Suite} @param suite: suite to get override for - @type binary: L{daklib.upload.Binary} + @type binary: L{daklib.upload.Binary} or L{daklib.packagelist.PackageListEntry} @param binary: binary to get override for @rtype: L{daklib.dbconn.Override} or C{None} @@ -817,7 +879,7 @@ class ArchiveUpload(object): if mapped_component is None: return None - query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \ + query = self.session.query(Override).filter_by(suite=suite, package=binary.name) \ .join(Component).filter(Component.component_name == mapped_component.component_name) \ .join(OverrideType).filter(OverrideType.overridetype == binary.type) @@ -841,10 +903,13 @@ class ArchiveUpload(object): if suite.overridesuite is not None: suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one() - # XXX: component for source? query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') + component = source_component_from_package_list(source.package_list, suite) + if component is not None: + query = query.filter(Override.component == component) + try: return query.one() except NoResultFound: @@ -889,6 +954,7 @@ class ArchiveUpload(object): # Validate signatures and hashes before we do any real work: for chk in ( checks.SignatureAndHashesCheck, + checks.SignatureTimestampCheck, checks.ChangesCheck, checks.ExternalHashesCheck, checks.SourceCheck, @@ -906,6 +972,7 @@ class ArchiveUpload(object): self.final_suites = final_suites for chk in ( + checks.SuiteCheck, checks.TransitionCheck, checks.ACLCheck, checks.NoSourceOnlyCheck, @@ -966,14 +1033,33 @@ class ArchiveUpload(object): source = self.changes.source if source is not None: component = source_component_func(source) - db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint) + db_source = self.transaction.install_source( + self.directory, + source, + suite, + component, + changed_by, + fingerprint=self.fingerprint + ) else: db_source = None db_binaries = [] for binary in self.changes.binaries: + copy_to_suite = suite + if utils.is_in_debug_section(binary.control) and suite.debug_suite is not None: + copy_to_suite = suite.debug_suite + component = binary_component_func(binary) - db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives) + db_binary = self.transaction.install_binary( + self.directory, + binary, + copy_to_suite, + component, + fingerprint=self.fingerprint, + source_suites=source_suites, + extra_source_archives=extra_source_archives + ) db_binaries.append(db_binary) if suite.copychanges: @@ -1095,7 +1181,7 @@ class ArchiveUpload(object): continue script = rule['Script'] - retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False) + retcode = daklib.daksubprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename), suite.suite_name], shell=False) if retcode != 0: print "W: error processing {0}.".format(f.filename) remaining.append(f) @@ -1197,7 +1283,7 @@ class ArchiveUpload(object): source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery() source_component_func = lambda source: self._source_override(overridesuite, source).component - binary_component_func = lambda binary: self._binary_component(overridesuite, binary) + binary_component_func = lambda binary: self._binary_component(overridesuite, binary, only_overrides=False) (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])