"""
from daklib.config import Config
+import daklib.daksubprocess
from daklib.dbconn import *
import daklib.dbconn as dbconn
from daklib.regexes import *
from daklib.textutils import fix_maintainer, ParseMaintError
import daklib.lintian as lintian
import daklib.utils as utils
-from daklib.upload import InvalidHashException
+import daklib.upload
import apt_inst
import apt_pkg
from apt_pkg import version_compare
+import datetime
import errno
import os
+import subprocess
+import textwrap
import time
import yaml
-# TODO: replace by subprocess
-import commands
+def check_fields_for_valid_utf8(filename, control):
+ """Check all fields of a control file for valid UTF-8"""
+ for field in control.keys():
+ try:
+ field.decode('utf-8')
+ control[field].decode('utf-8')
+ except UnicodeDecodeError:
+ raise Reject('{0}: The {1} field is not valid UTF-8'.format(filename, field))
class Reject(Exception):
"""exception raised by failing checks"""
pass
-class RejectStupidMaintainerException(Exception):
+class RejectExternalFilesMismatch(Reject):
"""exception raised by failing the external hashes check"""
def __str__(self):
return False
class SignatureAndHashesCheck(Check):
+ def check_replay(self, upload):
+ # Use private session as we want to remember having seen the .changes
+ # in all cases.
+ session = upload.session
+ history = SignatureHistory.from_signed_file(upload.changes)
+ r = history.query(session)
+ if r is not None:
+ raise Reject('Signature for changes file was already seen at {0}.\nPlease refresh the signature of the changes file if you want to upload it again.'.format(r.seen))
+ return True
+
"""Check signature of changes and dsc file (if included in upload)
Make sure the signature is valid and done by a known user.
changes = upload.changes
if not changes.valid_signature:
raise Reject("Signature for .changes not valid.")
+ self.check_replay(upload)
self._check_hashes(upload, changes.filename, changes.files.itervalues())
source = None
try:
for f in files:
f.check(upload.directory)
- except IOError as e:
- if e.errno == errno.ENOENT:
- raise Reject('{0} refers to non-existing file: {1}\n'
- 'Perhaps you need to include it in your upload?'
- .format(filename, os.path.basename(e.filename)))
- raise
- except InvalidHashException as e:
+ except daklib.upload.FileDoesNotExist as e:
+ raise Reject('{0}: {1}\n'
+ 'Perhaps you need to include the file in your upload?'
+ .format(filename, unicode(e)))
+ except daklib.upload.UploadException as e:
raise Reject('{0}: {1}'.format(filename, unicode(e)))
+class SignatureTimestampCheck(Check):
+ """Check timestamp of .changes signature"""
+ def check(self, upload):
+ changes = upload.changes
+
+ now = datetime.datetime.utcnow()
+ timestamp = changes.signature_timestamp
+ age = now - timestamp
+
+ age_max = datetime.timedelta(days=365)
+ age_min = datetime.timedelta(days=-7)
+
+ if age > age_max:
+ raise Reject('{0}: Signature from {1} is too old (maximum age is {2} days)'.format(changes.filename, timestamp, age_max.days))
+ if age < age_min:
+ raise Reject('{0}: Signature from {1} is too far in the future (tolerance is {2} days)'.format(changes.filename, timestamp, abs(age_min.days)))
+
+ return True
+
class ChangesCheck(Check):
"""Check changes file for syntax errors."""
def check(self, upload):
if field not in control:
raise Reject('{0}: misses mandatory field {1}'.format(fn, field))
+ check_fields_for_valid_utf8(fn, control)
+
source_match = re_field_source.match(control['Source'])
if not source_match:
raise Reject('{0}: Invalid Source field'.format(fn))
return
if ext_size != f.size:
- raise RejectStupidMaintainerException(f.filename, 'size', f.size, ext_size)
+ raise RejectExternalFilesMismatch(f.filename, 'size', f.size, ext_size)
if ext_md5sum != f.md5sum:
- raise RejectStupidMaintainerException(f.filename, 'md5sum', f.md5sum, ext_md5sum)
+ raise RejectExternalFilesMismatch(f.filename, 'md5sum', f.md5sum, ext_md5sum)
if ext_sha1sum != f.sha1sum:
- raise RejectStupidMaintainerException(f.filename, 'sha1sum', f.sha1sum, ext_sha1sum)
+ raise RejectExternalFilesMismatch(f.filename, 'sha1sum', f.sha1sum, ext_sha1sum)
if ext_sha256sum != f.sha256sum:
- raise RejectStupidMaintainerException(f.filename, 'sha256sum', f.sha256sum, ext_sha256sum)
+ raise RejectExternalFilesMismatch(f.filename, 'sha256sum', f.sha256sum, ext_sha256sum)
def check(self, upload):
cnf = Config()
class BinaryCheck(Check):
"""Check binary packages for syntax errors."""
def check(self, upload):
+ debug_deb_name_postfix = "-dbgsym"
+ # XXX: Handle dynamic debug section name here
+
for binary in upload.changes.binaries:
self.check_binary(upload, binary)
- binary_names = set([ binary.control['Package'] for binary in upload.changes.binaries ])
- for bn in binary_names:
- if bn not in upload.changes.binary_names:
- raise Reject('Package {0} is not mentioned in Binary field in changes'.format(bn))
+ binaries = {binary.control['Package']: binary
+ for binary in upload.changes.binaries}
+
+ for name, binary in binaries.items():
+ if name in upload.changes.binary_names:
+ # Package is listed in Binary field. Everything is good.
+ pass
+ elif daklib.utils.is_in_debug_section(binary.control):
+ # If we have a binary package in the debug section, we
+ # can allow it to not be present in the Binary field
+ # in the .changes file, so long as its name (without
+ # -dbgsym) is present in the Binary list.
+ if not name.endswith(debug_deb_name_postfix):
+ raise Reject('Package {0} is in the debug section, but '
+ 'does not end in {1}.'.format(name, debug_deb_name_postfix))
+
+ # Right, so, it's named properly, let's check that
+ # the corresponding package is in the Binary list
+ origin_package_name = name[:-len(debug_deb_name_postfix)]
+ if origin_package_name not in upload.changes.binary_names:
+ raise Reject(
+ "Debug package {debug}'s corresponding binary package "
+ "{origin} is not present in the Binary field.".format(
+ debug=name, origin=origin_package_name))
+ else:
+ # Someone was a nasty little hacker and put a package
+ # into the .changes that isn't in debian/control. Bad,
+ # bad person.
+ raise Reject('Package {0} is not mentioned in Binary field in changes'.format(name))
return True
fn = binary.hashed_file.filename
control = binary.control
- for field in ('Package', 'Architecture', 'Version', 'Description'):
+ for field in ('Package', 'Architecture', 'Version', 'Description', 'Section'):
if field not in control:
raise Reject('{0}: Missing mandatory field {0}.'.format(fn, field))
+ check_fields_for_valid_utf8(fn, control)
+
# check fields
package = control['Package']
except:
raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
+ # "Multi-Arch: no" breaks wanna-build, #768353
+ multi_arch = control.get("Multi-Arch")
+ if multi_arch == 'no':
+ raise Reject('{0}: Multi-Arch: no support in Debian is broken (#768353)'.format(fn))
+
class BinaryTimestampCheck(Check):
"""check timestamps of files in binary packages
def check(self, upload):
cnf = Config()
future_cutoff = time.time() + cnf.find_i('Dinstall::FutureTimeTravelGrace', 24*3600)
- past_cutoff = time.mktime(time.strptime(cnf.find('Dinstall::PastCutoffYear', '1984'), '%Y'))
+ past_cutoff = time.mktime(time.strptime(cnf.find('Dinstall::PastCutoffYear', '1975'), '%Y'))
class TarTime(object):
def __init__(self):
self.past_files = dict()
def callback(self, member, data):
if member.mtime > future_cutoff:
- future_files[member.name] = member.mtime
+ self.future_files[member.name] = member.mtime
elif member.mtime < past_cutoff:
- past_files[member.name] = member.mtime
+ self.past_files[member.name] = member.mtime
def format_reason(filename, direction, files):
reason = "{0}: has {1} file(s) with a timestamp too far in the {2}:\n".format(filename, len(files), direction)
control = source.dsc
dsc_fn = source._dsc_file.filename
+ check_fields_for_valid_utf8(dsc_fn, control)
+
# check fields
if not re_field_package.match(control['Source']):
raise Reject('{0}: Invalid Source field'.format(dsc_fn))
if transitions is None:
return True
+ session = upload.session
+
control = upload.changes.changes
source = re_field_source.match(control['Source']).group('package')
for trans in transitions:
t = transitions[trans]
- source = t["source"]
+ transition_source = t["source"]
expected = t["new"]
# Will be None if nothing is in testing.
- current = get_source_in_suite(source, "testing", session)
+ current = get_source_in_suite(transition_source, "testing", session)
if current is not None:
compare = apt_pkg.version_compare(current.version, expected)
Release Team, and {3} is the Release-Team member responsible for it.
Please mail debian-release@lists.debian.org or contact {3} directly if you
need further assistance. You might want to upload to experimental until this
-transition is done.""".format(source, currentlymsg, expected,t["rm"])))
+transition is done.""".format(transition_source, currentlymsg, expected,t["rm"])))
raise Reject(rejectmsg)
contents = file(path, 'r').read()
try:
- transitions = yaml.load(contents)
+ transitions = yaml.safe_load(contents)
return transitions
except yaml.YAMLError as msg:
utils.warn('Not checking transitions, the transitions file is broken: {0}'.format(msg))
return None
class NoSourceOnlyCheck(Check):
+ def is_source_only_upload(self, upload):
+ changes = upload.changes
+ if changes.source is not None and len(changes.binaries) == 0:
+ return True
+ return False
+
"""Check for source-only upload
Source-only uploads are only allowed if Dinstall::AllowSourceOnlyUploads is
set. Otherwise they are rejected.
+
+ Source-only uploads are only accepted for source packages having a
+ Package-List field that also lists architectures per package. This
+ check can be disabled via
+ Dinstall::AllowSourceOnlyUploadsWithoutPackageList.
+
+ Source-only uploads to NEW are only allowed if
+ Dinstall::AllowSourceOnlyNew is set.
+
+ Uploads not including architecture-independent packages are only
+ allowed if Dinstall::AllowNoArchIndepUploads is set.
+
"""
def check(self, upload):
- if Config().find_b("Dinstall::AllowSourceOnlyUploads"):
+ if not self.is_source_only_upload(upload):
return True
+
+ allow_source_only_uploads = Config().find_b('Dinstall::AllowSourceOnlyUploads')
+ allow_source_only_uploads_without_package_list = Config().find_b('Dinstall::AllowSourceOnlyUploadsWithoutPackageList')
+ allow_source_only_new = Config().find_b('Dinstall::AllowSourceOnlyNew')
+ allow_no_arch_indep_uploads = Config().find_b('Dinstall::AllowNoArchIndepUploads')
changes = upload.changes
- if changes.source is not None and len(changes.binaries) == 0:
+
+ if not allow_source_only_uploads:
raise Reject('Source-only uploads are not allowed.')
+ if not allow_source_only_uploads_without_package_list \
+ and changes.source.package_list.fallback:
+ raise Reject('Source-only uploads are only allowed if a Package-List field that also list architectures is included in the source package. dpkg (>= 1.17.7) includes this information.')
+ if not allow_source_only_new and upload.new:
+ raise Reject('Source-only uploads to NEW are not allowed.')
+
+ if not allow_no_arch_indep_uploads \
+ and 'all' not in changes.architectures \
+ and 'experimental' not in changes.distributions \
+ and 'unstable' not in changes.distributions \
+ and 'sid' not in changes.distributions \
+ and changes.source.package_list.has_arch_indep_packages():
+ raise Reject('Uploads not including architecture-independent packages are not allowed.')
+
return True
class LintianCheck(Check):
with open(tagfile, 'r') as sourcefile:
sourcecontent = sourcefile.read()
try:
- lintiantags = yaml.load(sourcecontent)['lintian']
+ lintiantags = yaml.safe_load(sourcecontent)['lintian']
except yaml.YAMLError as msg:
raise Exception('Could not read lintian tags file {0}, YAML error: {1}'.format(tagfile, msg))
changespath = os.path.join(upload.directory, changes.filename)
try:
cmd = []
+ result = 0
user = cnf.get('Dinstall::UnprivUser') or None
if user is not None:
cmd.extend(['sudo', '-H', '-u', user])
- cmd.extend(['LINTIAN_COLL_UNPACKED_SKIP_SIG=1', '/usr/bin/lintian', '--show-overrides', '--tags-from-file', temp_filename, changespath])
- result, output = commands.getstatusoutput(" ".join(cmd))
+ cmd.extend(['/usr/bin/lintian', '--show-overrides', '--tags-from-file', temp_filename, changespath])
+ output = daklib.daksubprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as e:
+ result = e.returncode
+ output = e.output
finally:
os.unlink(temp_filename)
if query.first() is None:
raise Reject('source format {0} is not allowed in suite {1}'.format(source_format, suite.suite_name))
+class SuiteCheck(Check):
+ def per_suite_check(self, upload, suite):
+ if not suite.accept_source_uploads and upload.changes.source is not None:
+ raise Reject('The suite "{0}" does not accept source uploads.'.format(suite.suite_name))
+ if not suite.accept_binary_uploads and len(upload.changes.binaries) != 0:
+ raise Reject('The suite "{0}" does not accept binary uploads.'.format(suite.suite_name))
+ return True
+
class SuiteArchitectureCheck(Check):
def per_suite_check(self, upload, suite):
session = upload.session
else:
return db_binary.version
- def _version_checks(self, upload, suite, op):
+ def _version_checks(self, upload, suite, other_suite, op, op_name):
session = upload.session
if upload.changes.source is not None:
source_name = upload.changes.source.dsc['Source']
source_version = upload.changes.source.dsc['Version']
- v = self._highest_source_version(session, source_name, suite)
+ v = self._highest_source_version(session, source_name, other_suite)
if v is not None and not op(version_compare(source_version, v)):
- raise Reject('Version check failed (source={0}, version={1}, other-version={2}, suite={3})'.format(source_name, source_version, v, suite.suite_name))
+ raise Reject("Version check failed:\n"
+ "Your upload included the source package {0}, version {1},\n"
+ "however {3} already has version {2}.\n"
+ "Uploads to {5} must have a {4} version than present in {3}."
+ .format(source_name, source_version, v, other_suite.suite_name, op_name, suite.suite_name))
for binary in upload.changes.binaries:
binary_name = binary.control['Package']
binary_version = binary.control['Version']
architecture = binary.control['Architecture']
- v = self._highest_binary_version(session, binary_name, suite, architecture)
+ v = self._highest_binary_version(session, binary_name, other_suite, architecture)
if v is not None and not op(version_compare(binary_version, v)):
- raise Reject('Version check failed (binary={0}, version={1}, other-version={2}, suite={3})'.format(binary_name, binary_version, v, suite.suite_name))
+ raise Reject("Version check failed:\n"
+ "Your upload included the binary package {0}, version {1}, for {2},\n"
+ "however {4} already has version {3}.\n"
+ "Uploads to {6} must have a {5} version than present in {4}."
+ .format(binary_name, binary_version, architecture, v, other_suite.suite_name, op_name, suite.suite_name))
def per_suite_check(self, upload, suite):
session = upload.session
must_be_newer_than.append(suite)
for s in must_be_newer_than:
- self._version_checks(upload, s, lambda result: result > 0)
+ self._version_checks(upload, suite, s, lambda result: result > 0, 'higher')
vc_older = session.query(dbconn.VersionCheck).filter_by(suite=suite, check='MustBeOlderThan')
must_be_older_than = [ vc.reference for vc in vc_older ]
for s in must_be_older_than:
- self._version_checks(upload, s, lambda result: result < 0)
+ self._version_checks(upload, suite, s, lambda result: result < 0, 'lower')
return True
projects / dak.git / blobdiff