daklib/checks.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # Parts based on code that is
   4 # Copyright (C) 2001-2006, James Troup <james@nocrew.org>
   5 # Copyright (C) 2009-2010, Joerg Jaspert <joerg@debian.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License along
  18 # with this program; if not, write to the Free Software Foundation, Inc.,
  19 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  20
  21 """module provided pre-acceptance tests
  22
  23 Please read the documentation for the `Check` class for the interface.
  24 """
  25
  26 from daklib.config import Config
  27 from .dbconn import *
  28 import daklib.dbconn as dbconn
  29 from .regexes import *
  30 from .textutils import fix_maintainer, ParseMaintError
  31 import daklib.lintian as lintian
  32 import daklib.utils as utils
  33
  34 import apt_pkg
  35 from apt_pkg import version_compare
  36 import os
  37 import yaml
  38
  39 # TODO: replace by subprocess
  40 import commands
  41
  42 class Reject(Exception):
  43     """exception raised by failing checks"""
  44     pass
  45
  46 class Check(object):
  47     """base class for checks
  48
  49     checks are called by daklib.archive.ArchiveUpload.  Failing tests should
  50     raise a `daklib.checks.Reject` exception including a human-readable
  51     description why the upload should be rejected.
  52     """
  53     def check(self, upload):
  54         """do checks
  55
  56         Args:
  57            upload (daklib.archive.ArchiveUpload): upload to check
  58
  59         Raises:
  60            daklib.checks.Reject
  61         """
  62         raise NotImplemented
  63     def per_suite_check(self, upload, suite):
  64         """do per-suite checks
  65
  66         Args:
  67            upload (daklib.archive.ArchiveUpload): upload to check
  68            suite (daklib.dbconn.Suite): suite to check
  69
  70         Raises:
  71            daklib.checks.Reject
  72         """
  73         raise NotImplemented
  74     @property
  75     def forcable(self):
  76         """allow to force ignore failing test
  77
  78         True if it is acceptable to force ignoring a failing test,
  79         False otherwise
  80         """
  81         return False
  82
  83 class SignatureCheck(Check):
  84     """Check signature of changes and dsc file (if included in upload)
  85
  86     Make sure the signature is valid and done by a known user.
  87     """
  88     def check(self, upload):
  89         changes = upload.changes
  90         if not changes.valid_signature:
  91             raise Reject("Signature for .changes not valid.")
  92         if changes.source is not None:
  93             if not changes.source.valid_signature:
  94                 raise Reject("Signature for .dsc not valid.")
  95             if changes.source.primary_fingerprint != changes.primary_fingerprint:
  96                 raise Reject(".changes and .dsc not signed by the same key.")
  97         if upload.fingerprint is None or upload.fingerprint.uid is None:
  98             raise Reject(".changes signed by unknown key.")
  99
 100 class ChangesCheck(Check):
 101     """Check changes file for syntax errors."""
 102     def check(self, upload):
 103         changes = upload.changes
 104         control = changes.changes
 105         fn = changes.filename
 106
 107         for field in ('Distribution', 'Source', 'Binary', 'Architecture', 'Version', 'Maintainer', 'Files', 'Changes', 'Description'):
 108             if field not in control:
 109                 raise Reject('{0}: misses mandatory field {1}'.format(fn, field))
 110
 111         source_match = re_field_source.match(control['Source'])
 112         if not source_match:
 113             raise Reject('{0}: Invalid Source field'.format(fn))
 114         version_match = re_field_version.match(control['Version'])
 115         if not version_match:
 116             raise Reject('{0}: Invalid Version field'.format(fn))
 117         version_without_epoch = version_match.group('without_epoch')
 118
 119         match = re_file_changes.match(fn)
 120         if not match:
 121             raise Reject('{0}: Does not match re_file_changes'.format(fn))
 122         if match.group('package') != source_match.group('package'):
 123             raise Reject('{0}: Filename does not match Source field'.format(fn))
 124         if match.group('version') != version_without_epoch:
 125             raise Reject('{0}: Filename does not match Version field'.format(fn))
 126
 127         for bn in changes.binary_names:
 128             if not re_field_package.match(bn):
 129                 raise Reject('{0}: Invalid binary package name {1}'.format(fn, bn))
 130
 131         if 'source' in changes.architectures and changes.source is None:
 132             raise Reject("Changes has architecture source, but no source found.")
 133         if changes.source is not None and 'source' not in changes.architectures:
 134             raise Reject("Upload includes source, but changes does not say so.")
 135
 136         try:
 137             fix_maintainer(changes.changes['Maintainer'])
 138         except ParseMaintError as e:
 139             raise Reject('{0}: Failed to parse Maintainer field: {1}'.format(changes.filename, e))
 140
 141         try:
 142             changed_by = changes.changes.get('Changed-By')
 143             if changed_by is not None:
 144                 fix_maintainer(changed_by)
 145         except ParseMaintError as e:
 146             raise Reject('{0}: Failed to parse Changed-By field: {1}'.format(changes.filename, e))
 147
 148         if len(changes.files) == 0:
 149             raise Reject("Changes includes no files.")
 150
 151         for bugnum in changes.closed_bugs:
 152             if not re_isanum.match(bugnum):
 153                 raise Reject('{0}: "{1}" in Closes field is not a number'.format(changes.filename, bugnum))
 154
 155         return True
 156
 157 class HashesCheck(Check):
 158     """Check hashes in .changes and .dsc are valid."""
 159     def check(self, upload):
 160         changes = upload.changes
 161         for f in changes.files.itervalues():
 162             f.check(upload.directory)
 163             source = changes.source
 164         if source is not None:
 165             for f in source.files.itervalues():
 166                 f.check(upload.directory)
 167
 168 class BinaryCheck(Check):
 169     """Check binary packages for syntax errors."""
 170     def check(self, upload):
 171         for binary in upload.changes.binaries:
 172             self.check_binary(upload, binary)
 173
 174         binary_names = set([ binary.control['Package'] for binary in upload.changes.binaries ])
 175         for bn in binary_names:
 176             if bn not in upload.changes.binary_names:
 177                 raise Reject('Package {0} is not mentioned in Binary field in changes'.format(bn))
 178
 179         return True
 180
 181     def check_binary(self, upload, binary):
 182         fn = binary.hashed_file.filename
 183         control = binary.control
 184
 185         for field in ('Package', 'Architecture', 'Version', 'Description'):
 186             if field not in control:
 187                 raise Reject('{0}: Missing mandatory field {0}.'.format(fn, field))
 188
 189         # check fields
 190
 191         package = control['Package']
 192         if not re_field_package.match(package):
 193             raise Reject('{0}: Invalid Package field'.format(fn))
 194
 195         version = control['Version']
 196         version_match = re_field_version.match(version)
 197         if not version_match:
 198             raise Reject('{0}: Invalid Version field'.format(fn))
 199         version_without_epoch = version_match.group('without_epoch')
 200
 201         architecture = control['Architecture']
 202         if architecture not in upload.changes.architectures:
 203             raise Reject('{0}: Architecture not in Architecture field in changes file'.format(fn))
 204         if architecture == 'source':
 205             raise Reject('{0}: Architecture "source" invalid for binary packages'.format(fn))
 206
 207         source = control.get('Source')
 208         if source is not None and not re_field_source.match(source):
 209             raise Reject('{0}: Invalid Source field'.format(fn))
 210
 211         # check filename
 212
 213         match = re_file_binary.match(fn)
 214         if package != match.group('package'):
 215             raise Reject('{0}: filename does not match Package field'.format(fn))
 216         if version_without_epoch != match.group('version'):
 217             raise Reject('{0}: filename does not match Version field'.format(fn))
 218         if architecture != match.group('architecture'):
 219             raise Reject('{0}: filename does not match Architecture field'.format(fn))
 220
 221         # check dependency field syntax
 222
 223         for field in ('Breaks', 'Conflicts', 'Depends', 'Enhances', 'Pre-Depends',
 224                       'Provides', 'Recommends', 'Replaces', 'Suggests'):
 225             value = control.get(field)
 226             if value is not None:
 227                 if value.strip() == '':
 228                     raise Reject('{0}: empty {1} field'.format(fn, field))
 229                 try:
 230                     apt_pkg.parse_depends(value)
 231                 except:
 232                     raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
 233
 234         for field in ('Built-Using',):
 235             value = control.get(field)
 236             if value is not None:
 237                 if value.strip() == '':
 238                     raise Reject('{0}: empty {1} field'.format(fn, field))
 239                 try:
 240                     apt_pkg.parse_src_depends(value)
 241                 except:
 242                     raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
 243
 244 class SourceCheck(Check):
 245     """Check source package for syntax errors."""
 246     def check_filename(self, control, filename, regex):
 247         # In case we have an .orig.tar.*, we have to strip the Debian revison
 248         # from the version number. So handle this special case first.
 249         is_orig = True
 250         match = re_file_orig.match(filename)
 251         if not match:
 252             is_orig = False
 253             match = regex.match(filename)
 254
 255         if not match:
 256             raise Reject('{0}: does not match regular expression for source filenames'.format(filename))
 257         if match.group('package') != control['Source']:
 258             raise Reject('{0}: filename does not match Source field'.format(filename))
 259
 260         version = control['Version']
 261         if is_orig:
 262             version = re_field_version_upstream.match(version).group('upstream')
 263         version_match =  re_field_version.match(version)
 264         version_without_epoch = version_match.group('without_epoch')
 265         if match.group('version') != version_without_epoch:
 266             raise Reject('{0}: filename does not match Version field'.format(filename))
 267
 268     def check(self, upload):
 269         if upload.changes.source is None:
 270             return True
 271
 272         changes = upload.changes.changes
 273         source = upload.changes.source
 274         control = source.dsc
 275         dsc_fn = source._dsc_file.filename
 276
 277         # check fields
 278         if not re_field_package.match(control['Source']):
 279             raise Reject('{0}: Invalid Source field'.format(dsc_fn))
 280         if control['Source'] != changes['Source']:
 281             raise Reject('{0}: Source field does not match Source field in changes'.format(dsc_fn))
 282         if control['Version'] != changes['Version']:
 283             raise Reject('{0}: Version field does not match Version field in changes'.format(dsc_fn))
 284
 285         # check filenames
 286         self.check_filename(control, dsc_fn, re_file_dsc)
 287         for f in source.files.itervalues():
 288             self.check_filename(control, f.filename, re_file_source)
 289
 290         # check dependency field syntax
 291         for field in ('Build-Conflicts', 'Build-Conflicts-Indep', 'Build-Depends', 'Build-Depends-Arch', 'Build-Depends-Indep'):
 292             value = control.get(field)
 293             if value is not None:
 294                 if value.strip() == '':
 295                     raise Reject('{0}: empty {1} field'.format(dsc_fn, field))
 296                 try:
 297                     apt_pkg.parse_src_depends(value)
 298                 except Exception as e:
 299                     raise Reject('{0}: APT could not parse {1} field: {2}'.format(dsc_fn, field, e))
 300
 301         # TODO: check all expected files for given source format are included
 302
 303 class SingleDistributionCheck(Check):
 304     """Check that the .changes targets only a single distribution."""
 305     def check(self, upload):
 306         if len(upload.changes.distributions) != 1:
 307             raise Reject("Only uploads to a single distribution are allowed.")
 308
 309 class ACLCheck(Check):
 310     """Check the uploader is allowed to upload the packages in .changes"""
 311     def _check_dm(self, upload):
 312         # This code is not very nice, but hopefully works until we can replace
 313         # DM-Upload-Allowed, cf. https://lists.debian.org/debian-project/2012/06/msg00029.html
 314         session = upload.session
 315
 316         if 'source' not in upload.changes.architectures:
 317             raise Reject('DM uploads must include source')
 318         distributions = upload.changes.distributions
 319         for dist in distributions:
 320             if dist not in ('unstable', 'experimental', 'squeeze-backports'):
 321                 raise Reject("Uploading to {0} is not allowed for DMs.".format(dist))
 322         for f in upload.changes.files.itervalues():
 323             if f.section == 'byhand' or f.section[:4] == "raw-":
 324                 raise Reject("Uploading byhand packages is not allowed for DMs.")
 325
 326         # Reject NEW packages
 327         assert len(distributions) == 1
 328         suite = session.query(Suite).filter_by(suite_name=distributions[0]).one()
 329         overridesuite = suite
 330         if suite.overridesuite is not None:
 331             overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 332         if upload._check_new(overridesuite):
 333             raise Reject('Uploading NEW packages is not allowed for DMs.')
 334
 335         # Check DM-Upload-Allowed
 336         last_suites = ['unstable', 'experimental']
 337         if suite.suite_name.endswith('-backports'):
 338             last_suites = [suite.suite_name]
 339         last = session.query(DBSource).filter_by(source=upload.changes.changes['Source']) \
 340             .join(DBSource.suites).filter(Suite.suite_name.in_(last_suites)) \
 341             .order_by(DBSource.version.desc()).limit(1).first()
 342         if last is None:
 343             raise Reject('No existing source found in {0}'.format(' or '.join(last_suites)))
 344         if not last.dm_upload_allowed:
 345             raise Reject('DM-Upload-Allowed is not set in {0}={1}'.format(last.source, last.version))
 346
 347         # check current Changed-by is in last Maintainer or Uploaders
 348         uploader_names = [ u.name for u in last.uploaders ]
 349         changed_by_field = upload.changes.changes.get('Changed-By', upload.changes.changes['Maintainer'])
 350         if changed_by_field not in uploader_names:
 351             raise Reject('{0} is not an uploader for {1}={2}'.format(changed_by_field, last.source, last.version))
 352
 353         # check Changed-by is the DM
 354         changed_by = fix_maintainer(changed_by_field)
 355         uid = upload.fingerprint.uid
 356         if uid is None:
 357             raise Reject('Unknown uid for fingerprint {0}'.format(upload.fingerprint.fingerprint))
 358         if uid.uid != changed_by[3] and uid.name != changed_by[2]:
 359             raise Reject('DMs are not allowed to sponsor uploads (expected {0} <{1}> as maintainer, but got {2})'.format(uid.name, uid.uid, changed_by_field))
 360
 361         # Try to catch hijacks.
 362         # This doesn't work correctly. Uploads to experimental can still
 363         # "hijack" binaries from unstable. Also one can hijack packages
 364         # via buildds (but people who try this should not be DMs).
 365         for binary_name in upload.changes.binary_names:
 366             binaries = session.query(DBBinary).join(DBBinary.source) \
 367                 .join(DBBinary.suites).filter(Suite.suite_name.in_(upload.changes.distributions)) \
 368                 .filter(DBBinary.package == binary_name)
 369             for binary in binaries:
 370                 if binary.source.source != upload.changes.changes['Source']:
 371                     raise Reject('DMs must not hijack binaries (binary={0}, other-source={1})'.format(binary_name, binary.source.source))
 372
 373         return True
 374
 375     def check(self, upload):
 376         fingerprint = upload.fingerprint
 377         source_acl = fingerprint.source_acl
 378         if source_acl is None:
 379             if 'source' in upload.changes.architectures:
 380                 raise Reject('Fingerprint {0} must not upload source'.format(fingerprint.fingerprint))
 381         elif source_acl.access_level == 'dm':
 382             self._check_dm(upload)
 383         elif source_acl.access_level != 'full':
 384             raise Reject('Unknown source_acl access level {0} for fingerprint {1}'.format(source_acl.access_level, fingerprint.fingerprint))
 385
 386         bin_architectures = set(upload.changes.architectures)
 387         bin_architectures.discard('source')
 388         binary_acl = fingerprint.binary_acl
 389         if binary_acl is None:
 390             if len(bin_architectures) > 0:
 391                 raise Reject('Fingerprint {0} must not upload binary packages'.format(fingerprint.fingerprint))
 392         elif binary_acl.access_level == 'map':
 393             query = upload.session.query(BinaryACLMap).filter_by(fingerprint=fingerprint)
 394             allowed_architectures = [ m.architecture.arch_string for m in query ]
 395
 396             for arch in upload.changes.architectures:
 397                 if arch not in allowed_architectures:
 398                     raise Reject('Fingerprint {0} must not  upload binaries for architecture {1}'.format(fingerprint.fingerprint, arch))
 399         elif binary_acl.access_level != 'full':
 400             raise Reject('Unknown binary_acl access level {0} for fingerprint {1}'.format(binary_acl.access_level, fingerprint.fingerprint))
 401
 402         return True
 403
 404 class NoSourceOnlyCheck(Check):
 405     """Check for source-only upload
 406
 407     Source-only uploads are only allowed if Dinstall::AllowSourceOnlyUploads is
 408     set. Otherwise they are rejected.
 409     """
 410     def check(self, upload):
 411         if Config().find_b("Dinstall::AllowSourceOnlyUploads"):
 412             return True
 413         changes = upload.changes
 414         if changes.source is not None and len(changes.binaries) == 0:
 415             raise Reject('Source-only uploads are not allowed.')
 416         return True
 417
 418 class LintianCheck(Check):
 419     """Check package using lintian"""
 420     def check(self, upload):
 421         changes = upload.changes
 422
 423         # Only check sourceful uploads.
 424         if changes.source is None:
 425             return True
 426         # Only check uploads to unstable or experimental.
 427         if 'unstable' not in changes.distributions and 'experimental' not in changes.distributions:
 428             return True
 429
 430         cnf = Config()
 431         if 'Dinstall::LintianTags' not in cnf:
 432             return True
 433         tagfile = cnf['Dinstall::LintianTags']
 434
 435         with open(tagfile, 'r') as sourcefile:
 436             sourcecontent = sourcefile.read()
 437         try:
 438             lintiantags = yaml.load(sourcecontent)['lintian']
 439         except yaml.YAMLError as msg:
 440             raise Exception('Could not read lintian tags file {0}, YAML error: {1}'.format(tagfile, msg))
 441
 442         fd, temp_filename = utils.temp_filename()
 443         temptagfile = os.fdopen(fd, 'w')
 444         for tags in lintiantags.itervalues():
 445             for tag in tags:
 446                 print >>temptagfile, tag
 447         temptagfile.close()
 448
 449         changespath = os.path.join(upload.directory, changes.filename)
 450         try:
 451             # FIXME: no shell
 452             cmd = "lintian --show-overrides --tags-from-file {0} {1}".format(temp_filename, changespath)
 453             result, output = commands.getstatusoutput(cmd)
 454         finally:
 455             os.unlink(temp_filename)
 456
 457         if result == 2:
 458             utils.warn("lintian failed for %s [return code: %s]." % \
 459                 (changespath, result))
 460             utils.warn(utils.prefix_multi_line_string(output, \
 461                 " [possible output:] "))
 462
 463         parsed_tags = lintian.parse_lintian_output(output)
 464         rejects = list(lintian.generate_reject_messages(parsed_tags, lintiantags))
 465         if len(rejects) != 0:
 466             raise Reject('\n'.join(rejects))
 467
 468         return True
 469
 470 class SourceFormatCheck(Check):
 471     """Check source format is allowed in the target suite"""
 472     def per_suite_check(self, upload, suite):
 473         source = upload.changes.source
 474         session = upload.session
 475         if source is None:
 476             return True
 477
 478         source_format = source.dsc['Format']
 479         query = session.query(SrcFormat).filter_by(format_name=source_format).filter(SrcFormat.suites.contains(suite))
 480         if query.first() is None:
 481             raise Reject('source format {0} is not allowed in suite {1}'.format(source_format, suite.suite_name))
 482
 483 class SuiteArchitectureCheck(Check):
 484     def per_suite_check(self, upload, suite):
 485         session = upload.session
 486         for arch in upload.changes.architectures:
 487             query = session.query(Architecture).filter_by(arch_string=arch).filter(Architecture.suites.contains(suite))
 488             if query.first() is None:
 489                 raise Reject('Architecture {0} is not allowed in suite {2}'.format(arch, suite.suite_name))
 490
 491         return True
 492
 493 class VersionCheck(Check):
 494     """Check version constraints"""
 495     def _highest_source_version(self, session, source_name, suite):
 496         db_source = session.query(DBSource).filter_by(source=source_name) \
 497             .filter(DBSource.suites.contains(suite)).order_by(DBSource.version.desc()).first()
 498         if db_source is None:
 499             return None
 500         else:
 501             return db_source.version
 502
 503     def _highest_binary_version(self, session, binary_name, suite, architecture):
 504         db_binary = session.query(DBBinary).filter_by(package=binary_name) \
 505             .filter(DBBinary.suites.contains(suite)) \
 506             .filter(Architecture.arch_string.in_(['all', architecture])) \
 507             .order_by(DBBinary.version.desc()).first()
 508         if db_binary is None:
 509             return None
 510         else:
 511             return db_binary.version
 512
 513     def _version_checks(self, upload, suite, expected_result):
 514         session = upload.session
 515
 516         if upload.changes.source is not None:
 517             source_name = upload.changes.source.dsc['Source']
 518             source_version = upload.changes.source.dsc['Version']
 519             v = self._highest_source_version(session, source_name, suite)
 520             if v is not None and version_compare(source_version, v) != expected_result:
 521                 raise Reject('Version check failed (source={0}, version={1}, suite={2})'.format(source_name, source_version, suite.suite_name))
 522
 523         for binary in upload.changes.binaries:
 524             binary_name = binary.control['Package']
 525             binary_version = binary.control['Version']
 526             architecture = binary.control['Architecture']
 527             v = self._highest_binary_version(session, binary_name, suite, architecture)
 528             if v is not None and version_compare(binary_version, v) != expected_result:
 529                 raise Reject('Version check failed (binary={0}, version={1}, suite={2})'.format(binary_name, binary_version, suite.suite_name))
 530
 531     def per_suite_check(self, upload, suite):
 532         session = upload.session
 533
 534         vc_newer = session.query(dbconn.VersionCheck).filter_by(suite=suite) \
 535             .filter(dbconn.VersionCheck.check.in_(['MustBeNewerThan', 'Enhances']))
 536         must_be_newer_than = [ vc.reference for vc in vc_newer ]
 537         # Must be newer than old versions in `suite`
 538         must_be_newer_than.append(suite)
 539
 540         for s in must_be_newer_than:
 541             self._version_checks(upload, s, 1)
 542
 543         vc_older = session.query(dbconn.VersionCheck).filter_by(suite=suite, check='MustBeOlderThan')
 544         must_be_older_than = [ vc.reference for vc in vc_older ]
 545
 546         for s in must_be_older_than:
 547             self._version_checks(upload, s, -1)
 548
 549         return True
 550
 551     @property
 552     def forcable(self):
 553         return True