daklib/checks.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # Parts based on code that is
   4 # Copyright (C) 2001-2006, James Troup <james@nocrew.org>
   5 # Copyright (C) 2009-2010, Joerg Jaspert <joerg@debian.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License along
  18 # with this program; if not, write to the Free Software Foundation, Inc.,
  19 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  20
  21 """module provided pre-acceptance tests
  22
  23 Please read the documentation for the `Check` class for the interface.
  24 """
  25
  26 from daklib.config import Config
  27 from .dbconn import *
  28 import daklib.dbconn as dbconn
  29 from .regexes import *
  30 from .textutils import fix_maintainer, ParseMaintError
  31 import daklib.lintian as lintian
  32 import daklib.utils as utils
  33
  34 import apt_pkg
  35 from apt_pkg import version_compare
  36 import os
  37 import yaml
  38
  39 # TODO: replace by subprocess
  40 import commands
  41
  42 class Reject(Exception):
  43     """exception raised by failing checks"""
  44     pass
  45
  46 class Check(object):
  47     """base class for checks
  48
  49     checks are called by daklib.archive.ArchiveUpload.  Failing tests should
  50     raise a `daklib.checks.Reject` exception including a human-readable
  51     description why the upload should be rejected.
  52     """
  53     def check(self, upload):
  54         """do checks
  55
  56         Args:
  57            upload (daklib.archive.ArchiveUpload): upload to check
  58
  59         Raises:
  60            daklib.checks.Reject
  61         """
  62         raise NotImplemented
  63     def per_suite_check(self, upload, suite):
  64         """do per-suite checks
  65
  66         Args:
  67            upload (daklib.archive.ArchiveUpload): upload to check
  68            suite (daklib.dbconn.Suite): suite to check
  69
  70         Raises:
  71            daklib.checks.Reject
  72         """
  73         raise NotImplemented
  74     @property
  75     def forcable(self):
  76         """allow to force ignore failing test
  77
  78         True if it is acceptable to force ignoring a failing test,
  79         False otherwise
  80         """
  81         return False
  82
  83 class SignatureCheck(Check):
  84     """Check signature of changes and dsc file (if included in upload)
  85
  86     Make sure the signature is valid and done by a known user.
  87     """
  88     def check(self, upload):
  89         changes = upload.changes
  90         if not changes.valid_signature:
  91             raise Reject("Signature for .changes not valid.")
  92         if changes.source is not None:
  93             if not changes.source.valid_signature:
  94                 raise Reject("Signature for .dsc not valid.")
  95             if changes.source.primary_fingerprint != changes.primary_fingerprint:
  96                 raise Reject(".changes and .dsc not signed by the same key.")
  97         if upload.fingerprint is None or upload.fingerprint.uid is None:
  98             raise Reject(".changes signed by unknown key.")
  99
 100 class ChangesCheck(Check):
 101     """Check changes file for syntax errors."""
 102     def check(self, upload):
 103         changes = upload.changes
 104         control = changes.changes
 105         fn = changes.filename
 106
 107         for field in ('Distribution', 'Source', 'Binary', 'Architecture', 'Version', 'Maintainer', 'Files', 'Changes', 'Description'):
 108             if field not in control:
 109                 raise Reject('{0}: misses mandatory field {1}'.format(fn, field))
 110
 111         source_match = re_field_source.match(control['Source'])
 112         if not source_match:
 113             raise Reject('{0}: Invalid Source field'.format(fn))
 114         version_match = re_field_version.match(control['Version'])
 115         if not version_match:
 116             raise Reject('{0}: Invalid Version field'.format(fn))
 117         version_without_epoch = version_match.group('without_epoch')
 118
 119         match = re_file_changes.match(fn)
 120         if not match:
 121             raise Reject('{0}: Does not match re_file_changes'.format(fn))
 122         if match.group('package') != source_match.group('package'):
 123             raise Reject('{0}: Filename does not match Source field'.format(fn))
 124         if match.group('version') != version_without_epoch:
 125             raise Reject('{0}: Filename does not match Version field'.format(fn))
 126
 127         for bn in changes.binary_names:
 128             if not re_field_package.match(bn):
 129                 raise Reject('{0}: Invalid binary package name {1}'.format(fn, bn))
 130
 131         if 'source' in changes.architectures and changes.source is None:
 132             raise Reject("Changes has architecture source, but no source found.")
 133         if changes.source is not None and 'source' not in changes.architectures:
 134             raise Reject("Upload includes source, but changes does not say so.")
 135
 136         try:
 137             fix_maintainer(changes.changes['Maintainer'])
 138         except ParseMaintError as e:
 139             raise Reject('{0}: Failed to parse Maintainer field: {1}'.format(changes.filename, e))
 140
 141         try:
 142             changed_by = changes.changes.get('Changed-By')
 143             if changed_by is not None:
 144                 fix_maintainer(changed_by)
 145         except ParseMaintError as e:
 146             raise Reject('{0}: Failed to parse Changed-By field: {1}'.format(changes.filename, e))
 147
 148         if len(changes.files) == 0:
 149             raise Reject("Changes includes no files.")
 150
 151         for bugnum in changes.closed_bugs:
 152             if not re_isanum.match(bugnum):
 153                 raise Reject('{0}: "{1}" in Closes field is not a number'.format(changes.filename, bugnum))
 154
 155         return True
 156
 157 class HashesCheck(Check):
 158     """Check hashes in .changes and .dsc are valid."""
 159     def check(self, upload):
 160         changes = upload.changes
 161         for f in changes.files.itervalues():
 162             f.check(upload.directory)
 163             source = changes.source
 164         if source is not None:
 165             for f in source.files.itervalues():
 166                 f.check(upload.directory)
 167
 168 class BinaryCheck(Check):
 169     """Check binary packages for syntax errors."""
 170     def check(self, upload):
 171         for binary in upload.changes.binaries:
 172             self.check_binary(upload, binary)
 173
 174         binary_names = set([ binary.control['Package'] for binary in upload.changes.binaries ])
 175         for bn in binary_names:
 176             if bn not in upload.changes.binary_names:
 177                 raise Reject('Package {0} is not mentioned in Binary field in changes'.format(bn))
 178         for bn in upload.changes.binary_names:
 179             if bn not in binary_names:
 180                 raise Reject('Binary field in changes has {0}, but was not found in upload'.format(bn))
 181
 182         return True
 183
 184     def check_binary(self, upload, binary):
 185         fn = binary.hashed_file.filename
 186         control = binary.control
 187
 188         for field in ('Package', 'Architecture', 'Version', 'Description'):
 189             if field not in control:
 190                 raise Reject('{0}: Missing mandatory field {0}.'.format(fn, field))
 191
 192         # check fields
 193
 194         package = control['Package']
 195         if not re_field_package.match(package):
 196             raise Reject('{0}: Invalid Package field'.format(fn))
 197
 198         version = control['Version']
 199         version_match = re_field_version.match(version)
 200         if not version_match:
 201             raise Reject('{0}: Invalid Version field'.format(fn))
 202         version_without_epoch = version_match.group('without_epoch')
 203
 204         architecture = control['Architecture']
 205         if architecture not in upload.changes.architectures:
 206             raise Reject('{0}: Architecture not in Architecture field in changes file'.format(fn))
 207         if architecture == 'source':
 208             raise Reject('{0}: Architecture "source" invalid for binary packages'.format(fn))
 209
 210         source = control.get('Source')
 211         if source is not None and not re_field_source.match(source):
 212             raise Reject('{0}: Invalid Source field'.format(fn))
 213
 214         # check filename
 215
 216         match = re_file_binary.match(fn)
 217         if package != match.group('package'):
 218             raise Reject('{0}: filename does not match Package field'.format(fn))
 219         if version_without_epoch != match.group('version'):
 220             raise Reject('{0}: filename does not match Version field'.format(fn))
 221         if architecture != match.group('architecture'):
 222             raise Reject('{0}: filename does not match Architecture field'.format(fn))
 223
 224         # check dependency field syntax
 225
 226         for field in ('Breaks', 'Conflicts', 'Depends', 'Enhances', 'Pre-Depends',
 227                       'Provides', 'Recommends', 'Replaces', 'Suggests'):
 228             value = control.get(field)
 229             if value is not None:
 230                 if value.strip() == '':
 231                     raise Reject('{0}: empty {1} field'.format(fn, field))
 232                 try:
 233                     apt_pkg.parse_depends(value)
 234                 except:
 235                     raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
 236
 237         for field in ('Built-Using',):
 238             value = control.get(field)
 239             if value is not None:
 240                 if value.strip() == '':
 241                     raise Reject('{0}: empty {1} field'.format(fn, field))
 242                 try:
 243                     apt_pkg.parse_src_depends(value)
 244                 except:
 245                     raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
 246
 247 class SourceCheck(Check):
 248     """Check source package for syntax errors."""
 249     def check_filename(self, control, filename, regex):
 250         # In case we have an .orig.tar.*, we have to strip the Debian revison
 251         # from the version number. So handle this special case first.
 252         is_orig = True
 253         match = re_file_orig.match(filename)
 254         if not match:
 255             is_orig = False
 256             match = regex.match(filename)
 257
 258         if not match:
 259             raise Reject('{0}: does not match regular expression for source filenames'.format(filename))
 260         if match.group('package') != control['Source']:
 261             raise Reject('{0}: filename does not match Source field'.format(filename))
 262
 263         version = control['Version']
 264         if is_orig:
 265             version = re_field_version_upstream.match(version).group('upstream')
 266         version_match =  re_field_version.match(version)
 267         version_without_epoch = version_match.group('without_epoch')
 268         if match.group('version') != version_without_epoch:
 269             raise Reject('{0}: filename does not match Version field'.format(filename))
 270
 271     def check(self, upload):
 272         if upload.changes.source is None:
 273             return True
 274
 275         changes = upload.changes.changes
 276         source = upload.changes.source
 277         control = source.dsc
 278         dsc_fn = source._dsc_file.filename
 279
 280         # check fields
 281         if not re_field_package.match(control['Source']):
 282             raise Reject('{0}: Invalid Source field'.format(dsc_fn))
 283         if control['Source'] != changes['Source']:
 284             raise Reject('{0}: Source field does not match Source field in changes'.format(dsc_fn))
 285         if control['Version'] != changes['Version']:
 286             raise Reject('{0}: Version field does not match Version field in changes'.format(dsc_fn))
 287
 288         # check filenames
 289         self.check_filename(control, dsc_fn, re_file_dsc)
 290         for f in source.files.itervalues():
 291             self.check_filename(control, f.filename, re_file_source)
 292
 293         # check dependency field syntax
 294         for field in ('Build-Conflicts', 'Build-Conflicts-Indep', 'Build-Depends', 'Build-Depends-Arch', 'Build-Depends-Indep'):
 295             value = control.get(field)
 296             if value is not None:
 297                 if value.strip() == '':
 298                     raise Reject('{0}: empty {1} field'.format(dsc_fn, field))
 299                 try:
 300                     apt_pkg.parse_src_depends(value)
 301                 except Exception as e:
 302                     raise Reject('{0}: APT could not parse {1} field: {2}'.format(dsc_fn, field, e))
 303
 304         # TODO: check all expected files for given source format are included
 305
 306 class SingleDistributionCheck(Check):
 307     """Check that the .changes targets only a single distribution."""
 308     def check(self, upload):
 309         if len(upload.changes.distributions) != 1:
 310             raise Reject("Only uploads to a single distribution are allowed.")
 311
 312 class ACLCheck(Check):
 313     """Check the uploader is allowed to upload the packages in .changes"""
 314     def _check_dm(self, upload):
 315         # This code is not very nice, but hopefully works until we can replace
 316         # DM-Upload-Allowed, cf. https://lists.debian.org/debian-project/2012/06/msg00029.html
 317         session = upload.session
 318
 319         if 'source' not in upload.changes.architectures:
 320             raise Reject('DM uploads must include source')
 321         distributions = upload.changes.distributions
 322         for dist in distributions:
 323             if dist not in ('unstable', 'experimental', 'squeeze-backports'):
 324                 raise Reject("Uploading to {0} is not allowed for DMs.".format(dist))
 325         for f in upload.changes.files.itervalues():
 326             if f.section == 'byhand' or f.section[:4] == "raw-":
 327                 raise Reject("Uploading byhand packages is not allowed for DMs.")
 328
 329         # Reject NEW packages
 330         assert len(distributions) == 1
 331         suite = session.query(Suite).filter_by(suite_name=distributions[0]).one()
 332         overridesuite = suite
 333         if suite.overridesuite is not None:
 334             overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 335         if upload._check_new(overridesuite):
 336             raise Reject('Uploading NEW packages is not allowed for DMs.')
 337
 338         # Check DM-Upload-Allowed
 339         last_suites = ['unstable', 'experimental']
 340         if suite.suite_name.endswith('-backports'):
 341             last_suites = [suite.suite_name]
 342         last = session.query(DBSource).filter_by(source=upload.changes.changes['Source']) \
 343             .join(DBSource.suites).filter(Suite.suite_name.in_(last_suites)) \
 344             .order_by(DBSource.version.desc()).limit(1).first()
 345         if last is None:
 346             raise Reject('No existing source found in {0}'.format(' or '.join(last_suites)))
 347         if not last.dm_upload_allowed:
 348             raise Reject('DM-Upload-Allowed is not set in {0}={1}'.format(last.source, last.version))
 349
 350         # check current Changed-by is in last Maintainer or Uploaders
 351         uploader_names = [ u.name for u in last.uploaders ]
 352         changed_by_field = upload.changes.changes.get('Changed-By', upload.changes.changes['Maintainer'])
 353         if changed_by_field not in uploader_names:
 354             raise Reject('{0} is not an uploader for {1}={2}'.format(changed_by_field, last.source, last.version))
 355
 356         # check Changed-by is the DM
 357         changed_by = fix_maintainer(changed_by_field)
 358         uid = upload.fingerprint.uid
 359         if uid is None:
 360             raise Reject('Unknown uid for fingerprint {0}'.format(upload.fingerprint.fingerprint))
 361         if uid.uid != changed_by[3] and uid.name != changed_by[2]:
 362             raise Reject('DMs are not allowed to sponsor uploads (expected {0} <{1}> as maintainer, but got {2})'.format(uid.name, uid.uid, changed_by_field))
 363
 364         # Try to catch hijacks.
 365         # This doesn't work correctly. Uploads to experimental can still
 366         # "hijack" binaries from unstable. Also one can hijack packages
 367         # via buildds (but people who try this should not be DMs).
 368         for binary_name in upload.changes.binary_names:
 369             binaries = session.query(DBBinary).join(DBBinary.source) \
 370                 .join(DBBinary.suites).filter(Suite.suite_name.in_(upload.changes.distributions)) \
 371                 .filter(DBBinary.package == binary_name)
 372             for binary in binaries:
 373                 if binary.source.source != upload.changes.changes['Source']:
 374                     raise Reject('DMs must not hijack binaries (binary={0}, other-source={1})'.format(binary_name, binary.source.source))
 375
 376         return True
 377
 378     def check(self, upload):
 379         fingerprint = upload.fingerprint
 380         source_acl = fingerprint.source_acl
 381         if source_acl is None:
 382             if 'source' in upload.changes.architectures:
 383                 raise Reject('Fingerprint {0} must not upload source'.format(fingerprint.fingerprint))
 384         elif source_acl.access_level == 'dm':
 385             self._check_dm(upload)
 386         elif source_acl.access_level != 'full':
 387             raise Reject('Unknown source_acl access level {0} for fingerprint {1}'.format(source_acl.access_level, fingerprint.fingerprint))
 388
 389         bin_architectures = set(upload.changes.architectures)
 390         bin_architectures.remove('source')
 391         binary_acl = fingerprint.binary_acl
 392         if binary_acl is None:
 393             if len(bin_architectures) > 0:
 394                 raise Reject('Fingerprint {0} must not upload binary packages'.format(fingerprint.fingerprint))
 395         elif binary_acl.access_level == 'map':
 396             query = upload.session.query(BinaryACLMap).filter_by(fingerprint=fingerprint)
 397             allowed_architectures = [ m.architecture.arch_string for m in query ]
 398
 399             for arch in upload.changes.architectures:
 400                 if arch not in allowed_architectures:
 401                     raise Reject('Fingerprint {0} must not  upload binaries for architecture {1}'.format(fingerprint.fingerprint, arch))
 402         elif binary_acl.access_level != 'full':
 403             raise Reject('Unknown binary_acl access level {0} for fingerprint {1}'.format(binary_acl.access_level, fingerprint.fingerprint))
 404
 405         return True
 406
 407 class NoSourceOnlyCheck(Check):
 408     """Check for source-only upload
 409
 410     Source-only uploads are only allowed if Dinstall::AllowSourceOnlyUploads is
 411     set. Otherwise they are rejected.
 412     """
 413     def check(self, upload):
 414         if Config().find_b("Dinstall::AllowSourceOnlyUploads"):
 415             return True
 416         changes = upload.changes
 417         if changes.source is not None and len(changes.binaries) == 0:
 418             raise Reject('Source-only uploads are not allowed.')
 419         return True
 420
 421 class LintianCheck(Check):
 422     """Check package using lintian"""
 423     def check(self, upload):
 424         changes = upload.changes
 425
 426         # Only check sourceful uploads.
 427         if changes.source is None:
 428             return True
 429         # Only check uploads to unstable or experimental.
 430         if 'unstable' not in changes.distributions and 'experimental' not in changes.distributions:
 431             return True
 432
 433         cnf = Config()
 434         if 'Dinstall::LintianTags' not in cnf:
 435             return True
 436         tagfile = cnf['Dinstall::LintianTags']
 437
 438         with open(tagfile, 'r') as sourcefile:
 439             sourcecontent = sourcefile.read()
 440         try:
 441             lintiantags = yaml.load(sourcecontent)['lintian']
 442         except yaml.YAMLError as msg:
 443             raise Exception('Could not read lintian tags file {0}, YAML error: {1}'.format(tagfile, msg))
 444
 445         fd, temp_filename = utils.temp_filename()
 446         temptagfile = os.fdopen(fd, 'w')
 447         for tags in lintiantags.itervalues():
 448             for tag in tags:
 449                 print >>temptagfile, tag
 450         temptagfile.close()
 451
 452         changespath = os.path.join(upload.directory, changes.filename)
 453         try:
 454             # FIXME: no shell
 455             cmd = "lintian --show-overrides --tags-from-file {0} {1}".format(temp_filename, changespath)
 456             result, output = commands.getstatusoutput(cmd)
 457         finally:
 458             os.unlink(temp_filename)
 459
 460         if result == 2:
 461             utils.warn("lintian failed for %s [return code: %s]." % \
 462                 (changespath, result))
 463             utils.warn(utils.prefix_multi_line_string(output, \
 464                 " [possible output:] "))
 465
 466         parsed_tags = lintian.parse_lintian_output(output)
 467         rejects = list(lintian.generate_reject_messages(parsed_tags, lintiantags))
 468         if len(rejects) != 0:
 469             raise Reject('\n'.join(rejects))
 470
 471         return True
 472
 473 class SourceFormatCheck(Check):
 474     """Check source format is allowed in the target suite"""
 475     def per_suite_check(self, upload, suite):
 476         source = upload.changes.source
 477         session = upload.session
 478         if source is None:
 479             return True
 480
 481         source_format = source.dsc['Format']
 482         query = session.query(SrcFormat).filter_by(format_name=source_format).filter(SrcFormat.suites.contains(suite))
 483         if query.first() is None:
 484             raise Reject('source format {0} is not allowed in suite {1}'.format(source_format, suite.suite_name))
 485
 486 class SuiteArchitectureCheck(Check):
 487     def per_suite_check(self, upload, suite):
 488         session = upload.session
 489         for arch in upload.changes.architectures:
 490             query = session.query(Architecture).filter_by(arch_string=arch).filter(Architecture.suites.contains(suite))
 491             if query.first() is None:
 492                 raise Reject('Architecture {0} is not allowed in suite {2}'.format(arch, suite.suite_name))
 493
 494         return True
 495
 496 class VersionCheck(Check):
 497     """Check version constraints"""
 498     def _highest_source_version(self, session, source_name, suite):
 499         db_source = session.query(DBSource).filter_by(source=source_name) \
 500             .filter(DBSource.suites.contains(suite)).order_by(DBSource.version.desc()).first()
 501         if db_source is None:
 502             return None
 503         else:
 504             return db_source.version
 505
 506     def _highest_binary_version(self, session, binary_name, suite, architecture):
 507         db_binary = session.query(DBBinary).filter_by(package=binary_name) \
 508             .filter(DBBinary.suites.contains(suite)) \
 509             .filter(Architecture.arch_string.in_(['all', architecture])) \
 510             .order_by(DBBinary.version.desc()).first()
 511         if db_binary is None:
 512             return None
 513         else:
 514             return db_binary.version
 515
 516     def _version_checks(self, upload, suite, expected_result):
 517         session = upload.session
 518
 519         if upload.changes.source is not None:
 520             source_name = upload.changes.source.dsc['Source']
 521             source_version = upload.changes.source.dsc['Version']
 522             v = self._highest_source_version(session, source_name, suite)
 523             if v is not None and version_compare(source_version, v) != expected_result:
 524                 raise Reject('Version check failed (source={0}, version={1}, suite={2})'.format(source_name, source_version, suite.suite_name))
 525
 526         for binary in upload.changes.binaries:
 527             binary_name = binary.control['Package']
 528             binary_version = binary.control['Version']
 529             architecture = binary.control['Architecture']
 530             v = self._highest_binary_version(session, binary_name, suite, architecture)
 531             if v is not None and version_compare(binary_version, v) != expected_result:
 532                 raise Reject('Version check failed (binary={0}, version={1}, suite={2})'.format(binary_name, binary_version, suite.suite_name))
 533
 534     def per_suite_check(self, upload, suite):
 535         session = upload.session
 536
 537         vc_newer = session.query(dbconn.VersionCheck).filter_by(suite=suite) \
 538             .filter(dbconn.VersionCheck.check.in_(['MustBeNewerThan', 'Enhances']))
 539         must_be_newer_than = [ vc.reference for vc in vc_newer ]
 540         # Must be newer than old versions in `suite`
 541         must_be_newer_than.append(suite)
 542
 543         for s in must_be_newer_than:
 544             self._version_checks(upload, s, 1)
 545
 546         vc_older = session.query(dbconn.VersionCheck).filter_by(suite=suite, check='MustBeOlderThan')
 547         must_be_older_than = [ vc.reference for vc in vc_older ]
 548
 549         for s in must_be_older_than:
 550             self._version_checks(upload, s, -1)
 551
 552         return True
 553
 554     @property
 555     def forcable(self):
 556         return True