]> git.decadent.org.uk Git - dak.git/blob - daklib/checks.py
Merge remote-tracking branch 'ansgar/pu/multiarchive-1' into merge
[dak.git] / daklib / checks.py
1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
2 #
3 # Parts based on code that is
4 # Copyright (C) 2001-2006, James Troup <james@nocrew.org>
5 # Copyright (C) 2009-2010, Joerg Jaspert <joerg@debian.org>
6 #
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License along
18 # with this program; if not, write to the Free Software Foundation, Inc.,
19 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21 """module provided pre-acceptance tests
22
23 Please read the documentation for the `Check` class for the interface.
24 """
25
26 from daklib.config import Config
27 from .dbconn import *
28 import daklib.dbconn as dbconn
29 from .regexes import *
30 from .textutils import fix_maintainer, ParseMaintError
31 import daklib.lintian as lintian
32 import daklib.utils as utils
33
34 import apt_inst
35 import apt_pkg
36 from apt_pkg import version_compare
37 import os
38 import time
39 import yaml
40
41 # TODO: replace by subprocess
42 import commands
43
44 class Reject(Exception):
45     """exception raised by failing checks"""
46     pass
47
48 class Check(object):
49     """base class for checks
50
51     checks are called by daklib.archive.ArchiveUpload.  Failing tests should
52     raise a `daklib.checks.Reject` exception including a human-readable
53     description why the upload should be rejected.
54     """
55     def check(self, upload):
56         """do checks
57
58         Args:
59            upload (daklib.archive.ArchiveUpload): upload to check
60
61         Raises:
62            daklib.checks.Reject
63         """
64         raise NotImplemented
65     def per_suite_check(self, upload, suite):
66         """do per-suite checks
67
68         Args:
69            upload (daklib.archive.ArchiveUpload): upload to check
70            suite (daklib.dbconn.Suite): suite to check
71
72         Raises:
73            daklib.checks.Reject
74         """
75         raise NotImplemented
76     @property
77     def forcable(self):
78         """allow to force ignore failing test
79
80         True if it is acceptable to force ignoring a failing test,
81         False otherwise
82         """
83         return False
84
85 class SignatureCheck(Check):
86     """Check signature of changes and dsc file (if included in upload)
87
88     Make sure the signature is valid and done by a known user.
89     """
90     def check(self, upload):
91         changes = upload.changes
92         if not changes.valid_signature:
93             raise Reject("Signature for .changes not valid.")
94         if changes.source is not None:
95             if not changes.source.valid_signature:
96                 raise Reject("Signature for .dsc not valid.")
97             if changes.source.primary_fingerprint != changes.primary_fingerprint:
98                 raise Reject(".changes and .dsc not signed by the same key.")
99         if upload.fingerprint is None or upload.fingerprint.uid is None:
100             raise Reject(".changes signed by unknown key.")
101
102 class ChangesCheck(Check):
103     """Check changes file for syntax errors."""
104     def check(self, upload):
105         changes = upload.changes
106         control = changes.changes
107         fn = changes.filename
108
109         for field in ('Distribution', 'Source', 'Binary', 'Architecture', 'Version', 'Maintainer', 'Files', 'Changes', 'Description'):
110             if field not in control:
111                 raise Reject('{0}: misses mandatory field {1}'.format(fn, field))
112
113         source_match = re_field_source.match(control['Source'])
114         if not source_match:
115             raise Reject('{0}: Invalid Source field'.format(fn))
116         version_match = re_field_version.match(control['Version'])
117         if not version_match:
118             raise Reject('{0}: Invalid Version field'.format(fn))
119         version_without_epoch = version_match.group('without_epoch')
120
121         match = re_file_changes.match(fn)
122         if not match:
123             raise Reject('{0}: Does not match re_file_changes'.format(fn))
124         if match.group('package') != source_match.group('package'):
125             raise Reject('{0}: Filename does not match Source field'.format(fn))
126         if match.group('version') != version_without_epoch:
127             raise Reject('{0}: Filename does not match Version field'.format(fn))
128
129         for bn in changes.binary_names:
130             if not re_field_package.match(bn):
131                 raise Reject('{0}: Invalid binary package name {1}'.format(fn, bn))
132
133         if 'source' in changes.architectures and changes.source is None:
134             raise Reject("Changes has architecture source, but no source found.")
135         if changes.source is not None and 'source' not in changes.architectures:
136             raise Reject("Upload includes source, but changes does not say so.")
137
138         try:
139             fix_maintainer(changes.changes['Maintainer'])
140         except ParseMaintError as e:
141             raise Reject('{0}: Failed to parse Maintainer field: {1}'.format(changes.filename, e))
142
143         try:
144             changed_by = changes.changes.get('Changed-By')
145             if changed_by is not None:
146                 fix_maintainer(changed_by)
147         except ParseMaintError as e:
148             raise Reject('{0}: Failed to parse Changed-By field: {1}'.format(changes.filename, e))
149
150         if len(changes.files) == 0:
151             raise Reject("Changes includes no files.")
152
153         for bugnum in changes.closed_bugs:
154             if not re_isanum.match(bugnum):
155                 raise Reject('{0}: "{1}" in Closes field is not a number'.format(changes.filename, bugnum))
156
157         return True
158
159 class HashesCheck(Check):
160     """Check hashes in .changes and .dsc are valid."""
161     def check(self, upload):
162         changes = upload.changes
163         for f in changes.files.itervalues():
164             f.check(upload.directory)
165             source = changes.source
166         if source is not None:
167             for f in source.files.itervalues():
168                 f.check(upload.directory)
169
170 class BinaryCheck(Check):
171     """Check binary packages for syntax errors."""
172     def check(self, upload):
173         for binary in upload.changes.binaries:
174             self.check_binary(upload, binary)
175
176         binary_names = set([ binary.control['Package'] for binary in upload.changes.binaries ])
177         for bn in binary_names:
178             if bn not in upload.changes.binary_names:
179                 raise Reject('Package {0} is not mentioned in Binary field in changes'.format(bn))
180
181         return True
182
183     def check_binary(self, upload, binary):
184         fn = binary.hashed_file.filename
185         control = binary.control
186
187         for field in ('Package', 'Architecture', 'Version', 'Description'):
188             if field not in control:
189                 raise Reject('{0}: Missing mandatory field {0}.'.format(fn, field))
190
191         # check fields
192
193         package = control['Package']
194         if not re_field_package.match(package):
195             raise Reject('{0}: Invalid Package field'.format(fn))
196
197         version = control['Version']
198         version_match = re_field_version.match(version)
199         if not version_match:
200             raise Reject('{0}: Invalid Version field'.format(fn))
201         version_without_epoch = version_match.group('without_epoch')
202
203         architecture = control['Architecture']
204         if architecture not in upload.changes.architectures:
205             raise Reject('{0}: Architecture not in Architecture field in changes file'.format(fn))
206         if architecture == 'source':
207             raise Reject('{0}: Architecture "source" invalid for binary packages'.format(fn))
208
209         source = control.get('Source')
210         if source is not None and not re_field_source.match(source):
211             raise Reject('{0}: Invalid Source field'.format(fn))
212
213         # check filename
214
215         match = re_file_binary.match(fn)
216         if package != match.group('package'):
217             raise Reject('{0}: filename does not match Package field'.format(fn))
218         if version_without_epoch != match.group('version'):
219             raise Reject('{0}: filename does not match Version field'.format(fn))
220         if architecture != match.group('architecture'):
221             raise Reject('{0}: filename does not match Architecture field'.format(fn))
222
223         # check dependency field syntax
224
225         for field in ('Breaks', 'Conflicts', 'Depends', 'Enhances', 'Pre-Depends',
226                       'Provides', 'Recommends', 'Replaces', 'Suggests'):
227             value = control.get(field)
228             if value is not None:
229                 if value.strip() == '':
230                     raise Reject('{0}: empty {1} field'.format(fn, field))
231                 try:
232                     apt_pkg.parse_depends(value)
233                 except:
234                     raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
235
236         for field in ('Built-Using',):
237             value = control.get(field)
238             if value is not None:
239                 if value.strip() == '':
240                     raise Reject('{0}: empty {1} field'.format(fn, field))
241                 try:
242                     apt_pkg.parse_src_depends(value)
243                 except:
244                     raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
245
246 class BinaryTimestampCheck(Check):
247     """check timestamps of files in binary packages
248
249     Files in the near future cause ugly warnings and extreme time travel
250     can cause errors on extraction.
251     """
252     def check(self, upload):
253         cnf = Config()
254         future_cutoff = time.time() + cnf.find_i('Dinstall::FutureTimeTravelGrace', 24*3600)
255         past_cutoff = time.mktime(time.strptime(cnf.find('Dinstall::PastCutoffYear', '1984'), '%Y'))
256
257         class TarTime(object):
258             def __init__(self):
259                 self.future_files = dict()
260                 self.past_files = dict()
261             def callback(self, member, data):
262                 if member.mtime > future_cutoff:
263                     future_files[member.name] = member.mtime
264                 elif member.mtime < past_cutoff:
265                     past_files[member.name] = member.mtime
266
267         def format_reason(filename, direction, files):
268             reason = "{0}: has {1} file(s) with a timestamp too far in the {2}:\n".format(filename, len(files), direction)
269             for fn, ts in files.iteritems():
270                 reason += "  {0} ({1})".format(fn, time.ctime(ts))
271             return reason
272
273         for binary in upload.changes.binaries:
274             filename = binary.hashed_file.filename
275             path = os.path.join(upload.directory, filename)
276             deb = apt_inst.DebFile(path)
277             tar = TarTime()
278             deb.control.go(tar.callback)
279             if tar.future_files:
280                 raise Reject(format_reason(filename, 'future', tar.future_files))
281             if tar.past_files:
282                 raise Reject(format_reason(filename, 'past', tar.past_files))
283
284 class SourceCheck(Check):
285     """Check source package for syntax errors."""
286     def check_filename(self, control, filename, regex):
287         # In case we have an .orig.tar.*, we have to strip the Debian revison
288         # from the version number. So handle this special case first.
289         is_orig = True
290         match = re_file_orig.match(filename)
291         if not match:
292             is_orig = False
293             match = regex.match(filename)
294
295         if not match:
296             raise Reject('{0}: does not match regular expression for source filenames'.format(filename))
297         if match.group('package') != control['Source']:
298             raise Reject('{0}: filename does not match Source field'.format(filename))
299
300         version = control['Version']
301         if is_orig:
302             version = re_field_version_upstream.match(version).group('upstream')
303         version_match =  re_field_version.match(version)
304         version_without_epoch = version_match.group('without_epoch')
305         if match.group('version') != version_without_epoch:
306             raise Reject('{0}: filename does not match Version field'.format(filename))
307
308     def check(self, upload):
309         if upload.changes.source is None:
310             return True
311
312         changes = upload.changes.changes
313         source = upload.changes.source
314         control = source.dsc
315         dsc_fn = source._dsc_file.filename
316
317         # check fields
318         if not re_field_package.match(control['Source']):
319             raise Reject('{0}: Invalid Source field'.format(dsc_fn))
320         if control['Source'] != changes['Source']:
321             raise Reject('{0}: Source field does not match Source field in changes'.format(dsc_fn))
322         if control['Version'] != changes['Version']:
323             raise Reject('{0}: Version field does not match Version field in changes'.format(dsc_fn))
324
325         # check filenames
326         self.check_filename(control, dsc_fn, re_file_dsc)
327         for f in source.files.itervalues():
328             self.check_filename(control, f.filename, re_file_source)
329
330         # check dependency field syntax
331         for field in ('Build-Conflicts', 'Build-Conflicts-Indep', 'Build-Depends', 'Build-Depends-Arch', 'Build-Depends-Indep'):
332             value = control.get(field)
333             if value is not None:
334                 if value.strip() == '':
335                     raise Reject('{0}: empty {1} field'.format(dsc_fn, field))
336                 try:
337                     apt_pkg.parse_src_depends(value)
338                 except Exception as e:
339                     raise Reject('{0}: APT could not parse {1} field: {2}'.format(dsc_fn, field, e))
340
341         # TODO: check all expected files for given source format are included
342
343 class SingleDistributionCheck(Check):
344     """Check that the .changes targets only a single distribution."""
345     def check(self, upload):
346         if len(upload.changes.distributions) != 1:
347             raise Reject("Only uploads to a single distribution are allowed.")
348
349 class ACLCheck(Check):
350     """Check the uploader is allowed to upload the packages in .changes"""
351     def _check_dm(self, upload):
352         # This code is not very nice, but hopefully works until we can replace
353         # DM-Upload-Allowed, cf. https://lists.debian.org/debian-project/2012/06/msg00029.html
354         session = upload.session
355
356         if 'source' not in upload.changes.architectures:
357             raise Reject('DM uploads must include source')
358         for f in upload.changes.files.itervalues():
359             if f.section == 'byhand' or f.section[:4] == "raw-":
360                 raise Reject("Uploading byhand packages is not allowed for DMs.")
361
362         # Reject NEW packages
363         distributions = upload.changes.distributions
364         assert len(distributions) == 1
365         suite = session.query(Suite).filter_by(suite_name=distributions[0]).one()
366         overridesuite = suite
367         if suite.overridesuite is not None:
368             overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
369         if upload._check_new(overridesuite):
370             raise Reject('Uploading NEW packages is not allowed for DMs.')
371
372         # Check DM-Upload-Allowed
373         last_suites = ['unstable', 'experimental']
374         if suite.suite_name.endswith('-backports'):
375             last_suites = [suite.suite_name]
376         last = session.query(DBSource).filter_by(source=upload.changes.changes['Source']) \
377             .join(DBSource.suites).filter(Suite.suite_name.in_(last_suites)) \
378             .order_by(DBSource.version.desc()).limit(1).first()
379         if last is None:
380             raise Reject('No existing source found in {0}'.format(' or '.join(last_suites)))
381         if not last.dm_upload_allowed:
382             raise Reject('DM-Upload-Allowed is not set in {0}={1}'.format(last.source, last.version))
383
384         # check current Changed-by is in last Maintainer or Uploaders
385         uploader_names = [ u.name for u in last.uploaders ]
386         changed_by_field = upload.changes.changes.get('Changed-By', upload.changes.changes['Maintainer'])
387         if changed_by_field not in uploader_names:
388             raise Reject('{0} is not an uploader for {1}={2}'.format(changed_by_field, last.source, last.version))
389
390         # check Changed-by is the DM
391         changed_by = fix_maintainer(changed_by_field)
392         uid = upload.fingerprint.uid
393         if uid is None:
394             raise Reject('Unknown uid for fingerprint {0}'.format(upload.fingerprint.fingerprint))
395         if uid.uid != changed_by[3] and uid.name != changed_by[2]:
396             raise Reject('DMs are not allowed to sponsor uploads (expected {0} <{1}> as maintainer, but got {2})'.format(uid.name, uid.uid, changed_by_field))
397
398         # Try to catch hijacks.
399         # This doesn't work correctly. Uploads to experimental can still
400         # "hijack" binaries from unstable. Also one can hijack packages
401         # via buildds (but people who try this should not be DMs).
402         for binary_name in upload.changes.binary_names:
403             binaries = session.query(DBBinary).join(DBBinary.source) \
404                 .join(DBBinary.suites).filter(Suite.suite_name.in_(upload.changes.distributions)) \
405                 .filter(DBBinary.package == binary_name)
406             for binary in binaries:
407                 if binary.source.source != upload.changes.changes['Source']:
408                     raise Reject('DMs must not hijack binaries (binary={0}, other-source={1})'.format(binary_name, binary.source.source))
409
410         return True
411
412     def check(self, upload):
413         fingerprint = upload.fingerprint
414         source_acl = fingerprint.source_acl
415         if source_acl is None:
416             if 'source' in upload.changes.architectures:
417                 raise Reject('Fingerprint {0} must not upload source'.format(fingerprint.fingerprint))
418         elif source_acl.access_level == 'dm':
419             self._check_dm(upload)
420         elif source_acl.access_level != 'full':
421             raise Reject('Unknown source_acl access level {0} for fingerprint {1}'.format(source_acl.access_level, fingerprint.fingerprint))
422
423         bin_architectures = set(upload.changes.architectures)
424         bin_architectures.discard('source')
425         binary_acl = fingerprint.binary_acl
426         if binary_acl is None:
427             if len(bin_architectures) > 0:
428                 raise Reject('Fingerprint {0} must not upload binary packages'.format(fingerprint.fingerprint))
429         elif binary_acl.access_level == 'map':
430             query = upload.session.query(BinaryACLMap).filter_by(fingerprint=fingerprint)
431             allowed_architectures = [ m.architecture.arch_string for m in query ]
432
433             for arch in upload.changes.architectures:
434                 if arch not in allowed_architectures:
435                     raise Reject('Fingerprint {0} must not  upload binaries for architecture {1}'.format(fingerprint.fingerprint, arch))
436         elif binary_acl.access_level != 'full':
437             raise Reject('Unknown binary_acl access level {0} for fingerprint {1}'.format(binary_acl.access_level, fingerprint.fingerprint))
438
439         return True
440
441 class NoSourceOnlyCheck(Check):
442     """Check for source-only upload
443
444     Source-only uploads are only allowed if Dinstall::AllowSourceOnlyUploads is
445     set. Otherwise they are rejected.
446     """
447     def check(self, upload):
448         if Config().find_b("Dinstall::AllowSourceOnlyUploads"):
449             return True
450         changes = upload.changes
451         if changes.source is not None and len(changes.binaries) == 0:
452             raise Reject('Source-only uploads are not allowed.')
453         return True
454
455 class LintianCheck(Check):
456     """Check package using lintian"""
457     def check(self, upload):
458         changes = upload.changes
459
460         # Only check sourceful uploads.
461         if changes.source is None:
462             return True
463         # Only check uploads to unstable or experimental.
464         if 'unstable' not in changes.distributions and 'experimental' not in changes.distributions:
465             return True
466
467         cnf = Config()
468         if 'Dinstall::LintianTags' not in cnf:
469             return True
470         tagfile = cnf['Dinstall::LintianTags']
471
472         with open(tagfile, 'r') as sourcefile:
473             sourcecontent = sourcefile.read()
474         try:
475             lintiantags = yaml.load(sourcecontent)['lintian']
476         except yaml.YAMLError as msg:
477             raise Exception('Could not read lintian tags file {0}, YAML error: {1}'.format(tagfile, msg))
478
479         fd, temp_filename = utils.temp_filename()
480         temptagfile = os.fdopen(fd, 'w')
481         for tags in lintiantags.itervalues():
482             for tag in tags:
483                 print >>temptagfile, tag
484         temptagfile.close()
485
486         changespath = os.path.join(upload.directory, changes.filename)
487         try:
488             # FIXME: no shell
489             cmd = "lintian --show-overrides --tags-from-file {0} {1}".format(temp_filename, changespath)
490             result, output = commands.getstatusoutput(cmd)
491         finally:
492             os.unlink(temp_filename)
493
494         if result == 2:
495             utils.warn("lintian failed for %s [return code: %s]." % \
496                 (changespath, result))
497             utils.warn(utils.prefix_multi_line_string(output, \
498                 " [possible output:] "))
499
500         parsed_tags = lintian.parse_lintian_output(output)
501         rejects = list(lintian.generate_reject_messages(parsed_tags, lintiantags))
502         if len(rejects) != 0:
503             raise Reject('\n'.join(rejects))
504
505         return True
506
507 class SourceFormatCheck(Check):
508     """Check source format is allowed in the target suite"""
509     def per_suite_check(self, upload, suite):
510         source = upload.changes.source
511         session = upload.session
512         if source is None:
513             return True
514
515         source_format = source.dsc['Format']
516         query = session.query(SrcFormat).filter_by(format_name=source_format).filter(SrcFormat.suites.contains(suite))
517         if query.first() is None:
518             raise Reject('source format {0} is not allowed in suite {1}'.format(source_format, suite.suite_name))
519
520 class SuiteArchitectureCheck(Check):
521     def per_suite_check(self, upload, suite):
522         session = upload.session
523         for arch in upload.changes.architectures:
524             query = session.query(Architecture).filter_by(arch_string=arch).filter(Architecture.suites.contains(suite))
525             if query.first() is None:
526                 raise Reject('Architecture {0} is not allowed in suite {2}'.format(arch, suite.suite_name))
527
528         return True
529
530 class VersionCheck(Check):
531     """Check version constraints"""
532     def _highest_source_version(self, session, source_name, suite):
533         db_source = session.query(DBSource).filter_by(source=source_name) \
534             .filter(DBSource.suites.contains(suite)).order_by(DBSource.version.desc()).first()
535         if db_source is None:
536             return None
537         else:
538             return db_source.version
539
540     def _highest_binary_version(self, session, binary_name, suite, architecture):
541         db_binary = session.query(DBBinary).filter_by(package=binary_name) \
542             .filter(DBBinary.suites.contains(suite)) \
543             .join(DBBinary.architecture) \
544             .filter(Architecture.arch_string.in_(['all', architecture])) \
545             .order_by(DBBinary.version.desc()).first()
546         if db_binary is None:
547             return None
548         else:
549             return db_binary.version
550
551     def _version_checks(self, upload, suite, op):
552         session = upload.session
553
554         if upload.changes.source is not None:
555             source_name = upload.changes.source.dsc['Source']
556             source_version = upload.changes.source.dsc['Version']
557             v = self._highest_source_version(session, source_name, suite)
558             if v is not None and not op(version_compare(source_version, v)):
559                 raise Reject('Version check failed (source={0}, version={1}, other-version={2}, suite={3})'.format(source_name, source_version, v, suite.suite_name))
560
561         for binary in upload.changes.binaries:
562             binary_name = binary.control['Package']
563             binary_version = binary.control['Version']
564             architecture = binary.control['Architecture']
565             v = self._highest_binary_version(session, binary_name, suite, architecture)
566             if v is not None and not op(version_compare(binary_version, v)):
567                 raise Reject('Version check failed (binary={0}, version={1}, other-version={2}, suite={3})'.format(binary_name, binary_version, v, suite.suite_name))
568
569     def per_suite_check(self, upload, suite):
570         session = upload.session
571
572         vc_newer = session.query(dbconn.VersionCheck).filter_by(suite=suite) \
573             .filter(dbconn.VersionCheck.check.in_(['MustBeNewerThan', 'Enhances']))
574         must_be_newer_than = [ vc.reference for vc in vc_newer ]
575         # Must be newer than old versions in `suite`
576         must_be_newer_than.append(suite)
577
578         for s in must_be_newer_than:
579             self._version_checks(upload, s, lambda result: result > 0)
580
581         vc_older = session.query(dbconn.VersionCheck).filter_by(suite=suite, check='MustBeOlderThan')
582         must_be_older_than = [ vc.reference for vc in vc_older ]
583
584         for s in must_be_older_than:
585             self._version_checks(upload, s, lambda result: result < 0)
586
587         return True
588
589     @property
590     def forcable(self):
591         return True