1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to handle uploads not yet installed to the archive
19 This module provides classes to handle uploads not yet installed to the
20 archive. Central is the L{Changes} class which represents a changes file.
21 It provides methods to access the included binary and source packages.
30 from daklib.gpg import SignedFile
31 from daklib.regexes import *
32 import daklib.packagelist
34 class UploadException(Exception):
37 class InvalidChangesException(UploadException):
40 class InvalidBinaryException(UploadException):
43 class InvalidSourceException(UploadException):
46 class InvalidHashException(UploadException):
47 def __init__(self, filename, hash_name, expected, actual):
48 self.filename = filename
49 self.hash_name = hash_name
50 self.expected = expected
53 return ("Invalid {0} hash for {1}:\n"
54 "According to the control file the {0} hash should be {2},\n"
57 "If you did not include {1} in your upload, a different version\n"
58 "might already be known to the archive software.") \
59 .format(self.hash_name, self.filename, self.expected, self.actual)
61 class InvalidFilenameException(UploadException):
62 def __init__(self, filename):
63 self.filename = filename
65 return "Invalid filename '{0}'.".format(self.filename)
67 class FileDoesNotExist(UploadException):
68 def __init__(self, filename):
69 self.filename = filename
71 return "Refers to non-existing file '{0}'".format(self.filename)
73 class HashedFile(object):
74 """file with checksums
76 def __init__(self, filename, size, md5sum, sha1sum, sha256sum, section=None, priority=None, input_filename=None):
77 self.filename = filename
82 if input_filename is None:
83 input_filename = filename
84 self.input_filename = input_filename
85 """name of the file on disk
87 Used for temporary files that should not be installed using their on-disk name.
97 """MD5 hash in hexdigits
101 self.sha1sum = sha1sum
102 """SHA1 hash in hexdigits
106 self.sha256sum = sha256sum
107 """SHA256 hash in hexdigits
111 self.section = section
112 """section or C{None}
113 @type: str or C{None}
116 self.priority = priority
117 """priority or C{None}
118 @type: str of C{None}
122 def from_file(cls, directory, filename, section=None, priority=None):
123 """create with values for an existing file
125 Create a C{HashedFile} object that refers to an already existing file.
128 @param directory: directory the file is located in
131 @param filename: filename
133 @type section: str or C{None}
134 @param section: optional section as given in .changes files
136 @type priority: str or C{None}
137 @param priority: optional priority as given in .changes files
139 @rtype: L{HashedFile}
140 @return: C{HashedFile} object for the given file
142 path = os.path.join(directory, filename)
143 with open(path, 'r') as fh:
144 size = os.fstat(fh.fileno()).st_size
145 hashes = apt_pkg.Hashes(fh)
146 return cls(filename, size, hashes.md5, hashes.sha1, hashes.sha256, section, priority)
148 def check(self, directory):
151 Check if size and hashes match the expected value.
154 @param directory: directory the file is located in
156 @raise InvalidHashException: hash mismatch
158 path = os.path.join(directory, self.input_filename)
160 with open(path) as fh:
163 if e.errno == errno.ENOENT:
164 raise FileDoesNotExist(self.input_filename)
167 def check_fh(self, fh):
168 size = os.fstat(fh.fileno()).st_size
170 hashes = apt_pkg.Hashes(fh)
172 if size != self.size:
173 raise InvalidHashException(self.filename, 'size', self.size, size)
175 if hashes.md5 != self.md5sum:
176 raise InvalidHashException(self.filename, 'md5sum', self.md5sum, hashes.md5)
178 if hashes.sha1 != self.sha1sum:
179 raise InvalidHashException(self.filename, 'sha1sum', self.sha1sum, hashes.sha1)
181 if hashes.sha256 != self.sha256sum:
182 raise InvalidHashException(self.filename, 'sha256sum', self.sha256sum, hashes.sha256)
184 def parse_file_list(control, has_priority_and_section, safe_file_regexp = re_file_safe, fields = ('Files', 'Checksums-Sha1', 'Checksums-Sha256')):
185 """Parse Files and Checksums-* fields
187 @type control: dict-like
188 @param control: control file to take fields from
190 @type has_priority_and_section: bool
191 @param has_priority_and_section: Files field include section and priority
194 @raise InvalidChangesException: missing fields or other grave errors
197 @return: dict mapping filenames to L{daklib.upload.HashedFile} objects
201 for line in control.get(fields[0], "").split('\n'):
205 if has_priority_and_section:
206 (md5sum, size, section, priority, filename) = line.split()
207 entry = dict(md5sum=md5sum, size=long(size), section=section, priority=priority, filename=filename)
209 (md5sum, size, filename) = line.split()
210 entry = dict(md5sum=md5sum, size=long(size), filename=filename)
212 entries[filename] = entry
214 for line in control.get(fields[1], "").split('\n'):
217 (sha1sum, size, filename) = line.split()
218 entry = entries.get(filename, None)
220 raise InvalidChangesException('{0} is listed in {1}, but not in {2}.'.format(filename, fields[1], fields[0]))
221 if entry is not None and entry.get('size', None) != long(size):
222 raise InvalidChangesException('Size for {0} in {1} and {2} fields differ.'.format(filename, fields[0], fields[1]))
223 entry['sha1sum'] = sha1sum
225 for line in control.get(fields[2], "").split('\n'):
228 (sha256sum, size, filename) = line.split()
229 entry = entries.get(filename, None)
231 raise InvalidChangesException('{0} is listed in {1}, but not in {2}.'.format(filename, fields[2], fields[0]))
232 if entry is not None and entry.get('size', None) != long(size):
233 raise InvalidChangesException('Size for {0} in {1} and {2} fields differ.'.format(filename, fields[0], fields[2]))
234 entry['sha256sum'] = sha256sum
237 for entry in entries.itervalues():
238 filename = entry['filename']
239 if 'size' not in entry:
240 raise InvalidChangesException('No size for {0}.'.format(filename))
241 if 'md5sum' not in entry:
242 raise InvalidChangesException('No md5sum for {0}.'.format(filename))
243 if 'sha1sum' not in entry:
244 raise InvalidChangesException('No sha1sum for {0}.'.format(filename))
245 if 'sha256sum' not in entry:
246 raise InvalidChangesException('No sha256sum for {0}.'.format(filename))
247 if safe_file_regexp is not None and not safe_file_regexp.match(filename):
248 raise InvalidChangesException("{0}: References file with unsafe filename {1}.".format(self.filename, filename))
249 f = files[filename] = HashedFile(**entry)
253 class Changes(object):
254 """Representation of a .changes file
256 def __init__(self, directory, filename, keyrings, require_signature=True):
257 if not re_file_safe.match(filename):
258 raise InvalidChangesException('{0}: unsafe filename'.format(filename))
260 self.directory = directory
261 """directory the .changes is located in
265 self.filename = filename
266 """name of the .changes file
270 data = open(self.path).read()
271 self._signed_file = SignedFile(data, keyrings, require_signature)
272 self.changes = apt_pkg.TagSection(self._signed_file.contents)
273 """dict to access fields of the .changes file
277 self._binaries = None
280 self._keyrings = keyrings
281 self._require_signature = require_signature
285 """path to the .changes file
288 return os.path.join(self.directory, self.filename)
291 def primary_fingerprint(self):
292 """fingerprint of the key used for signing the .changes file
295 return self._signed_file.primary_fingerprint
298 def valid_signature(self):
299 """C{True} if the .changes has a valid signature
302 return self._signed_file.valid
305 def signature_timestamp(self):
306 return self._signed_file.signature_timestamp
309 def contents_sha1(self):
310 return self._signed_file.contents_sha1
313 def architectures(self):
314 """list of architectures included in the upload
317 return self.changes.get('Architecture', '').split()
320 def distributions(self):
321 """list of target distributions for the upload
324 return self.changes['Distribution'].split()
328 """included source or C{None}
329 @type: L{daklib.upload.Source} or C{None}
331 if self._source is None:
333 for f in self.files.itervalues():
334 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename):
335 source_files.append(f)
336 if len(source_files) > 0:
337 self._source = Source(self.directory, source_files, self._keyrings, self._require_signature)
342 """C{True} if the upload includes source
345 return "source" in self.architectures
348 def source_name(self):
349 """source package name
352 return re_field_source.match(self.changes['Source']).group('package')
356 """included binary packages
357 @type: list of L{daklib.upload.Binary}
359 if self._binaries is None:
361 for f in self.files.itervalues():
362 if re_file_binary.match(f.filename):
363 binaries.append(Binary(self.directory, f))
364 self._binaries = binaries
365 return self._binaries
368 def byhand_files(self):
369 """included byhand files
370 @type: list of L{daklib.upload.HashedFile}
374 for f in self.files.itervalues():
375 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename) or re_file_binary.match(f.filename):
377 if re_file_buildinfo.match(f.filename):
379 if f.section != 'byhand' and f.section[:4] != 'raw-':
380 raise InvalidChangesException("{0}: {1} looks like a byhand package, but is in section {2}".format(self.filename, f.filename, f.section))
386 def binary_names(self):
387 """names of included binary packages
390 return self.changes['Binary'].split()
393 def closed_bugs(self):
394 """bugs closed by this upload
397 return self.changes.get('Closes', '').split()
401 """dict mapping filenames to L{daklib.upload.HashedFile} objects
404 if self._files is None:
405 self._files = parse_file_list(self.changes, True)
410 """total size of files included in this upload in bytes
414 for f in self.files.itervalues():
418 def __cmp__(self, other):
419 """compare two changes files
421 We sort by source name and version first. If these are identical,
422 we sort changes that include source before those without source (so
423 that sourceful uploads get processed first), and finally fall back
424 to the filename (this should really never happen).
427 @return: n where n < 0 if self < other, n = 0 if self == other, n > 0 if self > other
429 ret = cmp(self.changes.get('Source'), other.changes.get('Source'))
433 ret = apt_pkg.version_compare(self.changes.get('Version', ''), other.changes.get('Version', ''))
436 # sort changes with source before changes without source
437 if 'source' in self.architectures and 'source' not in other.architectures:
439 elif 'source' not in self.architectures and 'source' in other.architectures:
445 # fall back to filename
446 ret = cmp(self.filename, other.filename)
450 class Binary(object):
451 """Representation of a binary package
453 def __init__(self, directory, hashed_file):
454 self.hashed_file = hashed_file
455 """file object for the .deb
459 path = os.path.join(directory, hashed_file.input_filename)
460 data = apt_inst.DebFile(path).control.extractdata("control")
462 self.control = apt_pkg.TagSection(data)
463 """dict to access fields in DEBIAN/control
468 def from_file(cls, directory, filename):
469 hashed_file = HashedFile.from_file(directory, filename)
470 return cls(directory, hashed_file)
474 """get tuple with source package name and version
477 source = self.control.get("Source", None)
479 return (self.control["Package"], self.control["Version"])
480 match = re_field_source.match(source)
482 raise InvalidBinaryException('{0}: Invalid Source field.'.format(self.hashed_file.filename))
483 version = match.group('version')
485 version = self.control['Version']
486 return (match.group('package'), version)
490 return self.control['Package']
494 """package type ('deb' or 'udeb')
497 match = re_file_binary.match(self.hashed_file.filename)
499 raise InvalidBinaryException('{0}: Does not match re_file_binary'.format(self.hashed_file.filename))
500 return match.group('type')
507 fields = self.control['Section'].split('/')
512 class Source(object):
513 """Representation of a source package
515 def __init__(self, directory, hashed_files, keyrings, require_signature=True):
516 self.hashed_files = hashed_files
517 """list of source files (including the .dsc itself)
518 @type: list of L{HashedFile}
521 self._dsc_file = None
522 for f in hashed_files:
523 if re_file_dsc.match(f.filename):
524 if self._dsc_file is not None:
525 raise InvalidSourceException("Multiple .dsc found ({0} and {1})".format(self._dsc_file.filename, f.filename))
529 # make sure the hash for the dsc is valid before we use it
530 self._dsc_file.check(directory)
532 dsc_file_path = os.path.join(directory, self._dsc_file.input_filename)
533 data = open(dsc_file_path, 'r').read()
534 self._signed_file = SignedFile(data, keyrings, require_signature)
535 self.dsc = apt_pkg.TagSection(self._signed_file.contents)
536 """dict to access fields in the .dsc file
540 self.package_list = daklib.packagelist.PackageList(self.dsc)
541 """Information about packages built by the source.
542 @type: daklib.packagelist.PackageList
548 def from_file(cls, directory, filename, keyrings, require_signature=True):
549 hashed_file = HashedFile.from_file(directory, filename)
550 return cls(directory, [hashed_file], keyrings, require_signature)
554 """dict mapping filenames to L{HashedFile} objects for additional source files
556 This list does not include the .dsc itself.
560 if self._files is None:
561 self._files = parse_file_list(self.dsc, False)
565 def primary_fingerprint(self):
566 """fingerprint of the key used to sign the .dsc
569 return self._signed_file.primary_fingerprint
572 def valid_signature(self):
573 """C{True} if the .dsc has a valid signature
576 return self._signed_file.valid
580 """guessed component name
582 Might be wrong. Don't rely on this.
586 if 'Section' not in self.dsc:
588 fields = self.dsc['Section'].split('/')
595 """filename of .dsc file
598 return self._dsc_file.filename