1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to handle uploads not yet installed to the archive
19 This module provides classes to handle uploads not yet installed to the
20 archive. Central is the L{Changes} class which represents a changes file.
21 It provides methods to access the included binary and source packages.
30 from daklib.gpg import SignedFile
31 from daklib.regexes import *
32 import daklib.packagelist
34 class UploadException(Exception):
37 class InvalidChangesException(UploadException):
40 class InvalidBinaryException(UploadException):
43 class InvalidSourceException(UploadException):
46 class InvalidHashException(UploadException):
47 def __init__(self, filename, hash_name, expected, actual):
48 self.filename = filename
49 self.hash_name = hash_name
50 self.expected = expected
53 return ("Invalid {0} hash for {1}:\n"
54 "According to the control file the {0} hash should be {2},\n"
57 "If you did not include {1} in your upload, a different version\n"
58 "might already be known to the archive software.") \
59 .format(self.hash_name, self.filename, self.expected, self.actual)
61 class InvalidFilenameException(UploadException):
62 def __init__(self, filename):
63 self.filename = filename
65 return "Invalid filename '{0}'.".format(self.filename)
67 class FileDoesNotExist(UploadException):
68 def __init__(self, filename):
69 self.filename = filename
71 return "Refers to non-existing file '{0}'".format(self.filename)
73 class HashedFile(object):
74 """file with checksums
76 def __init__(self, filename, size, md5sum, sha1sum, sha256sum, section=None, priority=None):
77 self.filename = filename
88 """MD5 hash in hexdigits
92 self.sha1sum = sha1sum
93 """SHA1 hash in hexdigits
97 self.sha256sum = sha256sum
98 """SHA256 hash in hexdigits
102 self.section = section
103 """section or C{None}
104 @type: str or C{None}
107 self.priority = priority
108 """priority or C{None}
109 @type: str of C{None}
113 def from_file(cls, directory, filename, section=None, priority=None):
114 """create with values for an existing file
116 Create a C{HashedFile} object that refers to an already existing file.
119 @param directory: directory the file is located in
122 @param filename: filename
124 @type section: str or C{None}
125 @param section: optional section as given in .changes files
127 @type priority: str or C{None}
128 @param priority: optional priority as given in .changes files
130 @rtype: L{HashedFile}
131 @return: C{HashedFile} object for the given file
133 path = os.path.join(directory, filename)
134 with open(path, 'r') as fh:
135 size = os.fstat(fh.fileno()).st_size
136 hashes = apt_pkg.Hashes(fh)
137 return cls(filename, size, hashes.md5, hashes.sha1, hashes.sha256, section, priority)
139 def check(self, directory):
142 Check if size and hashes match the expected value.
145 @param directory: directory the file is located in
147 @raise InvalidHashException: hash mismatch
149 path = os.path.join(directory, self.filename)
151 with open(path) as fh:
154 if e.errno == errno.ENOENT:
155 raise FileDoesNotExist(self.filename)
158 def check_fh(self, fh):
159 size = os.fstat(fh.fileno()).st_size
161 hashes = apt_pkg.Hashes(fh)
163 if size != self.size:
164 raise InvalidHashException(self.filename, 'size', self.size, size)
166 if hashes.md5 != self.md5sum:
167 raise InvalidHashException(self.filename, 'md5sum', self.md5sum, hashes.md5)
169 if hashes.sha1 != self.sha1sum:
170 raise InvalidHashException(self.filename, 'sha1sum', self.sha1sum, hashes.sha1)
172 if hashes.sha256 != self.sha256sum:
173 raise InvalidHashException(self.filename, 'sha256sum', self.sha256sum, hashes.sha256)
175 def parse_file_list(control, has_priority_and_section, safe_file_regexp = re_file_safe, fields = ('Files', 'Checksums-Sha1', 'Checksums-Sha256')):
176 """Parse Files and Checksums-* fields
178 @type control: dict-like
179 @param control: control file to take fields from
181 @type has_priority_and_section: bool
182 @param has_priority_and_section: Files field include section and priority
185 @raise InvalidChangesException: missing fields or other grave errors
188 @return: dict mapping filenames to L{daklib.upload.HashedFile} objects
192 for line in control.get(fields[0], "").split('\n'):
196 if has_priority_and_section:
197 (md5sum, size, section, priority, filename) = line.split()
198 entry = dict(md5sum=md5sum, size=long(size), section=section, priority=priority, filename=filename)
200 (md5sum, size, filename) = line.split()
201 entry = dict(md5sum=md5sum, size=long(size), filename=filename)
203 entries[filename] = entry
205 for line in control.get(fields[1], "").split('\n'):
208 (sha1sum, size, filename) = line.split()
209 entry = entries.get(filename, None)
211 raise InvalidChangesException('{0} is listed in {1}, but not in {2}.'.format(filename, fields[1], fields[0]))
212 if entry is not None and entry.get('size', None) != long(size):
213 raise InvalidChangesException('Size for {0} in {1} and {2} fields differ.'.format(filename, fields[0], fields[1]))
214 entry['sha1sum'] = sha1sum
216 for line in control.get(fields[2], "").split('\n'):
219 (sha256sum, size, filename) = line.split()
220 entry = entries.get(filename, None)
222 raise InvalidChangesException('{0} is listed in {1}, but not in {2}.'.format(filename, fields[2], fields[0]))
223 if entry is not None and entry.get('size', None) != long(size):
224 raise InvalidChangesException('Size for {0} in {1} and {2} fields differ.'.format(filename, fields[0], fields[2]))
225 entry['sha256sum'] = sha256sum
228 for entry in entries.itervalues():
229 filename = entry['filename']
230 if 'size' not in entry:
231 raise InvalidChangesException('No size for {0}.'.format(filename))
232 if 'md5sum' not in entry:
233 raise InvalidChangesException('No md5sum for {0}.'.format(filename))
234 if 'sha1sum' not in entry:
235 raise InvalidChangesException('No sha1sum for {0}.'.format(filename))
236 if 'sha256sum' not in entry:
237 raise InvalidChangesException('No sha256sum for {0}.'.format(filename))
238 if safe_file_regexp is not None and not safe_file_regexp.match(filename):
239 raise InvalidChangesException("{0}: References file with unsafe filename {1}.".format(self.filename, filename))
240 f = files[filename] = HashedFile(**entry)
244 class Changes(object):
245 """Representation of a .changes file
247 def __init__(self, directory, filename, keyrings, require_signature=True):
248 if not re_file_safe.match(filename):
249 raise InvalidChangesException('{0}: unsafe filename'.format(filename))
251 self.directory = directory
252 """directory the .changes is located in
256 self.filename = filename
257 """name of the .changes file
261 data = open(self.path).read()
262 self._signed_file = SignedFile(data, keyrings, require_signature)
263 self.changes = apt_pkg.TagSection(self._signed_file.contents)
264 """dict to access fields of the .changes file
268 self._binaries = None
271 self._keyrings = keyrings
272 self._require_signature = require_signature
276 """path to the .changes file
279 return os.path.join(self.directory, self.filename)
282 def primary_fingerprint(self):
283 """fingerprint of the key used for signing the .changes file
286 return self._signed_file.primary_fingerprint
289 def valid_signature(self):
290 """C{True} if the .changes has a valid signature
293 return self._signed_file.valid
296 def signature_timestamp(self):
297 return self._signed_file.signature_timestamp
300 def contents_sha1(self):
301 return self._signed_file.contents_sha1
304 def architectures(self):
305 """list of architectures included in the upload
308 return self.changes.get('Architecture', '').split()
311 def distributions(self):
312 """list of target distributions for the upload
315 return self.changes['Distribution'].split()
319 """included source or C{None}
320 @type: L{daklib.upload.Source} or C{None}
322 if self._source is None:
324 for f in self.files.itervalues():
325 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename):
326 source_files.append(f)
327 if len(source_files) > 0:
328 self._source = Source(self.directory, source_files, self._keyrings, self._require_signature)
333 """C{True} if the upload includes source
336 return "source" in self.architectures
339 def source_name(self):
340 """source package name
343 return re_field_source.match(self.changes['Source']).group('package')
347 """included binary packages
348 @type: list of L{daklib.upload.Binary}
350 if self._binaries is None:
352 for f in self.files.itervalues():
353 if re_file_binary.match(f.filename):
354 binaries.append(Binary(self.directory, f))
355 self._binaries = binaries
356 return self._binaries
359 def byhand_files(self):
360 """included byhand files
361 @type: list of L{daklib.upload.HashedFile}
365 for f in self.files.itervalues():
366 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename) or re_file_binary.match(f.filename):
368 if f.section != 'byhand' and f.section[:4] != 'raw-':
369 raise InvalidChangesException("{0}: {1} looks like a byhand package, but is in section {2}".format(self.filename, f.filename, f.section))
375 def binary_names(self):
376 """names of included binary packages
379 return self.changes['Binary'].split()
382 def closed_bugs(self):
383 """bugs closed by this upload
386 return self.changes.get('Closes', '').split()
390 """dict mapping filenames to L{daklib.upload.HashedFile} objects
393 if self._files is None:
394 self._files = parse_file_list(self.changes, True)
399 """total size of files included in this upload in bytes
403 for f in self.files.itervalues():
407 def __cmp__(self, other):
408 """compare two changes files
410 We sort by source name and version first. If these are identical,
411 we sort changes that include source before those without source (so
412 that sourceful uploads get processed first), and finally fall back
413 to the filename (this should really never happen).
416 @return: n where n < 0 if self < other, n = 0 if self == other, n > 0 if self > other
418 ret = cmp(self.changes.get('Source'), other.changes.get('Source'))
422 ret = apt_pkg.version_compare(self.changes.get('Version', ''), other.changes.get('Version', ''))
425 # sort changes with source before changes without source
426 if 'source' in self.architectures and 'source' not in other.architectures:
428 elif 'source' not in self.architectures and 'source' in other.architectures:
434 # fall back to filename
435 ret = cmp(self.filename, other.filename)
439 class Binary(object):
440 """Representation of a binary package
442 def __init__(self, directory, hashed_file):
443 self.hashed_file = hashed_file
444 """file object for the .deb
448 path = os.path.join(directory, hashed_file.filename)
449 data = apt_inst.DebFile(path).control.extractdata("control")
451 self.control = apt_pkg.TagSection(data)
452 """dict to access fields in DEBIAN/control
457 def from_file(cls, directory, filename):
458 hashed_file = HashedFile.from_file(directory, filename)
459 return cls(directory, hashed_file)
463 """get tuple with source package name and version
466 source = self.control.get("Source", None)
468 return (self.control["Package"], self.control["Version"])
469 match = re_field_source.match(source)
471 raise InvalidBinaryException('{0}: Invalid Source field.'.format(self.hashed_file.filename))
472 version = match.group('version')
474 version = self.control['Version']
475 return (match.group('package'), version)
479 return self.control['Package']
483 """package type ('deb' or 'udeb')
486 match = re_file_binary.match(self.hashed_file.filename)
488 raise InvalidBinaryException('{0}: Does not match re_file_binary'.format(self.hashed_file.filename))
489 return match.group('type')
496 fields = self.control['Section'].split('/')
501 class Source(object):
502 """Representation of a source package
504 def __init__(self, directory, hashed_files, keyrings, require_signature=True):
505 self.hashed_files = hashed_files
506 """list of source files (including the .dsc itself)
507 @type: list of L{HashedFile}
510 self._dsc_file = None
511 for f in hashed_files:
512 if re_file_dsc.match(f.filename):
513 if self._dsc_file is not None:
514 raise InvalidSourceException("Multiple .dsc found ({0} and {1})".format(self._dsc_file.filename, f.filename))
518 # make sure the hash for the dsc is valid before we use it
519 self._dsc_file.check(directory)
521 dsc_file_path = os.path.join(directory, self._dsc_file.filename)
522 data = open(dsc_file_path, 'r').read()
523 self._signed_file = SignedFile(data, keyrings, require_signature)
524 self.dsc = apt_pkg.TagSection(self._signed_file.contents)
525 """dict to access fields in the .dsc file
529 self.package_list = daklib.packagelist.PackageList(self.dsc)
530 """Information about packages built by the source.
531 @type: daklib.packagelist.PackageList
537 def from_file(cls, directory, filename, keyrings, require_signature=True):
538 hashed_file = HashedFile.from_file(directory, filename)
539 return cls(directory, [hashed_file], keyrings, require_signature)
543 """dict mapping filenames to L{HashedFile} objects for additional source files
545 This list does not include the .dsc itself.
549 if self._files is None:
550 self._files = parse_file_list(self.dsc, False)
554 def primary_fingerprint(self):
555 """fingerprint of the key used to sign the .dsc
558 return self._signed_file.primary_fingerprint
561 def valid_signature(self):
562 """C{True} if the .dsc has a valid signature
565 return self._signed_file.valid
569 """guessed component name
571 Might be wrong. Don't rely on this.
575 if 'Section' not in self.dsc:
577 fields = self.dsc['Section'].split('/')
584 """filename of .dsc file
587 return self._dsc_file.filename