1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to handle uploads not yet installed to the archive
19 This module provides classes to handle uploads not yet installed to the
20 archive. Central is the L{Changes} class which represents a changes file.
21 It provides methods to access the included binary and source packages.
29 from daklib.gpg import SignedFile
30 from daklib.regexes import *
32 class InvalidChangesException(Exception):
35 class InvalidBinaryException(Exception):
38 class InvalidSourceException(Exception):
41 class InvalidHashException(Exception):
42 def __init__(self, filename, hash_name, expected, actual):
43 self.filename = filename
44 self.hash_name = hash_name
45 self.expected = expected
48 return ("Invalid {0} hash for {1}:\n"
49 "According to the control file the {0} hash should be {2},\n"
52 "If you did not include {1} in you upload, a different version\n"
53 "might already be known to the archive software.") \
54 .format(self.hash_name, self.filename, self.expected, self.actual)
56 class InvalidFilenameException(Exception):
57 def __init__(self, filename):
58 self.filename = filename
60 return "Invalid filename '{0}'.".format(self.filename)
62 class HashedFile(object):
63 """file with checksums
65 def __init__(self, filename, size, md5sum, sha1sum, sha256sum, section=None, priority=None):
66 self.filename = filename
77 """MD5 hash in hexdigits
81 self.sha1sum = sha1sum
82 """SHA1 hash in hexdigits
86 self.sha256sum = sha256sum
87 """SHA256 hash in hexdigits
91 self.section = section
96 self.priority = priority
97 """priority or C{None}
102 def from_file(cls, directory, filename, section=None, priority=None):
103 """create with values for an existing file
105 Create a C{HashedFile} object that refers to an already existing file.
108 @param directory: directory the file is located in
111 @param filename: filename
113 @type section: str or C{None}
114 @param section: optional section as given in .changes files
116 @type priority: str or C{None}
117 @param priority: optional priority as given in .changes files
119 @rtype: L{HashedFile}
120 @return: C{HashedFile} object for the given file
122 path = os.path.join(directory, filename)
123 size = os.stat(path).st_size
124 with open(path, 'r') as fh:
125 hashes = apt_pkg.Hashes(fh)
126 return cls(filename, size, hashes.md5, hashes.sha1, hashes.sha256, section, priority)
128 def check(self, directory):
131 Check if size and hashes match the expected value.
134 @param directory: directory the file is located in
136 @raise InvalidHashException: hash mismatch
138 path = os.path.join(directory, self.filename)
141 size = os.stat(path).st_size
142 if size != self.size:
143 raise InvalidHashException(self.filename, 'size', self.size, size)
145 md5sum = apt_pkg.md5sum(fh)
146 if md5sum != self.md5sum:
147 raise InvalidHashException(self.filename, 'md5sum', self.md5sum, md5sum)
150 sha1sum = apt_pkg.sha1sum(fh)
151 if sha1sum != self.sha1sum:
152 raise InvalidHashException(self.filename, 'sha1sum', self.sha1sum, sha1sum)
155 sha256sum = apt_pkg.sha256sum(fh)
156 if sha256sum != self.sha256sum:
157 raise InvalidHashException(self.filename, 'sha256sum', self.sha256sum, sha256sum)
159 def parse_file_list(control, has_priority_and_section):
160 """Parse Files and Checksums-* fields
162 @type control: dict-like
163 @param control: control file to take fields from
165 @type has_priority_and_section: bool
166 @param has_priority_and_section: Files field include section and priority
169 @raise InvalidChangesException: missing fields or other grave errors
172 @return: dict mapping filenames to L{daklib.upload.HashedFile} objects
176 for line in control["Files"].split('\n'):
180 if has_priority_and_section:
181 (md5sum, size, section, priority, filename) = line.split()
182 entry = dict(md5sum=md5sum, size=long(size), section=section, priority=priority, filename=filename)
184 (md5sum, size, filename) = line.split()
185 entry = dict(md5sum=md5sum, size=long(size), filename=filename)
187 entries[filename] = entry
189 for line in control["Checksums-Sha1"].split('\n'):
192 (sha1sum, size, filename) = line.split()
193 entry = entries.get(filename, None)
195 raise InvalidChangesException('{0} is listed in Checksums-Sha1, but not in Files.'.format(filename))
196 if entry is not None and entry.get('size', None) != long(size):
197 raise InvalidChangesException('Size for {0} in Files and Checksum-Sha1 fields differ.'.format(filename))
198 entry['sha1sum'] = sha1sum
200 for line in control["Checksums-Sha256"].split('\n'):
203 (sha256sum, size, filename) = line.split()
204 entry = entries.get(filename, None)
206 raise InvalidChangesException('{0} is listed in Checksums-Sha256, but not in Files.'.format(filename))
207 if entry is not None and entry.get('size', None) != long(size):
208 raise InvalidChangesException('Size for {0} in Files and Checksum-Sha256 fields differ.'.format(filename))
209 entry['sha256sum'] = sha256sum
212 for entry in entries.itervalues():
213 filename = entry['filename']
214 if 'size' not in entry:
215 raise InvalidChangesException('No size for {0}.'.format(filename))
216 if 'md5sum' not in entry:
217 raise InvalidChangesException('No md5sum for {0}.'.format(filename))
218 if 'sha1sum' not in entry:
219 raise InvalidChangesException('No sha1sum for {0}.'.format(filename))
220 if 'sha256sum' not in entry:
221 raise InvalidChangesException('No sha256sum for {0}.'.format(filename))
222 if not re_file_safe.match(filename):
223 raise InvalidChangesException("{0}: References file with unsafe filename {1}.".format(self.filename, filename))
224 f = files[filename] = HashedFile(**entry)
228 class Changes(object):
229 """Representation of a .changes file
231 def __init__(self, directory, filename, keyrings, require_signature=True):
232 if not re_file_safe.match(filename):
233 raise InvalidChangesException('{0}: unsafe filename'.format(filename))
235 self.directory = directory
236 """directory the .changes is located in
240 self.filename = filename
241 """name of the .changes file
245 data = open(self.path).read()
246 self._signed_file = SignedFile(data, keyrings, require_signature)
247 self.changes = apt_pkg.TagSection(self._signed_file.contents)
248 """dict to access fields of the .changes file
252 self._binaries = None
255 self._keyrings = keyrings
256 self._require_signature = require_signature
260 """path to the .changes file
263 return os.path.join(self.directory, self.filename)
266 def primary_fingerprint(self):
267 """fingerprint of the key used for signing the .changes file
270 return self._signed_file.primary_fingerprint
273 def valid_signature(self):
274 """C{True} if the .changes has a valid signature
277 return self._signed_file.valid
280 def architectures(self):
281 """list of architectures included in the upload
284 return self.changes['Architecture'].split()
287 def distributions(self):
288 """list of target distributions for the upload
291 return self.changes['Distribution'].split()
295 """included source or C{None}
296 @type: L{daklib.upload.Source} or C{None}
298 if self._source is None:
300 for f in self.files.itervalues():
301 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename):
302 source_files.append(f)
303 if len(source_files) > 0:
304 self._source = Source(self.directory, source_files, self._keyrings, self._require_signature)
309 """C{True} if the upload includes source
312 return "source" in self.architectures
315 def source_name(self):
316 """source package name
319 return re_field_source.match(self.changes['Source']).group('package')
323 """included binary packages
324 @type: list of L{daklib.upload.Binary}
326 if self._binaries is None:
328 for f in self.files.itervalues():
329 if re_file_binary.match(f.filename):
330 binaries.append(Binary(self.directory, f))
331 self._binaries = binaries
332 return self._binaries
335 def byhand_files(self):
336 """included byhand files
337 @type: list of L{daklib.upload.HashedFile}
341 for f in self.files.itervalues():
342 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename) or re_file_binary.match(f.filename):
344 if f.section != 'byhand' and f.section[:4] != 'raw-':
345 raise InvalidChangesException("{0}: {1} looks like a byhand package, but is in section {2}".format(self.filename, f.filename, f.section))
351 def binary_names(self):
352 """names of included binary packages
355 return self.changes['Binary'].split()
358 def closed_bugs(self):
359 """bugs closed by this upload
362 return self.changes.get('Closes', '').split()
366 """dict mapping filenames to L{daklib.upload.HashedFile} objects
369 if self._files is None:
370 self._files = parse_file_list(self.changes, True)
375 """total size of files included in this upload in bytes
379 for f in self.files.itervalues():
383 def __cmp__(self, other):
384 """compare two changes files
386 We sort by source name and version first. If these are identical,
387 we sort changes that include source before those without source (so
388 that sourceful uploads get processed first), and finally fall back
389 to the filename (this should really never happen).
392 @return: n where n < 0 if self < other, n = 0 if self == other, n > 0 if self > other
394 ret = cmp(self.changes.get('Source'), other.changes.get('Source'))
398 ret = apt_pkg.version_compare(self.changes.get('Version', ''), other.changes.get('Version', ''))
401 # sort changes with source before changes without source
402 if 'source' in self.architectures and 'source' not in other.architectures:
404 elif 'source' not in self.architectures and 'source' in other.architectures:
410 # fall back to filename
411 ret = cmp(self.filename, other.filename)
415 class Binary(object):
416 """Representation of a binary package
418 def __init__(self, directory, hashed_file):
419 self.hashed_file = hashed_file
420 """file object for the .deb
424 path = os.path.join(directory, hashed_file.filename)
425 data = apt_inst.DebFile(path).control.extractdata("control")
427 self.control = apt_pkg.TagSection(data)
428 """dict to access fields in DEBIAN/control
433 def from_file(cls, directory, filename):
434 hashed_file = HashedFile.from_file(directory, filename)
435 return cls(directory, hashed_file)
439 """get tuple with source package name and version
442 source = self.control.get("Source", None)
444 return (self.control["Package"], self.control["Version"])
445 match = re_field_source.match(source)
447 raise InvalidBinaryException('{0}: Invalid Source field.'.format(self.hashed_file.filename))
448 version = match.group('version')
450 version = self.control['Version']
451 return (match.group('package'), version)
455 """package type ('deb' or 'udeb')
458 match = re_file_binary.match(self.hashed_file.filename)
460 raise InvalidBinaryException('{0}: Does not match re_file_binary'.format(self.hashed_file.filename))
461 return match.group('type')
468 fields = self.control['Section'].split('/')
473 class Source(object):
474 """Representation of a source package
476 def __init__(self, directory, hashed_files, keyrings, require_signature=True):
477 self.hashed_files = hashed_files
478 """list of source files (including the .dsc itself)
479 @type: list of L{HashedFile}
482 self._dsc_file = None
483 for f in hashed_files:
484 if re_file_dsc.match(f.filename):
485 if self._dsc_file is not None:
486 raise InvalidSourceException("Multiple .dsc found ({0} and {1})".format(self._dsc_file.filename, f.filename))
490 # make sure the hash for the dsc is valid before we use it
491 self._dsc_file.check(directory)
493 dsc_file_path = os.path.join(directory, self._dsc_file.filename)
494 data = open(dsc_file_path, 'r').read()
495 self._signed_file = SignedFile(data, keyrings, require_signature)
496 self.dsc = apt_pkg.TagSection(self._signed_file.contents)
497 """dict to access fields in the .dsc file
504 def from_file(cls, directory, filename, keyrings, require_signature=True):
505 hashed_file = HashedFile.from_file(directory, filename)
506 return cls(directory, [hashed_file], keyrings, require_signature)
510 """dict mapping filenames to L{HashedFile} objects for additional source files
512 This list does not include the .dsc itself.
516 if self._files is None:
517 self._files = parse_file_list(self.dsc, False)
521 def primary_fingerprint(self):
522 """fingerprint of the key used to sign the .dsc
525 return self._signed_file.primary_fingerprint
528 def valid_signature(self):
529 """C{True} if the .dsc has a valid signature
532 return self._signed_file.valid
536 """guessed component name
538 Might be wrong. Don't rely on this.
542 if 'Section' not in self.dsc:
544 fields = self.dsc['Section'].split('/')
551 """filename of .dsc file
554 return self._dsc_file.filename