1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to handle uploads not yet installed to the archive
19 This module provides classes to handle uploads not yet installed to the
20 archive. Central is the L{Changes} class which represents a changes file.
21 It provides methods to access the included binary and source packages.
29 from daklib.gpg import SignedFile
30 from daklib.regexes import *
31 import daklib.packagelist
33 class UploadException(Exception):
36 class InvalidChangesException(UploadException):
39 class InvalidBinaryException(UploadException):
42 class InvalidSourceException(UploadException):
45 class InvalidHashException(UploadException):
46 def __init__(self, filename, hash_name, expected, actual):
47 self.filename = filename
48 self.hash_name = hash_name
49 self.expected = expected
52 return ("Invalid {0} hash for {1}:\n"
53 "According to the control file the {0} hash should be {2},\n"
56 "If you did not include {1} in you upload, a different version\n"
57 "might already be known to the archive software.") \
58 .format(self.hash_name, self.filename, self.expected, self.actual)
60 class InvalidFilenameException(UploadException):
61 def __init__(self, filename):
62 self.filename = filename
64 return "Invalid filename '{0}'.".format(self.filename)
66 class FileDoesNotExist(UploadException):
67 def __init__(self, filename):
68 self.filename = filename
70 return "Refers to non-existing file '{0}'".format(self.filename)
72 class HashedFile(object):
73 """file with checksums
75 def __init__(self, filename, size, md5sum, sha1sum, sha256sum, section=None, priority=None):
76 self.filename = filename
87 """MD5 hash in hexdigits
91 self.sha1sum = sha1sum
92 """SHA1 hash in hexdigits
96 self.sha256sum = sha256sum
97 """SHA256 hash in hexdigits
101 self.section = section
102 """section or C{None}
103 @type: str or C{None}
106 self.priority = priority
107 """priority or C{None}
108 @type: str of C{None}
112 def from_file(cls, directory, filename, section=None, priority=None):
113 """create with values for an existing file
115 Create a C{HashedFile} object that refers to an already existing file.
118 @param directory: directory the file is located in
121 @param filename: filename
123 @type section: str or C{None}
124 @param section: optional section as given in .changes files
126 @type priority: str or C{None}
127 @param priority: optional priority as given in .changes files
129 @rtype: L{HashedFile}
130 @return: C{HashedFile} object for the given file
132 path = os.path.join(directory, filename)
133 with open(path, 'r') as fh:
134 size = os.fstat(fh.fileno()).st_size
135 hashes = apt_pkg.Hashes(fh)
136 return cls(filename, size, hashes.md5, hashes.sha1, hashes.sha256, section, priority)
138 def check(self, directory):
141 Check if size and hashes match the expected value.
144 @param directory: directory the file is located in
146 @raise InvalidHashException: hash mismatch
148 path = os.path.join(directory, self.filename)
151 with open(path) as fh:
152 size = os.fstat(fh.fileno()).st_size
153 hashes = apt_pkg.Hashes(fh)
155 if e.errno == errno.ENOENT:
156 raise FileDoesNotExist(self.filename)
159 if size != self.size:
160 raise InvalidHashException(self.filename, 'size', self.size, size)
162 if hashes.md5 != self.md5sum:
163 raise InvalidHashException(self.filename, 'md5sum', self.md5sum, hashes.md5)
165 if hashes.sha1 != self.sha1sum:
166 raise InvalidHashException(self.filename, 'sha1sum', self.sha1sum, hashes.sha1)
168 if hashes.sha256 != self.sha256sum:
169 raise InvalidHashException(self.filename, 'sha256sum', self.sha256sum, hashes.sha256)
171 def parse_file_list(control, has_priority_and_section):
172 """Parse Files and Checksums-* fields
174 @type control: dict-like
175 @param control: control file to take fields from
177 @type has_priority_and_section: bool
178 @param has_priority_and_section: Files field include section and priority
181 @raise InvalidChangesException: missing fields or other grave errors
184 @return: dict mapping filenames to L{daklib.upload.HashedFile} objects
188 for line in control.get("Files", "").split('\n'):
192 if has_priority_and_section:
193 (md5sum, size, section, priority, filename) = line.split()
194 entry = dict(md5sum=md5sum, size=long(size), section=section, priority=priority, filename=filename)
196 (md5sum, size, filename) = line.split()
197 entry = dict(md5sum=md5sum, size=long(size), filename=filename)
199 entries[filename] = entry
201 for line in control.get("Checksums-Sha1", "").split('\n'):
204 (sha1sum, size, filename) = line.split()
205 entry = entries.get(filename, None)
207 raise InvalidChangesException('{0} is listed in Checksums-Sha1, but not in Files.'.format(filename))
208 if entry is not None and entry.get('size', None) != long(size):
209 raise InvalidChangesException('Size for {0} in Files and Checksum-Sha1 fields differ.'.format(filename))
210 entry['sha1sum'] = sha1sum
212 for line in control.get("Checksums-Sha256", "").split('\n'):
215 (sha256sum, size, filename) = line.split()
216 entry = entries.get(filename, None)
218 raise InvalidChangesException('{0} is listed in Checksums-Sha256, but not in Files.'.format(filename))
219 if entry is not None and entry.get('size', None) != long(size):
220 raise InvalidChangesException('Size for {0} in Files and Checksum-Sha256 fields differ.'.format(filename))
221 entry['sha256sum'] = sha256sum
224 for entry in entries.itervalues():
225 filename = entry['filename']
226 if 'size' not in entry:
227 raise InvalidChangesException('No size for {0}.'.format(filename))
228 if 'md5sum' not in entry:
229 raise InvalidChangesException('No md5sum for {0}.'.format(filename))
230 if 'sha1sum' not in entry:
231 raise InvalidChangesException('No sha1sum for {0}.'.format(filename))
232 if 'sha256sum' not in entry:
233 raise InvalidChangesException('No sha256sum for {0}.'.format(filename))
234 if not re_file_safe.match(filename):
235 raise InvalidChangesException("{0}: References file with unsafe filename {1}.".format(self.filename, filename))
236 f = files[filename] = HashedFile(**entry)
240 class Changes(object):
241 """Representation of a .changes file
243 def __init__(self, directory, filename, keyrings, require_signature=True):
244 if not re_file_safe.match(filename):
245 raise InvalidChangesException('{0}: unsafe filename'.format(filename))
247 self.directory = directory
248 """directory the .changes is located in
252 self.filename = filename
253 """name of the .changes file
257 data = open(self.path).read()
258 self._signed_file = SignedFile(data, keyrings, require_signature)
259 self.changes = apt_pkg.TagSection(self._signed_file.contents)
260 """dict to access fields of the .changes file
264 self._binaries = None
267 self._keyrings = keyrings
268 self._require_signature = require_signature
272 """path to the .changes file
275 return os.path.join(self.directory, self.filename)
278 def primary_fingerprint(self):
279 """fingerprint of the key used for signing the .changes file
282 return self._signed_file.primary_fingerprint
285 def valid_signature(self):
286 """C{True} if the .changes has a valid signature
289 return self._signed_file.valid
292 def signature_timestamp(self):
293 return self._signed_file.signature_timestamp
296 def contents_sha1(self):
297 return self._signed_file.contents_sha1
300 def architectures(self):
301 """list of architectures included in the upload
304 return self.changes.get('Architecture', '').split()
307 def distributions(self):
308 """list of target distributions for the upload
311 return self.changes['Distribution'].split()
315 """included source or C{None}
316 @type: L{daklib.upload.Source} or C{None}
318 if self._source is None:
320 for f in self.files.itervalues():
321 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename):
322 source_files.append(f)
323 if len(source_files) > 0:
324 self._source = Source(self.directory, source_files, self._keyrings, self._require_signature)
329 """C{True} if the upload includes source
332 return "source" in self.architectures
335 def source_name(self):
336 """source package name
339 return re_field_source.match(self.changes['Source']).group('package')
343 """included binary packages
344 @type: list of L{daklib.upload.Binary}
346 if self._binaries is None:
348 for f in self.files.itervalues():
349 if re_file_binary.match(f.filename):
350 binaries.append(Binary(self.directory, f))
351 self._binaries = binaries
352 return self._binaries
355 def byhand_files(self):
356 """included byhand files
357 @type: list of L{daklib.upload.HashedFile}
361 for f in self.files.itervalues():
362 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename) or re_file_binary.match(f.filename):
364 if f.section != 'byhand' and f.section[:4] != 'raw-':
365 raise InvalidChangesException("{0}: {1} looks like a byhand package, but is in section {2}".format(self.filename, f.filename, f.section))
371 def binary_names(self):
372 """names of included binary packages
375 return self.changes['Binary'].split()
378 def closed_bugs(self):
379 """bugs closed by this upload
382 return self.changes.get('Closes', '').split()
386 """dict mapping filenames to L{daklib.upload.HashedFile} objects
389 if self._files is None:
390 self._files = parse_file_list(self.changes, True)
395 """total size of files included in this upload in bytes
399 for f in self.files.itervalues():
403 def __cmp__(self, other):
404 """compare two changes files
406 We sort by source name and version first. If these are identical,
407 we sort changes that include source before those without source (so
408 that sourceful uploads get processed first), and finally fall back
409 to the filename (this should really never happen).
412 @return: n where n < 0 if self < other, n = 0 if self == other, n > 0 if self > other
414 ret = cmp(self.changes.get('Source'), other.changes.get('Source'))
418 ret = apt_pkg.version_compare(self.changes.get('Version', ''), other.changes.get('Version', ''))
421 # sort changes with source before changes without source
422 if 'source' in self.architectures and 'source' not in other.architectures:
424 elif 'source' not in self.architectures and 'source' in other.architectures:
430 # fall back to filename
431 ret = cmp(self.filename, other.filename)
435 class Binary(object):
436 """Representation of a binary package
438 def __init__(self, directory, hashed_file):
439 self.hashed_file = hashed_file
440 """file object for the .deb
444 path = os.path.join(directory, hashed_file.filename)
445 data = apt_inst.DebFile(path).control.extractdata("control")
447 self.control = apt_pkg.TagSection(data)
448 """dict to access fields in DEBIAN/control
453 def from_file(cls, directory, filename):
454 hashed_file = HashedFile.from_file(directory, filename)
455 return cls(directory, hashed_file)
459 """get tuple with source package name and version
462 source = self.control.get("Source", None)
464 return (self.control["Package"], self.control["Version"])
465 match = re_field_source.match(source)
467 raise InvalidBinaryException('{0}: Invalid Source field.'.format(self.hashed_file.filename))
468 version = match.group('version')
470 version = self.control['Version']
471 return (match.group('package'), version)
475 return self.control['Package']
479 """package type ('deb' or 'udeb')
482 match = re_file_binary.match(self.hashed_file.filename)
484 raise InvalidBinaryException('{0}: Does not match re_file_binary'.format(self.hashed_file.filename))
485 return match.group('type')
492 fields = self.control['Section'].split('/')
497 class Source(object):
498 """Representation of a source package
500 def __init__(self, directory, hashed_files, keyrings, require_signature=True):
501 self.hashed_files = hashed_files
502 """list of source files (including the .dsc itself)
503 @type: list of L{HashedFile}
506 self._dsc_file = None
507 for f in hashed_files:
508 if re_file_dsc.match(f.filename):
509 if self._dsc_file is not None:
510 raise InvalidSourceException("Multiple .dsc found ({0} and {1})".format(self._dsc_file.filename, f.filename))
514 # make sure the hash for the dsc is valid before we use it
515 self._dsc_file.check(directory)
517 dsc_file_path = os.path.join(directory, self._dsc_file.filename)
518 data = open(dsc_file_path, 'r').read()
519 self._signed_file = SignedFile(data, keyrings, require_signature)
520 self.dsc = apt_pkg.TagSection(self._signed_file.contents)
521 """dict to access fields in the .dsc file
525 self.package_list = daklib.packagelist.PackageList(self.dsc)
526 """Information about packages built by the source.
527 @type: daklib.packagelist.PackageList
533 def from_file(cls, directory, filename, keyrings, require_signature=True):
534 hashed_file = HashedFile.from_file(directory, filename)
535 return cls(directory, [hashed_file], keyrings, require_signature)
539 """dict mapping filenames to L{HashedFile} objects for additional source files
541 This list does not include the .dsc itself.
545 if self._files is None:
546 self._files = parse_file_list(self.dsc, False)
550 def primary_fingerprint(self):
551 """fingerprint of the key used to sign the .dsc
554 return self._signed_file.primary_fingerprint
557 def valid_signature(self):
558 """C{True} if the .dsc has a valid signature
561 return self._signed_file.valid
565 """guessed component name
567 Might be wrong. Don't rely on this.
571 if 'Section' not in self.dsc:
573 fields = self.dsc['Section'].split('/')
580 """filename of .dsc file
583 return self._dsc_file.filename