1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to handle uploads not yet installed to the archive
19 This module provides classes to handle uploads not yet installed to the
20 archive. Central is the L{Changes} class which represents a changes file.
21 It provides methods to access the included binary and source packages.
30 from daklib.gpg import SignedFile
31 from daklib.regexes import *
32 import daklib.packagelist
34 class UploadException(Exception):
37 class InvalidChangesException(UploadException):
40 class InvalidBinaryException(UploadException):
43 class InvalidSourceException(UploadException):
46 class InvalidHashException(UploadException):
47 def __init__(self, filename, hash_name, expected, actual):
48 self.filename = filename
49 self.hash_name = hash_name
50 self.expected = expected
53 return ("Invalid {0} hash for {1}:\n"
54 "According to the control file the {0} hash should be {2},\n"
57 "If you did not include {1} in you upload, a different version\n"
58 "might already be known to the archive software.") \
59 .format(self.hash_name, self.filename, self.expected, self.actual)
61 class InvalidFilenameException(UploadException):
62 def __init__(self, filename):
63 self.filename = filename
65 return "Invalid filename '{0}'.".format(self.filename)
67 class FileDoesNotExist(UploadException):
68 def __init__(self, filename):
69 self.filename = filename
71 return "Refers to non-existing file '{0}'".format(self.filename)
73 class HashedFile(object):
74 """file with checksums
76 def __init__(self, filename, size, md5sum, sha1sum, sha256sum, section=None, priority=None):
77 self.filename = filename
88 """MD5 hash in hexdigits
92 self.sha1sum = sha1sum
93 """SHA1 hash in hexdigits
97 self.sha256sum = sha256sum
98 """SHA256 hash in hexdigits
102 self.section = section
103 """section or C{None}
104 @type: str or C{None}
107 self.priority = priority
108 """priority or C{None}
109 @type: str of C{None}
113 def from_file(cls, directory, filename, section=None, priority=None):
114 """create with values for an existing file
116 Create a C{HashedFile} object that refers to an already existing file.
119 @param directory: directory the file is located in
122 @param filename: filename
124 @type section: str or C{None}
125 @param section: optional section as given in .changes files
127 @type priority: str or C{None}
128 @param priority: optional priority as given in .changes files
130 @rtype: L{HashedFile}
131 @return: C{HashedFile} object for the given file
133 path = os.path.join(directory, filename)
134 with open(path, 'r') as fh:
135 size = os.fstat(fh.fileno()).st_size
136 hashes = apt_pkg.Hashes(fh)
137 return cls(filename, size, hashes.md5, hashes.sha1, hashes.sha256, section, priority)
139 def check(self, directory):
142 Check if size and hashes match the expected value.
145 @param directory: directory the file is located in
147 @raise InvalidHashException: hash mismatch
149 path = os.path.join(directory, self.filename)
152 with open(path) as fh:
153 size = os.fstat(fh.fileno()).st_size
154 hashes = apt_pkg.Hashes(fh)
156 if e.errno == errno.ENOENT:
157 raise FileDoesNotExist(self.filename)
160 if size != self.size:
161 raise InvalidHashException(self.filename, 'size', self.size, size)
163 if hashes.md5 != self.md5sum:
164 raise InvalidHashException(self.filename, 'md5sum', self.md5sum, hashes.md5)
166 if hashes.sha1 != self.sha1sum:
167 raise InvalidHashException(self.filename, 'sha1sum', self.sha1sum, hashes.sha1)
169 if hashes.sha256 != self.sha256sum:
170 raise InvalidHashException(self.filename, 'sha256sum', self.sha256sum, hashes.sha256)
172 def parse_file_list(control, has_priority_and_section):
173 """Parse Files and Checksums-* fields
175 @type control: dict-like
176 @param control: control file to take fields from
178 @type has_priority_and_section: bool
179 @param has_priority_and_section: Files field include section and priority
182 @raise InvalidChangesException: missing fields or other grave errors
185 @return: dict mapping filenames to L{daklib.upload.HashedFile} objects
189 for line in control.get("Files", "").split('\n'):
193 if has_priority_and_section:
194 (md5sum, size, section, priority, filename) = line.split()
195 entry = dict(md5sum=md5sum, size=long(size), section=section, priority=priority, filename=filename)
197 (md5sum, size, filename) = line.split()
198 entry = dict(md5sum=md5sum, size=long(size), filename=filename)
200 entries[filename] = entry
202 for line in control.get("Checksums-Sha1", "").split('\n'):
205 (sha1sum, size, filename) = line.split()
206 entry = entries.get(filename, None)
208 raise InvalidChangesException('{0} is listed in Checksums-Sha1, but not in Files.'.format(filename))
209 if entry is not None and entry.get('size', None) != long(size):
210 raise InvalidChangesException('Size for {0} in Files and Checksum-Sha1 fields differ.'.format(filename))
211 entry['sha1sum'] = sha1sum
213 for line in control.get("Checksums-Sha256", "").split('\n'):
216 (sha256sum, size, filename) = line.split()
217 entry = entries.get(filename, None)
219 raise InvalidChangesException('{0} is listed in Checksums-Sha256, but not in Files.'.format(filename))
220 if entry is not None and entry.get('size', None) != long(size):
221 raise InvalidChangesException('Size for {0} in Files and Checksum-Sha256 fields differ.'.format(filename))
222 entry['sha256sum'] = sha256sum
225 for entry in entries.itervalues():
226 filename = entry['filename']
227 if 'size' not in entry:
228 raise InvalidChangesException('No size for {0}.'.format(filename))
229 if 'md5sum' not in entry:
230 raise InvalidChangesException('No md5sum for {0}.'.format(filename))
231 if 'sha1sum' not in entry:
232 raise InvalidChangesException('No sha1sum for {0}.'.format(filename))
233 if 'sha256sum' not in entry:
234 raise InvalidChangesException('No sha256sum for {0}.'.format(filename))
235 if not re_file_safe.match(filename):
236 raise InvalidChangesException("{0}: References file with unsafe filename {1}.".format(self.filename, filename))
237 f = files[filename] = HashedFile(**entry)
241 class Changes(object):
242 """Representation of a .changes file
244 def __init__(self, directory, filename, keyrings, require_signature=True):
245 if not re_file_safe.match(filename):
246 raise InvalidChangesException('{0}: unsafe filename'.format(filename))
248 self.directory = directory
249 """directory the .changes is located in
253 self.filename = filename
254 """name of the .changes file
258 data = open(self.path).read()
259 self._signed_file = SignedFile(data, keyrings, require_signature)
260 self.changes = apt_pkg.TagSection(self._signed_file.contents)
261 """dict to access fields of the .changes file
265 self._binaries = None
268 self._keyrings = keyrings
269 self._require_signature = require_signature
273 """path to the .changes file
276 return os.path.join(self.directory, self.filename)
279 def primary_fingerprint(self):
280 """fingerprint of the key used for signing the .changes file
283 return self._signed_file.primary_fingerprint
286 def valid_signature(self):
287 """C{True} if the .changes has a valid signature
290 return self._signed_file.valid
293 def signature_timestamp(self):
294 return self._signed_file.signature_timestamp
297 def contents_sha1(self):
298 return self._signed_file.contents_sha1
301 def architectures(self):
302 """list of architectures included in the upload
305 return self.changes.get('Architecture', '').split()
308 def distributions(self):
309 """list of target distributions for the upload
312 return self.changes['Distribution'].split()
316 """included source or C{None}
317 @type: L{daklib.upload.Source} or C{None}
319 if self._source is None:
321 for f in self.files.itervalues():
322 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename):
323 source_files.append(f)
324 if len(source_files) > 0:
325 self._source = Source(self.directory, source_files, self._keyrings, self._require_signature)
330 """C{True} if the upload includes source
333 return "source" in self.architectures
336 def source_name(self):
337 """source package name
340 return re_field_source.match(self.changes['Source']).group('package')
344 """included binary packages
345 @type: list of L{daklib.upload.Binary}
347 if self._binaries is None:
349 for f in self.files.itervalues():
350 if re_file_binary.match(f.filename):
351 binaries.append(Binary(self.directory, f))
352 self._binaries = binaries
353 return self._binaries
356 def byhand_files(self):
357 """included byhand files
358 @type: list of L{daklib.upload.HashedFile}
362 for f in self.files.itervalues():
363 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename) or re_file_binary.match(f.filename):
365 if f.section != 'byhand' and f.section[:4] != 'raw-':
366 raise InvalidChangesException("{0}: {1} looks like a byhand package, but is in section {2}".format(self.filename, f.filename, f.section))
372 def binary_names(self):
373 """names of included binary packages
376 return self.changes['Binary'].split()
379 def closed_bugs(self):
380 """bugs closed by this upload
383 return self.changes.get('Closes', '').split()
387 """dict mapping filenames to L{daklib.upload.HashedFile} objects
390 if self._files is None:
391 self._files = parse_file_list(self.changes, True)
396 """total size of files included in this upload in bytes
400 for f in self.files.itervalues():
404 def __cmp__(self, other):
405 """compare two changes files
407 We sort by source name and version first. If these are identical,
408 we sort changes that include source before those without source (so
409 that sourceful uploads get processed first), and finally fall back
410 to the filename (this should really never happen).
413 @return: n where n < 0 if self < other, n = 0 if self == other, n > 0 if self > other
415 ret = cmp(self.changes.get('Source'), other.changes.get('Source'))
419 ret = apt_pkg.version_compare(self.changes.get('Version', ''), other.changes.get('Version', ''))
422 # sort changes with source before changes without source
423 if 'source' in self.architectures and 'source' not in other.architectures:
425 elif 'source' not in self.architectures and 'source' in other.architectures:
431 # fall back to filename
432 ret = cmp(self.filename, other.filename)
436 class Binary(object):
437 """Representation of a binary package
439 def __init__(self, directory, hashed_file):
440 self.hashed_file = hashed_file
441 """file object for the .deb
445 path = os.path.join(directory, hashed_file.filename)
446 data = apt_inst.DebFile(path).control.extractdata("control")
448 self.control = apt_pkg.TagSection(data)
449 """dict to access fields in DEBIAN/control
454 def from_file(cls, directory, filename):
455 hashed_file = HashedFile.from_file(directory, filename)
456 return cls(directory, hashed_file)
460 """get tuple with source package name and version
463 source = self.control.get("Source", None)
465 return (self.control["Package"], self.control["Version"])
466 match = re_field_source.match(source)
468 raise InvalidBinaryException('{0}: Invalid Source field.'.format(self.hashed_file.filename))
469 version = match.group('version')
471 version = self.control['Version']
472 return (match.group('package'), version)
476 return self.control['Package']
480 """package type ('deb' or 'udeb')
483 match = re_file_binary.match(self.hashed_file.filename)
485 raise InvalidBinaryException('{0}: Does not match re_file_binary'.format(self.hashed_file.filename))
486 return match.group('type')
493 fields = self.control['Section'].split('/')
498 class Source(object):
499 """Representation of a source package
501 def __init__(self, directory, hashed_files, keyrings, require_signature=True):
502 self.hashed_files = hashed_files
503 """list of source files (including the .dsc itself)
504 @type: list of L{HashedFile}
507 self._dsc_file = None
508 for f in hashed_files:
509 if re_file_dsc.match(f.filename):
510 if self._dsc_file is not None:
511 raise InvalidSourceException("Multiple .dsc found ({0} and {1})".format(self._dsc_file.filename, f.filename))
515 # make sure the hash for the dsc is valid before we use it
516 self._dsc_file.check(directory)
518 dsc_file_path = os.path.join(directory, self._dsc_file.filename)
519 data = open(dsc_file_path, 'r').read()
520 self._signed_file = SignedFile(data, keyrings, require_signature)
521 self.dsc = apt_pkg.TagSection(self._signed_file.contents)
522 """dict to access fields in the .dsc file
526 self.package_list = daklib.packagelist.PackageList(self.dsc)
527 """Information about packages built by the source.
528 @type: daklib.packagelist.PackageList
534 def from_file(cls, directory, filename, keyrings, require_signature=True):
535 hashed_file = HashedFile.from_file(directory, filename)
536 return cls(directory, [hashed_file], keyrings, require_signature)
540 """dict mapping filenames to L{HashedFile} objects for additional source files
542 This list does not include the .dsc itself.
546 if self._files is None:
547 self._files = parse_file_list(self.dsc, False)
551 def primary_fingerprint(self):
552 """fingerprint of the key used to sign the .dsc
555 return self._signed_file.primary_fingerprint
558 def valid_signature(self):
559 """C{True} if the .dsc has a valid signature
562 return self._signed_file.valid
566 """guessed component name
568 Might be wrong. Don't rely on this.
572 if 'Section' not in self.dsc:
574 fields = self.dsc['Section'].split('/')
581 """filename of .dsc file
584 return self._dsc_file.filename