1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to manipulate the archive
19 This module provides classes to manipulate the archive.
22 from daklib.dbconn import *
23 import daklib.checks as checks
24 from daklib.config import Config
25 import daklib.upload as upload
26 import daklib.utils as utils
27 from daklib.fstransactions import FilesystemTransaction
28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
31 from datetime import datetime
35 from sqlalchemy.orm.exc import NoResultFound
39 class ArchiveException(Exception):
42 class HashMismatchException(ArchiveException):
45 class ArchiveTransaction(object):
46 """manipulate the archive in a transaction
49 self.fs = FilesystemTransaction()
50 self.session = DBConn().session()
52 def get_file(self, hashed_file, source_name, check_hashes=True):
53 """Look for file C{hashed_file} in database
55 @type hashed_file: L{daklib.upload.HashedFile}
56 @param hashed_file: file to look for in the database
58 @type source_name: str
59 @param source_name: source package name
61 @type check_hashes: bool
62 @param check_hashes: check size and hashes match
64 @raise KeyError: file was not found in the database
65 @raise HashMismatchException: hash mismatch
67 @rtype: L{daklib.dbconn.PoolFile}
68 @return: database entry for the file
70 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
72 poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
73 if check_hashes and (poolfile.filesize != hashed_file.size
74 or poolfile.md5sum != hashed_file.md5sum
75 or poolfile.sha1sum != hashed_file.sha1sum
76 or poolfile.sha256sum != hashed_file.sha256sum):
77 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
80 raise KeyError('{0} not found in database.'.format(poolname))
82 def _install_file(self, directory, hashed_file, archive, component, source_name):
85 Will not give an error when the file is already present.
87 @rtype: L{daklib.dbconn.PoolFile}
88 @return: batabase object for the new file
90 session = self.session
92 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
94 poolfile = self.get_file(hashed_file, source_name)
96 poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
97 poolfile.md5sum = hashed_file.md5sum
98 poolfile.sha1sum = hashed_file.sha1sum
99 poolfile.sha256sum = hashed_file.sha256sum
100 session.add(poolfile)
104 session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
105 except NoResultFound:
106 archive_file = ArchiveFile(archive, component, poolfile)
107 session.add(archive_file)
110 path = os.path.join(archive.path, 'pool', component.component_name, poolname)
111 hashed_file_path = os.path.join(directory, hashed_file.filename)
112 self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
116 def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
117 """Install a binary package
120 @param directory: directory the binary package is located in
122 @type binary: L{daklib.upload.Binary}
123 @param binary: binary package to install
125 @type suite: L{daklib.dbconn.Suite}
126 @param suite: target suite
128 @type component: L{daklib.dbconn.Component}
129 @param component: target component
131 @type allow_tainted: bool
132 @param allow_tainted: allow to copy additional files from tainted archives
134 @type fingerprint: L{daklib.dbconn.Fingerprint}
135 @param fingerprint: optional fingerprint
137 @type source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
138 @param source_suites: suites to copy the source from if they are not
139 in C{suite} or C{True} to allow copying from any
142 @type extra_source_archives: list of L{daklib.dbconn.Archive}
143 @param extra_source_archives: extra archives to copy Built-Using sources from
145 @rtype: L{daklib.dbconn.DBBinary}
146 @return: databse object for the new package
148 session = self.session
149 control = binary.control
150 maintainer = get_or_set_maintainer(control['Maintainer'], session)
151 architecture = get_architecture(control['Architecture'], session)
153 (source_name, source_version) = binary.source
154 source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
155 source = source_query.filter(DBSource.suites.contains(suite)).first()
157 if source_suites != True:
158 source_query = source_query.join(DBSource.suites) \
159 .filter(Suite.suite_id == source_suites.c.id)
160 source = source_query.first()
162 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
163 self.copy_source(source, suite, component)
165 db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
168 package=control['Package'],
169 version=control['Version'],
170 architecture=architecture,
174 maintainer=maintainer,
176 binarytype=binary.type,
177 fingerprint=fingerprint,
181 db_binary = session.query(DBBinary).filter_by(**unique).one()
182 for key, value in rest.iteritems():
183 if getattr(db_binary, key) != value:
184 raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
185 except NoResultFound:
186 db_binary = DBBinary(**unique)
187 for key, value in rest.iteritems():
188 setattr(db_binary, key, value)
189 session.add(db_binary)
191 import_metadata_into_db(db_binary, session)
193 self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
195 if suite not in db_binary.suites:
196 db_binary.suites.append(suite)
202 def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
203 """ensure source exists in the given archive
205 This is intended to be used to check that Built-Using sources exist.
208 @param filename: filename to use in error messages
210 @type source: L{daklib.dbconn.DBSource}
211 @param source: source to look for
213 @type archive: L{daklib.dbconn.Archive}
214 @param archive: archive to look in
216 @type extra_archives: list of L{daklib.dbconn.Archive}
217 @param extra_archives: list of archives to copy the source package from
218 if it is not yet present in C{archive}
220 session = self.session
221 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
222 if db_file is not None:
225 # Try to copy file from one extra archive
226 if extra_archives is None:
228 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
230 raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
232 source_archive = db_file.archive
233 for dsc_file in source.srcfiles:
234 af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
235 # We were given an explicit list of archives so it is okay to copy from tainted archives.
236 self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
238 def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
239 """Add Built-Using sources to C{db_binary.extra_sources}
241 session = self.session
242 built_using = control.get('Built-Using', None)
244 if built_using is not None:
245 for dep in apt_pkg.parse_depends(built_using):
246 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
247 bu_source_name, bu_source_version, comp = dep[0]
248 assert comp == '=', 'Built-Using must contain strict dependencies'
250 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
251 if bu_source is None:
252 raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
254 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
256 db_binary.extra_sources.append(bu_source)
258 def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
259 """Install a source package
262 @param directory: directory the source package is located in
264 @type source: L{daklib.upload.Source}
265 @param source: source package to install
267 @type suite: L{daklib.dbconn.Suite}
268 @param suite: target suite
270 @type component: L{daklib.dbconn.Component}
271 @param component: target component
273 @type changed_by: L{daklib.dbconn.Maintainer}
274 @param changed_by: person who prepared this version of the package
276 @type allow_tainted: bool
277 @param allow_tainted: allow to copy additional files from tainted archives
279 @type fingerprint: L{daklib.dbconn.Fingerprint}
280 @param fingerprint: optional fingerprint
282 @rtype: L{daklib.dbconn.DBSource}
283 @return: database object for the new source
285 session = self.session
286 archive = suite.archive
288 maintainer = get_or_set_maintainer(control['Maintainer'], session)
289 source_name = control['Source']
291 ### Add source package to database
293 # We need to install the .dsc first as the DBSource object refers to it.
294 db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
298 version=control['Version'],
301 maintainer=maintainer,
302 changedby=changed_by,
303 #install_date=datetime.now().date(),
304 poolfile=db_file_dsc,
305 fingerprint=fingerprint,
306 dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
311 db_source = session.query(DBSource).filter_by(**unique).one()
312 for key, value in rest.iteritems():
313 if getattr(db_source, key) != value:
314 raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
315 except NoResultFound:
317 db_source = DBSource(**unique)
318 for key, value in rest.iteritems():
319 setattr(db_source, key, value)
320 # XXX: set as default in postgres?
321 db_source.install_date = datetime.now().date()
322 session.add(db_source)
325 # Add .dsc file. Other files will be added later.
326 db_dsc_file = DSCFile()
327 db_dsc_file.source = db_source
328 db_dsc_file.poolfile = db_file_dsc
329 session.add(db_dsc_file)
332 if suite in db_source.suites:
335 db_source.suites.append(suite)
338 for f in db_source.srcfiles:
339 self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
342 ### Now add remaining files and copy them to the archive.
344 for hashed_file in source.files.itervalues():
345 hashed_file_path = os.path.join(directory, hashed_file.filename)
346 if os.path.exists(hashed_file_path):
347 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
350 db_file = self.get_file(hashed_file, source_name)
351 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
353 db_dsc_file = DSCFile()
354 db_dsc_file.source = db_source
355 db_dsc_file.poolfile = db_file
356 session.add(db_dsc_file)
360 # Importing is safe as we only arrive here when we did not find the source already installed earlier.
361 import_metadata_into_db(db_source, session)
363 # Uploaders are the maintainer and co-maintainers from the Uploaders field
364 db_source.uploaders.append(maintainer)
365 if 'Uploaders' in control:
366 from daklib.textutils import split_uploaders
367 for u in split_uploaders(control['Uploaders']):
368 db_source.uploaders.append(get_or_set_maintainer(u, session))
373 def _copy_file(self, db_file, archive, component, allow_tainted=False):
374 """Copy a file to the given archive and component
376 @type db_file: L{daklib.dbconn.PoolFile}
377 @param db_file: file to copy
379 @type archive: L{daklib.dbconn.Archive}
380 @param archive: target archive
382 @type component: L{daklib.dbconn.Archive}
383 @param component: target component
385 @type allow_tainted: bool
386 @param allow_tainted: allow to copy from tainted archives (such as NEW)
388 session = self.session
390 if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
391 query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
392 if not allow_tainted:
393 query = query.join(Archive).filter(Archive.tainted == False)
395 source_af = query.first()
396 if source_af is None:
397 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
398 target_af = ArchiveFile(archive, component, db_file)
399 session.add(target_af)
401 self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
403 def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
404 """Copy a binary package to the given suite and component
406 @type db_binary: L{daklib.dbconn.DBBinary}
407 @param db_binary: binary to copy
409 @type suite: L{daklib.dbconn.Suite}
410 @param suite: target suite
412 @type component: L{daklib.dbconn.Component}
413 @param component: target component
415 @type allow_tainted: bool
416 @param allow_tainted: allow to copy from tainted archives (such as NEW)
418 @type extra_archives: list of L{daklib.dbconn.Archive}
419 @param extra_archives: extra archives to copy Built-Using sources from
421 session = self.session
422 archive = suite.archive
426 filename = db_binary.poolfile.filename
428 # make sure source is present in target archive
429 db_source = db_binary.source
430 if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
431 raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
433 # make sure built-using packages are present in target archive
434 for db_source in db_binary.extra_sources:
435 self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
438 db_file = db_binary.poolfile
439 self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
440 if suite not in db_binary.suites:
441 db_binary.suites.append(suite)
444 def copy_source(self, db_source, suite, component, allow_tainted=False):
445 """Copy a source package to the given suite and component
447 @type db_source: L{daklib.dbconn.DBSource}
448 @param db_source: source to copy
450 @type suite: L{daklib.dbconn.Suite}
451 @param suite: target suite
453 @type component: L{daklib.dbconn.Component}
454 @param component: target component
456 @type allow_tainted: bool
457 @param allow_tainted: allow to copy from tainted archives (such as NEW)
459 archive = suite.archive
462 for db_dsc_file in db_source.srcfiles:
463 self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
464 if suite not in db_source.suites:
465 db_source.suites.append(suite)
468 def remove_file(self, db_file, archive, component):
469 """Remove a file from a given archive and component
471 @type db_file: L{daklib.dbconn.PoolFile}
472 @param db_file: file to remove
474 @type archive: L{daklib.dbconn.Archive}
475 @param archive: archive to remove the file from
477 @type component: L{daklib.dbconn.Component}
478 @param component: component to remove the file from
480 af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
481 self.fs.unlink(af.path)
482 self.session.delete(af)
484 def remove_binary(self, binary, suite):
485 """Remove a binary from a given suite and component
487 @type binary: L{daklib.dbconn.DBBinary}
488 @param binary: binary to remove
490 @type suite: L{daklib.dbconn.Suite}
491 @param suite: suite to remove the package from
493 binary.suites.remove(suite)
496 def remove_source(self, source, suite):
497 """Remove a source from a given suite and component
499 @type source: L{daklib.dbconn.DBSource}
500 @param source: source to remove
502 @type suite: L{daklib.dbconn.Suite}
503 @param suite: suite to remove the package from
505 @raise ArchiveException: source package is still referenced by other
506 binaries in the suite
508 session = self.session
510 query = session.query(DBBinary).filter_by(source=source) \
511 .filter(DBBinary.suites.contains(suite))
512 if query.first() is not None:
513 raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
515 source.suites.remove(suite)
521 self.session.commit()
524 self.session.rollback()
528 """rollback changes"""
529 self.session.rollback()
535 def __exit__(self, type, value, traceback):
542 class ArchiveUpload(object):
545 This class can be used in a with-statement::
547 with ArchiveUpload(...) as upload:
550 Doing so will automatically run any required cleanup and also rollback the
551 transaction if it was not committed.
553 def __init__(self, directory, changes, keyrings):
554 self.transaction = ArchiveTransaction()
555 """transaction used to handle the upload
556 @type: L{daklib.archive.ArchiveTransaction}
559 self.session = self.transaction.session
560 """database session"""
562 self.original_directory = directory
563 self.original_changes = changes
567 @type: L{daklib.upload.Changes}
570 self.directory = None
571 """directory with temporary copy of files. set by C{prepare}
575 self.keyrings = keyrings
577 self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
578 """fingerprint of the key used to sign the upload
579 @type: L{daklib.dbconn.Fingerprint}
582 self.reject_reasons = []
583 """reasons why the upload cannot by accepted
593 self.final_suites = None
596 """upload is NEW. set by C{check}
600 self._checked = False
601 """checks passes. set by C{check}
605 self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
606 self._new = self._new_queue.suite
608 def warn(self, message):
609 """add a warning message
611 Adds a warning message that can later be seen in C{self.warnings}
613 @type message: string
614 @param message: warning message
616 self.warnings.append(message)
619 """prepare upload for further processing
621 This copies the files involved to a temporary directory. If you use
622 this method directly, you have to remove the directory given by the
623 C{directory} attribute later on your own.
625 Instead of using the method directly, you can also use a with-statement::
627 with ArchiveUpload(...) as upload:
630 This will automatically handle any required cleanup.
632 assert self.directory is None
633 assert self.original_changes.valid_signature
636 session = self.transaction.session
638 self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
639 mode=0o2750, group=cnf.unprivgroup)
640 with FilesystemTransaction() as fs:
641 src = os.path.join(self.original_directory, self.original_changes.filename)
642 dst = os.path.join(self.directory, self.original_changes.filename)
643 fs.copy(src, dst, mode=0o640)
645 self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
647 for f in self.changes.files.itervalues():
648 src = os.path.join(self.original_directory, f.filename)
649 dst = os.path.join(self.directory, f.filename)
650 if not os.path.exists(src):
652 fs.copy(src, dst, mode=0o640)
654 source = self.changes.source
655 if source is not None:
656 for f in source.files.itervalues():
657 src = os.path.join(self.original_directory, f.filename)
658 dst = os.path.join(self.directory, f.filename)
659 if not os.path.exists(dst):
661 db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
662 db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
663 fs.copy(db_archive_file.path, dst, symlink=True)
665 # Ignore if get_file could not find it. Upload will
666 # probably be rejected later.
669 def unpacked_source(self):
670 """Path to unpacked source
672 Get path to the unpacked source. This method does unpack the source
673 into a temporary directory under C{self.directory} if it has not
674 been done so already.
676 @rtype: str or C{None}
677 @return: string giving the path to the unpacked source directory
678 or C{None} if no source was included in the upload.
680 assert self.directory is not None
682 source = self.changes.source
685 dsc_path = os.path.join(self.directory, source._dsc_file.filename)
687 sourcedir = os.path.join(self.directory, 'source')
688 if not os.path.exists(sourcedir):
689 devnull = open('/dev/null', 'w')
690 subprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
691 if not os.path.isdir(sourcedir):
692 raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
695 def _map_suite(self, suite_name):
696 for rule in Config().value_list("SuiteMappings"):
697 fields = rule.split()
699 if rtype == "map" or rtype == "silent-map":
700 (src, dst) = fields[1:3]
701 if src == suite_name:
703 if rtype != "silent-map":
704 self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
705 elif rtype == "ignore":
707 if suite_name == ignored:
708 self.warnings.append('Ignoring target suite {0}.'.format(ignored))
710 elif rtype == "reject":
712 if suite_name == rejected:
713 self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
714 ## XXX: propup-version and map-unreleased not yet implemented
717 def _mapped_suites(self):
718 """Get target suites after mappings
720 @rtype: list of L{daklib.dbconn.Suite}
721 @return: list giving the mapped target suites of this upload
723 session = self.session
726 for dist in self.changes.distributions:
727 suite_name = self._map_suite(dist)
728 if suite_name is not None:
729 suite_names.append(suite_name)
731 suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
734 def _check_new(self, suite):
735 """Check if upload is NEW
737 An upload is NEW if it has binary or source packages that do not have
738 an override in C{suite} OR if it references files ONLY in a tainted
739 archive (eg. when it references files in NEW).
742 @return: C{True} if the upload is NEW, C{False} otherwise
744 session = self.session
747 # Check for missing overrides
748 for b in self.changes.binaries:
749 override = self._binary_override(suite, b)
751 self.warnings.append('binary:{0} is NEW.'.format(b.control['Package']))
754 if self.changes.source is not None:
755 override = self._source_override(suite, self.changes.source)
757 self.warnings.append('source:{0} is NEW.'.format(self.changes.source.control['Source']))
760 # Check if we reference a file only in a tainted archive
761 files = self.changes.files.values()
762 if self.changes.source is not None:
763 files.extend(self.changes.source.files.values())
765 query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
766 query_untainted = query.join(Archive).filter(Archive.tainted == False)
768 in_archive = (query.first() is not None)
769 in_untainted_archive = (query_untainted.first() is not None)
771 if in_archive and not in_untainted_archive:
772 self.warnings.append('{0} is only available in NEW.'.format(f.filename))
777 def _final_suites(self):
778 session = self.session
780 mapped_suites = self._mapped_suites()
783 for suite in mapped_suites:
784 overridesuite = suite
785 if suite.overridesuite is not None:
786 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
787 if self._check_new(overridesuite):
789 final_suites.add(suite)
793 def _binary_override(self, suite, binary):
794 """Get override entry for a binary
796 @type suite: L{daklib.dbconn.Suite}
797 @param suite: suite to get override for
799 @type binary: L{daklib.upload.Binary}
800 @param binary: binary to get override for
802 @rtype: L{daklib.dbconn.Override} or C{None}
803 @return: override for the given binary or C{None}
805 if suite.overridesuite is not None:
806 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
808 mapped_component = get_mapped_component(binary.component)
809 if mapped_component is None:
812 query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
813 .join(Component).filter(Component.component_name == mapped_component.component_name) \
814 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
818 except NoResultFound:
821 def _source_override(self, suite, source):
822 """Get override entry for a source
824 @type suite: L{daklib.dbconn.Suite}
825 @param suite: suite to get override for
827 @type source: L{daklib.upload.Source}
828 @param source: source to get override for
830 @rtype: L{daklib.dbconn.Override} or C{None}
831 @return: override for the given source or C{None}
833 if suite.overridesuite is not None:
834 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
836 # XXX: component for source?
837 query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
838 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
842 except NoResultFound:
845 def _binary_component(self, suite, binary, only_overrides=True):
846 """get component for a binary
848 By default this will only look at overrides to get the right component;
849 if C{only_overrides} is C{False} this method will also look at the
852 @type suite: L{daklib.dbconn.Suite}
854 @type binary: L{daklib.upload.Binary}
856 @type only_overrides: bool
857 @param only_overrides: only use overrides to get the right component
859 @rtype: L{daklib.dbconn.Component} or C{None}
861 override = self._binary_override(suite, binary)
862 if override is not None:
863 return override.component
866 return get_mapped_component(binary.component, self.session)
868 def check(self, force=False):
869 """run checks against the upload
872 @param force: ignore failing forcable checks
875 @return: C{True} if all checks passed, C{False} otherwise
877 # XXX: needs to be better structured.
878 assert self.changes.valid_signature
881 # Validate signatures and hashes before we do any real work:
883 checks.SignatureCheck,
886 checks.ExternalHashesCheck,
889 checks.BinaryTimestampCheck,
890 checks.SingleDistributionCheck,
894 final_suites = self._final_suites()
895 if len(final_suites) == 0:
896 self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
899 self.final_suites = final_suites
902 checks.TransitionCheck,
904 checks.NoSourceOnlyCheck,
911 checks.SourceFormatCheck,
912 checks.SuiteArchitectureCheck,
915 for suite in final_suites:
916 chk().per_suite_check(self, suite)
918 if len(self.reject_reasons) != 0:
923 except checks.Reject as e:
924 self.reject_reasons.append(unicode(e))
925 except Exception as e:
926 self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
929 def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
930 """Install upload to the given suite
932 @type suite: L{daklib.dbconn.Suite}
933 @param suite: suite to install the package into. This is the real suite,
934 ie. after any redirection to NEW or a policy queue
936 @param source_component_func: function to get the L{daklib.dbconn.Component}
937 for a L{daklib.upload.Source} object
939 @param binary_component_func: function to get the L{daklib.dbconn.Component}
940 for a L{daklib.upload.Binary} object
942 @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
944 @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
946 @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
947 object for the install source or C{None} if no source was
948 included. The second is a list of L{daklib.dbconn.DBBinary}
949 objects for the installed binary packages.
951 # XXX: move this function to ArchiveTransaction?
953 control = self.changes.changes
954 changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
956 if source_suites is None:
957 source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()
959 source = self.changes.source
960 if source is not None:
961 component = source_component_func(source)
962 db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
967 for binary in self.changes.binaries:
968 component = binary_component_func(binary)
969 db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
970 db_binaries.append(db_binary)
972 if suite.copychanges:
973 src = os.path.join(self.directory, self.changes.filename)
974 dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
975 self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
977 return (db_source, db_binaries)
979 def _install_changes(self):
980 assert self.changes.valid_signature
981 control = self.changes.changes
982 session = self.transaction.session
986 # Only add changelog for sourceful uploads and binNMUs
987 if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
988 query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
989 changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
990 assert changelog_id is not None
992 db_changes = DBChange()
993 db_changes.changesname = self.changes.filename
994 db_changes.source = control['Source']
995 db_changes.binaries = control.get('Binary', None)
996 db_changes.architecture = control['Architecture']
997 db_changes.version = control['Version']
998 db_changes.distribution = control['Distribution']
999 db_changes.urgency = control['Urgency']
1000 db_changes.maintainer = control['Maintainer']
1001 db_changes.changedby = control.get('Changed-By', control['Maintainer'])
1002 db_changes.date = control['Date']
1003 db_changes.fingerprint = self.fingerprint.fingerprint
1004 db_changes.changelog_id = changelog_id
1005 db_changes.closes = self.changes.closed_bugs
1007 self.transaction.session.add(db_changes)
1008 self.transaction.session.flush()
1012 def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
1013 u = PolicyQueueUpload()
1014 u.policy_queue = policy_queue
1015 u.target_suite = target_suite
1016 u.changes = db_changes
1017 u.source = db_source
1018 u.binaries = db_binaries
1019 self.transaction.session.add(u)
1020 self.transaction.session.flush()
1022 dst = os.path.join(policy_queue.path, self.changes.filename)
1023 self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)
1027 def try_autobyhand(self):
1030 Try to handle byhand packages automatically.
1032 @rtype: list of L{daklib.upload.HashedFile}
1033 @return: list of remaining byhand files
1035 assert len(self.reject_reasons) == 0
1036 assert self.changes.valid_signature
1037 assert self.final_suites is not None
1038 assert self._checked
1040 byhand = self.changes.byhand_files
1041 if len(byhand) == 0:
1044 suites = list(self.final_suites)
1045 assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1049 control = self.changes.changes
1050 automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1054 parts = f.filename.split('_', 2)
1056 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1060 package, version, archext = parts
1061 arch, ext = archext.split('.', 1)
1064 rule = automatic_byhand_packages.subtree(package)
1069 if rule['Source'] != self.changes.source_name or rule['Section'] != f.section or rule['Extension'] != ext:
1073 script = rule['Script']
1074 retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1076 print "W: error processing {0}.".format(f.filename)
1079 return len(remaining) == 0
1081 def _install_byhand(self, policy_queue_upload, hashed_file):
1082 """install byhand file
1084 @type policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1086 @type hashed_file: L{daklib.upload.HashedFile}
1088 fs = self.transaction.fs
1089 session = self.transaction.session
1090 policy_queue = policy_queue_upload.policy_queue
1092 byhand_file = PolicyQueueByhandFile()
1093 byhand_file.upload = policy_queue_upload
1094 byhand_file.filename = hashed_file.filename
1095 session.add(byhand_file)
1098 src = os.path.join(self.directory, hashed_file.filename)
1099 dst = os.path.join(policy_queue.path, hashed_file.filename)
1100 fs.copy(src, dst, mode=policy_queue.change_perms)
1104 def _do_bts_versiontracking(self):
1106 fs = self.transaction.fs
1108 btsdir = cnf.get('Dir::BTSVersionTrack')
1109 if btsdir is None or btsdir == '':
1112 base = os.path.join(btsdir, self.changes.filename[:-8])
1115 sourcedir = self.unpacked_source()
1116 if sourcedir is not None:
1117 fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1118 versions = fs.create("{0}.versions".format(base), mode=0o644)
1119 for line in fh.readlines():
1120 if re_changelog_versions.match(line):
1121 versions.write(line)
1125 # binary -> source mapping
1126 debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1127 for binary in self.changes.binaries:
1128 control = binary.control
1129 source_package, source_version = binary.source
1130 line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
1131 print >>debinfo, line
1134 def _policy_queue(self, suite):
1135 if suite.policy_queue is not None:
1136 return suite.policy_queue
1142 Install upload to a suite or policy queue. This method does B{not}
1143 handle uploads to NEW.
1145 You need to have called the C{check} method before calling this method.
1147 assert len(self.reject_reasons) == 0
1148 assert self.changes.valid_signature
1149 assert self.final_suites is not None
1150 assert self._checked
1153 db_changes = self._install_changes()
1155 for suite in self.final_suites:
1156 overridesuite = suite
1157 if suite.overridesuite is not None:
1158 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1160 policy_queue = self._policy_queue(suite)
1162 redirected_suite = suite
1163 if policy_queue is not None:
1164 redirected_suite = policy_queue.suite
1166 # source can be in the suite we install to or any suite we enhance
1167 source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
1168 for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
1169 .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
1170 .filter(VersionCheck.check == 'Enhances'):
1171 source_suite_ids.add(enhanced_suite_id)
1173 source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()
1175 source_component_func = lambda source: self._source_override(overridesuite, source).component
1176 binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1178 (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1180 if policy_queue is not None:
1181 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1183 # copy to build queues
1184 if policy_queue is None or policy_queue.send_to_build_queues:
1185 for build_queue in suite.copy_queues:
1186 self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1188 self._do_bts_versiontracking()
1190 def install_to_new(self):
1191 """install upload to NEW
1193 Install upload to NEW. This method does B{not} handle regular uploads
1194 to suites or policy queues.
1196 You need to have called the C{check} method before calling this method.
1198 # Uploads to NEW are special as we don't have overrides.
1199 assert len(self.reject_reasons) == 0
1200 assert self.changes.valid_signature
1201 assert self.final_suites is not None
1203 source = self.changes.source
1204 binaries = self.changes.binaries
1205 byhand = self.changes.byhand_files
1207 # we need a suite to guess components
1208 suites = list(self.final_suites)
1209 assert len(suites) == 1, "NEW uploads must be to a single suite"
1212 # decide which NEW queue to use
1213 if suite.new_queue is None:
1214 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1216 new_queue = suite.new_queue
1218 # There is only one global BYHAND queue
1219 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1220 new_suite = new_queue.suite
1223 def binary_component_func(binary):
1224 return self._binary_component(suite, binary, only_overrides=False)
1226 # guess source component
1227 # XXX: should be moved into an extra method
1228 binary_component_names = set()
1229 for binary in binaries:
1230 component = binary_component_func(binary)
1231 binary_component_names.add(component.component_name)
1232 source_component_name = None
1233 for c in self.session.query(Component).order_by(Component.component_id):
1234 guess = c.component_name
1235 if guess in binary_component_names:
1236 source_component_name = guess
1238 if source_component_name is None:
1239 source_component = self.session.query(Component).order_by(Component.component_id).first()
1241 source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1242 source_component_func = lambda source: source_component
1244 db_changes = self._install_changes()
1245 (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1246 policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1249 self._install_byhand(policy_upload, f)
1251 self._do_bts_versiontracking()
1254 """commit changes"""
1255 self.transaction.commit()
1258 """rollback changes"""
1259 self.transaction.rollback()
1261 def __enter__(self):
1265 def __exit__(self, type, value, traceback):
1266 if self.directory is not None:
1267 shutil.rmtree(self.directory)
1268 self.directory = None
1270 self.transaction.rollback()