1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to manipulate the archive
19 This module provides classes to manipulate the archive.
23 import daklib.checks as checks
24 from daklib.config import Config
25 import daklib.upload as upload
26 import daklib.utils as utils
27 from .fstransactions import FilesystemTransaction
28 from .regexes import re_changelog_versions, re_bin_only_nmu
31 from datetime import datetime
35 from sqlalchemy.orm.exc import NoResultFound
39 class ArchiveException(Exception):
42 class HashMismatchException(ArchiveException):
45 class ArchiveTransaction(object):
46 """manipulate the archive in a transaction
49 self.fs = FilesystemTransaction()
50 self.session = DBConn().session()
52 def get_file(self, hashed_file, source_name):
53 """Look for file `hashed_file` in database
56 hashed_file (daklib.upload.HashedFile): file to look for in the database
59 KeyError: file was not found in the database
60 HashMismatchException: hash mismatch
63 `daklib.dbconn.PoolFile` object for the database
65 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
67 poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
68 if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
69 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
72 raise KeyError('{0} not found in database.'.format(poolname))
74 def _install_file(self, directory, hashed_file, archive, component, source_name):
77 Will not give an error when the file is already present.
80 `daklib.dbconn.PoolFile` object for the new file
82 session = self.session
84 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
86 poolfile = self.get_file(hashed_file, source_name)
88 poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
89 poolfile.md5sum = hashed_file.md5sum
90 poolfile.sha1sum = hashed_file.sha1sum
91 poolfile.sha256sum = hashed_file.sha256sum
96 session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
98 archive_file = ArchiveFile(archive, component, poolfile)
99 session.add(archive_file)
102 path = os.path.join(archive.path, 'pool', component.component_name, poolname)
103 hashed_file_path = os.path.join(directory, hashed_file.filename)
104 self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
108 def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
109 """Install a binary package
112 directory (str): directory the binary package is located in
113 binary (daklib.upload.Binary): binary package to install
114 suite (daklib.dbconn.Suite): target suite
115 component (daklib.dbconn.Component): target component
118 allow_tainted (bool): allow to copy additional files from tainted archives
119 fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
120 source_suites (list of daklib.dbconn.Suite or True): suites to copy
121 the source from if they are not in `suite` or True to allow
122 copying from any suite.
123 This can also be a SQLAlchemy (sub)query object.
124 extra_source_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
127 `daklib.dbconn.DBBinary` object for the new package
129 session = self.session
130 control = binary.control
131 maintainer = get_or_set_maintainer(control['Maintainer'], session)
132 architecture = get_architecture(control['Architecture'], session)
134 (source_name, source_version) = binary.source
135 source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
136 source = source_query.filter(DBSource.suites.contains(suite)).first()
138 if source_suites != True:
139 source_query = source_query.join(DBSource.suites) \
140 .filter(Suite.suite_id == source_suites.c.id)
141 source = source_query.first()
143 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
144 self.copy_source(source, suite, component)
146 db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
149 package=control['Package'],
150 version=control['Version'],
151 architecture=architecture,
155 maintainer=maintainer,
157 binarytype=binary.type,
158 fingerprint=fingerprint,
162 db_binary = session.query(DBBinary).filter_by(**unique).one()
163 for key, value in rest.iteritems():
164 if getattr(db_binary, key) != value:
165 raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
166 except NoResultFound:
167 db_binary = DBBinary(**unique)
168 for key, value in rest.iteritems():
169 setattr(db_binary, key, value)
170 session.add(db_binary)
172 import_metadata_into_db(db_binary, session)
174 self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
176 if suite not in db_binary.suites:
177 db_binary.suites.append(suite)
183 def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
184 """ensure source exists in the given archive
186 This is intended to be used to check that Built-Using sources exist.
189 filename (str): filename to use in error messages
190 source (daklib.dbconn.DBSource): source to look for
191 archive (daklib.dbconn.Archive): archive to look in
194 extra_archives (list of daklib.dbconn.Archive): list of archives to copy
195 the source package from if it is not yet present in `archive`
197 session = self.session
198 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
199 if db_file is not None:
202 # Try to copy file from one extra archive
203 if extra_archives is None:
205 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
207 raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
209 source_archive = db_file.archive
210 for dsc_file in source.srcfiles:
211 af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
212 # We were given an explicit list of archives so it is okay to copy from tainted archives.
213 self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
215 def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
216 """Add Built-Using sources to `db_binary.extra_sources`
218 session = self.session
219 built_using = control.get('Built-Using', None)
221 if built_using is not None:
222 for dep in apt_pkg.parse_depends(built_using):
223 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
224 bu_source_name, bu_source_version, comp = dep[0]
225 assert comp == '=', 'Built-Using must contain strict dependencies'
227 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
228 if bu_source is None:
229 raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
231 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
233 db_binary.extra_sources.append(bu_source)
235 def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
236 """Install a source package
239 directory (str): directory the source package is located in
240 source (daklib.upload.Source): source package to install
241 suite (daklib.dbconn.Suite): target suite
242 component (daklib.dbconn.Component): target component
243 changed_by (daklib.dbconn.Maintainer): person who prepared this version of the package
246 allow_tainted (bool): allow to copy additional files from tainted archives
247 fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
250 `daklib.dbconn.DBSource` object for the new source
252 session = self.session
253 archive = suite.archive
255 maintainer = get_or_set_maintainer(control['Maintainer'], session)
256 source_name = control['Source']
258 ### Add source package to database
260 # We need to install the .dsc first as the DBSource object refers to it.
261 db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
265 version=control['Version'],
268 maintainer=maintainer,
269 changedby=changed_by,
270 #install_date=datetime.now().date(),
271 poolfile=db_file_dsc,
272 fingerprint=fingerprint,
273 dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
278 db_source = session.query(DBSource).filter_by(**unique).one()
279 for key, value in rest.iteritems():
280 if getattr(db_source, key) != value:
281 raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
282 except NoResultFound:
284 db_source = DBSource(**unique)
285 for key, value in rest.iteritems():
286 setattr(db_source, key, value)
287 # XXX: set as default in postgres?
288 db_source.install_date = datetime.now().date()
289 session.add(db_source)
292 # Add .dsc file. Other files will be added later.
293 db_dsc_file = DSCFile()
294 db_dsc_file.source = db_source
295 db_dsc_file.poolfile = db_file_dsc
296 session.add(db_dsc_file)
299 if suite in db_source.suites:
302 db_source.suites.append(suite)
307 ### Now add remaining files and copy them to the archive.
309 for hashed_file in source.files.itervalues():
310 hashed_file_path = os.path.join(directory, hashed_file.filename)
311 if os.path.exists(hashed_file_path):
312 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
315 db_file = self.get_file(hashed_file, source_name)
316 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
318 db_dsc_file = DSCFile()
319 db_dsc_file.source = db_source
320 db_dsc_file.poolfile = db_file
321 session.add(db_dsc_file)
325 # Importing is safe as we only arrive here when we did not find the source already installed earlier.
326 import_metadata_into_db(db_source, session)
328 # Uploaders are the maintainer and co-maintainers from the Uploaders field
329 db_source.uploaders.append(maintainer)
330 if 'Uploaders' in control:
331 def split_uploaders(field):
333 for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
336 for u in split_uploaders(control['Uploaders']):
337 db_source.uploaders.append(get_or_set_maintainer(u, session))
342 def _copy_file(self, db_file, archive, component, allow_tainted=False):
343 """Copy a file to the given archive and component
346 db_file (daklib.dbconn.PoolFile): file to copy
347 archive (daklib.dbconn.Archive): target archive
348 component (daklib.dbconn.Component): target component
351 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
353 session = self.session
355 if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
356 query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
357 if not allow_tainted:
358 query = query.join(Archive).filter(Archive.tainted == False)
360 source_af = query.first()
361 if source_af is None:
362 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
363 target_af = ArchiveFile(archive, component, db_file)
364 session.add(target_af)
366 self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
368 def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
369 """Copy a binary package to the given suite and component
372 db_binary (daklib.dbconn.DBBinary): binary to copy
373 suite (daklib.dbconn.Suite): target suite
374 component (daklib.dbconn.Component): target component
377 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
378 extra_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
380 session = self.session
381 archive = suite.archive
385 filename = db_binary.poolfile.filename
387 # make sure source is present in target archive
388 db_source = db_binary.source
389 if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
390 raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
392 # make sure built-using packages are present in target archive
393 for db_source in db_binary.extra_sources:
394 self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
397 db_file = db_binary.poolfile
398 self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
399 if suite not in db_binary.suites:
400 db_binary.suites.append(suite)
403 def copy_source(self, db_source, suite, component, allow_tainted=False):
404 """Copy a source package to the given suite and component
407 db_source (daklib.dbconn.DBSource): source to copy
408 suite (daklib.dbconn.Suite): target suite
409 component (daklib.dbconn.Component): target component
412 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
414 archive = suite.archive
417 for db_dsc_file in db_source.srcfiles:
418 self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
419 if suite not in db_source.suites:
420 db_source.suites.append(suite)
423 def remove_file(self, db_file, archive, component):
424 """Remove a file from a given archive and component
427 db_file (daklib.dbconn.PoolFile): file to remove
428 archive (daklib.dbconn.Archive): archive to remove the file from
429 component (daklib.dbconn.Component): component to remove the file from
431 af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
432 self.fs.unlink(af.path)
433 self.session.delete(af)
435 def remove_binary(self, binary, suite):
436 """Remove a binary from a given suite and component
439 binary (daklib.dbconn.DBBinary): binary to remove
440 suite (daklib.dbconn.Suite): suite to remove the package from
442 binary.suites.remove(suite)
445 def remove_source(self, source, suite):
446 """Remove a source from a given suite and component
449 ArchiveException: source package is still referenced by other
450 binaries in the suite
453 binary (daklib.dbconn.DBSource): source to remove
454 suite (daklib.dbconn.Suite): suite to remove the package from
456 session = self.session
458 query = session.query(DBBinary).filter_by(source=source) \
459 .filter(DBBinary.suites.contains(suite))
460 if query.first() is not None:
461 raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
463 source.suites.remove(suite)
469 self.session.commit()
472 self.session.rollback()
476 """rollback changes"""
477 self.session.rollback()
483 def __exit__(self, type, value, traceback):
490 class ArchiveUpload(object):
493 This class can be used in a with-statement:
495 with ArchiveUpload(...) as upload:
498 Doing so will automatically run any required cleanup and also rollback the
499 transaction if it was not committed.
502 changes (daklib.upload.Changes): upload to process
503 directory (str): directory with temporary copy of files. set by `prepare`
504 fingerprint (daklib.dbconn.Fingerprint): fingerprint used to sign the upload
505 new (bool): upload is NEW. set by `check`
506 reject_reasons (list of str): reasons why the upload cannot be accepted
507 session: database session
508 transaction (daklib.archive.ArchiveTransaction): transaction used to handle the upload
509 warnings (list of str): warnings (NOT USED YET)
511 def __init__(self, directory, changes, keyrings):
512 self.transaction = ArchiveTransaction()
513 self.session = self.transaction.session
515 self.original_directory = directory
516 self.original_changes = changes
518 self.directory = None
519 self.keyrings = keyrings
521 self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
523 self.reject_reasons = []
525 self.final_suites = None
528 self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
529 self._new = self._new_queue.suite
532 """prepare upload for further processing
534 This copies the files involved to a temporary directory. If you use
535 this method directly, you have to remove the directory given by the
536 `directory` attribute later on your own.
538 Instead of using the method directly, you can also use a with-statement:
540 with ArchiveUpload(...) as upload:
543 This will automatically handle any required cleanup.
545 assert self.directory is None
546 assert self.original_changes.valid_signature
549 session = self.transaction.session
551 self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
552 with FilesystemTransaction() as fs:
553 src = os.path.join(self.original_directory, self.original_changes.filename)
554 dst = os.path.join(self.directory, self.original_changes.filename)
557 self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
559 for f in self.changes.files.itervalues():
560 src = os.path.join(self.original_directory, f.filename)
561 dst = os.path.join(self.directory, f.filename)
564 source = self.changes.source
565 if source is not None:
566 for f in source.files.itervalues():
567 src = os.path.join(self.original_directory, f.filename)
568 dst = os.path.join(self.directory, f.filename)
569 if f.filename not in self.changes.files:
570 db_file = self.transaction.get_file(f, source.dsc['Source'])
571 db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
572 fs.copy(db_archive_file.path, dst, symlink=True)
574 def unpacked_source(self):
575 """Path to unpacked source
577 Get path to the unpacked source. This method does unpack the source
578 into a temporary directory under `self.directory` if it has not
579 been done so already.
582 String giving the path to the unpacked source directory
583 or None if no source was included in the upload.
585 assert self.directory is not None
587 source = self.changes.source
590 dsc_path = os.path.join(self.directory, source._dsc_file.filename)
592 sourcedir = os.path.join(self.directory, 'source')
593 if not os.path.exists(sourcedir):
594 subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
595 if not os.path.isdir(sourcedir):
596 raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
599 def _map_suite(self, suite_name):
600 for rule in Config().value_list("SuiteMappings"):
601 fields = rule.split()
603 if rtype == "map" or rtype == "silent-map":
604 (src, dst) = fields[1:3]
605 if src == suite_name:
607 if rtype != "silent-map":
608 self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
609 elif rtype == "ignore":
611 if suite_name == ignored:
612 self.warnings.append('Ignoring target suite {0}.'.format(ignored))
614 elif rtype == "reject":
616 if suite_name == rejected:
617 self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
618 ## XXX: propup-version and map-unreleased not yet implemented
621 def _mapped_suites(self):
622 """Get target suites after mappings
625 list of daklib.dbconn.Suite giving the mapped target suites of this upload
627 session = self.session
630 for dist in self.changes.distributions:
631 suite_name = self._map_suite(dist)
632 if suite_name is not None:
633 suite_names.append(suite_name)
635 suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
638 def _mapped_component(self, component_name):
639 """get component after mappings
641 Evaluate component mappings from ComponentMappings in dak.conf for the
642 given component name.
644 NOTE: ansgar wants to get rid of this. It's currently only used for
648 component_name (str): component name
651 `daklib.dbconn.Component` object
654 for m in cnf.value_list("ComponentMappings"):
655 (src, dst) = m.split()
656 if component_name == src:
658 component = self.session.query(Component).filter_by(component_name=component_name).one()
661 def _check_new(self, suite):
662 """Check if upload is NEW
664 An upload is NEW if it has binary or source packages that do not have
665 an override in `suite` OR if it references files ONLY in a tainted
666 archive (eg. when it references files in NEW).
669 True if the upload is NEW, False otherwise
671 session = self.session
673 # Check for missing overrides
674 for b in self.changes.binaries:
675 override = self._binary_override(suite, b)
679 if self.changes.source is not None:
680 override = self._source_override(suite, self.changes.source)
684 # Check if we reference a file only in a tainted archive
685 files = self.changes.files.values()
686 if self.changes.source is not None:
687 files.extend(self.changes.source.files.values())
689 query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
690 query_untainted = query.join(Archive).filter(Archive.tainted == False)
692 in_archive = (query.first() is not None)
693 in_untainted_archive = (query_untainted.first() is not None)
695 if in_archive and not in_untainted_archive:
698 def _final_suites(self):
699 session = self.session
701 mapped_suites = self._mapped_suites()
704 for suite in mapped_suites:
705 overridesuite = suite
706 if suite.overridesuite is not None:
707 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
708 if self._check_new(overridesuite):
710 final_suites.add(suite)
714 def _binary_override(self, suite, binary):
715 """Get override entry for a binary
718 suite (daklib.dbconn.Suite)
719 binary (daklib.upload.Binary)
722 daklib.dbconn.Override or None
724 if suite.overridesuite is not None:
725 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
727 query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
728 .join(Component).filter(Component.component_name == binary.component) \
729 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
733 except NoResultFound:
736 def _source_override(self, suite, source):
737 """Get override entry for a source
740 suite (daklib.dbconn.Suite)
741 source (daklib.upload.Source)
744 daklib.dbconn.Override or None
746 if suite.overridesuite is not None:
747 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
749 # XXX: component for source?
750 query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
751 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
755 except NoResultFound:
758 def _binary_component(self, suite, binary, only_overrides=True):
759 """get component for a binary
761 By default this will only look at overrides to get the right component;
762 if `only_overrides` is False this method will also look at the Section field.
765 suite (daklib.dbconn.Suite)
766 binary (daklib.upload.Binary)
769 only_overrides (bool): only use overrides to get the right component.
773 `daklib.dbconn.Component` object or None
775 override = self._binary_override(suite, binary)
776 if override is not None:
777 return override.component
780 return self._mapped_component(binary.component)
782 def check(self, force=False):
783 """run checks against the upload
786 force (bool): ignore failing forcable checks
789 True if all checks passed, False otherwise
791 # XXX: needs to be better structured.
792 assert self.changes.valid_signature
796 checks.SignatureCheck,
801 checks.BinaryTimestampCheck,
803 checks.SingleDistributionCheck,
804 checks.NoSourceOnlyCheck,
809 final_suites = self._final_suites()
810 if len(final_suites) == 0:
811 self.reject_reasons.append('Ended with no suite to install to.')
815 checks.SourceFormatCheck,
816 checks.SuiteArchitectureCheck,
819 for suite in final_suites:
820 chk().per_suite_check(self, suite)
822 if len(self.reject_reasons) != 0:
825 self.final_suites = final_suites
827 except checks.Reject as e:
828 self.reject_reasons.append(unicode(e))
829 except Exception as e:
830 self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
833 def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
834 """Install upload to the given suite
837 suite (daklib.dbconn.Suite): suite to install the package into.
838 This is the real suite, ie. after any redirection to NEW or a policy queue
839 source_component_func: function to get the `daklib.dbconn.Component`
840 for a `daklib.upload.Source` object
841 binary_component_func: function to get the `daklib.dbconn.Component`
842 for a `daklib.upload.Binary` object
845 source_suites: see `daklib.archive.ArchiveTransaction.install_binary`
846 extra_source_archives: see `daklib.archive.ArchiveTransaction.install_binary`
849 tuple with two elements. The first is a `daklib.dbconn.DBSource`
850 object for the install source or None if no source was included.
851 The second is a list of `daklib.dbconn.DBBinary` objects for the
852 installed binary packages.
854 # XXX: move this function to ArchiveTransaction?
856 control = self.changes.changes
857 changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
859 if source_suites is None:
860 source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
862 source = self.changes.source
863 if source is not None:
864 component = source_component_func(source)
865 db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
870 for binary in self.changes.binaries:
871 component = binary_component_func(binary)
872 db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
873 db_binaries.append(db_binary)
875 if suite.copychanges:
876 src = os.path.join(self.directory, self.changes.filename)
877 dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
878 self.transaction.fs.copy(src, dst)
880 return (db_source, db_binaries)
882 def _install_changes(self):
883 assert self.changes.valid_signature
884 control = self.changes.changes
885 session = self.transaction.session
889 # Only add changelog for sourceful uploads and binNMUs
890 if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
891 query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
892 changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
893 assert changelog_id is not None
895 db_changes = DBChange()
896 db_changes.changesname = self.changes.filename
897 db_changes.source = control['Source']
898 db_changes.binaries = control.get('Binary', None)
899 db_changes.architecture = control['Architecture']
900 db_changes.version = control['Version']
901 db_changes.distribution = control['Distribution']
902 db_changes.urgency = control['Urgency']
903 db_changes.maintainer = control['Maintainer']
904 db_changes.changedby = control.get('Changed-By', control['Maintainer'])
905 db_changes.date = control['Date']
906 db_changes.fingerprint = self.fingerprint.fingerprint
907 db_changes.changelog_id = changelog_id
908 db_changes.closes = self.changes.closed_bugs
910 self.transaction.session.add(db_changes)
911 self.transaction.session.flush()
915 def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
916 u = PolicyQueueUpload()
917 u.policy_queue = policy_queue
918 u.target_suite = target_suite
919 u.changes = db_changes
921 u.binaries = db_binaries
922 self.transaction.session.add(u)
923 self.transaction.session.flush()
925 dst = os.path.join(policy_queue.path, self.changes.filename)
926 self.transaction.fs.copy(self.changes.path, dst)
930 def try_autobyhand(self):
933 Try to handle byhand packages automatically.
936 list of `daklib.upload.hashed_file` for the remaining byhand packages
938 assert len(self.reject_reasons) == 0
939 assert self.changes.valid_signature
940 assert self.final_suites is not None
942 byhand = self.changes.byhand_files
946 suites = list(self.final_suites)
947 assert len(suites) == 1, "BYHAND uploads must be to a single suite"
951 control = self.changes.changes
952 automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
956 parts = f.filename.split('_', 2)
958 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
962 package, version, archext = parts
963 arch, ext = archext.split('.', 1)
965 rule = automatic_byhand_packages.get(package)
970 if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
974 script = rule['Script']
975 retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
977 print "W: error processing {0}.".format(f.filename)
980 return len(remaining) == 0
982 def _install_byhand(self, policy_queue_upload, hashed_file):
985 policy_queue_upload (daklib.dbconn.PolicyQueueUpload): XXX
986 hashed_file (daklib.upload.HashedFile): XXX
988 fs = self.transaction.fs
989 session = self.transaction.session
990 policy_queue = policy_queue_upload.policy_queue
992 byhand_file = PolicyQueueByhandFile()
993 byhand_file.upload = policy_queue_upload
994 byhand_file.filename = hashed_file.filename
995 session.add(byhand_file)
998 src = os.path.join(self.directory, hashed_file.filename)
999 dst = os.path.join(policy_queue.path, hashed_file.filename)
1004 def _do_bts_versiontracking(self):
1006 fs = self.transaction.fs
1008 btsdir = cnf.get('Dir::BTSVersionTrack')
1009 if btsdir is None or btsdir == '':
1012 base = os.path.join(btsdir, self.changes.filename[:-8])
1015 sourcedir = self.unpacked_source()
1016 if sourcedir is not None:
1017 fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1018 versions = fs.create("{0}.versions".format(base), mode=0o644)
1019 for line in fh.readlines():
1020 if re_changelog_versions.match(line):
1021 versions.write(line)
1025 # binary -> source mapping
1026 debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1027 for binary in self.changes.binaries:
1028 control = binary.control
1029 source_package, source_version = binary.source
1030 line = " ".join([control['Package'], control['Version'], source_package, source_version])
1031 print >>debinfo, line
1037 Install upload to a suite or policy queue. This method does *not*
1038 handle uploads to NEW.
1040 You need to have called the `check` method before calling this method.
1042 assert len(self.reject_reasons) == 0
1043 assert self.changes.valid_signature
1044 assert self.final_suites is not None
1047 db_changes = self._install_changes()
1049 for suite in self.final_suites:
1050 overridesuite = suite
1051 if suite.overridesuite is not None:
1052 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1054 redirected_suite = suite
1055 if suite.policy_queue is not None:
1056 redirected_suite = suite.policy_queue.suite
1058 source_component_func = lambda source: self._source_override(overridesuite, source).component
1059 binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1061 (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1063 if suite.policy_queue is not None:
1064 self._install_policy(suite.policy_queue, suite, db_changes, db_source, db_binaries)
1066 # copy to build queues
1067 if suite.policy_queue is None or suite.policy_queue.send_to_build_queues:
1068 for build_queue in suite.copy_queues:
1069 self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1071 self._do_bts_versiontracking()
1073 def install_to_new(self):
1074 """install upload to NEW
1076 Install upload to NEW. This method does *not* handle regular uploads
1077 to suites or policy queues.
1079 You need to have called the `check` method before calling this method.
1081 # Uploads to NEW are special as we don't have overrides.
1082 assert len(self.reject_reasons) == 0
1083 assert self.changes.valid_signature
1084 assert self.final_suites is not None
1086 source = self.changes.source
1087 binaries = self.changes.binaries
1088 byhand = self.changes.byhand_files
1090 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1092 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1093 new_suite = new_queue.suite
1095 # we need a suite to guess components
1096 suites = list(self.final_suites)
1097 assert len(suites) == 1, "NEW uploads must be to a single suite"
1100 def binary_component_func(binary):
1101 return self._binary_component(suite, binary, only_overrides=False)
1103 # guess source component
1104 # XXX: should be moved into an extra method
1105 binary_component_names = set()
1106 for binary in binaries:
1107 component = binary_component_func(binary)
1108 binary_component_names.add(component.component_name)
1109 source_component_name = None
1110 for c in self.session.query(Component).order_by(Component.component_id):
1111 guess = c.component_name
1112 if guess in binary_component_names:
1113 source_component_name = guess
1115 if source_component_name is None:
1116 raise Exception('Could not guess source component.')
1117 source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1118 source_component_func = lambda source: source_component
1120 db_changes = self._install_changes()
1121 (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1122 policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1125 self._install_byhand(policy_upload, f)
1127 self._do_bts_versiontracking()
1130 """commit changes"""
1131 self.transaction.commit()
1134 """rollback changes"""
1135 self.transaction.rollback()
1137 def __enter__(self):
1141 def __exit__(self, type, value, traceback):
1142 if self.directory is not None:
1143 shutil.rmtree(self.directory)
1144 self.directory = None
1146 self.transaction.rollback()