1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to manipulate the archive
19 This module provides classes to manipulate the archive.
23 import daklib.checks as checks
24 from daklib.config import Config
25 import daklib.upload as upload
26 import daklib.utils as utils
27 from .fstransactions import FilesystemTransaction
28 from .regexes import re_changelog_versions, re_bin_only_nmu
31 from datetime import datetime
35 from sqlalchemy.orm.exc import NoResultFound
39 class ArchiveException(Exception):
42 class HashMismatchException(ArchiveException):
45 class ArchiveTransaction(object):
46 """manipulate the archive in a transaction
49 self.fs = FilesystemTransaction()
50 self.session = DBConn().session()
52 def get_file(self, hashed_file, source_name):
53 """Look for file `hashed_file` in database
56 hashed_file (daklib.upload.HashedFile): file to look for in the database
59 KeyError: file was not found in the database
60 HashMismatchException: hash mismatch
63 `daklib.dbconn.PoolFile` object for the database
65 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
67 poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
68 if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
69 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
72 raise KeyError('{0} not found in database.'.format(poolname))
74 def _install_file(self, directory, hashed_file, archive, component, source_name):
77 Will not give an error when the file is already present.
80 `daklib.dbconn.PoolFile` object for the new file
82 session = self.session
84 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
86 poolfile = self.get_file(hashed_file, source_name)
88 poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
89 poolfile.md5sum = hashed_file.md5sum
90 poolfile.sha1sum = hashed_file.sha1sum
91 poolfile.sha256sum = hashed_file.sha256sum
96 session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
98 archive_file = ArchiveFile(archive, component, poolfile)
99 session.add(archive_file)
102 path = os.path.join(archive.path, 'pool', component.component_name, poolname)
103 hashed_file_path = os.path.join(directory, hashed_file.filename)
104 self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
108 def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
109 """Install a binary package
112 directory (str): directory the binary package is located in
113 binary (daklib.upload.Binary): binary package to install
114 suite (daklib.dbconn.Suite): target suite
115 component (daklib.dbconn.Component): target component
118 allow_tainted (bool): allow to copy additional files from tainted archives
119 fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
120 source_suites (list of daklib.dbconn.Suite or True): suites to copy
121 the source from if they are not in `suite` or True to allow
122 copying from any suite.
123 This can also be a SQLAlchemy (sub)query object.
124 extra_source_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
127 `daklib.dbconn.DBBinary` object for the new package
129 session = self.session
130 control = binary.control
131 maintainer = get_or_set_maintainer(control['Maintainer'], session)
132 architecture = get_architecture(control['Architecture'], session)
134 (source_name, source_version) = binary.source
135 source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
136 source = source_query.filter(DBSource.suites.contains(suite)).first()
138 if source_suites != True:
139 source_query = source_query.join(DBSource.suites) \
140 .filter(Suite.suite_id == source_suites.c.id)
141 source = source_query.first()
143 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
144 self.copy_source(source, suite, component)
146 db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
149 package=control['Package'],
150 version=control['Version'],
151 architecture=architecture,
155 maintainer=maintainer,
157 binarytype=binary.type,
158 fingerprint=fingerprint,
162 db_binary = session.query(DBBinary).filter_by(**unique).one()
163 for key, value in rest.iteritems():
164 if getattr(db_binary, key) != value:
165 raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
166 except NoResultFound:
167 db_binary = DBBinary(**unique)
168 for key, value in rest.iteritems():
169 setattr(db_binary, key, value)
170 session.add(db_binary)
172 import_metadata_into_db(db_binary, session)
174 self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
176 if suite not in db_binary.suites:
177 db_binary.suites.append(suite)
183 def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
184 """ensure source exists in the given archive
186 This is intended to be used to check that Built-Using sources exist.
189 filename (str): filename to use in error messages
190 source (daklib.dbconn.DBSource): source to look for
191 archive (daklib.dbconn.Archive): archive to look in
194 extra_archives (list of daklib.dbconn.Archive): list of archives to copy
195 the source package from if it is not yet present in `archive`
197 session = self.session
198 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
199 if db_file is not None:
202 # Try to copy file from one extra archive
203 if extra_archives is None:
205 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
207 raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
209 source_archive = db_file.archive
210 for dsc_file in source.srcfiles:
211 af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
212 # We were given an explicit list of archives so it is okay to copy from tainted archives.
213 self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
215 def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
216 """Add Built-Using sources to `db_binary.extra_sources`
218 session = self.session
219 built_using = control.get('Built-Using', None)
221 if built_using is not None:
222 for dep in apt_pkg.parse_depends(built_using):
223 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
224 bu_source_name, bu_source_version, comp = dep[0]
225 assert comp == '=', 'Built-Using must contain strict dependencies'
227 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
228 if bu_source is None:
229 raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
231 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
233 db_binary.extra_sources.append(bu_source)
235 def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
236 """Install a source package
239 directory (str): directory the source package is located in
240 source (daklib.upload.Source): source package to install
241 suite (daklib.dbconn.Suite): target suite
242 component (daklib.dbconn.Component): target component
243 changed_by (daklib.dbconn.Maintainer): person who prepared this version of the package
246 allow_tainted (bool): allow to copy additional files from tainted archives
247 fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
250 `daklib.dbconn.DBSource` object for the new source
252 session = self.session
253 archive = suite.archive
255 maintainer = get_or_set_maintainer(control['Maintainer'], session)
256 source_name = control['Source']
258 ### Add source package to database
260 # We need to install the .dsc first as the DBSource object refers to it.
261 db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
265 version=control['Version'],
268 maintainer=maintainer,
269 changedby=changed_by,
270 #install_date=datetime.now().date(),
271 poolfile=db_file_dsc,
272 fingerprint=fingerprint,
273 dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
278 db_source = session.query(DBSource).filter_by(**unique).one()
279 for key, value in rest.iteritems():
280 if getattr(db_source, key) != value:
281 raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
282 except NoResultFound:
284 db_source = DBSource(**unique)
285 for key, value in rest.iteritems():
286 setattr(db_source, key, value)
287 # XXX: set as default in postgres?
288 db_source.install_date = datetime.now().date()
289 session.add(db_source)
292 # Add .dsc file. Other files will be added later.
293 db_dsc_file = DSCFile()
294 db_dsc_file.source = db_source
295 db_dsc_file.poolfile = db_file_dsc
296 session.add(db_dsc_file)
299 if suite in db_source.suites:
302 db_source.suites.append(suite)
307 ### Now add remaining files and copy them to the archive.
309 for hashed_file in source.files.itervalues():
310 hashed_file_path = os.path.join(directory, hashed_file.filename)
311 if os.path.exists(hashed_file_path):
312 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
315 db_file = self.get_file(hashed_file, source_name)
316 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
318 db_dsc_file = DSCFile()
319 db_dsc_file.source = db_source
320 db_dsc_file.poolfile = db_file
321 session.add(db_dsc_file)
325 # Importing is safe as we only arrive here when we did not find the source already installed earlier.
326 import_metadata_into_db(db_source, session)
328 # Uploaders are the maintainer and co-maintainers from the Uploaders field
329 db_source.uploaders.append(maintainer)
330 if 'Uploaders' in control:
331 def split_uploaders(field):
333 for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
336 for u in split_uploaders(control['Uploaders']):
337 db_source.uploaders.append(get_or_set_maintainer(u, session))
342 def _copy_file(self, db_file, archive, component, allow_tainted=False):
343 """Copy a file to the given archive and component
346 db_file (daklib.dbconn.PoolFile): file to copy
347 archive (daklib.dbconn.Archive): target archive
348 component (daklib.dbconn.Component): target component
351 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
353 session = self.session
355 if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
356 query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
357 if not allow_tainted:
358 query = query.join(Archive).filter(Archive.tainted == False)
360 source_af = query.first()
361 if source_af is None:
362 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
363 target_af = ArchiveFile(archive, component, db_file)
364 session.add(target_af)
366 self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
368 def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
369 """Copy a binary package to the given suite and component
372 db_binary (daklib.dbconn.DBBinary): binary to copy
373 suite (daklib.dbconn.Suite): target suite
374 component (daklib.dbconn.Component): target component
377 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
378 extra_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
380 session = self.session
381 archive = suite.archive
385 # make sure built-using packages are present in target archive
386 filename = db_binary.poolfile.filename
387 for db_source in db_binary.extra_sources:
388 self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
391 db_file = db_binary.poolfile
392 self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
393 if suite not in db_binary.suites:
394 db_binary.suites.append(suite)
397 def copy_source(self, db_source, suite, component, allow_tainted=False):
398 """Copy a source package to the given suite and component
401 db_source (daklib.dbconn.DBSource): source to copy
402 suite (daklib.dbconn.Suite): target suite
403 component (daklib.dbconn.Component): target component
406 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
408 archive = suite.archive
411 for db_dsc_file in db_source.srcfiles:
412 self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
413 if suite not in db_source.suites:
414 db_source.suites.append(suite)
417 def remove_file(self, db_file, archive, component):
418 """Remove a file from a given archive and component
421 db_file (daklib.dbconn.PoolFile): file to remove
422 archive (daklib.dbconn.Archive): archive to remove the file from
423 component (daklib.dbconn.Component): component to remove the file from
425 af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
426 self.fs.unlink(af.path)
427 self.session.delete(af)
429 def remove_binary(self, binary, suite):
430 """Remove a binary from a given suite and component
433 binary (daklib.dbconn.DBBinary): binary to remove
434 suite (daklib.dbconn.Suite): suite to remove the package from
436 binary.suites.remove(suite)
439 def remove_source(self, source, suite):
440 """Remove a source from a given suite and component
443 ArchiveException: source package is still referenced by other
444 binaries in the suite
447 binary (daklib.dbconn.DBSource): source to remove
448 suite (daklib.dbconn.Suite): suite to remove the package from
450 session = self.session
452 query = session.query(DBBinary).filter_by(source=source) \
453 .filter(DBBinary.suites.contains(suite))
454 if query.first() is not None:
455 raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
457 source.suites.remove(suite)
463 self.session.commit()
466 self.session.rollback()
470 """rollback changes"""
471 self.session.rollback()
477 def __exit__(self, type, value, traceback):
484 class ArchiveUpload(object):
487 This class can be used in a with-statement:
489 with ArchiveUpload(...) as upload:
492 Doing so will automatically run any required cleanup and also rollback the
493 transaction if it was not committed.
496 changes (daklib.upload.Changes): upload to process
497 directory (str): directory with temporary copy of files. set by `prepare`
498 fingerprint (daklib.dbconn.Fingerprint): fingerprint used to sign the upload
499 new (bool): upload is NEW. set by `check`
500 reject_reasons (list of str): reasons why the upload cannot be accepted
501 session: database session
502 transaction (daklib.archive.ArchiveTransaction): transaction used to handle the upload
503 warnings (list of str): warnings (NOT USED YET)
505 def __init__(self, directory, changes, keyrings):
506 self.transaction = ArchiveTransaction()
507 self.session = self.transaction.session
509 self.original_directory = directory
510 self.original_changes = changes
512 self.directory = None
513 self.keyrings = keyrings
515 self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
517 self.reject_reasons = []
519 self.final_suites = None
522 self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
523 self._new = self._new_queue.suite
526 """prepare upload for further processing
528 This copies the files involved to a temporary directory. If you use
529 this method directly, you have to remove the directory given by the
530 `directory` attribute later on your own.
532 Instead of using the method directly, you can also use a with-statement:
534 with ArchiveUpload(...) as upload:
537 This will automatically handle any required cleanup.
539 assert self.directory is None
540 assert self.original_changes.valid_signature
543 session = self.transaction.session
545 self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
546 with FilesystemTransaction() as fs:
547 src = os.path.join(self.original_directory, self.original_changes.filename)
548 dst = os.path.join(self.directory, self.original_changes.filename)
551 self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
553 for f in self.changes.files.itervalues():
554 src = os.path.join(self.original_directory, f.filename)
555 dst = os.path.join(self.directory, f.filename)
558 source = self.changes.source
559 if source is not None:
560 for f in source.files.itervalues():
561 src = os.path.join(self.original_directory, f.filename)
562 dst = os.path.join(self.directory, f.filename)
563 if f.filename not in self.changes.files:
564 db_file = self.transaction.get_file(f, source.dsc['Source'])
565 db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
566 fs.copy(db_archive_file.path, dst, symlink=True)
568 def unpacked_source(self):
569 """Path to unpacked source
571 Get path to the unpacked source. This method does unpack the source
572 into a temporary directory under `self.directory` if it has not
573 been done so already.
576 String giving the path to the unpacked source directory
577 or None if no source was included in the upload.
579 assert self.directory is not None
581 source = self.changes.source
584 dsc_path = os.path.join(self.directory, source._dsc_file.filename)
586 sourcedir = os.path.join(self.directory, 'source')
587 if not os.path.exists(sourcedir):
588 subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
589 if not os.path.isdir(sourcedir):
590 raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
593 def _map_suite(self, suite_name):
594 for rule in Config().value_list("SuiteMappings"):
595 fields = rule.split()
597 if rtype == "map" or rtype == "silent-map":
598 (src, dst) = fields[1:3]
599 if src == suite_name:
601 if rtype != "silent-map":
602 self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
603 elif rtype == "ignore":
605 if suite_name == ignored:
606 self.warnings.append('Ignoring target suite {0}.'.format(ignored))
608 elif rtype == "reject":
610 if suite_name == rejected:
611 self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
612 ## XXX: propup-version and map-unreleased not yet implemented
615 def _mapped_suites(self):
616 """Get target suites after mappings
619 list of daklib.dbconn.Suite giving the mapped target suites of this upload
621 session = self.session
624 for dist in self.changes.distributions:
625 suite_name = self._map_suite(dist)
626 if suite_name is not None:
627 suite_names.append(suite_name)
629 suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
632 def _check_new(self, suite):
633 """Check if upload is NEW
635 An upload is NEW if it has binary or source packages that do not have
636 an override in `suite` OR if it references files ONLY in a tainted
637 archive (eg. when it references files in NEW).
640 True if the upload is NEW, False otherwise
642 session = self.session
644 # Check for missing overrides
645 for b in self.changes.binaries:
646 override = self._binary_override(suite, b)
650 if self.changes.source is not None:
651 override = self._source_override(suite, self.changes.source)
655 # Check if we reference a file only in a tainted archive
656 files = self.changes.files.values()
657 if self.changes.source is not None:
658 files.extend(self.changes.source.files.values())
660 query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
661 query_untainted = query.join(Archive).filter(Archive.tainted == False)
663 in_archive = (query.first() is not None)
664 in_untainted_archive = (query_untainted.first() is not None)
666 if in_archive and not in_untainted_archive:
669 def _final_suites(self):
670 session = self.session
672 mapped_suites = self._mapped_suites()
675 for suite in mapped_suites:
676 overridesuite = suite
677 if suite.overridesuite is not None:
678 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
679 if self._check_new(overridesuite):
681 final_suites.add(suite)
685 def _binary_override(self, suite, binary):
686 """Get override entry for a binary
689 suite (daklib.dbconn.Suite)
690 binary (daklib.upload.Binary)
693 daklib.dbconn.Override or None
695 if suite.overridesuite is not None:
696 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
698 query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
699 .join(Component).filter(Component.component_name == binary.component) \
700 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
704 except NoResultFound:
707 def _source_override(self, suite, source):
708 """Get override entry for a source
711 suite (daklib.dbconn.Suite)
712 source (daklib.upload.Source)
715 daklib.dbconn.Override or None
717 if suite.overridesuite is not None:
718 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
720 # XXX: component for source?
721 query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
722 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
726 except NoResultFound:
729 def check(self, force=False):
730 """run checks against the upload
733 force (bool): ignore failing forcable checks
736 True if all checks passed, False otherwise
738 # XXX: needs to be better structured.
739 assert self.changes.valid_signature
743 checks.SignatureCheck,
749 checks.SingleDistributionCheck,
750 checks.NoSourceOnlyCheck,
755 final_suites = self._final_suites()
756 if len(final_suites) == 0:
757 self.reject_reasons.append('Ended with no suite to install to.')
761 checks.SourceFormatCheck,
762 checks.SuiteArchitectureCheck,
765 for suite in final_suites:
766 chk().per_suite_check(self, suite)
768 if len(self.reject_reasons) != 0:
771 self.final_suites = final_suites
773 except checks.Reject as e:
774 self.reject_reasons.append(unicode(e))
775 except Exception as e:
776 self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
779 def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
780 """Install upload to the given suite
783 suite (daklib.dbconn.Suite): suite to install the package into.
784 This is the real suite, ie. after any redirection to NEW or a policy queue
785 source_component_func: function to get the `daklib.dbconn.Component`
786 for a `daklib.upload.Source` object
787 binary_component_func: function to get the `daklib.dbconn.Component`
788 for a `daklib.upload.Binary` object
791 source_suites: see `daklib.archive.ArchiveTransaction.install_binary`
792 extra_source_archives: see `daklib.archive.ArchiveTransaction.install_binary`
795 tuple with two elements. The first is a `daklib.dbconn.DBSource`
796 object for the install source or None if no source was included.
797 The second is a list of `daklib.dbconn.DBBinary` objects for the
798 installed binary packages.
800 # XXX: move this function to ArchiveTransaction?
802 control = self.changes.changes
803 changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
805 if source_suites is None:
806 source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
808 source = self.changes.source
809 if source is not None:
810 component = source_component_func(source)
811 db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
816 for binary in self.changes.binaries:
817 component = binary_component_func(binary)
818 db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
819 db_binaries.append(db_binary)
821 if suite.copychanges:
822 src = os.path.join(self.directory, self.changes.filename)
823 dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
824 self.transaction.fs.copy(src, dst)
826 return (db_source, db_binaries)
828 def _install_changes(self):
829 assert self.changes.valid_signature
830 control = self.changes.changes
831 session = self.transaction.session
835 # Only add changelog for sourceful uploads and binNMUs
836 if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
837 query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
838 changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
839 assert changelog_id is not None
841 db_changes = DBChange()
842 db_changes.changesname = self.changes.filename
843 db_changes.source = control['Source']
844 db_changes.binaries = control.get('Binary', None)
845 db_changes.architecture = control['Architecture']
846 db_changes.version = control['Version']
847 db_changes.distribution = control['Distribution']
848 db_changes.urgency = control['Urgency']
849 db_changes.maintainer = control['Maintainer']
850 db_changes.changedby = control.get('Changed-By', control['Maintainer'])
851 db_changes.date = control['Date']
852 db_changes.fingerprint = self.fingerprint.fingerprint
853 db_changes.changelog_id = changelog_id
854 db_changes.closes = self.changes.closed_bugs
856 self.transaction.session.add(db_changes)
857 self.transaction.session.flush()
861 def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
862 u = PolicyQueueUpload()
863 u.policy_queue = policy_queue
864 u.target_suite = target_suite
865 u.changes = db_changes
867 u.binaries = db_binaries
868 self.transaction.session.add(u)
869 self.transaction.session.flush()
871 dst = os.path.join(policy_queue.path, self.changes.filename)
872 self.transaction.fs.copy(self.changes.path, dst)
876 def try_autobyhand(self):
879 Try to handle byhand packages automatically.
882 list of `daklib.upload.hashed_file` for the remaining byhand packages
884 assert len(self.reject_reasons) == 0
885 assert self.changes.valid_signature
886 assert self.final_suites is not None
888 byhand = self.changes.byhand_files
892 suites = list(self.final_suites)
893 assert len(suites) == 1, "BYHAND uploads must be to a single suite"
897 control = self.changes.changes
898 automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
902 parts = f.filename.split('_', 2)
904 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
908 package, version, archext = parts
909 arch, ext = archext.split('.', 1)
911 rule = automatic_byhand_packages.get(package)
916 if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
920 script = rule['Script']
921 retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
923 print "W: error processing {0}.".format(f.filename)
926 return len(remaining) == 0
928 def _install_byhand(self, policy_queue_upload, hashed_file):
931 policy_queue_upload (daklib.dbconn.PolicyQueueUpload): XXX
932 hashed_file (daklib.upload.HashedFile): XXX
934 fs = self.transaction.fs
935 session = self.transaction.session
936 policy_queue = policy_queue_upload.policy_queue
938 byhand_file = PolicyQueueByhandFile()
939 byhand_file.upload = policy_queue_upload
940 byhand_file.filename = hashed_file.filename
941 session.add(byhand_file)
944 src = os.path.join(self.directory, hashed_file.filename)
945 dst = os.path.join(policy_queue.path, hashed_file.filename)
950 def _do_bts_versiontracking(self):
952 fs = self.transaction.fs
954 btsdir = cnf.get('Dir::BTSVersionTrack')
955 if btsdir is None or btsdir == '':
958 base = os.path.join(btsdir, self.changes.filename[:-8])
961 sourcedir = self.unpacked_source()
962 if sourcedir is not None:
963 fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
964 versions = fs.create("{0}.versions".format(base), mode=0o644)
965 for line in fh.readlines():
966 if re_changelog_versions.match(line):
971 # binary -> source mapping
972 debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
973 for binary in self.changes.binaries:
974 control = binary.control
975 source_package, source_version = binary.source
976 line = " ".join([control['Package'], control['Version'], source_package, source_version])
977 print >>debinfo, line
983 Install upload to a suite or policy queue. This method does *not*
984 handle uploads to NEW.
986 You need to have called the `check` method before calling this method.
988 assert len(self.reject_reasons) == 0
989 assert self.changes.valid_signature
990 assert self.final_suites is not None
993 db_changes = self._install_changes()
995 for suite in self.final_suites:
996 overridesuite = suite
997 if suite.overridesuite is not None:
998 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1000 redirected_suite = suite
1001 if suite.policy_queue is not None:
1002 redirected_suite = suite.policy_queue.suite
1004 source_component_func = lambda source: self._source_override(overridesuite, source).component
1005 binary_component_func = lambda binary: self._binary_override(overridesuite, binary).component
1007 (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1009 if suite.policy_queue is not None:
1010 self._install_policy(suite.policy_queue, suite, db_changes, db_source, db_binaries)
1012 # copy to build queues
1013 if suite.policy_queue is None or suite.policy_queue.send_to_build_queues:
1014 for build_queue in suite.copy_queues:
1015 self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1017 self._do_bts_versiontracking()
1019 def install_to_new(self):
1020 """install upload to NEW
1022 Install upload to NEW. This method does *not* handle regular uploads
1023 to suites or policy queues.
1025 You need to have called the `check` method before calling this method.
1027 # Uploads to NEW are special as we don't have overrides.
1028 assert len(self.reject_reasons) == 0
1029 assert self.changes.valid_signature
1030 assert self.final_suites is not None
1032 source = self.changes.source
1033 binaries = self.changes.binaries
1034 byhand = self.changes.byhand_files
1036 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1038 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1039 new_suite = new_queue.suite
1041 # we need a suite to guess components
1042 suites = list(self.final_suites)
1043 assert len(suites) == 1, "NEW uploads must be to a single suite"
1046 def binary_component_func(binary):
1047 override = self._binary_override(suite, binary)
1048 if override is not None:
1049 return override.component
1050 component_name = binary.component
1051 component = self.session.query(Component).filter_by(component_name=component_name).one()
1054 # guess source component
1055 # XXX: should be moved into an extra method
1056 binary_component_names = set()
1057 for binary in binaries:
1058 component = binary_component_func(binary)
1059 binary_component_names.add(component.component_name)
1060 source_component_name = None
1061 for guess in ('main', 'contrib', 'non-free'):
1062 if guess in binary_component_names:
1063 source_component_name = guess
1065 if source_component_name is None:
1066 raise Exception('Could not guess source component.')
1067 source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1068 source_component_func = lambda source: source_component
1070 db_changes = self._install_changes()
1071 (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1072 policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1075 self._install_byhand(policy_upload, f)
1077 self._do_bts_versiontracking()
1080 """commit changes"""
1081 self.transaction.commit()
1084 """rollback changes"""
1085 self.transaction.rollback()
1087 def __enter__(self):
1091 def __exit__(self, type, value, traceback):
1092 if self.directory is not None:
1093 shutil.rmtree(self.directory)
1094 self.directory = None
1096 self.transaction.rollback()