1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to manipulate the archive
19 This module provides classes to manipulate the archive.
23 import daklib.checks as checks
24 from daklib.config import Config
25 import daklib.upload as upload
26 import daklib.utils as utils
27 from .fstransactions import FilesystemTransaction
28 from .regexes import re_changelog_versions, re_bin_only_nmu
31 from datetime import datetime
35 from sqlalchemy.orm.exc import NoResultFound
39 class ArchiveException(Exception):
42 class HashMismatchException(ArchiveException):
45 class ArchiveTransaction(object):
46 """manipulate the archive in a transaction
49 self.fs = FilesystemTransaction()
50 self.session = DBConn().session()
52 def get_file(self, hashed_file, source_name):
53 """Look for file `hashed_file` in database
56 hashed_file (daklib.upload.HashedFile): file to look for in the database
59 KeyError: file was not found in the database
60 HashMismatchException: hash mismatch
63 `daklib.dbconn.PoolFile` object for the database
65 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
67 poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
68 if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
69 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
72 raise KeyError('{0} not found in database.'.format(poolname))
74 def _install_file(self, directory, hashed_file, archive, component, source_name):
77 Will not give an error when the file is already present.
80 `daklib.dbconn.PoolFile` object for the new file
82 session = self.session
84 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
86 poolfile = self.get_file(hashed_file, source_name)
88 poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
89 poolfile.md5sum = hashed_file.md5sum
90 poolfile.sha1sum = hashed_file.sha1sum
91 poolfile.sha256sum = hashed_file.sha256sum
96 session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
98 archive_file = ArchiveFile(archive, component, poolfile)
99 session.add(archive_file)
102 path = os.path.join(archive.path, 'pool', component.component_name, poolname)
103 hashed_file_path = os.path.join(directory, hashed_file.filename)
104 self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
108 def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
109 """Install a binary package
112 directory (str): directory the binary package is located in
113 binary (daklib.upload.Binary): binary package to install
114 suite (daklib.dbconn.Suite): target suite
115 component (daklib.dbconn.Component): target component
118 allow_tainted (bool): allow to copy additional files from tainted archives
119 fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
120 source_suites (list of daklib.dbconn.Suite or True): suites to copy
121 the source from if they are not in `suite` or True to allow
122 copying from any suite.
123 This can also be a SQLAlchemy (sub)query object.
124 extra_source_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
127 `daklib.dbconn.DBBinary` object for the new package
129 session = self.session
130 control = binary.control
131 maintainer = get_or_set_maintainer(control['Maintainer'], session)
132 architecture = get_architecture(control['Architecture'], session)
134 (source_name, source_version) = binary.source
135 source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
136 source = source_query.filter(DBSource.suites.contains(suite)).first()
138 if source_suites != True:
139 source_query = source_query.join(DBSource.suites) \
140 .filter(Suite.suite_id == source_suites.c.id)
141 source = source_query.first()
143 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
144 self.copy_source(source, suite, component)
146 db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
149 package=control['Package'],
150 version=control['Version'],
151 architecture=architecture,
155 maintainer=maintainer,
157 binarytype=binary.type,
158 fingerprint=fingerprint,
162 db_binary = session.query(DBBinary).filter_by(**unique).one()
163 for key, value in rest.iteritems():
164 if getattr(db_binary, key) != value:
165 raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
166 except NoResultFound:
167 db_binary = DBBinary(**unique)
168 for key, value in rest.iteritems():
169 setattr(db_binary, key, value)
170 session.add(db_binary)
172 import_metadata_into_db(db_binary, session)
174 self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
176 if suite not in db_binary.suites:
177 db_binary.suites.append(suite)
183 def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
184 """ensure source exists in the given archive
186 This is intended to be used to check that Built-Using sources exist.
189 filename (str): filename to use in error messages
190 source (daklib.dbconn.DBSource): source to look for
191 archive (daklib.dbconn.Archive): archive to look in
194 extra_archives (list of daklib.dbconn.Archive): list of archives to copy
195 the source package from if it is not yet present in `archive`
197 session = self.session
198 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
199 if db_file is not None:
202 # Try to copy file from one extra archive
203 if extra_archives is None:
205 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
207 raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
209 source_archive = db_file.archive
210 for dsc_file in source.srcfiles:
211 af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
212 # We were given an explicit list of archives so it is okay to copy from tainted archives.
213 self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
215 def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
216 """Add Built-Using sources to `db_binary.extra_sources`
218 session = self.session
219 built_using = control.get('Built-Using', None)
221 if built_using is not None:
222 for dep in apt_pkg.parse_depends(built_using):
223 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
224 bu_source_name, bu_source_version, comp = dep[0]
225 assert comp == '=', 'Built-Using must contain strict dependencies'
227 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
228 if bu_source is None:
229 raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
231 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
233 db_binary.extra_sources.append(bu_source)
235 def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
236 """Install a source package
239 directory (str): directory the source package is located in
240 source (daklib.upload.Source): source package to install
241 suite (daklib.dbconn.Suite): target suite
242 component (daklib.dbconn.Component): target component
243 changed_by (daklib.dbconn.Maintainer): person who prepared this version of the package
246 allow_tainted (bool): allow to copy additional files from tainted archives
247 fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
250 `daklib.dbconn.DBSource` object for the new source
252 session = self.session
253 archive = suite.archive
255 maintainer = get_or_set_maintainer(control['Maintainer'], session)
256 source_name = control['Source']
258 ### Add source package to database
260 # We need to install the .dsc first as the DBSource object refers to it.
261 db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
265 version=control['Version'],
268 maintainer=maintainer,
269 changedby=changed_by,
270 #install_date=datetime.now().date(),
271 poolfile=db_file_dsc,
272 fingerprint=fingerprint,
273 dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
278 db_source = session.query(DBSource).filter_by(**unique).one()
279 for key, value in rest.iteritems():
280 if getattr(db_source, key) != value:
281 raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
282 except NoResultFound:
284 db_source = DBSource(**unique)
285 for key, value in rest.iteritems():
286 setattr(db_source, key, value)
287 # XXX: set as default in postgres?
288 db_source.install_date = datetime.now().date()
289 session.add(db_source)
292 # Add .dsc file. Other files will be added later.
293 db_dsc_file = DSCFile()
294 db_dsc_file.source = db_source
295 db_dsc_file.poolfile = db_file_dsc
296 session.add(db_dsc_file)
299 if suite in db_source.suites:
302 db_source.suites.append(suite)
307 ### Now add remaining files and copy them to the archive.
309 for hashed_file in source.files.itervalues():
310 hashed_file_path = os.path.join(directory, hashed_file.filename)
311 if os.path.exists(hashed_file_path):
312 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
315 db_file = self.get_file(hashed_file, source_name)
316 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
318 db_dsc_file = DSCFile()
319 db_dsc_file.source = db_source
320 db_dsc_file.poolfile = db_file
321 session.add(db_dsc_file)
325 # Importing is safe as we only arrive here when we did not find the source already installed earlier.
326 import_metadata_into_db(db_source, session)
328 # Uploaders are the maintainer and co-maintainers from the Uploaders field
329 db_source.uploaders.append(maintainer)
330 if 'Uploaders' in control:
331 def split_uploaders(field):
333 for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
336 for u in split_uploaders(control['Uploaders']):
337 db_source.uploaders.append(get_or_set_maintainer(u, session))
342 def _copy_file(self, db_file, archive, component, allow_tainted=False):
343 """Copy a file to the given archive and component
346 db_file (daklib.dbconn.PoolFile): file to copy
347 archive (daklib.dbconn.Archive): target archive
348 component (daklib.dbconn.Component): target component
351 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
353 session = self.session
355 if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
356 query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
357 if not allow_tainted:
358 query = query.join(Archive).filter(Archive.tainted == False)
360 source_af = query.first()
361 if source_af is None:
362 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
363 target_af = ArchiveFile(archive, component, db_file)
364 session.add(target_af)
366 self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
368 def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
369 """Copy a binary package to the given suite and component
372 db_binary (daklib.dbconn.DBBinary): binary to copy
373 suite (daklib.dbconn.Suite): target suite
374 component (daklib.dbconn.Component): target component
377 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
378 extra_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
380 session = self.session
381 archive = suite.archive
385 # make sure built-using packages are present in target archive
386 filename = db_binary.poolfile.filename
387 for db_source in db_binary.extra_sources:
388 self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
391 db_file = db_binary.poolfile
392 self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
393 if suite not in db_binary.suites:
394 db_binary.suites.append(suite)
397 def copy_source(self, db_source, suite, component, allow_tainted=False):
398 """Copy a source package to the given suite and component
401 db_source (daklib.dbconn.DBSource): source to copy
402 suite (daklib.dbconn.Suite): target suite
403 component (daklib.dbconn.Component): target component
406 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
408 archive = suite.archive
411 for db_dsc_file in db_source.srcfiles:
412 self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
413 if suite not in db_source.suites:
414 db_source.suites.append(suite)
417 def remove_file(self, db_file, archive, component):
418 """Remove a file from a given archive and component
421 db_file (daklib.dbconn.PoolFile): file to remove
422 archive (daklib.dbconn.Archive): archive to remove the file from
423 component (daklib.dbconn.Component): component to remove the file from
425 af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
426 self.fs.unlink(af.path)
427 self.session.delete(af)
429 def remove_binary(self, binary, suite):
430 """Remove a binary from a given suite and component
433 binary (daklib.dbconn.DBBinary): binary to remove
434 suite (daklib.dbconn.Suite): suite to remove the package from
436 binary.suites.remove(suite)
439 def remove_source(self, source, suite):
440 """Remove a source from a given suite and component
443 ArchiveException: source package is still referenced by other
444 binaries in the suite
447 binary (daklib.dbconn.DBSource): source to remove
448 suite (daklib.dbconn.Suite): suite to remove the package from
450 session = self.session
452 query = session.query(DBBinary).filter_by(source=source) \
453 .filter(DBBinary.suites.contains(suite))
454 if query.first() is not None:
455 raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
457 source.suites.remove(suite)
463 self.session.commit()
466 self.session.rollback()
470 """rollback changes"""
471 self.session.rollback()
477 def __exit__(self, type, value, traceback):
484 class ArchiveUpload(object):
487 This class can be used in a with-statement:
489 with ArchiveUpload(...) as upload:
492 Doing so will automatically run any required cleanup and also rollback the
493 transaction if it was not committed.
496 changes (daklib.upload.Changes): upload to process
497 directory (str): directory with temporary copy of files. set by `prepare`
498 fingerprint (daklib.dbconn.Fingerprint): fingerprint used to sign the upload
499 new (bool): upload is NEW. set by `check`
500 reject_reasons (list of str): reasons why the upload cannot be accepted
501 session: database session
502 transaction (daklib.archive.ArchiveTransaction): transaction used to handle the upload
503 warnings (list of str): warnings (NOT USED YET)
505 def __init__(self, directory, changes, keyrings):
506 self.transaction = ArchiveTransaction()
507 self.session = self.transaction.session
509 self.original_directory = directory
510 self.original_changes = changes
512 self.directory = None
513 self.keyrings = keyrings
515 self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
517 self.reject_reasons = []
519 self.final_suites = None
522 self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
523 self._new = self._new_queue.suite
526 """prepare upload for further processing
528 This copies the files involved to a temporary directory. If you use
529 this method directly, you have to remove the directory given by the
530 `directory` attribute later on your own.
532 Instead of using the method directly, you can also use a with-statement:
534 with ArchiveUpload(...) as upload:
537 This will automatically handle any required cleanup.
539 assert self.directory is None
540 assert self.original_changes.valid_signature
543 session = self.transaction.session
545 self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
546 with FilesystemTransaction() as fs:
547 src = os.path.join(self.original_directory, self.original_changes.filename)
548 dst = os.path.join(self.directory, self.original_changes.filename)
551 self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
553 for f in self.changes.files.itervalues():
554 src = os.path.join(self.original_directory, f.filename)
555 dst = os.path.join(self.directory, f.filename)
558 source = self.changes.source
559 if source is not None:
560 for f in source.files.itervalues():
561 src = os.path.join(self.original_directory, f.filename)
562 dst = os.path.join(self.directory, f.filename)
563 if f.filename not in self.changes.files:
564 db_file = self.transaction.get_file(f, source.dsc['Source'])
565 db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
566 fs.copy(db_archive_file.path, dst, symlink=True)
568 def unpacked_source(self):
569 """Path to unpacked source
571 Get path to the unpacked source. This method does unpack the source
572 into a temporary directory under `self.directory` if it has not
573 been done so already.
576 String giving the path to the unpacked source directory
577 or None if no source was included in the upload.
579 assert self.directory is not None
581 source = self.changes.source
584 dsc_path = os.path.join(self.directory, source._dsc_file.filename)
586 sourcedir = os.path.join(self.directory, 'source')
587 if not os.path.exists(sourcedir):
588 subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
589 if not os.path.isdir(sourcedir):
590 raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
593 def _map_suite(self, suite_name):
594 for rule in Config().value_list("SuiteMappings"):
595 fields = rule.split()
597 if rtype == "map" or rtype == "silent-map":
598 (src, dst) = fields[1:3]
599 if src == suite_name:
601 if rtype != "silent-map":
602 self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
603 elif rtype == "ignore":
605 if suite_name == ignored:
606 self.warnings.append('Ignoring target suite {0}.'.format(ignored))
608 elif rtype == "reject":
610 if suite_name == rejected:
611 self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
612 ## XXX: propup-version and map-unreleased not yet implemented
615 def _mapped_suites(self):
616 """Get target suites after mappings
619 list of daklib.dbconn.Suite giving the mapped target suites of this upload
621 session = self.session
624 for dist in self.changes.distributions:
625 suite_name = self._map_suite(dist)
626 if suite_name is not None:
627 suite_names.append(suite_name)
629 suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
632 def _mapped_component(self, component_name):
633 """get component after mappings
635 Evaluate component mappings from ComponentMappings in dak.conf for the
636 given component name.
638 NOTE: ansgar wants to get rid of this. It's currently only used for
642 component_name (str): component name
645 `daklib.dbconn.Component` object
648 for m in cnf.value_list("ComponentMappings"):
649 (src, dst) = m.split()
650 if component_name == src:
652 component = self.session.query(Component).filter_by(component_name=component_name).one()
655 def _check_new(self, suite):
656 """Check if upload is NEW
658 An upload is NEW if it has binary or source packages that do not have
659 an override in `suite` OR if it references files ONLY in a tainted
660 archive (eg. when it references files in NEW).
663 True if the upload is NEW, False otherwise
665 session = self.session
667 # Check for missing overrides
668 for b in self.changes.binaries:
669 override = self._binary_override(suite, b)
673 if self.changes.source is not None:
674 override = self._source_override(suite, self.changes.source)
678 # Check if we reference a file only in a tainted archive
679 files = self.changes.files.values()
680 if self.changes.source is not None:
681 files.extend(self.changes.source.files.values())
683 query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
684 query_untainted = query.join(Archive).filter(Archive.tainted == False)
686 in_archive = (query.first() is not None)
687 in_untainted_archive = (query_untainted.first() is not None)
689 if in_archive and not in_untainted_archive:
692 def _final_suites(self):
693 session = self.session
695 mapped_suites = self._mapped_suites()
698 for suite in mapped_suites:
699 overridesuite = suite
700 if suite.overridesuite is not None:
701 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
702 if self._check_new(overridesuite):
704 final_suites.add(suite)
708 def _binary_override(self, suite, binary):
709 """Get override entry for a binary
712 suite (daklib.dbconn.Suite)
713 binary (daklib.upload.Binary)
716 daklib.dbconn.Override or None
718 if suite.overridesuite is not None:
719 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
721 query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
722 .join(Component).filter(Component.component_name == binary.component) \
723 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
727 except NoResultFound:
730 def _source_override(self, suite, source):
731 """Get override entry for a source
734 suite (daklib.dbconn.Suite)
735 source (daklib.upload.Source)
738 daklib.dbconn.Override or None
740 if suite.overridesuite is not None:
741 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
743 # XXX: component for source?
744 query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
745 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
749 except NoResultFound:
752 def _binary_component(self, suite, binary, only_overrides=True):
753 """get component for a binary
755 By default this will only look at overrides to get the right component;
756 if `only_overrides` is False this method will also look at the Section field.
759 suite (daklib.dbconn.Suite)
760 binary (daklib.upload.Binary)
763 only_overrides (bool): only use overrides to get the right component.
767 `daklib.dbconn.Component` object or None
769 override = self._binary_override(suite, binary)
770 if override is not None:
771 return override.component
774 return self._mapped_component(binary.component)
776 def check(self, force=False):
777 """run checks against the upload
780 force (bool): ignore failing forcable checks
783 True if all checks passed, False otherwise
785 # XXX: needs to be better structured.
786 assert self.changes.valid_signature
790 checks.SignatureCheck,
795 checks.BinaryTimestampCheck,
797 checks.SingleDistributionCheck,
798 checks.NoSourceOnlyCheck,
803 final_suites = self._final_suites()
804 if len(final_suites) == 0:
805 self.reject_reasons.append('Ended with no suite to install to.')
809 checks.SourceFormatCheck,
810 checks.SuiteArchitectureCheck,
813 for suite in final_suites:
814 chk().per_suite_check(self, suite)
816 if len(self.reject_reasons) != 0:
819 self.final_suites = final_suites
821 except checks.Reject as e:
822 self.reject_reasons.append(unicode(e))
823 except Exception as e:
824 self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
827 def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
828 """Install upload to the given suite
831 suite (daklib.dbconn.Suite): suite to install the package into.
832 This is the real suite, ie. after any redirection to NEW or a policy queue
833 source_component_func: function to get the `daklib.dbconn.Component`
834 for a `daklib.upload.Source` object
835 binary_component_func: function to get the `daklib.dbconn.Component`
836 for a `daklib.upload.Binary` object
839 source_suites: see `daklib.archive.ArchiveTransaction.install_binary`
840 extra_source_archives: see `daklib.archive.ArchiveTransaction.install_binary`
843 tuple with two elements. The first is a `daklib.dbconn.DBSource`
844 object for the install source or None if no source was included.
845 The second is a list of `daklib.dbconn.DBBinary` objects for the
846 installed binary packages.
848 # XXX: move this function to ArchiveTransaction?
850 control = self.changes.changes
851 changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
853 if source_suites is None:
854 source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
856 source = self.changes.source
857 if source is not None:
858 component = source_component_func(source)
859 db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
864 for binary in self.changes.binaries:
865 component = binary_component_func(binary)
866 db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
867 db_binaries.append(db_binary)
869 if suite.copychanges:
870 src = os.path.join(self.directory, self.changes.filename)
871 dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
872 self.transaction.fs.copy(src, dst)
874 return (db_source, db_binaries)
876 def _install_changes(self):
877 assert self.changes.valid_signature
878 control = self.changes.changes
879 session = self.transaction.session
883 # Only add changelog for sourceful uploads and binNMUs
884 if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
885 query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
886 changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
887 assert changelog_id is not None
889 db_changes = DBChange()
890 db_changes.changesname = self.changes.filename
891 db_changes.source = control['Source']
892 db_changes.binaries = control.get('Binary', None)
893 db_changes.architecture = control['Architecture']
894 db_changes.version = control['Version']
895 db_changes.distribution = control['Distribution']
896 db_changes.urgency = control['Urgency']
897 db_changes.maintainer = control['Maintainer']
898 db_changes.changedby = control.get('Changed-By', control['Maintainer'])
899 db_changes.date = control['Date']
900 db_changes.fingerprint = self.fingerprint.fingerprint
901 db_changes.changelog_id = changelog_id
902 db_changes.closes = self.changes.closed_bugs
904 self.transaction.session.add(db_changes)
905 self.transaction.session.flush()
909 def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
910 u = PolicyQueueUpload()
911 u.policy_queue = policy_queue
912 u.target_suite = target_suite
913 u.changes = db_changes
915 u.binaries = db_binaries
916 self.transaction.session.add(u)
917 self.transaction.session.flush()
919 dst = os.path.join(policy_queue.path, self.changes.filename)
920 self.transaction.fs.copy(self.changes.path, dst)
924 def try_autobyhand(self):
927 Try to handle byhand packages automatically.
930 list of `daklib.upload.hashed_file` for the remaining byhand packages
932 assert len(self.reject_reasons) == 0
933 assert self.changes.valid_signature
934 assert self.final_suites is not None
936 byhand = self.changes.byhand_files
940 suites = list(self.final_suites)
941 assert len(suites) == 1, "BYHAND uploads must be to a single suite"
945 control = self.changes.changes
946 automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
950 parts = f.filename.split('_', 2)
952 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
956 package, version, archext = parts
957 arch, ext = archext.split('.', 1)
959 rule = automatic_byhand_packages.get(package)
964 if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
968 script = rule['Script']
969 retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
971 print "W: error processing {0}.".format(f.filename)
974 return len(remaining) == 0
976 def _install_byhand(self, policy_queue_upload, hashed_file):
979 policy_queue_upload (daklib.dbconn.PolicyQueueUpload): XXX
980 hashed_file (daklib.upload.HashedFile): XXX
982 fs = self.transaction.fs
983 session = self.transaction.session
984 policy_queue = policy_queue_upload.policy_queue
986 byhand_file = PolicyQueueByhandFile()
987 byhand_file.upload = policy_queue_upload
988 byhand_file.filename = hashed_file.filename
989 session.add(byhand_file)
992 src = os.path.join(self.directory, hashed_file.filename)
993 dst = os.path.join(policy_queue.path, hashed_file.filename)
998 def _do_bts_versiontracking(self):
1000 fs = self.transaction.fs
1002 btsdir = cnf.get('Dir::BTSVersionTrack')
1003 if btsdir is None or btsdir == '':
1006 base = os.path.join(btsdir, self.changes.filename[:-8])
1009 sourcedir = self.unpacked_source()
1010 if sourcedir is not None:
1011 fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1012 versions = fs.create("{0}.versions".format(base), mode=0o644)
1013 for line in fh.readlines():
1014 if re_changelog_versions.match(line):
1015 versions.write(line)
1019 # binary -> source mapping
1020 debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1021 for binary in self.changes.binaries:
1022 control = binary.control
1023 source_package, source_version = binary.source
1024 line = " ".join([control['Package'], control['Version'], source_package, source_version])
1025 print >>debinfo, line
1031 Install upload to a suite or policy queue. This method does *not*
1032 handle uploads to NEW.
1034 You need to have called the `check` method before calling this method.
1036 assert len(self.reject_reasons) == 0
1037 assert self.changes.valid_signature
1038 assert self.final_suites is not None
1041 db_changes = self._install_changes()
1043 for suite in self.final_suites:
1044 overridesuite = suite
1045 if suite.overridesuite is not None:
1046 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1048 redirected_suite = suite
1049 if suite.policy_queue is not None:
1050 redirected_suite = suite.policy_queue.suite
1052 source_component_func = lambda source: self._source_override(overridesuite, source).component
1053 binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1055 (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1057 if suite.policy_queue is not None:
1058 self._install_policy(suite.policy_queue, suite, db_changes, db_source, db_binaries)
1060 # copy to build queues
1061 if suite.policy_queue is None or suite.policy_queue.send_to_build_queues:
1062 for build_queue in suite.copy_queues:
1063 self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1065 self._do_bts_versiontracking()
1067 def install_to_new(self):
1068 """install upload to NEW
1070 Install upload to NEW. This method does *not* handle regular uploads
1071 to suites or policy queues.
1073 You need to have called the `check` method before calling this method.
1075 # Uploads to NEW are special as we don't have overrides.
1076 assert len(self.reject_reasons) == 0
1077 assert self.changes.valid_signature
1078 assert self.final_suites is not None
1080 source = self.changes.source
1081 binaries = self.changes.binaries
1082 byhand = self.changes.byhand_files
1084 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1086 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1087 new_suite = new_queue.suite
1089 # we need a suite to guess components
1090 suites = list(self.final_suites)
1091 assert len(suites) == 1, "NEW uploads must be to a single suite"
1094 def binary_component_func(binary):
1095 return self._binary_component(suite, binary, only_overrides=False)
1097 # guess source component
1098 # XXX: should be moved into an extra method
1099 binary_component_names = set()
1100 for binary in binaries:
1101 component = binary_component_func(binary)
1102 binary_component_names.add(component.component_name)
1103 source_component_name = None
1104 for c in self.session.query(Component).order_by(Component.component_id):
1105 guess = c.component_name
1106 if guess in binary_component_names:
1107 source_component_name = guess
1109 if source_component_name is None:
1110 raise Exception('Could not guess source component.')
1111 source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1112 source_component_func = lambda source: source_component
1114 db_changes = self._install_changes()
1115 (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1116 policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1119 self._install_byhand(policy_upload, f)
1121 self._do_bts_versiontracking()
1124 """commit changes"""
1125 self.transaction.commit()
1128 """rollback changes"""
1129 self.transaction.rollback()
1131 def __enter__(self):
1135 def __exit__(self, type, value, traceback):
1136 if self.directory is not None:
1137 shutil.rmtree(self.directory)
1138 self.directory = None
1140 self.transaction.rollback()