]> git.decadent.org.uk Git - dak.git/blob - archive.py
29a5621d3a4fbf9409063fac5004f18fbab52671
[dak.git] / archive.py
1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
2 #
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
7 #
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 # GNU General Public License for more details.
12 #
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
16
17 """module to manipulate the archive
18
19 This module provides classes to manipulate the archive.
20 """
21
22 from daklib.dbconn import *
23 import daklib.checks as checks
24 from daklib.config import Config
25 import daklib.upload as upload
26 import daklib.utils as utils
27 from daklib.fstransactions import FilesystemTransaction
28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
29
30 import apt_pkg
31 from datetime import datetime
32 import os
33 import shutil
34 import subprocess
35 from sqlalchemy.orm.exc import NoResultFound
36 import tempfile
37 import traceback
38
39 class ArchiveException(Exception):
40     pass
41
42 class HashMismatchException(ArchiveException):
43     pass
44
45 class ArchiveTransaction(object):
46     """manipulate the archive in a transaction
47     """
48     def __init__(self):
49         self.fs = FilesystemTransaction()
50         self.session = DBConn().session()
51
52     def get_file(self, hashed_file, source_name):
53         """Look for file C{hashed_file} in database
54
55         @type  hashed_file: L{daklib.upload.HashedFile}
56         @param hashed_file: file to look for in the database
57
58         @raise KeyError: file was not found in the database
59         @raise HashMismatchException: hash mismatch
60
61         @rtype:  L{daklib.dbconn.PoolFile}
62         @return: database entry for the file
63         """
64         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
65         try:
66             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
67             if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
68                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
69             return poolfile
70         except NoResultFound:
71             raise KeyError('{0} not found in database.'.format(poolname))
72
73     def _install_file(self, directory, hashed_file, archive, component, source_name):
74         """Install a file
75
76         Will not give an error when the file is already present.
77
78         @rtype:  L{daklib.dbconn.PoolFile}
79         @return: batabase object for the new file
80         """
81         session = self.session
82
83         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
84         try:
85             poolfile = self.get_file(hashed_file, source_name)
86         except KeyError:
87             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
88             poolfile.md5sum = hashed_file.md5sum
89             poolfile.sha1sum = hashed_file.sha1sum
90             poolfile.sha256sum = hashed_file.sha256sum
91             session.add(poolfile)
92             session.flush()
93
94         try:
95             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
96         except NoResultFound:
97             archive_file = ArchiveFile(archive, component, poolfile)
98             session.add(archive_file)
99             session.flush()
100
101             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
102             hashed_file_path = os.path.join(directory, hashed_file.filename)
103             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
104
105         return poolfile
106
107     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
108         """Install a binary package
109
110         @type  directory: str
111         @param directory: directory the binary package is located in
112
113         @type  binary: L{daklib.upload.Binary}
114         @param binary: binary package to install
115
116         @type  suite: L{daklib.dbconn.Suite}
117         @param suite: target suite
118
119         @type  component: L{daklib.dbconn.Component}
120         @param component: target component
121
122         @type  allow_tainted: bool
123         @param allow_tainted: allow to copy additional files from tainted archives
124
125         @type  fingerprint: L{daklib.dbconn.Fingerprint}
126         @param fingerprint: optional fingerprint
127
128         @type  source_suites: list of L{daklib.dbconn.Suite} or C{True}
129         @param source_suites: suites to copy the source from if they are not
130                               in C{suite} or C{True} to allow copying from any
131                               suite.
132                               This can also be a SQLAlchemy (sub)query object.
133
134         @type  extra_source_archives: list of L{daklib.dbconn.Archive}
135         @param extra_source_archives: extra archives to copy Built-Using sources from
136
137         @rtype:  L{daklib.dbconn.DBBinary}
138         @return: databse object for the new package
139         """
140         session = self.session
141         control = binary.control
142         maintainer = get_or_set_maintainer(control['Maintainer'], session)
143         architecture = get_architecture(control['Architecture'], session)
144
145         (source_name, source_version) = binary.source
146         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
147         source = source_query.filter(DBSource.suites.contains(suite)).first()
148         if source is None:
149             if source_suites != True:
150                 source_query = source_query.join(DBSource.suites) \
151                     .filter(Suite.suite_id == source_suites.c.id)
152             source = source_query.first()
153             if source is None:
154                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
155             self.copy_source(source, suite, component)
156
157         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
158
159         unique = dict(
160             package=control['Package'],
161             version=control['Version'],
162             architecture=architecture,
163             )
164         rest = dict(
165             source=source,
166             maintainer=maintainer,
167             poolfile=db_file,
168             binarytype=binary.type,
169             fingerprint=fingerprint,
170             )
171
172         try:
173             db_binary = session.query(DBBinary).filter_by(**unique).one()
174             for key, value in rest.iteritems():
175                 if getattr(db_binary, key) != value:
176                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
177         except NoResultFound:
178             db_binary = DBBinary(**unique)
179             for key, value in rest.iteritems():
180                 setattr(db_binary, key, value)
181             session.add(db_binary)
182             session.flush()
183             import_metadata_into_db(db_binary, session)
184
185             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
186
187         if suite not in db_binary.suites:
188             db_binary.suites.append(suite)
189
190         session.flush()
191
192         return db_binary
193
194     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
195         """ensure source exists in the given archive
196
197         This is intended to be used to check that Built-Using sources exist.
198
199         @type  filename: str
200         @param filename: filename to use in error messages
201
202         @type  source: L{daklib.dbconn.DBSource}
203         @param source: source to look for
204
205         @type  archive: L{daklib.dbconn.Archive}
206         @param archive: archive to look in
207
208         @type  extra_archives: list of L{daklib.dbconn.Archive}
209         @param extra_archives: list of archives to copy the source package from
210                                if it is not yet present in C{archive}
211         """
212         session = self.session
213         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
214         if db_file is not None:
215             return True
216
217         # Try to copy file from one extra archive
218         if extra_archives is None:
219             extra_archives = []
220         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
221         if db_file is None:
222             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
223
224         source_archive = db_file.archive
225         for dsc_file in source.srcfiles:
226             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
227             # We were given an explicit list of archives so it is okay to copy from tainted archives.
228             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
229
230     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
231         """Add Built-Using sources to C{db_binary.extra_sources}
232         """
233         session = self.session
234         built_using = control.get('Built-Using', None)
235
236         if built_using is not None:
237             for dep in apt_pkg.parse_depends(built_using):
238                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
239                 bu_source_name, bu_source_version, comp = dep[0]
240                 assert comp == '=', 'Built-Using must contain strict dependencies'
241
242                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
243                 if bu_source is None:
244                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
245
246                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
247
248                 db_binary.extra_sources.append(bu_source)
249
250     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
251         """Install a source package
252
253         @type  directory: str
254         @param directory: directory the source package is located in
255
256         @type  source: L{daklib.upload.Source}
257         @param source: source package to install
258
259         @type  suite: L{daklib.dbconn.Suite}
260         @param suite: target suite
261
262         @type  component: L{daklib.dbconn.Component}
263         @param component: target component
264
265         @type  changed_by: L{daklib.dbconn.Maintainer}
266         @param changed_by: person who prepared this version of the package
267
268         @type  allow_tainted: bool
269         @param allow_tainted: allow to copy additional files from tainted archives
270
271         @type  fingerprint: L{daklib.dbconn.Fingerprint}
272         @param fingerprint: optional fingerprint
273
274         @rtype:  L{daklib.dbconn.DBSource}
275         @return: database object for the new source
276         """
277         session = self.session
278         archive = suite.archive
279         control = source.dsc
280         maintainer = get_or_set_maintainer(control['Maintainer'], session)
281         source_name = control['Source']
282
283         ### Add source package to database
284
285         # We need to install the .dsc first as the DBSource object refers to it.
286         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
287
288         unique = dict(
289             source=source_name,
290             version=control['Version'],
291             )
292         rest = dict(
293             maintainer=maintainer,
294             changedby=changed_by,
295             #install_date=datetime.now().date(),
296             poolfile=db_file_dsc,
297             fingerprint=fingerprint,
298             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
299             )
300
301         created = False
302         try:
303             db_source = session.query(DBSource).filter_by(**unique).one()
304             for key, value in rest.iteritems():
305                 if getattr(db_source, key) != value:
306                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
307         except NoResultFound:
308             created = True
309             db_source = DBSource(**unique)
310             for key, value in rest.iteritems():
311                 setattr(db_source, key, value)
312             # XXX: set as default in postgres?
313             db_source.install_date = datetime.now().date()
314             session.add(db_source)
315             session.flush()
316
317             # Add .dsc file. Other files will be added later.
318             db_dsc_file = DSCFile()
319             db_dsc_file.source = db_source
320             db_dsc_file.poolfile = db_file_dsc
321             session.add(db_dsc_file)
322             session.flush()
323
324         if suite in db_source.suites:
325             return db_source
326
327         db_source.suites.append(suite)
328
329         if not created:
330             return db_source
331
332         ### Now add remaining files and copy them to the archive.
333
334         for hashed_file in source.files.itervalues():
335             hashed_file_path = os.path.join(directory, hashed_file.filename)
336             if os.path.exists(hashed_file_path):
337                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
338                 session.add(db_file)
339             else:
340                 db_file = self.get_file(hashed_file, source_name)
341                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
342
343             db_dsc_file = DSCFile()
344             db_dsc_file.source = db_source
345             db_dsc_file.poolfile = db_file
346             session.add(db_dsc_file)
347
348         session.flush()
349
350         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
351         import_metadata_into_db(db_source, session)
352
353         # Uploaders are the maintainer and co-maintainers from the Uploaders field
354         db_source.uploaders.append(maintainer)
355         if 'Uploaders' in control:
356             def split_uploaders(field):
357                 import re
358                 for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
359                     yield u.strip()
360
361             for u in split_uploaders(control['Uploaders']):
362                 db_source.uploaders.append(get_or_set_maintainer(u, session))
363         session.flush()
364
365         return db_source
366
367     def _copy_file(self, db_file, archive, component, allow_tainted=False):
368         """Copy a file to the given archive and component
369
370         @type  db_file: L{daklib.dbconn.PoolFile}
371         @param db_file: file to copy
372
373         @type  archive: L{daklib.dbconn.Archive}
374         @param archive: target archive
375
376         @type  component: L{daklib.dbconn.Archive}
377         @param component: target component
378
379         @type  allow_tainted: bool
380         @param allow_tainted: allow to copy from tainted archives (such as NEW)
381         """
382         session = self.session
383
384         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
385             query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
386             if not allow_tainted:
387                 query = query.join(Archive).filter(Archive.tainted == False)
388
389             source_af = query.first()
390             if source_af is None:
391                 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
392             target_af = ArchiveFile(archive, component, db_file)
393             session.add(target_af)
394             session.flush()
395             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
396
397     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
398         """Copy a binary package to the given suite and component
399
400         @type  db_binary: L{daklib.dbconn.DBBinary}
401         @param db_binary: binary to copy
402
403         @type  suite: L{daklib.dbconn.Suite}
404         @param suite: target suite
405
406         @type  component: L{daklib.dbconn.Component}
407         @param component: target component
408
409         @type  allow_tainted: bool
410         @param allow_tainted: allow to copy from tainted archives (such as NEW)
411
412         @type  extra_archives: list of L{daklib.dbconn.Archive}
413         @param extra_archives: extra archives to copy Built-Using sources from
414         """
415         session = self.session
416         archive = suite.archive
417         if archive.tainted:
418             allow_tainted = True
419
420         filename = db_binary.poolfile.filename
421
422         # make sure source is present in target archive
423         db_source = db_binary.source
424         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
425             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
426
427         # make sure built-using packages are present in target archive
428         for db_source in db_binary.extra_sources:
429             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
430
431         # copy binary
432         db_file = db_binary.poolfile
433         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
434         if suite not in db_binary.suites:
435             db_binary.suites.append(suite)
436         self.session.flush()
437
438     def copy_source(self, db_source, suite, component, allow_tainted=False):
439         """Copy a source package to the given suite and component
440
441         @type  db_source: L{daklib.dbconn.DBSource}
442         @param db_source: source to copy
443
444         @type  suite: L{daklib.dbconn.Suite}
445         @param suite: target suite
446
447         @type  component: L{daklib.dbconn.Component}
448         @param component: target component
449
450         @type  allow_tainted: bool
451         @param allow_tainted: allow to copy from tainted archives (such as NEW)
452         """
453         archive = suite.archive
454         if archive.tainted:
455             allow_tainted = True
456         for db_dsc_file in db_source.srcfiles:
457             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
458         if suite not in db_source.suites:
459             db_source.suites.append(suite)
460         self.session.flush()
461
462     def remove_file(self, db_file, archive, component):
463         """Remove a file from a given archive and component
464
465         @type  db_file: L{daklib.dbconn.PoolFile}
466         @param db_file: file to remove
467
468         @type  archive: L{daklib.dbconn.Archive}
469         @param archive: archive to remove the file from
470
471         @type  component: L{daklib.dbconn.Component}
472         @param component: component to remove the file from
473         """
474         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
475         self.fs.unlink(af.path)
476         self.session.delete(af)
477
478     def remove_binary(self, binary, suite):
479         """Remove a binary from a given suite and component
480
481         @type  binary: L{daklib.dbconn.DBBinary}
482         @param binary: binary to remove
483
484         @type  suite: L{daklib.dbconn.Suite}
485         @param suite: suite to remove the package from
486         """
487         binary.suites.remove(suite)
488         self.session.flush()
489
490     def remove_source(self, source, suite):
491         """Remove a source from a given suite and component
492
493         @type  source: L{daklib.dbconn.DBSource}
494         @param source: source to remove
495
496         @type  suite: L{daklib.dbconn.Suite}
497         @param suite: suite to remove the package from
498
499         @raise ArchiveException: source package is still referenced by other
500                                  binaries in the suite
501         """
502         session = self.session
503
504         query = session.query(DBBinary).filter_by(source=source) \
505             .filter(DBBinary.suites.contains(suite))
506         if query.first() is not None:
507             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
508
509         source.suites.remove(suite)
510         session.flush()
511
512     def commit(self):
513         """commit changes"""
514         try:
515             self.session.commit()
516             self.fs.commit()
517         finally:
518             self.session.rollback()
519             self.fs.rollback()
520
521     def rollback(self):
522         """rollback changes"""
523         self.session.rollback()
524         self.fs.rollback()
525
526     def __enter__(self):
527         return self
528
529     def __exit__(self, type, value, traceback):
530         if type is None:
531             self.commit()
532         else:
533             self.rollback()
534         return None
535
536 class ArchiveUpload(object):
537     """handle an upload
538
539     This class can be used in a with-statement::
540
541        with ArchiveUpload(...) as upload:
542           ...
543
544     Doing so will automatically run any required cleanup and also rollback the
545     transaction if it was not committed.
546     """
547     def __init__(self, directory, changes, keyrings):
548         self.transaction = ArchiveTransaction()
549         """transaction used to handle the upload
550         @type: L{daklib.archive.ArchiveTransaction}
551         """
552
553         self.session = self.transaction.session
554         """database session"""
555
556         self.original_directory = directory
557         self.original_changes = changes
558
559         self.changes = None
560         """upload to process
561         @type: L{daklib.upload.Changes}
562         """
563
564         self.directory = None
565         """directory with temporary copy of files. set by C{prepare}
566         @type: str
567         """
568
569         self.keyrings = keyrings
570
571         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
572         """fingerprint of the key used to sign the upload
573         @type: L{daklib.dbconn.Fingerprint}
574         """
575
576         self.reject_reasons = []
577         """reasons why the upload cannot by accepted
578         @type: list of str
579         """
580
581         self.warnings = []
582         """warnings
583         @note: Not used yet.
584         @type: list of str
585         """
586
587         self.final_suites = None
588
589         self.new = False
590         """upload is NEW. set by C{check}
591         @type: bool
592         """
593
594         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
595         self._new = self._new_queue.suite
596
597     def prepare(self):
598         """prepare upload for further processing
599
600         This copies the files involved to a temporary directory.  If you use
601         this method directly, you have to remove the directory given by the
602         C{directory} attribute later on your own.
603
604         Instead of using the method directly, you can also use a with-statement::
605
606            with ArchiveUpload(...) as upload:
607               ...
608
609         This will automatically handle any required cleanup.
610         """
611         assert self.directory is None
612         assert self.original_changes.valid_signature
613
614         cnf = Config()
615         session = self.transaction.session
616
617         self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
618         with FilesystemTransaction() as fs:
619             src = os.path.join(self.original_directory, self.original_changes.filename)
620             dst = os.path.join(self.directory, self.original_changes.filename)
621             fs.copy(src, dst)
622
623             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
624
625             for f in self.changes.files.itervalues():
626                 src = os.path.join(self.original_directory, f.filename)
627                 dst = os.path.join(self.directory, f.filename)
628                 fs.copy(src, dst)
629
630             source = self.changes.source
631             if source is not None:
632                 for f in source.files.itervalues():
633                     src = os.path.join(self.original_directory, f.filename)
634                     dst = os.path.join(self.directory, f.filename)
635                     if f.filename not in self.changes.files:
636                         db_file = self.transaction.get_file(f, source.dsc['Source'])
637                         db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
638                         fs.copy(db_archive_file.path, dst, symlink=True)
639
640     def unpacked_source(self):
641         """Path to unpacked source
642
643         Get path to the unpacked source. This method does unpack the source
644         into a temporary directory under C{self.directory} if it has not
645         been done so already.
646
647         @rtype:  str or C{None}
648         @return: string giving the path to the unpacked source directory
649                  or C{None} if no source was included in the upload.
650         """
651         assert self.directory is not None
652
653         source = self.changes.source
654         if source is None:
655             return None
656         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
657
658         sourcedir = os.path.join(self.directory, 'source')
659         if not os.path.exists(sourcedir):
660             subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
661         if not os.path.isdir(sourcedir):
662             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
663         return sourcedir
664
665     def _map_suite(self, suite_name):
666         for rule in Config().value_list("SuiteMappings"):
667             fields = rule.split()
668             rtype = fields[0]
669             if rtype == "map" or rtype == "silent-map":
670                 (src, dst) = fields[1:3]
671                 if src == suite_name:
672                     suite_name = dst
673                     if rtype != "silent-map":
674                         self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
675             elif rtype == "ignore":
676                 ignored = fields[1]
677                 if suite_name == ignored:
678                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
679                     suite_name = None
680             elif rtype == "reject":
681                 rejected = fields[1]
682                 if suite_name == rejected:
683                     self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
684             ## XXX: propup-version and map-unreleased not yet implemented
685         return suite_name
686
687     def _mapped_suites(self):
688         """Get target suites after mappings
689
690         @rtype:  list of L{daklib.dbconn.Suite}
691         @return: list giving the mapped target suites of this upload
692         """
693         session = self.session
694
695         suite_names = []
696         for dist in self.changes.distributions:
697             suite_name = self._map_suite(dist)
698             if suite_name is not None:
699                 suite_names.append(suite_name)
700
701         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
702         return suites
703
704     def _mapped_component(self, component_name):
705         """get component after mappings
706
707         Evaluate component mappings from ComponentMappings in dak.conf for the
708         given component name.
709
710         @todo: ansgar wants to get rid of this. It's currently only used for
711                the security archive
712
713         @type  component_name: str
714         @param component_name: component name
715
716         @rtype:  L{daklib.dbconn.Component}
717         @return: component after applying maps
718         """
719         cnf = Config()
720         for m in cnf.value_list("ComponentMappings"):
721             (src, dst) = m.split()
722             if component_name == src:
723                 component_name = dst
724         component = self.session.query(Component).filter_by(component_name=component_name).one()
725         return component
726
727     def _check_new(self, suite):
728         """Check if upload is NEW
729
730         An upload is NEW if it has binary or source packages that do not have
731         an override in C{suite} OR if it references files ONLY in a tainted
732         archive (eg. when it references files in NEW).
733
734         @rtype:  bool
735         @return: C{True} if the upload is NEW, C{False} otherwise
736         """
737         session = self.session
738
739         # Check for missing overrides
740         for b in self.changes.binaries:
741             override = self._binary_override(suite, b)
742             if override is None:
743                 return True
744
745         if self.changes.source is not None:
746             override = self._source_override(suite, self.changes.source)
747             if override is None:
748                 return True
749
750         # Check if we reference a file only in a tainted archive
751         files = self.changes.files.values()
752         if self.changes.source is not None:
753             files.extend(self.changes.source.files.values())
754         for f in files:
755             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
756             query_untainted = query.join(Archive).filter(Archive.tainted == False)
757
758             in_archive = (query.first() is not None)
759             in_untainted_archive = (query_untainted.first() is not None)
760
761             if in_archive and not in_untainted_archive:
762                 return True
763
764     def _final_suites(self):
765         session = self.session
766
767         mapped_suites = self._mapped_suites()
768         final_suites = set()
769
770         for suite in mapped_suites:
771             overridesuite = suite
772             if suite.overridesuite is not None:
773                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
774             if self._check_new(overridesuite):
775                 self.new = True
776             final_suites.add(suite)
777
778         return final_suites
779
780     def _binary_override(self, suite, binary):
781         """Get override entry for a binary
782
783         @type  suite: L{daklib.dbconn.Suite}
784         @param suite: suite to get override for
785
786         @type  binary: L{daklib.upload.Binary}
787         @param binary: binary to get override for
788
789         @rtype:  L{daklib.dbconn.Override} or C{None}
790         @return: override for the given binary or C{None}
791         """
792         if suite.overridesuite is not None:
793             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
794
795         query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
796                 .join(Component).filter(Component.component_name == binary.component) \
797                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
798
799         try:
800             return query.one()
801         except NoResultFound:
802             return None
803
804     def _source_override(self, suite, source):
805         """Get override entry for a source
806
807         @type  suite: L{daklib.dbconn.Suite}
808         @param suite: suite to get override for
809
810         @type  source: L{daklib.upload.Source}
811         @param source: source to get override for
812
813         @rtype:  L{daklib.dbconn.Override} or C{None}
814         @return: override for the given source or C{None}
815         """
816         if suite.overridesuite is not None:
817             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
818
819         # XXX: component for source?
820         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
821                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
822
823         try:
824             return query.one()
825         except NoResultFound:
826             return None
827
828     def _binary_component(self, suite, binary, only_overrides=True):
829         """get component for a binary
830
831         By default this will only look at overrides to get the right component;
832         if C{only_overrides} is C{False} this method will also look at the
833         Section field.
834
835         @type  suite: L{daklib.dbconn.Suite}
836
837         @type  binary: L{daklib.upload.Binary}
838
839         @type  only_overrides: bool
840         @param only_overrides: only use overrides to get the right component
841
842         @rtype: L{daklib.dbconn.Component} or C{None}
843         """
844         override = self._binary_override(suite, binary)
845         if override is not None:
846             return override.component
847         if only_overrides:
848             return None
849         return self._mapped_component(binary.component)
850
851     def check(self, force=False):
852         """run checks against the upload
853
854         @type  force: bool
855         @param force: ignore failing forcable checks
856
857         @rtype:  bool
858         @return: C{True} if all checks passed, C{False} otherwise
859         """
860         # XXX: needs to be better structured.
861         assert self.changes.valid_signature
862
863         try:
864             for chk in (
865                     checks.SignatureCheck,
866                     checks.ChangesCheck,
867                     checks.TransitionCheck,
868                     checks.UploadBlockCheck,
869                     checks.HashesCheck,
870                     checks.SourceCheck,
871                     checks.BinaryCheck,
872                     checks.BinaryTimestampCheck,
873                     checks.ACLCheck,
874                     checks.SingleDistributionCheck,
875                     checks.NoSourceOnlyCheck,
876                     checks.LintianCheck,
877                     ):
878                 chk().check(self)
879
880             final_suites = self._final_suites()
881             if len(final_suites) == 0:
882                 self.reject_reasons.append('Ended with no suite to install to.')
883                 return False
884
885             for chk in (
886                     checks.SourceFormatCheck,
887                     checks.SuiteArchitectureCheck,
888                     checks.VersionCheck,
889                     ):
890                 for suite in final_suites:
891                     chk().per_suite_check(self, suite)
892
893             if len(self.reject_reasons) != 0:
894                 return False
895
896             self.final_suites = final_suites
897             return True
898         except checks.Reject as e:
899             self.reject_reasons.append(unicode(e))
900         except Exception as e:
901             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
902         return False
903
904     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
905         """Install upload to the given suite
906
907         @type  suite: L{daklib.dbconn.Suite}
908         @param suite: suite to install the package into. This is the real suite,
909                       ie. after any redirection to NEW or a policy queue
910
911         @param source_component_func: function to get the L{daklib.dbconn.Component}
912                                       for a L{daklib.upload.Source} object
913
914         @param binary_component_func: function to get the L{daklib.dbconn.Component}
915                                       for a L{daklib.upload.Binary} object
916
917         @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
918
919         @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
920
921         @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
922                  object for the install source or C{None} if no source was
923                  included. The second is a list of L{daklib.dbconn.DBBinary}
924                  objects for the installed binary packages.
925         """
926         # XXX: move this function to ArchiveTransaction?
927
928         control = self.changes.changes
929         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
930
931         if source_suites is None:
932             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
933
934         source = self.changes.source
935         if source is not None:
936             component = source_component_func(source)
937             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
938         else:
939             db_source = None
940
941         db_binaries = []
942         for binary in self.changes.binaries:
943             component = binary_component_func(binary)
944             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
945             db_binaries.append(db_binary)
946
947         if suite.copychanges:
948             src = os.path.join(self.directory, self.changes.filename)
949             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
950             self.transaction.fs.copy(src, dst)
951
952         return (db_source, db_binaries)
953
954     def _install_changes(self):
955         assert self.changes.valid_signature
956         control = self.changes.changes
957         session = self.transaction.session
958         config = Config()
959
960         changelog_id = None
961         # Only add changelog for sourceful uploads and binNMUs
962         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
963             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
964             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
965             assert changelog_id is not None
966
967         db_changes = DBChange()
968         db_changes.changesname = self.changes.filename
969         db_changes.source = control['Source']
970         db_changes.binaries = control.get('Binary', None)
971         db_changes.architecture = control['Architecture']
972         db_changes.version = control['Version']
973         db_changes.distribution = control['Distribution']
974         db_changes.urgency = control['Urgency']
975         db_changes.maintainer = control['Maintainer']
976         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
977         db_changes.date = control['Date']
978         db_changes.fingerprint = self.fingerprint.fingerprint
979         db_changes.changelog_id = changelog_id
980         db_changes.closes = self.changes.closed_bugs
981
982         self.transaction.session.add(db_changes)
983         self.transaction.session.flush()
984
985         return db_changes
986
987     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
988         u = PolicyQueueUpload()
989         u.policy_queue = policy_queue
990         u.target_suite = target_suite
991         u.changes = db_changes
992         u.source = db_source
993         u.binaries = db_binaries
994         self.transaction.session.add(u)
995         self.transaction.session.flush()
996
997         dst = os.path.join(policy_queue.path, self.changes.filename)
998         self.transaction.fs.copy(self.changes.path, dst)
999
1000         return u
1001
1002     def try_autobyhand(self):
1003         """Try AUTOBYHAND
1004
1005         Try to handle byhand packages automatically.
1006
1007         @rtype:  list of L{daklib.upload.HashedFile}
1008         @return: list of remaining byhand files
1009         """
1010         assert len(self.reject_reasons) == 0
1011         assert self.changes.valid_signature
1012         assert self.final_suites is not None
1013
1014         byhand = self.changes.byhand_files
1015         if len(byhand) == 0:
1016             return True
1017
1018         suites = list(self.final_suites)
1019         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1020         suite = suites[0]
1021
1022         cnf = Config()
1023         control = self.changes.changes
1024         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1025
1026         remaining = []
1027         for f in byhand:
1028             parts = f.filename.split('_', 2)
1029             if len(parts) != 3:
1030                 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1031                 remaining.append(f)
1032                 continue
1033
1034             package, version, archext = parts
1035             arch, ext = archext.split('.', 1)
1036
1037             rule = automatic_byhand_packages.get(package)
1038             if rule is None:
1039                 remaining.append(f)
1040                 continue
1041
1042             if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
1043                 remaining.append(f)
1044                 continue
1045
1046             script = rule['Script']
1047             retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1048             if retcode != 0:
1049                 print "W: error processing {0}.".format(f.filename)
1050                 remaining.append(f)
1051
1052         return len(remaining) == 0
1053
1054     def _install_byhand(self, policy_queue_upload, hashed_file):
1055         """install byhand file
1056
1057         @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1058
1059         @type  hashed_file: L{daklib.upload.HashedFile}
1060         """
1061         fs = self.transaction.fs
1062         session = self.transaction.session
1063         policy_queue = policy_queue_upload.policy_queue
1064
1065         byhand_file = PolicyQueueByhandFile()
1066         byhand_file.upload = policy_queue_upload
1067         byhand_file.filename = hashed_file.filename
1068         session.add(byhand_file)
1069         session.flush()
1070
1071         src = os.path.join(self.directory, hashed_file.filename)
1072         dst = os.path.join(policy_queue.path, hashed_file.filename)
1073         fs.copy(src, dst)
1074
1075         return byhand_file
1076
1077     def _do_bts_versiontracking(self):
1078         cnf = Config()
1079         fs = self.transaction.fs
1080
1081         btsdir = cnf.get('Dir::BTSVersionTrack')
1082         if btsdir is None or btsdir == '':
1083             return
1084
1085         base = os.path.join(btsdir, self.changes.filename[:-8])
1086
1087         # version history
1088         sourcedir = self.unpacked_source()
1089         if sourcedir is not None:
1090             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1091             versions = fs.create("{0}.versions".format(base), mode=0o644)
1092             for line in fh.readlines():
1093                 if re_changelog_versions.match(line):
1094                     versions.write(line)
1095             fh.close()
1096             versions.close()
1097
1098         # binary -> source mapping
1099         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1100         for binary in self.changes.binaries:
1101             control = binary.control
1102             source_package, source_version = binary.source
1103             line = " ".join([control['Package'], control['Version'], source_package, source_version])
1104             print >>debinfo, line
1105         debinfo.close()
1106
1107     def _policy_queue(self, suite):
1108         if suite.policy_queue is not None:
1109             return suite.policy_queue
1110         return None
1111
1112     def install(self):
1113         """install upload
1114
1115         Install upload to a suite or policy queue.  This method does B{not}
1116         handle uploads to NEW.
1117
1118         You need to have called the C{check} method before calling this method.
1119         """
1120         assert len(self.reject_reasons) == 0
1121         assert self.changes.valid_signature
1122         assert self.final_suites is not None
1123         assert not self.new
1124
1125         db_changes = self._install_changes()
1126
1127         for suite in self.final_suites:
1128             overridesuite = suite
1129             if suite.overridesuite is not None:
1130                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1131
1132             policy_queue = self._policy_queue(suite)
1133
1134             redirected_suite = suite
1135             if policy_queue is not None:
1136                 redirected_suite = policy_queue.suite
1137
1138             source_component_func = lambda source: self._source_override(overridesuite, source).component
1139             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1140
1141             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1142
1143             if policy_queue is not None:
1144                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1145
1146             # copy to build queues
1147             if policy_queue is None or policy_queue.send_to_build_queues:
1148                 for build_queue in suite.copy_queues:
1149                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1150
1151         self._do_bts_versiontracking()
1152
1153     def install_to_new(self):
1154         """install upload to NEW
1155
1156         Install upload to NEW.  This method does B{not} handle regular uploads
1157         to suites or policy queues.
1158
1159         You need to have called the C{check} method before calling this method.
1160         """
1161         # Uploads to NEW are special as we don't have overrides.
1162         assert len(self.reject_reasons) == 0
1163         assert self.changes.valid_signature
1164         assert self.final_suites is not None
1165
1166         source = self.changes.source
1167         binaries = self.changes.binaries
1168         byhand = self.changes.byhand_files
1169
1170         new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1171         if len(byhand) > 0:
1172             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1173         new_suite = new_queue.suite
1174
1175         # we need a suite to guess components
1176         suites = list(self.final_suites)
1177         assert len(suites) == 1, "NEW uploads must be to a single suite"
1178         suite = suites[0]
1179
1180         def binary_component_func(binary):
1181             return self._binary_component(suite, binary, only_overrides=False)
1182
1183         # guess source component
1184         # XXX: should be moved into an extra method
1185         binary_component_names = set()
1186         for binary in binaries:
1187             component = binary_component_func(binary)
1188             binary_component_names.add(component.component_name)
1189         source_component_name = None
1190         for c in self.session.query(Component).order_by(Component.component_id):
1191             guess = c.component_name
1192             if guess in binary_component_names:
1193                 source_component_name = guess
1194                 break
1195         if source_component_name is None:
1196             raise Exception('Could not guess source component.')
1197         source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1198         source_component_func = lambda source: source_component
1199
1200         db_changes = self._install_changes()
1201         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1202         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1203
1204         for f in byhand:
1205             self._install_byhand(policy_upload, f)
1206
1207         self._do_bts_versiontracking()
1208
1209     def commit(self):
1210         """commit changes"""
1211         self.transaction.commit()
1212
1213     def rollback(self):
1214         """rollback changes"""
1215         self.transaction.rollback()
1216
1217     def __enter__(self):
1218         self.prepare()
1219         return self
1220
1221     def __exit__(self, type, value, traceback):
1222         if self.directory is not None:
1223             shutil.rmtree(self.directory)
1224             self.directory = None
1225         self.changes = None
1226         self.transaction.rollback()
1227         return None