]> git.decadent.org.uk Git - dak.git/blob - daklib/archive.py
match_architecture: Fix raising exception if an architecture is unknown
[dak.git] / daklib / archive.py
1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
2 #
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
7 #
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 # GNU General Public License for more details.
12 #
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
16
17 """module to manipulate the archive
18
19 This module provides classes to manipulate the archive.
20 """
21
22 from daklib.dbconn import *
23 import daklib.checks as checks
24 from daklib.config import Config
25 import daklib.upload as upload
26 import daklib.utils as utils
27 from daklib.fstransactions import FilesystemTransaction
28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
29 import daklib.daksubprocess
30
31 import apt_pkg
32 from datetime import datetime
33 import os
34 import shutil
35 from sqlalchemy.orm.exc import NoResultFound
36 import sqlalchemy.exc
37 import tempfile
38 import traceback
39
40 class ArchiveException(Exception):
41     pass
42
43 class HashMismatchException(ArchiveException):
44     pass
45
46 class ArchiveTransaction(object):
47     """manipulate the archive in a transaction
48     """
49     def __init__(self):
50         self.fs = FilesystemTransaction()
51         self.session = DBConn().session()
52
53     def get_file(self, hashed_file, source_name, check_hashes=True):
54         """Look for file C{hashed_file} in database
55
56         @type  hashed_file: L{daklib.upload.HashedFile}
57         @param hashed_file: file to look for in the database
58
59         @type  source_name: str
60         @param source_name: source package name
61
62         @type  check_hashes: bool
63         @param check_hashes: check size and hashes match
64
65         @raise KeyError: file was not found in the database
66         @raise HashMismatchException: hash mismatch
67
68         @rtype:  L{daklib.dbconn.PoolFile}
69         @return: database entry for the file
70         """
71         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
72         try:
73             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
74             if check_hashes and (poolfile.filesize != hashed_file.size
75                                  or poolfile.md5sum != hashed_file.md5sum
76                                  or poolfile.sha1sum != hashed_file.sha1sum
77                                  or poolfile.sha256sum != hashed_file.sha256sum):
78                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
79             return poolfile
80         except NoResultFound:
81             raise KeyError('{0} not found in database.'.format(poolname))
82
83     def _install_file(self, directory, hashed_file, archive, component, source_name):
84         """Install a file
85
86         Will not give an error when the file is already present.
87
88         @rtype:  L{daklib.dbconn.PoolFile}
89         @return: database object for the new file
90         """
91         session = self.session
92
93         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
94         try:
95             poolfile = self.get_file(hashed_file, source_name)
96         except KeyError:
97             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
98             poolfile.md5sum = hashed_file.md5sum
99             poolfile.sha1sum = hashed_file.sha1sum
100             poolfile.sha256sum = hashed_file.sha256sum
101             session.add(poolfile)
102             session.flush()
103
104         try:
105             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
106         except NoResultFound:
107             archive_file = ArchiveFile(archive, component, poolfile)
108             session.add(archive_file)
109             session.flush()
110
111             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
112             hashed_file_path = os.path.join(directory, hashed_file.filename)
113             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
114
115         return poolfile
116
117     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
118         """Install a binary package
119
120         @type  directory: str
121         @param directory: directory the binary package is located in
122
123         @type  binary: L{daklib.upload.Binary}
124         @param binary: binary package to install
125
126         @type  suite: L{daklib.dbconn.Suite}
127         @param suite: target suite
128
129         @type  component: L{daklib.dbconn.Component}
130         @param component: target component
131
132         @type  allow_tainted: bool
133         @param allow_tainted: allow to copy additional files from tainted archives
134
135         @type  fingerprint: L{daklib.dbconn.Fingerprint}
136         @param fingerprint: optional fingerprint
137
138         @type  source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
139         @param source_suites: suites to copy the source from if they are not
140                               in C{suite} or C{True} to allow copying from any
141                               suite.
142
143         @type  extra_source_archives: list of L{daklib.dbconn.Archive}
144         @param extra_source_archives: extra archives to copy Built-Using sources from
145
146         @rtype:  L{daklib.dbconn.DBBinary}
147         @return: databse object for the new package
148         """
149         session = self.session
150         control = binary.control
151         maintainer = get_or_set_maintainer(control['Maintainer'], session)
152         architecture = get_architecture(control['Architecture'], session)
153
154         (source_name, source_version) = binary.source
155         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
156         source = source_query.filter(DBSource.suites.contains(suite)).first()
157         if source is None:
158             if source_suites != True:
159                 source_query = source_query.join(DBSource.suites) \
160                     .filter(Suite.suite_id == source_suites.c.id)
161             source = source_query.first()
162             if source is None:
163                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
164             self.copy_source(source, suite, component)
165
166         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
167
168         unique = dict(
169             package=control['Package'],
170             version=control['Version'],
171             architecture=architecture,
172             )
173         rest = dict(
174             source=source,
175             maintainer=maintainer,
176             poolfile=db_file,
177             binarytype=binary.type,
178             fingerprint=fingerprint,
179             )
180
181         try:
182             db_binary = session.query(DBBinary).filter_by(**unique).one()
183             for key, value in rest.iteritems():
184                 if getattr(db_binary, key) != value:
185                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
186         except NoResultFound:
187             db_binary = DBBinary(**unique)
188             for key, value in rest.iteritems():
189                 setattr(db_binary, key, value)
190             session.add(db_binary)
191             session.flush()
192             import_metadata_into_db(db_binary, session)
193
194             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
195
196         if suite not in db_binary.suites:
197             db_binary.suites.append(suite)
198
199         session.flush()
200
201         return db_binary
202
203     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
204         """ensure source exists in the given archive
205
206         This is intended to be used to check that Built-Using sources exist.
207
208         @type  filename: str
209         @param filename: filename to use in error messages
210
211         @type  source: L{daklib.dbconn.DBSource}
212         @param source: source to look for
213
214         @type  archive: L{daklib.dbconn.Archive}
215         @param archive: archive to look in
216
217         @type  extra_archives: list of L{daklib.dbconn.Archive}
218         @param extra_archives: list of archives to copy the source package from
219                                if it is not yet present in C{archive}
220         """
221         session = self.session
222         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
223         if db_file is not None:
224             return True
225
226         # Try to copy file from one extra archive
227         if extra_archives is None:
228             extra_archives = []
229         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
230         if db_file is None:
231             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
232
233         source_archive = db_file.archive
234         for dsc_file in source.srcfiles:
235             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
236             # We were given an explicit list of archives so it is okay to copy from tainted archives.
237             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
238
239     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
240         """Add Built-Using sources to C{db_binary.extra_sources}
241         """
242         session = self.session
243         built_using = control.get('Built-Using', None)
244
245         if built_using is not None:
246             for dep in apt_pkg.parse_depends(built_using):
247                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
248                 bu_source_name, bu_source_version, comp = dep[0]
249                 assert comp == '=', 'Built-Using must contain strict dependencies'
250
251                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
252                 if bu_source is None:
253                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
254
255                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
256
257                 db_binary.extra_sources.append(bu_source)
258
259     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
260         """Install a source package
261
262         @type  directory: str
263         @param directory: directory the source package is located in
264
265         @type  source: L{daklib.upload.Source}
266         @param source: source package to install
267
268         @type  suite: L{daklib.dbconn.Suite}
269         @param suite: target suite
270
271         @type  component: L{daklib.dbconn.Component}
272         @param component: target component
273
274         @type  changed_by: L{daklib.dbconn.Maintainer}
275         @param changed_by: person who prepared this version of the package
276
277         @type  allow_tainted: bool
278         @param allow_tainted: allow to copy additional files from tainted archives
279
280         @type  fingerprint: L{daklib.dbconn.Fingerprint}
281         @param fingerprint: optional fingerprint
282
283         @rtype:  L{daklib.dbconn.DBSource}
284         @return: database object for the new source
285         """
286         session = self.session
287         archive = suite.archive
288         control = source.dsc
289         maintainer = get_or_set_maintainer(control['Maintainer'], session)
290         source_name = control['Source']
291
292         ### Add source package to database
293
294         # We need to install the .dsc first as the DBSource object refers to it.
295         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
296
297         unique = dict(
298             source=source_name,
299             version=control['Version'],
300             )
301         rest = dict(
302             maintainer=maintainer,
303             changedby=changed_by,
304             #install_date=datetime.now().date(),
305             poolfile=db_file_dsc,
306             fingerprint=fingerprint,
307             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
308             )
309
310         created = False
311         try:
312             db_source = session.query(DBSource).filter_by(**unique).one()
313             for key, value in rest.iteritems():
314                 if getattr(db_source, key) != value:
315                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
316         except NoResultFound:
317             created = True
318             db_source = DBSource(**unique)
319             for key, value in rest.iteritems():
320                 setattr(db_source, key, value)
321             # XXX: set as default in postgres?
322             db_source.install_date = datetime.now().date()
323             session.add(db_source)
324             session.flush()
325
326             # Add .dsc file. Other files will be added later.
327             db_dsc_file = DSCFile()
328             db_dsc_file.source = db_source
329             db_dsc_file.poolfile = db_file_dsc
330             session.add(db_dsc_file)
331             session.flush()
332
333         if suite in db_source.suites:
334             return db_source
335
336         db_source.suites.append(suite)
337
338         if not created:
339             for f in db_source.srcfiles:
340                 self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
341             return db_source
342
343         ### Now add remaining files and copy them to the archive.
344
345         for hashed_file in source.files.itervalues():
346             hashed_file_path = os.path.join(directory, hashed_file.filename)
347             if os.path.exists(hashed_file_path):
348                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
349                 session.add(db_file)
350             else:
351                 db_file = self.get_file(hashed_file, source_name)
352                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
353
354             db_dsc_file = DSCFile()
355             db_dsc_file.source = db_source
356             db_dsc_file.poolfile = db_file
357             session.add(db_dsc_file)
358
359         session.flush()
360
361         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
362         import_metadata_into_db(db_source, session)
363
364         # Uploaders are the maintainer and co-maintainers from the Uploaders field
365         db_source.uploaders.append(maintainer)
366         if 'Uploaders' in control:
367             from daklib.textutils import split_uploaders
368             for u in split_uploaders(control['Uploaders']):
369                 db_source.uploaders.append(get_or_set_maintainer(u, session))
370         session.flush()
371
372         return db_source
373
374     def _copy_file(self, db_file, archive, component, allow_tainted=False):
375         """Copy a file to the given archive and component
376
377         @type  db_file: L{daklib.dbconn.PoolFile}
378         @param db_file: file to copy
379
380         @type  archive: L{daklib.dbconn.Archive}
381         @param archive: target archive
382
383         @type  component: L{daklib.dbconn.Archive}
384         @param component: target component
385
386         @type  allow_tainted: bool
387         @param allow_tainted: allow to copy from tainted archives (such as NEW)
388         """
389         session = self.session
390
391         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
392             query = session.query(ArchiveFile).filter_by(file=db_file)
393             if not allow_tainted:
394                 query = query.join(Archive).filter(Archive.tainted == False)
395
396             source_af = query.first()
397             if source_af is None:
398                 raise ArchiveException('cp: Could not find {0} in any archive.'.format(db_file.filename))
399             target_af = ArchiveFile(archive, component, db_file)
400             session.add(target_af)
401             session.flush()
402             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
403
404     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
405         """Copy a binary package to the given suite and component
406
407         @type  db_binary: L{daklib.dbconn.DBBinary}
408         @param db_binary: binary to copy
409
410         @type  suite: L{daklib.dbconn.Suite}
411         @param suite: target suite
412
413         @type  component: L{daklib.dbconn.Component}
414         @param component: target component
415
416         @type  allow_tainted: bool
417         @param allow_tainted: allow to copy from tainted archives (such as NEW)
418
419         @type  extra_archives: list of L{daklib.dbconn.Archive}
420         @param extra_archives: extra archives to copy Built-Using sources from
421         """
422         session = self.session
423         archive = suite.archive
424         if archive.tainted:
425             allow_tainted = True
426
427         filename = db_binary.poolfile.filename
428
429         # make sure source is present in target archive
430         db_source = db_binary.source
431         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
432             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
433
434         # make sure built-using packages are present in target archive
435         for db_source in db_binary.extra_sources:
436             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
437
438         # copy binary
439         db_file = db_binary.poolfile
440         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
441         if suite not in db_binary.suites:
442             db_binary.suites.append(suite)
443         self.session.flush()
444
445     def copy_source(self, db_source, suite, component, allow_tainted=False):
446         """Copy a source package to the given suite and component
447
448         @type  db_source: L{daklib.dbconn.DBSource}
449         @param db_source: source to copy
450
451         @type  suite: L{daklib.dbconn.Suite}
452         @param suite: target suite
453
454         @type  component: L{daklib.dbconn.Component}
455         @param component: target component
456
457         @type  allow_tainted: bool
458         @param allow_tainted: allow to copy from tainted archives (such as NEW)
459         """
460         archive = suite.archive
461         if archive.tainted:
462             allow_tainted = True
463         for db_dsc_file in db_source.srcfiles:
464             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
465         if suite not in db_source.suites:
466             db_source.suites.append(suite)
467         self.session.flush()
468
469     def remove_file(self, db_file, archive, component):
470         """Remove a file from a given archive and component
471
472         @type  db_file: L{daklib.dbconn.PoolFile}
473         @param db_file: file to remove
474
475         @type  archive: L{daklib.dbconn.Archive}
476         @param archive: archive to remove the file from
477
478         @type  component: L{daklib.dbconn.Component}
479         @param component: component to remove the file from
480         """
481         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
482         self.fs.unlink(af.path)
483         self.session.delete(af)
484
485     def remove_binary(self, binary, suite):
486         """Remove a binary from a given suite and component
487
488         @type  binary: L{daklib.dbconn.DBBinary}
489         @param binary: binary to remove
490
491         @type  suite: L{daklib.dbconn.Suite}
492         @param suite: suite to remove the package from
493         """
494         binary.suites.remove(suite)
495         self.session.flush()
496
497     def remove_source(self, source, suite):
498         """Remove a source from a given suite and component
499
500         @type  source: L{daklib.dbconn.DBSource}
501         @param source: source to remove
502
503         @type  suite: L{daklib.dbconn.Suite}
504         @param suite: suite to remove the package from
505
506         @raise ArchiveException: source package is still referenced by other
507                                  binaries in the suite
508         """
509         session = self.session
510
511         query = session.query(DBBinary).filter_by(source=source) \
512             .filter(DBBinary.suites.contains(suite))
513         if query.first() is not None:
514             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
515
516         source.suites.remove(suite)
517         session.flush()
518
519     def commit(self):
520         """commit changes"""
521         try:
522             self.session.commit()
523             self.fs.commit()
524         finally:
525             self.session.rollback()
526             self.fs.rollback()
527
528     def rollback(self):
529         """rollback changes"""
530         self.session.rollback()
531         self.fs.rollback()
532
533     def __enter__(self):
534         return self
535
536     def __exit__(self, type, value, traceback):
537         if type is None:
538             self.commit()
539         else:
540             self.rollback()
541         return None
542
543 class ArchiveUpload(object):
544     """handle an upload
545
546     This class can be used in a with-statement::
547
548        with ArchiveUpload(...) as upload:
549           ...
550
551     Doing so will automatically run any required cleanup and also rollback the
552     transaction if it was not committed.
553     """
554     def __init__(self, directory, changes, keyrings):
555         self.transaction = ArchiveTransaction()
556         """transaction used to handle the upload
557         @type: L{daklib.archive.ArchiveTransaction}
558         """
559
560         self.session = self.transaction.session
561         """database session"""
562
563         self.original_directory = directory
564         self.original_changes = changes
565
566         self.changes = None
567         """upload to process
568         @type: L{daklib.upload.Changes}
569         """
570
571         self.directory = None
572         """directory with temporary copy of files. set by C{prepare}
573         @type: str
574         """
575
576         self.keyrings = keyrings
577
578         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
579         """fingerprint of the key used to sign the upload
580         @type: L{daklib.dbconn.Fingerprint}
581         """
582
583         self.reject_reasons = []
584         """reasons why the upload cannot by accepted
585         @type: list of str
586         """
587
588         self.warnings = []
589         """warnings
590         @note: Not used yet.
591         @type: list of str
592         """
593
594         self.final_suites = None
595
596         self.new = False
597         """upload is NEW. set by C{check}
598         @type: bool
599         """
600
601         self._checked = False
602         """checks passes. set by C{check}
603         @type: bool
604         """
605
606         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
607         self._new = self._new_queue.suite
608
609     def warn(self, message):
610         """add a warning message
611
612         Adds a warning message that can later be seen in C{self.warnings}
613
614         @type  message: string
615         @param message: warning message
616         """
617         self.warnings.append(message)
618
619     def prepare(self):
620         """prepare upload for further processing
621
622         This copies the files involved to a temporary directory.  If you use
623         this method directly, you have to remove the directory given by the
624         C{directory} attribute later on your own.
625
626         Instead of using the method directly, you can also use a with-statement::
627
628            with ArchiveUpload(...) as upload:
629               ...
630
631         This will automatically handle any required cleanup.
632         """
633         assert self.directory is None
634         assert self.original_changes.valid_signature
635
636         cnf = Config()
637         session = self.transaction.session
638
639         group = cnf.get('Dinstall::UnprivGroup') or None
640         self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
641                                             mode=0o2750, group=group)
642         with FilesystemTransaction() as fs:
643             src = os.path.join(self.original_directory, self.original_changes.filename)
644             dst = os.path.join(self.directory, self.original_changes.filename)
645             fs.copy(src, dst, mode=0o640)
646
647             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
648
649             for f in self.changes.files.itervalues():
650                 src = os.path.join(self.original_directory, f.filename)
651                 dst = os.path.join(self.directory, f.filename)
652                 if not os.path.exists(src):
653                     continue
654                 fs.copy(src, dst, mode=0o640)
655
656             source = None
657             try:
658                 source = self.changes.source
659             except Exception:
660                 # Do not raise an exception here if the .dsc is invalid.
661                 pass
662
663             if source is not None:
664                 for f in source.files.itervalues():
665                     src = os.path.join(self.original_directory, f.filename)
666                     dst = os.path.join(self.directory, f.filename)
667                     if not os.path.exists(dst):
668                         try:
669                             db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
670                             db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
671                             fs.copy(db_archive_file.path, dst, mode=0o640)
672                         except KeyError:
673                             # Ignore if get_file could not find it. Upload will
674                             # probably be rejected later.
675                             pass
676
677     def unpacked_source(self):
678         """Path to unpacked source
679
680         Get path to the unpacked source. This method does unpack the source
681         into a temporary directory under C{self.directory} if it has not
682         been done so already.
683
684         @rtype:  str or C{None}
685         @return: string giving the path to the unpacked source directory
686                  or C{None} if no source was included in the upload.
687         """
688         assert self.directory is not None
689
690         source = self.changes.source
691         if source is None:
692             return None
693         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
694
695         sourcedir = os.path.join(self.directory, 'source')
696         if not os.path.exists(sourcedir):
697             devnull = open('/dev/null', 'w')
698             daklib.daksubprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
699         if not os.path.isdir(sourcedir):
700             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
701         return sourcedir
702
703     def _map_suite(self, suite_name):
704         for rule in Config().value_list("SuiteMappings"):
705             fields = rule.split()
706             rtype = fields[0]
707             if rtype == "map" or rtype == "silent-map":
708                 (src, dst) = fields[1:3]
709                 if src == suite_name:
710                     suite_name = dst
711                     if rtype != "silent-map":
712                         self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
713             elif rtype == "ignore":
714                 ignored = fields[1]
715                 if suite_name == ignored:
716                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
717                     suite_name = None
718             elif rtype == "reject":
719                 rejected = fields[1]
720                 if suite_name == rejected:
721                     raise checks.Reject('Uploads to {0} are not accepted.'.format(rejected))
722             ## XXX: propup-version and map-unreleased not yet implemented
723         return suite_name
724
725     def _mapped_suites(self):
726         """Get target suites after mappings
727
728         @rtype:  list of L{daklib.dbconn.Suite}
729         @return: list giving the mapped target suites of this upload
730         """
731         session = self.session
732
733         suite_names = []
734         for dist in self.changes.distributions:
735             suite_name = self._map_suite(dist)
736             if suite_name is not None:
737                 suite_names.append(suite_name)
738
739         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
740         return suites
741
742     def _check_new_binary_overrides(self, suite):
743         new = False
744
745         binaries = self.changes.binaries
746         source = self.changes.source
747         if source is not None and not source.package_list.fallback:
748             packages = source.package_list.packages_for_suite(suite)
749             binaries = [ entry for entry in packages ]
750
751         for b in binaries:
752             override = self._binary_override(suite, b)
753             if override is None:
754                 self.warnings.append('binary:{0} is NEW.'.format(b.name))
755                 new = True
756
757         return new
758
759     def _check_new(self, suite):
760         """Check if upload is NEW
761
762         An upload is NEW if it has binary or source packages that do not have
763         an override in C{suite} OR if it references files ONLY in a tainted
764         archive (eg. when it references files in NEW).
765
766         @rtype:  bool
767         @return: C{True} if the upload is NEW, C{False} otherwise
768         """
769         session = self.session
770         new = False
771
772         # Check for missing overrides
773         if self._check_new_binary_overrides(suite):
774             new = True
775         if self.changes.source is not None:
776             override = self._source_override(suite, self.changes.source)
777             if override is None:
778                 self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
779                 new = True
780
781         # Check if we reference a file only in a tainted archive
782         files = self.changes.files.values()
783         if self.changes.source is not None:
784             files.extend(self.changes.source.files.values())
785         for f in files:
786             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
787             query_untainted = query.join(Archive).filter(Archive.tainted == False)
788
789             in_archive = (query.first() is not None)
790             in_untainted_archive = (query_untainted.first() is not None)
791
792             if in_archive and not in_untainted_archive:
793                 self.warnings.append('{0} is only available in NEW.'.format(f.filename))
794                 new = True
795
796         return new
797
798     def _final_suites(self):
799         session = self.session
800
801         mapped_suites = self._mapped_suites()
802         final_suites = set()
803
804         for suite in mapped_suites:
805             overridesuite = suite
806             if suite.overridesuite is not None:
807                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
808             if self._check_new(overridesuite):
809                 self.new = True
810             final_suites.add(suite)
811
812         return final_suites
813
814     def _binary_override(self, suite, binary):
815         """Get override entry for a binary
816
817         @type  suite: L{daklib.dbconn.Suite}
818         @param suite: suite to get override for
819
820         @type  binary: L{daklib.upload.Binary} or L{daklib.packagelist.PackageListEntry}
821         @param binary: binary to get override for
822
823         @rtype:  L{daklib.dbconn.Override} or C{None}
824         @return: override for the given binary or C{None}
825         """
826         if suite.overridesuite is not None:
827             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
828
829         mapped_component = get_mapped_component(binary.component)
830         if mapped_component is None:
831             return None
832
833         query = self.session.query(Override).filter_by(suite=suite, package=binary.name) \
834                 .join(Component).filter(Component.component_name == mapped_component.component_name) \
835                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
836
837         try:
838             return query.one()
839         except NoResultFound:
840             return None
841
842     def _source_override(self, suite, source):
843         """Get override entry for a source
844
845         @type  suite: L{daklib.dbconn.Suite}
846         @param suite: suite to get override for
847
848         @type  source: L{daklib.upload.Source}
849         @param source: source to get override for
850
851         @rtype:  L{daklib.dbconn.Override} or C{None}
852         @return: override for the given source or C{None}
853         """
854         if suite.overridesuite is not None:
855             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
856
857         # XXX: component for source?
858         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
859                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
860
861         try:
862             return query.one()
863         except NoResultFound:
864             return None
865
866     def _binary_component(self, suite, binary, only_overrides=True):
867         """get component for a binary
868
869         By default this will only look at overrides to get the right component;
870         if C{only_overrides} is C{False} this method will also look at the
871         Section field.
872
873         @type  suite: L{daklib.dbconn.Suite}
874
875         @type  binary: L{daklib.upload.Binary}
876
877         @type  only_overrides: bool
878         @param only_overrides: only use overrides to get the right component
879
880         @rtype: L{daklib.dbconn.Component} or C{None}
881         """
882         override = self._binary_override(suite, binary)
883         if override is not None:
884             return override.component
885         if only_overrides:
886             return None
887         return get_mapped_component(binary.component, self.session)
888
889     def check(self, force=False):
890         """run checks against the upload
891
892         @type  force: bool
893         @param force: ignore failing forcable checks
894
895         @rtype:  bool
896         @return: C{True} if all checks passed, C{False} otherwise
897         """
898         # XXX: needs to be better structured.
899         assert self.changes.valid_signature
900
901         try:
902             # Validate signatures and hashes before we do any real work:
903             for chk in (
904                     checks.SignatureAndHashesCheck,
905                     checks.ChangesCheck,
906                     checks.ExternalHashesCheck,
907                     checks.SourceCheck,
908                     checks.BinaryCheck,
909                     checks.BinaryTimestampCheck,
910                     checks.SingleDistributionCheck,
911                     ):
912                 chk().check(self)
913
914             final_suites = self._final_suites()
915             if len(final_suites) == 0:
916                 self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
917                 return False
918
919             self.final_suites = final_suites
920
921             for chk in (
922                     checks.TransitionCheck,
923                     checks.ACLCheck,
924                     checks.NoSourceOnlyCheck,
925                     checks.LintianCheck,
926                     ):
927                 chk().check(self)
928
929             for chk in (
930                     checks.ACLCheck,
931                     checks.SourceFormatCheck,
932                     checks.SuiteArchitectureCheck,
933                     checks.VersionCheck,
934                     ):
935                 for suite in final_suites:
936                     chk().per_suite_check(self, suite)
937
938             if len(self.reject_reasons) != 0:
939                 return False
940
941             self._checked = True
942             return True
943         except checks.Reject as e:
944             self.reject_reasons.append(unicode(e))
945         except Exception as e:
946             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
947         return False
948
949     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
950         """Install upload to the given suite
951
952         @type  suite: L{daklib.dbconn.Suite}
953         @param suite: suite to install the package into. This is the real suite,
954                       ie. after any redirection to NEW or a policy queue
955
956         @param source_component_func: function to get the L{daklib.dbconn.Component}
957                                       for a L{daklib.upload.Source} object
958
959         @param binary_component_func: function to get the L{daklib.dbconn.Component}
960                                       for a L{daklib.upload.Binary} object
961
962         @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
963
964         @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
965
966         @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
967                  object for the install source or C{None} if no source was
968                  included. The second is a list of L{daklib.dbconn.DBBinary}
969                  objects for the installed binary packages.
970         """
971         # XXX: move this function to ArchiveTransaction?
972
973         control = self.changes.changes
974         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
975
976         if source_suites is None:
977             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()
978
979         source = self.changes.source
980         if source is not None:
981             component = source_component_func(source)
982             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
983         else:
984             db_source = None
985
986         db_binaries = []
987         for binary in self.changes.binaries:
988             component = binary_component_func(binary)
989             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
990             db_binaries.append(db_binary)
991
992         if suite.copychanges:
993             src = os.path.join(self.directory, self.changes.filename)
994             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
995             self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
996
997         return (db_source, db_binaries)
998
999     def _install_changes(self):
1000         assert self.changes.valid_signature
1001         control = self.changes.changes
1002         session = self.transaction.session
1003         config = Config()
1004
1005         changelog_id = None
1006         # Only add changelog for sourceful uploads and binNMUs
1007         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
1008             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
1009             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
1010             assert changelog_id is not None
1011
1012         db_changes = DBChange()
1013         db_changes.changesname = self.changes.filename
1014         db_changes.source = control['Source']
1015         db_changes.binaries = control.get('Binary', None)
1016         db_changes.architecture = control['Architecture']
1017         db_changes.version = control['Version']
1018         db_changes.distribution = control['Distribution']
1019         db_changes.urgency = control['Urgency']
1020         db_changes.maintainer = control['Maintainer']
1021         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
1022         db_changes.date = control['Date']
1023         db_changes.fingerprint = self.fingerprint.fingerprint
1024         db_changes.changelog_id = changelog_id
1025         db_changes.closes = self.changes.closed_bugs
1026
1027         try:
1028             self.transaction.session.add(db_changes)
1029             self.transaction.session.flush()
1030         except sqlalchemy.exc.IntegrityError:
1031             raise ArchiveException('{0} is already known.'.format(self.changes.filename))
1032
1033         return db_changes
1034
1035     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
1036         u = PolicyQueueUpload()
1037         u.policy_queue = policy_queue
1038         u.target_suite = target_suite
1039         u.changes = db_changes
1040         u.source = db_source
1041         u.binaries = db_binaries
1042         self.transaction.session.add(u)
1043         self.transaction.session.flush()
1044
1045         dst = os.path.join(policy_queue.path, self.changes.filename)
1046         self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)
1047
1048         return u
1049
1050     def try_autobyhand(self):
1051         """Try AUTOBYHAND
1052
1053         Try to handle byhand packages automatically.
1054
1055         @rtype:  list of L{daklib.upload.HashedFile}
1056         @return: list of remaining byhand files
1057         """
1058         assert len(self.reject_reasons) == 0
1059         assert self.changes.valid_signature
1060         assert self.final_suites is not None
1061         assert self._checked
1062
1063         byhand = self.changes.byhand_files
1064         if len(byhand) == 0:
1065             return True
1066
1067         suites = list(self.final_suites)
1068         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1069         suite = suites[0]
1070
1071         cnf = Config()
1072         control = self.changes.changes
1073         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1074
1075         remaining = []
1076         for f in byhand:
1077             if '_' in f.filename:
1078                 parts = f.filename.split('_', 2)
1079                 if len(parts) != 3:
1080                     print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1081                     remaining.append(f)
1082                     continue
1083
1084                 package, version, archext = parts
1085                 arch, ext = archext.split('.', 1)
1086             else:
1087                 parts = f.filename.split('.')
1088                 if len(parts) < 2:
1089                     print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1090                     remaining.append(f)
1091                     continue
1092
1093                 package = parts[0]
1094                 version = '0'
1095                 arch = 'all'
1096                 ext = parts[-1]
1097
1098             try:
1099                 rule = automatic_byhand_packages.subtree(package)
1100             except KeyError:
1101                 remaining.append(f)
1102                 continue
1103
1104             if rule['Source'] != self.changes.source_name \
1105                     or rule['Section'] != f.section \
1106                     or ('Extension' in rule and rule['Extension'] != ext):
1107                 remaining.append(f)
1108                 continue
1109
1110             script = rule['Script']
1111             retcode = daklib.daksubprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1112             if retcode != 0:
1113                 print "W: error processing {0}.".format(f.filename)
1114                 remaining.append(f)
1115
1116         return len(remaining) == 0
1117
1118     def _install_byhand(self, policy_queue_upload, hashed_file):
1119         """install byhand file
1120
1121         @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1122
1123         @type  hashed_file: L{daklib.upload.HashedFile}
1124         """
1125         fs = self.transaction.fs
1126         session = self.transaction.session
1127         policy_queue = policy_queue_upload.policy_queue
1128
1129         byhand_file = PolicyQueueByhandFile()
1130         byhand_file.upload = policy_queue_upload
1131         byhand_file.filename = hashed_file.filename
1132         session.add(byhand_file)
1133         session.flush()
1134
1135         src = os.path.join(self.directory, hashed_file.filename)
1136         dst = os.path.join(policy_queue.path, hashed_file.filename)
1137         fs.copy(src, dst, mode=policy_queue.change_perms)
1138
1139         return byhand_file
1140
1141     def _do_bts_versiontracking(self):
1142         cnf = Config()
1143         fs = self.transaction.fs
1144
1145         btsdir = cnf.get('Dir::BTSVersionTrack')
1146         if btsdir is None or btsdir == '':
1147             return
1148
1149         base = os.path.join(btsdir, self.changes.filename[:-8])
1150
1151         # version history
1152         sourcedir = self.unpacked_source()
1153         if sourcedir is not None:
1154             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1155             versions = fs.create("{0}.versions".format(base), mode=0o644)
1156             for line in fh.readlines():
1157                 if re_changelog_versions.match(line):
1158                     versions.write(line)
1159             fh.close()
1160             versions.close()
1161
1162         # binary -> source mapping
1163         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1164         for binary in self.changes.binaries:
1165             control = binary.control
1166             source_package, source_version = binary.source
1167             line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
1168             print >>debinfo, line
1169         debinfo.close()
1170
1171     def _policy_queue(self, suite):
1172         if suite.policy_queue is not None:
1173             return suite.policy_queue
1174         return None
1175
1176     def install(self):
1177         """install upload
1178
1179         Install upload to a suite or policy queue.  This method does B{not}
1180         handle uploads to NEW.
1181
1182         You need to have called the C{check} method before calling this method.
1183         """
1184         assert len(self.reject_reasons) == 0
1185         assert self.changes.valid_signature
1186         assert self.final_suites is not None
1187         assert self._checked
1188         assert not self.new
1189
1190         db_changes = self._install_changes()
1191
1192         for suite in self.final_suites:
1193             overridesuite = suite
1194             if suite.overridesuite is not None:
1195                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1196
1197             policy_queue = self._policy_queue(suite)
1198
1199             redirected_suite = suite
1200             if policy_queue is not None:
1201                 redirected_suite = policy_queue.suite
1202
1203             # source can be in the suite we install to or any suite we enhance
1204             source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
1205             for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
1206                     .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
1207                     .filter(VersionCheck.check == 'Enhances'):
1208                 source_suite_ids.add(enhanced_suite_id)
1209
1210             source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()
1211
1212             source_component_func = lambda source: self._source_override(overridesuite, source).component
1213             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1214
1215             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1216
1217             if policy_queue is not None:
1218                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1219
1220             # copy to build queues
1221             if policy_queue is None or policy_queue.send_to_build_queues:
1222                 for build_queue in suite.copy_queues:
1223                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1224
1225         self._do_bts_versiontracking()
1226
1227     def install_to_new(self):
1228         """install upload to NEW
1229
1230         Install upload to NEW.  This method does B{not} handle regular uploads
1231         to suites or policy queues.
1232
1233         You need to have called the C{check} method before calling this method.
1234         """
1235         # Uploads to NEW are special as we don't have overrides.
1236         assert len(self.reject_reasons) == 0
1237         assert self.changes.valid_signature
1238         assert self.final_suites is not None
1239
1240         source = self.changes.source
1241         binaries = self.changes.binaries
1242         byhand = self.changes.byhand_files
1243
1244         # we need a suite to guess components
1245         suites = list(self.final_suites)
1246         assert len(suites) == 1, "NEW uploads must be to a single suite"
1247         suite = suites[0]
1248
1249         # decide which NEW queue to use
1250         if suite.new_queue is None:
1251             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1252         else:
1253             new_queue = suite.new_queue
1254         if len(byhand) > 0:
1255             # There is only one global BYHAND queue
1256             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1257         new_suite = new_queue.suite
1258
1259
1260         def binary_component_func(binary):
1261             return self._binary_component(suite, binary, only_overrides=False)
1262
1263         # guess source component
1264         # XXX: should be moved into an extra method
1265         binary_component_names = set()
1266         for binary in binaries:
1267             component = binary_component_func(binary)
1268             binary_component_names.add(component.component_name)
1269         source_component_name = None
1270         for c in self.session.query(Component).order_by(Component.component_id):
1271             guess = c.component_name
1272             if guess in binary_component_names:
1273                 source_component_name = guess
1274                 break
1275         if source_component_name is None:
1276             source_component = self.session.query(Component).order_by(Component.component_id).first()
1277         else:
1278             source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1279         source_component_func = lambda source: source_component
1280
1281         db_changes = self._install_changes()
1282         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1283         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1284
1285         for f in byhand:
1286             self._install_byhand(policy_upload, f)
1287
1288         self._do_bts_versiontracking()
1289
1290     def commit(self):
1291         """commit changes"""
1292         self.transaction.commit()
1293
1294     def rollback(self):
1295         """rollback changes"""
1296         self.transaction.rollback()
1297
1298     def __enter__(self):
1299         self.prepare()
1300         return self
1301
1302     def __exit__(self, type, value, traceback):
1303         if self.directory is not None:
1304             shutil.rmtree(self.directory)
1305             self.directory = None
1306         self.changes = None
1307         self.transaction.rollback()
1308         return None