dak/import.py

   1 #! /usr/bin/env python
   2 #
   3 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc.,
  17 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19 import apt_pkg
  20 import os
  21 import sys
  22
  23 from daklib.dbconn import *
  24 import daklib.archive
  25 import daklib.config
  26 import daklib.daklog
  27 import daklib.upload
  28 import daklib.regexes
  29
  30 def usage():
  31     print """Usage:
  32
  33 dak import <suite> <component> <files...>
  34 dak import -D|--dump <file> <suite> <component>
  35 dak import -E|--export-dump <suite> <component>
  36
  37 WARNING: This command does no sanity checks. Only use it on trusted packages.
  38
  39 Options:
  40   -h, --help:             show this help message
  41   -a, --add-overrides:    add missing overrides automatically
  42   -c, --changed-by:       Changed-By for imported source packages
  43                           (default: maintainer)
  44   -D, --dump <file>:      Import all files listed in <file>. The format
  45                           is described below.
  46   -E, --export-dump:      Export list of files in the format required
  47                           by dak import --dump.
  48   -s, --ignore-signature: ignore signature for imported source packages
  49
  50 File format used by --dump:
  51
  52   <filename>:<md5>:<sha1>:<sha256>:[<fingerprint>]:[<changed-by>]
  53 """
  54
  55 def import_source(log, transaction, suite, component, directory, hashed_file,
  56                   fingerprint=None, changed_by=None,
  57                   keyrings=None, require_signature=True, add_overrides=False):
  58     if keyrings is None:
  59         keyrings = []
  60     filename = hashed_file.filename
  61     session = transaction.session
  62
  63     source = daklib.upload.Source(directory, [hashed_file], keyrings, require_signature)
  64     if source.valid_signature:
  65         fingerprint = session.query(Fingerprint).filter_by(fingerprint=source.primary_fingerprint).first()
  66     if changed_by is None:
  67         changed_by = source.dsc['Maintainer']
  68     db_changed_by = get_or_set_maintainer(changed_by, session)
  69
  70     transaction.install_source(directory, source, suite, component, db_changed_by, fingerprint=fingerprint)
  71     log.log(['import-source', suite.suite_name, component.component_name, filename])
  72
  73     if add_overrides and not session.query(Override).filter_by(suite=suite.get_overridesuite(), component=component, package=source.dsc['Source']).join(OverrideType).filter(OverrideType.overridetype == 'dsc').first():
  74         overridetype = session.query(OverrideType).filter_by(overridetype='dsc').one()
  75         overridesuite = suite.get_overridesuite()
  76         section_name = 'misc'
  77         if component.component_name != 'main':
  78             section_name = "{0}/{1}".format(component.component_name, section_name)
  79         section = session.query(Section).filter_by(section=section_name).one()
  80         priority = session.query(Priority).filter_by(priority='extra').one()
  81
  82         override = Override(package=source.dsc['Source'], suite=overridesuite, component=component,
  83                             section=section, priority=priority, overridetype=overridetype)
  84         session.add(override)
  85         log.log(['add-source-override', suite.suite_name, component.component_name, source.dsc['Source'], section.section, priority.priority])
  86
  87 def import_binary(log, transaction, suite, component, directory, hashed_file, fingerprint=None, add_overrides=False):
  88     filename = hashed_file.filename
  89     session = transaction.session
  90
  91     binary = daklib.upload.Binary(directory, hashed_file)
  92     transaction.install_binary(directory, binary, suite, component, fingerprint=fingerprint)
  93     log.log(['import-binary', suite.suite_name, component.component_name, filename])
  94
  95     if add_overrides and not session.query(Override).filter_by(suite=suite.get_overridesuite(), component=component, package=binary.control['Package']).join(OverrideType).filter(OverrideType.overridetype == binary.type).first():
  96         overridetype = session.query(OverrideType).filter_by(overridetype=binary.type).one()
  97         overridesuite = suite.get_overridesuite()
  98         section = session.query(Section).filter_by(section=binary.control['Section']).one()
  99         priority = session.query(Priority).filter_by(priority=binary.control['Priority']).one()
 100
 101         override = Override(package=binary.control['Package'], suite=overridesuite, component=component,
 102                             section=section, priority=priority, overridetype=overridetype)
 103         session.add(override)
 104         log.log(['add-binary-override', suite.suite_name, component.component_name, binary.control['Package'], section.section, priority.priority])
 105
 106 def import_file(log, transaction, suite, component, directory, hashed_file,
 107                 fingerprint=None, changed_by=None, keyrings=None, require_signature=True,
 108                 add_overrides = False):
 109     filename = hashed_file.filename
 110     if daklib.regexes.re_file_binary.match(filename):
 111         import_binary(log, transaction, suite, component, directory, hashed_file,
 112                       fingerprint=fingerprint, add_overrides=add_overrides)
 113     elif daklib.regexes.re_file_dsc.match(filename):
 114         import_source(log, transaction, suite, component, directory, hashed_file,
 115                       fingerprint=fingerprint, changed_by=changed_by, keyrings=keyrings,
 116                       require_signature=require_signature, add_overrides=add_overrides)
 117     else:
 118         raise Exception('File is neither source nor binary package: {0}'.format(filename))
 119
 120 def import_dump(log, transaction, suite, component, fh,
 121                 keyrings=None, require_signature=True, add_overrides=False):
 122     session = transaction.session
 123     for line in fh:
 124         path, size, md5, sha1, sha256, fpr, changed_by = line.strip().split(':', 6)
 125
 126         if not changed_by:
 127             changed_by = None
 128         fingerprint = None
 129         if fpr:
 130             fingerprint = session.query(Fingerprint).filter_by(fingerprint=fpr).first()
 131             if fingerprint is None:
 132                 print 'W: {0}: unknown fingerprint {1}'.format(filename, fpr)
 133
 134         directory, filename = os.path.split(os.path.abspath(path))
 135         hashed_file = daklib.upload.HashedFile(filename, long(size), md5, sha1, sha256)
 136         hashed_file.check(directory)
 137
 138         import_file(log, transaction, suite, component, directory, hashed_file,
 139                     fingerprint=fingerprint, changed_by=changed_by,
 140                     keyrings=keyrings, require_signature=require_signature, add_overrides=add_overrides)
 141
 142         transaction.commit()
 143
 144 _export_query = r"""
 145 WITH
 146 tmp AS
 147   (SELECT 1 AS order, s.file AS file_id, s.sig_fpr AS fingerprint_id, s.changedby AS changed_by, sa.suite AS suite_id
 148      FROM source s
 149      JOIN src_associations sa ON sa.source = s.id
 150    UNION
 151    SELECT 2 AS order, b.file AS file_id, b.sig_fpr AS fingerprint_id, NULL, ba.suite AS suite_id
 152      FROM binaries b
 153      JOIN bin_associations ba ON ba.bin = b.id
 154   )
 155
 156 SELECT
 157   f.filename, f.size::TEXT, f.md5sum, f.sha1sum, f.sha256sum, COALESCE(fpr.fingerprint, ''), COALESCE(m.name, '')
 158 FROM files f
 159 JOIN tmp ON f.id = tmp.file_id
 160 JOIN suite ON suite.id = tmp.suite_id
 161 JOIN files_archive_map fam ON fam.file_id = f.id AND fam.archive_id = suite.archive_id
 162 LEFT JOIN fingerprint fpr ON fpr.id = tmp.fingerprint_id
 163 LEFT JOIN maintainer m ON m.id = tmp.changed_by
 164
 165 WHERE
 166   suite.id = :suite_id
 167   AND fam.component_id = :component_id
 168
 169 ORDER BY tmp.order, f.filename;
 170 """
 171
 172 def export_dump(transaction, suite, component):
 173     session = transaction.session
 174     query = session.execute(_export_query,
 175                             {'suite_id': suite.suite_id,
 176                              'component_id': component.component_id})
 177     for row in query:
 178         print ":".join(row)
 179
 180 def main(argv=None):
 181     if argv is None:
 182         argv = sys.argv
 183
 184     arguments = [
 185         ('h', 'help', 'Import::Options::Help'),
 186         ('a', 'add-overrides', 'Import::Options::AddOverrides'),
 187         ('c', 'changed-by', 'Import::Options::ChangedBy', 'HasArg'),
 188         ('D', 'dump', 'Import::Options::Dump', 'HasArg'),
 189         ('E', 'export-dump', 'Import::Options::Export'),
 190         ('s', 'ignore-signature', 'Import::Options::IgnoreSignature'),
 191         ]
 192
 193     cnf = daklib.config.Config()
 194     cnf['Import::Options::Dummy'] = ''
 195     argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
 196     options = cnf.subtree('Import::Options')
 197
 198     if 'Help' in options or len(argv) < 2:
 199         usage()
 200         sys.exit(0)
 201
 202     suite_name = argv[0]
 203     component_name = argv[1]
 204
 205     add_overrides = options.find_b('AddOverrides')
 206     require_signature = not options.find_b('IgnoreSignature')
 207     changed_by = options.find('ChangedBy') or None
 208
 209     log = daklib.daklog.Logger('import')
 210
 211     with daklib.archive.ArchiveTransaction() as transaction:
 212         session = transaction.session
 213         suite = session.query(Suite).filter_by(suite_name=suite_name).one()
 214         component = session.query(Component).filter_by(component_name=component_name).one()
 215         keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
 216         keyring_files = [ k.keyring_name for k in keyrings ]
 217
 218         dump = options.find('Dump') or None
 219         if options.find_b('Export'):
 220             export_dump(transaction, suite, component)
 221             transaction.rollback()
 222         elif dump is not None:
 223             with open(dump, 'r') as fh:
 224                 import_dump(log, transaction, suite, component, fh, keyring_files,
 225                             require_signature=require_signature, add_overrides=add_overrides)
 226             transaction.commit()
 227         else:
 228             files = argv[2:]
 229             for f in files:
 230                 directory, filename = os.path.split(os.path.abspath(f))
 231                 hashed_file = daklib.upload.HashedFile.from_file(directory, filename)
 232                 import_file(log, transaction, suite, component, directory, hashed_file,
 233                             changed_by=changed_by,
 234                             keyrings=keyring_files, require_signature=require_signature,
 235                             add_overrides=add_overrides)
 236             transaction.commit()
 237
 238     log.close()
 239
 240 if __name__ == '__main__':
 241     main()