From: Ansgar Burchardt Date: Tue, 18 Sep 2012 08:58:18 +0000 (+0200) Subject: Add process-commands program and activate new DM implementation. X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=dbd3ceac44904f678181a189f34dc75c56178953;p=dak.git Add process-commands program and activate new DM implementation. For now packages that use DMUA in the source package are still accepted. --- diff --git a/dak/dak.py b/dak/dak.py index 306137a1..c2a502cd 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -71,6 +71,8 @@ def init(): "Process NEW and BYHAND packages"), ("process-upload", "Process packages in queue/unchecked"), + ("process-commands", + "Process command files (*.dak-commands)"), ("process-policy", "Process packages in policy queues from COMMENTS files"), diff --git a/dak/dakdb/update89.py b/dak/dakdb/update89.py new file mode 100644 index 00000000..2ecccf52 --- /dev/null +++ b/dak/dakdb/update89.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +add table to keep track of seen signatures + +@contact: Debian FTP Master +@copyright: 2012 Ansgar Burchardt +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError +from daklib.config import Config + +################################################################################ +def do_update(self): + print __doc__ + try: + cnf = Config() + + c = self.db.cursor() + + c.execute("""CREATE TABLE signature_history ( + fingerprint TEXT NOT NULL, + signature_timestamp TIMESTAMP NOT NULL, + contents_sha1 TEXT NOT NULL, + seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (signature_timestamp, fingerprint, contents_sha1) + )""") + + c.execute("UPDATE config SET value = '89' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError as msg: + self.db.rollback() + raise DBUpdateError('Unable to apply sick update 89, rollback issued. Error message: {0}'.format(msg)) diff --git a/dak/process-commands.py b/dak/process-commands.py new file mode 100644 index 00000000..9bc2d3f1 --- /dev/null +++ b/dak/process-commands.py @@ -0,0 +1,82 @@ +#! /usr/bin/env python +# +# Copyright (C) 2012, Ansgar Burchardt +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import apt_pkg +import datetime +import os +import sys + +from daklib.config import Config +from daklib.command import CommandError, CommandFile +from daklib.daklog import Logger +from daklib.fstransactions import FilesystemTransaction +from daklib.utils import find_next_free + +def usage(): + print """Usage: dak command ... + +process command files +""" + +def main(argv=None): + if argv is None: + argv = sys.argv + + arguments = [('h', 'help', 'Process-Commands::Options::Help'), + ('d', 'directory', 'Process-Commands::Options::Directory', 'HasArg')] + + cnf = Config() + cnf['Command::Options::Dummy'] = '' + filenames = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) + options = cnf.subtree('Command::Options') + + if 'Help' in options or (len(filenames) == 0 and 'Directory' not in options): + usage() + sys.exit(0) + + log = Logger('command') + + now = datetime.datetime.now() + donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) + rejectdir = cnf['Dir::Reject'] + + if len(filenames) == 0: + filenames = [ fn for fn in os.listdir(options['Directory']) if fn.endswith('.dak-commands') ] + + for fn in filenames: + basename = os.path.basename(fn) + if not fn.endswith('.dak-commands'): + log.log(['unexpected filename', basename]) + continue + + command = CommandFile(fn, log) + if command.evaluate(): + log.log(['moving to done', basename]) + dst = find_next_free(os.path.join(donedir, basename)) + else: + log.log(['moving to reject', basename]) + dst = find_next_free(os.path.join(rejectdir, basename)) + + with FilesystemTransaction() as fs: + fs.move(fn, dst, mode=0o644) + fs.commit() + + log.close() + +if __name__ == '__main__': + main() diff --git a/dak/update_db.py b/dak/update_db.py index a27c32e7..97efd3fb 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -46,7 +46,7 @@ from daklib.daklog import Logger ################################################################################ Cnf = None -required_database_schema = 88 +required_database_schema = 89 ################################################################################ diff --git a/daklib/checks.py b/daklib/checks.py index 63ea10f2..cae801e4 100644 --- a/daklib/checks.py +++ b/daklib/checks.py @@ -464,8 +464,11 @@ class ACLCheck(Check): # XXX: Drop DMUA part here and switch to new implementation. # XXX: Send warning mail once users can set the new DMUA flag dmua_status, dmua_reason = self._check_dmua(upload) - if not dmua_status: - return False, dmua_reason + if acl_per_source is None: + if not dmua_status: + return False, dmua_reason + else: + upload.warn('DM flag not set, but accepted as DMUA was set.') #if acl_per_source is None: # return False, "not allowed to upload source package '{0}'".format(source_name) if acl.deny_per_source and acl_per_source is not None: diff --git a/daklib/command.py b/daklib/command.py new file mode 100644 index 00000000..31551d75 --- /dev/null +++ b/daklib/command.py @@ -0,0 +1,222 @@ +"""module to handle command files + +@contact: Debian FTP Master +@copyright: 2012, Ansgar Burchardt +@license: GPL-2+ +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import apt_pkg +import os +import re +import tempfile + +from daklib.config import Config +from daklib.dbconn import * +from daklib.gpg import SignedFile +from daklib.regexes import re_field_package +from daklib.textutils import fix_maintainer +from daklib.utils import gpg_get_key_addresses, send_mail, TemplateSubst + +class CommandError(Exception): + pass + +class CommandFile(object): + def __init__(self, path, log=None): + if log is None: + from daklib.daklog import Logger + log = Logger() + self.cc = [] + self.result = [] + self.log = log + self.path = path + self.filename = os.path.basename(path) + + def _check_replay(self, signed_file, session): + """check for replays + + @note: Will commit changes to the database. + + @type signed_file: L{daklib.gpg.SignedFile} + + @param session: database session + """ + # Mark commands file as seen to prevent replays. + signature_history = SignatureHistory.from_signed_file(signed_file) + session.add(signature_history) + session.commit() + + def _evaluate_sections(self, sections, session): + session.rollback() + try: + sections.next() + section = sections.section + + action = section.get('Action', None) + if action is None: + raise CommandError('Encountered section without Action field') + self.result.append('Action: {0}'.format(action)) + + if action == 'dm': + self.action_dm(self.fingerprint, section, session) + else: + raise CommandError('Unknown action: {0}'.format(action)) + except StopIteration: + pass + finally: + session.rollback() + + def _notify_uploader(self): + cnf = Config() + + bcc = 'X-DAK: dak process-command' + if 'Dinstall::Bcc' in cnf: + bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc']) + + cc = set(fix_maintainer(address)[1] for address in self.cc) + + subst = { + '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'], + '__MAINTAINER_TO__': fix_maintainer(self.uploader)[1], + '__CC__': ", ".join(cc), + '__BCC__': bcc, + '__RESULTS__': "\n".join(self.result), + '__FILENAME__': self.filename, + } + + message = TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-command.processed')) + + send_mail(message) + + def evaluate(self): + """evaluate commands file + + @rtype: bool + @returns: C{True} if the file was processed sucessfully, + C{False} otherwise + """ + result = True + + session = DBConn().session() + + keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority) + keyring_files = [ k.keyring_name for k in keyrings ] + + raw_contents = open(self.path, 'r').read() + signed_file = SignedFile(raw_contents, keyring_files) + if not signed_file.valid: + self.log.log(['invalid signature', self.filename]) + return False + + self.fingerprint = session.query(Fingerprint).filter_by(fingerprint=signed_file.primary_fingerprint).one() + if self.fingerprint.keyring is None: + self.log.log(['singed by key in unknown keyring', self.filename]) + return False + assert self.fingerprint.keyring.active + + self.log.log(['processing', self.filename, 'signed-by={0}'.format(self.fingerprint.fingerprint)]) + + with tempfile.TemporaryFile() as fh: + fh.write(signed_file.contents) + fh.seek(0) + sections = apt_pkg.TagFile(fh) + + self.uploader = None + addresses = gpg_get_key_addresses(self.fingerprint.fingerprint) + if len(addresses) > 0: + self.uploader = addresses[0] + + try: + sections.next() + section = sections.section + if 'Uploader' in section: + self.uploader = section['Uploader'] + # TODO: Verify first section has valid Archive field + if 'Archive' not in section: + raise CommandError('No Archive field in first section.') + + # TODO: send mail when we detected a replay. + self._check_replay(signed_file, session) + + self._evaluate_sections(sections, session) + self.result.append('') + except Exception as e: + self.log.log(['ERROR', e]) + self.result.append("There was an error processing this section:\n{0}".format(e)) + result = False + + self._notify_uploader() + + session.close() + self.log.log(['done', self.filename]) + + return result + + def _split_packages(self, value): + names = value.split() + for name in names: + if not re_field_package.match(name): + raise CommandError('Invalid package name "{0}"'.format(name)) + return names + + def action_dm(self, fingerprint, section, session): + cnf = Config() + + if 'Command::DM::AdminKeyrings' not in cnf \ + or 'Command::DM::ACL' not in cnf \ + or 'Command::DM::Keyrings' not in cnf: + raise CommandError('DM command is not configured for this archive.') + + allowed_keyrings = cnf.value_list('Command::DM::AdminKeyrings') + if fingerprint.keyring.keyring_name not in allowed_keyrings: + raise CommandError('Key {0} is not allowed to set DM'.format(fingerprint.fingerprint)) + + acl_name = cnf.get('Command::DM::ACL', 'dm') + acl = session.query(ACL).filter_by(name=acl_name).one() + + fpr = session.query(Fingerprint).filter_by(fingerprint=section['Fingerprint']).one() + if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list('Command::DM::Keyrings'): + raise CommandError('Key {0} is not in DM keyring.'.format(fpr.fingerprint)) + addresses = gpg_get_key_addresses(fpr.fingerprint) + if len(addresses) > 0: + self.cc.append(addresses[0]) + + self.log.log(['dm', 'fingerprint', fpr.fingerprint]) + self.result.append('Fingerprint: {0}'.format(fpr.fingerprint)) + if len(addresses) > 0: + self.log.log(['dm', 'uid', addresses[0]]) + self.result.append('Uid: {0}'.format(addresses[0])) + + for source in self._split_packages(section.get('Allow', '')): + if session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).first() is None: + aps = ACLPerSource() + aps.acl = acl + aps.fingerprint = fpr + aps.source = source + session.add(aps) + self.log.log(['dm', 'allow', fpr.fingerprint, source]) + self.result.append('Allowed: {0}'.format(source)) + else: + self.result.append('Already-Allowed: {0}'.format(source)) + + session.flush() + + for source in self._split_packages(section.get('Deny', '')): + session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).delete() + self.log.log(['dm', 'deny', fpr.fingerprint, source]) + self.result.append('Denied: {0}'.format(source)) + + session.commit() diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 41121fff..5217462a 100644 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -1850,6 +1850,26 @@ __all__.append('get_sections') ################################################################################ +class SignatureHistory(ORMObject): + @classmethod + def from_signed_file(cls, signed_file): + """signature history entry from signed file + + @type signed_file: L{daklib.gpg.SignedFile} + @param signed_file: signed file + + @rtype: L{SignatureHistory} + """ + self = cls() + self.fingerprint = signed_file.primary_fingerprint + self.signature_timestamp = signed_file.signature_timestamp + self.contents_sha1 = signed_file.contents_sha1() + return self + +__all__.append('SignatureHistory') + +################################################################################ + class SrcContents(ORMObject): def __init__(self, file = None, source = None): self.file = file @@ -2544,6 +2564,7 @@ class DBConn(object): 'policy_queue_byhand_file', 'priority', 'section', + 'signature_history', 'source', 'source_metadata', 'src_associations', @@ -2762,6 +2783,8 @@ class DBConn(object): properties = dict(section_id = self.tbl_section.c.id, section=self.tbl_section.c.section)) + mapper(SignatureHistory, self.tbl_signature_history) + mapper(DBSource, self.tbl_source, properties = dict(source_id = self.tbl_source.c.id, version = self.tbl_source.c.version, diff --git a/daklib/gpg.py b/daklib/gpg.py index 865e9bd6..828bf649 100644 --- a/daklib/gpg.py +++ b/daklib/gpg.py @@ -19,6 +19,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import apt_pkg +import datetime import errno import fcntl import os @@ -141,6 +143,17 @@ class SignedFile(object): return dict( (fd, "".join(read_lines[fd])) for fd in read_lines.keys() ) + def _parse_date(self, value): + """parse date string in YYYY-MM-DD format + + @rtype: L{datetime.datetime} + @returns: datetime objects for 0:00 on the given day + """ + year, month, day = value.split('-') + date = datetime.date(int(year), int(month), int(day)) + time = datetime.time(0, 0) + return datetime.datetime.combine(date, time) + def _parse_status(self, line): fields = line.split() if fields[0] != "[GNUPG:]": @@ -153,6 +166,7 @@ class SignedFile(object): self.valid = True self.fingerprint = fields[2] self.primary_fingerprint = fields[11] + self.signature_timestamp = self._parse_date(fields[3]) if fields[1] == "BADARMOR": raise GpgException("Bad armor.") @@ -190,4 +204,7 @@ class SignedFile(object): finally: os._exit(1) + def contents_sha1(self): + return apt_pkg.sha1sum(self.contents) + # vim: set sw=4 et: diff --git a/templates/process-command.processed b/templates/process-command.processed new file mode 100644 index 00000000..979032f2 --- /dev/null +++ b/templates/process-command.processed @@ -0,0 +1,12 @@ +From: __DAK_ADDRESS__ +To: __MAINTAINER_TO__ +Cc: __CC__ +__BCC__ +X-Debian: DAK +Precedence: bulk +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 8bit +Subject: Results of processing __FILENAME__ + +__RESULTS__