]> git.decadent.org.uk Git - dak.git/commitdiff
Add process-commands program and activate new DM implementation.
authorAnsgar Burchardt <ansgar@debian.org>
Tue, 18 Sep 2012 08:58:18 +0000 (10:58 +0200)
committerAnsgar Burchardt <ansgar@debian.org>
Wed, 19 Sep 2012 10:24:54 +0000 (12:24 +0200)
For now packages that use DMUA in the source package are still accepted.

dak/dak.py
dak/dakdb/update89.py [new file with mode: 0644]
dak/process-commands.py [new file with mode: 0644]
dak/update_db.py
daklib/checks.py
daklib/command.py [new file with mode: 0644]
daklib/dbconn.py
daklib/gpg.py
templates/process-command.processed [new file with mode: 0644]

index 306137a183fa14e1106fd75a00b64f0c29cf8722..c2a502cdcd16dcbe4d806fe46472d73431fae4ef 100755 (executable)
@@ -71,6 +71,8 @@ def init():
          "Process NEW and BYHAND packages"),
         ("process-upload",
          "Process packages in queue/unchecked"),
+        ("process-commands",
+         "Process command files (*.dak-commands)"),
         ("process-policy",
          "Process packages in policy queues from COMMENTS files"),
 
diff --git a/dak/dakdb/update89.py b/dak/dakdb/update89.py
new file mode 100644 (file)
index 0000000..2ecccf5
--- /dev/null
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+add table to keep track of seen signatures
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2012 Ansgar Burchardt <ansgar@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+################################################################################
+
+import psycopg2
+from daklib.dak_exceptions import DBUpdateError
+from daklib.config import Config
+
+################################################################################
+def do_update(self):
+    print __doc__
+    try:
+        cnf = Config()
+
+        c = self.db.cursor()
+
+        c.execute("""CREATE TABLE signature_history (
+          fingerprint TEXT NOT NULL,
+          signature_timestamp TIMESTAMP NOT NULL,
+          contents_sha1 TEXT NOT NULL,
+          seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+          PRIMARY KEY (signature_timestamp, fingerprint, contents_sha1)
+        )""")
+
+        c.execute("UPDATE config SET value = '89' WHERE name = 'db_revision'")
+        self.db.commit()
+
+    except psycopg2.ProgrammingError as msg:
+        self.db.rollback()
+        raise DBUpdateError('Unable to apply sick update 89, rollback issued. Error message: {0}'.format(msg))
diff --git a/dak/process-commands.py b/dak/process-commands.py
new file mode 100644 (file)
index 0000000..9bc2d3f
--- /dev/null
@@ -0,0 +1,82 @@
+#! /usr/bin/env python
+#
+# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import apt_pkg
+import datetime
+import os
+import sys
+
+from daklib.config import Config
+from daklib.command import CommandError, CommandFile
+from daklib.daklog import Logger
+from daklib.fstransactions import FilesystemTransaction
+from daklib.utils import find_next_free
+
+def usage():
+    print """Usage: dak command <command-file>...
+
+process command files
+"""
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+
+    arguments = [('h', 'help', 'Process-Commands::Options::Help'),
+                 ('d', 'directory', 'Process-Commands::Options::Directory', 'HasArg')]
+
+    cnf = Config()
+    cnf['Command::Options::Dummy'] = ''
+    filenames = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
+    options = cnf.subtree('Command::Options')
+
+    if 'Help' in options or (len(filenames) == 0 and 'Directory' not in options):
+        usage()
+        sys.exit(0)
+
+    log = Logger('command')
+
+    now = datetime.datetime.now()
+    donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
+    rejectdir = cnf['Dir::Reject']
+
+    if len(filenames) == 0:
+        filenames = [ fn for fn in os.listdir(options['Directory']) if fn.endswith('.dak-commands') ]
+
+    for fn in filenames:
+        basename = os.path.basename(fn)
+        if not fn.endswith('.dak-commands'):
+            log.log(['unexpected filename', basename])
+            continue
+
+        command = CommandFile(fn, log)
+        if command.evaluate():
+            log.log(['moving to done', basename])
+            dst = find_next_free(os.path.join(donedir, basename))
+        else:
+            log.log(['moving to reject', basename])
+            dst = find_next_free(os.path.join(rejectdir, basename))
+
+        with FilesystemTransaction() as fs:
+            fs.move(fn, dst, mode=0o644)
+            fs.commit()
+
+    log.close()
+
+if __name__ == '__main__':
+    main()
index a27c32e751f65c5b0b6f2c171c79592fa28242ee..97efd3fb68ecc8bbd786f177d150c8d069d743c3 100755 (executable)
@@ -46,7 +46,7 @@ from daklib.daklog import Logger
 ################################################################################
 
 Cnf = None
-required_database_schema = 88
+required_database_schema = 89
 
 ################################################################################
 
index 63ea10f2a8ee01d2aa8bf5c3646741bebf6ebf79..cae801e489f341c370d22ad982313ff0553fe4b6 100644 (file)
@@ -464,8 +464,11 @@ class ACLCheck(Check):
             # XXX: Drop DMUA part here and switch to new implementation.
             # XXX: Send warning mail once users can set the new DMUA flag
             dmua_status, dmua_reason = self._check_dmua(upload)
-            if not dmua_status:
-                return False, dmua_reason
+            if acl_per_source is None:
+                if not dmua_status:
+                    return False, dmua_reason
+                else:
+                    upload.warn('DM flag not set, but accepted as DMUA was set.')
             #if acl_per_source is None:
             #    return False, "not allowed to upload source package '{0}'".format(source_name)
         if acl.deny_per_source and acl_per_source is not None:
diff --git a/daklib/command.py b/daklib/command.py
new file mode 100644 (file)
index 0000000..31551d7
--- /dev/null
@@ -0,0 +1,222 @@
+"""module to handle command files
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2012, Ansgar Burchardt <ansgar@debian.org>
+@license: GPL-2+
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import apt_pkg
+import os
+import re
+import tempfile
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib.gpg import SignedFile
+from daklib.regexes import re_field_package
+from daklib.textutils import fix_maintainer
+from daklib.utils import gpg_get_key_addresses, send_mail, TemplateSubst
+
+class CommandError(Exception):
+    pass
+
+class CommandFile(object):
+    def __init__(self, path, log=None):
+        if log is None:
+            from daklib.daklog import Logger
+            log = Logger()
+        self.cc = []
+        self.result = []
+        self.log = log
+        self.path = path
+        self.filename = os.path.basename(path)
+
+    def _check_replay(self, signed_file, session):
+        """check for replays
+
+        @note: Will commit changes to the database.
+
+        @type signed_file: L{daklib.gpg.SignedFile}
+
+        @param session: database session
+        """
+        # Mark commands file as seen to prevent replays.
+        signature_history = SignatureHistory.from_signed_file(signed_file)
+        session.add(signature_history)
+        session.commit()
+
+    def _evaluate_sections(self, sections, session):
+        session.rollback()
+        try:
+            sections.next()
+            section = sections.section
+
+            action = section.get('Action', None)
+            if action is None:
+                raise CommandError('Encountered section without Action field')
+            self.result.append('Action: {0}'.format(action))
+
+            if action == 'dm':
+                self.action_dm(self.fingerprint, section, session)
+            else:
+                raise CommandError('Unknown action: {0}'.format(action))
+        except StopIteration:
+            pass
+        finally:
+            session.rollback()
+
+    def _notify_uploader(self):
+        cnf = Config()
+
+        bcc = 'X-DAK: dak process-command'
+        if 'Dinstall::Bcc' in cnf:
+            bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
+
+        cc = set(fix_maintainer(address)[1] for address in self.cc)
+
+        subst = {
+            '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
+            '__MAINTAINER_TO__': fix_maintainer(self.uploader)[1],
+            '__CC__': ", ".join(cc),
+            '__BCC__': bcc,
+            '__RESULTS__': "\n".join(self.result),
+            '__FILENAME__': self.filename,
+            }
+
+        message = TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-command.processed'))
+
+        send_mail(message)
+
+    def evaluate(self):
+        """evaluate commands file
+
+        @rtype:   bool
+        @returns: C{True} if the file was processed sucessfully,
+                  C{False} otherwise
+        """
+        result = True
+
+        session = DBConn().session()
+
+        keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
+        keyring_files = [ k.keyring_name for k in keyrings ]
+
+        raw_contents = open(self.path, 'r').read()
+        signed_file = SignedFile(raw_contents, keyring_files)
+        if not signed_file.valid:
+            self.log.log(['invalid signature', self.filename])
+            return False
+
+        self.fingerprint = session.query(Fingerprint).filter_by(fingerprint=signed_file.primary_fingerprint).one()
+        if self.fingerprint.keyring is None:
+            self.log.log(['singed by key in unknown keyring', self.filename])
+            return False
+        assert self.fingerprint.keyring.active
+
+        self.log.log(['processing', self.filename, 'signed-by={0}'.format(self.fingerprint.fingerprint)])
+
+        with tempfile.TemporaryFile() as fh:
+            fh.write(signed_file.contents)
+            fh.seek(0)
+            sections = apt_pkg.TagFile(fh)
+
+        self.uploader = None
+        addresses = gpg_get_key_addresses(self.fingerprint.fingerprint)
+        if len(addresses) > 0:
+            self.uploader = addresses[0]
+
+        try:
+            sections.next()
+            section = sections.section
+            if 'Uploader' in section:
+                self.uploader = section['Uploader']
+            # TODO: Verify first section has valid Archive field
+            if 'Archive' not in section:
+                raise CommandError('No Archive field in first section.')
+
+            # TODO: send mail when we detected a replay.
+            self._check_replay(signed_file, session)
+
+            self._evaluate_sections(sections, session)
+            self.result.append('')
+        except Exception as e:
+            self.log.log(['ERROR', e])
+            self.result.append("There was an error processing this section:\n{0}".format(e))
+            result = False
+
+        self._notify_uploader()
+
+        session.close()
+        self.log.log(['done', self.filename])
+
+        return result
+
+    def _split_packages(self, value):
+        names = value.split()
+        for name in names:
+            if not re_field_package.match(name):
+                raise CommandError('Invalid package name "{0}"'.format(name))
+        return names
+
+    def action_dm(self, fingerprint, section, session):
+        cnf = Config()
+
+        if 'Command::DM::AdminKeyrings' not in cnf \
+                or 'Command::DM::ACL' not in cnf \
+                or 'Command::DM::Keyrings' not in cnf:
+            raise CommandError('DM command is not configured for this archive.')
+
+        allowed_keyrings = cnf.value_list('Command::DM::AdminKeyrings')
+        if fingerprint.keyring.keyring_name not in allowed_keyrings:
+            raise CommandError('Key {0} is not allowed to set DM'.format(fingerprint.fingerprint))
+
+        acl_name = cnf.get('Command::DM::ACL', 'dm')
+        acl = session.query(ACL).filter_by(name=acl_name).one()
+
+        fpr = session.query(Fingerprint).filter_by(fingerprint=section['Fingerprint']).one()
+        if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list('Command::DM::Keyrings'):
+            raise CommandError('Key {0} is not in DM keyring.'.format(fpr.fingerprint))
+        addresses = gpg_get_key_addresses(fpr.fingerprint)
+        if len(addresses) > 0:
+            self.cc.append(addresses[0])
+
+        self.log.log(['dm', 'fingerprint', fpr.fingerprint])
+        self.result.append('Fingerprint: {0}'.format(fpr.fingerprint))
+        if len(addresses) > 0:
+            self.log.log(['dm', 'uid', addresses[0]])
+            self.result.append('Uid: {0}'.format(addresses[0]))
+
+        for source in self._split_packages(section.get('Allow', '')):
+            if session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).first() is None:
+                aps = ACLPerSource()
+                aps.acl = acl
+                aps.fingerprint = fpr
+                aps.source = source
+                session.add(aps)
+                self.log.log(['dm', 'allow', fpr.fingerprint, source])
+                self.result.append('Allowed: {0}'.format(source))
+            else:
+                self.result.append('Already-Allowed: {0}'.format(source))
+
+        session.flush()
+
+        for source in self._split_packages(section.get('Deny', '')):
+            session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).delete()
+            self.log.log(['dm', 'deny', fpr.fingerprint, source])
+            self.result.append('Denied: {0}'.format(source))
+
+        session.commit()
index 41121fff1699014a6db5e3ffd1ba43bc01a83f35..5217462af4688754d960126e887b4203084e2766 100644 (file)
@@ -1850,6 +1850,26 @@ __all__.append('get_sections')
 
 ################################################################################
 
+class SignatureHistory(ORMObject):
+    @classmethod
+    def from_signed_file(cls, signed_file):
+        """signature history entry from signed file
+
+        @type  signed_file: L{daklib.gpg.SignedFile}
+        @param signed_file: signed file
+
+        @rtype: L{SignatureHistory}
+        """
+        self = cls()
+        self.fingerprint = signed_file.primary_fingerprint
+        self.signature_timestamp = signed_file.signature_timestamp
+        self.contents_sha1 = signed_file.contents_sha1()
+        return self
+
+__all__.append('SignatureHistory')
+
+################################################################################
+
 class SrcContents(ORMObject):
     def __init__(self, file = None, source = None):
         self.file = file
@@ -2544,6 +2564,7 @@ class DBConn(object):
             'policy_queue_byhand_file',
             'priority',
             'section',
+            'signature_history',
             'source',
             'source_metadata',
             'src_associations',
@@ -2762,6 +2783,8 @@ class DBConn(object):
                properties = dict(section_id = self.tbl_section.c.id,
                                  section=self.tbl_section.c.section))
 
+        mapper(SignatureHistory, self.tbl_signature_history)
+
         mapper(DBSource, self.tbl_source,
                properties = dict(source_id = self.tbl_source.c.id,
                                  version = self.tbl_source.c.version,
index 865e9bd6dccc13b9b47354c510fa9c5503f2ab8c..828bf64906278f86db0aa13547b91356c0ba6568 100644 (file)
@@ -19,6 +19,8 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+import apt_pkg
+import datetime
 import errno
 import fcntl
 import os
@@ -141,6 +143,17 @@ class SignedFile(object):
 
         return dict( (fd, "".join(read_lines[fd])) for fd in read_lines.keys() )
 
+    def _parse_date(self, value):
+        """parse date string in YYYY-MM-DD format
+
+        @rtype:   L{datetime.datetime}
+        @returns: datetime objects for 0:00 on the given day
+        """
+        year, month, day = value.split('-')
+        date = datetime.date(int(year), int(month), int(day))
+        time = datetime.time(0, 0)
+        return datetime.datetime.combine(date, time)
+
     def _parse_status(self, line):
         fields = line.split()
         if fields[0] != "[GNUPG:]":
@@ -153,6 +166,7 @@ class SignedFile(object):
             self.valid = True
             self.fingerprint = fields[2]
             self.primary_fingerprint = fields[11]
+            self.signature_timestamp = self._parse_date(fields[3])
 
         if fields[1] == "BADARMOR":
             raise GpgException("Bad armor.")
@@ -190,4 +204,7 @@ class SignedFile(object):
         finally:
             os._exit(1)
 
+    def contents_sha1(self):
+        return apt_pkg.sha1sum(self.contents)
+
 # vim: set sw=4 et:
diff --git a/templates/process-command.processed b/templates/process-command.processed
new file mode 100644 (file)
index 0000000..979032f
--- /dev/null
@@ -0,0 +1,12 @@
+From: __DAK_ADDRESS__
+To: __MAINTAINER_TO__
+Cc: __CC__
+__BCC__
+X-Debian: DAK
+Precedence: bulk
+MIME-Version: 1.0
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 8bit
+Subject: Results of processing __FILENAME__
+
+__RESULTS__