echo "$timestamp": ${changes:-"Nothing to do"} >> $report
dak process-upload -a ${UNCHECKED_WITHOUT_LOCK} -d "$unchecked" >> $report
+ dak process-commands -d "$unchecked" >> $report
}
# process NEW policy queue
ftpteam "";
backports "/srv/ftp-master.debian.org/dak/config/debian/dak.conf-backports";
};
+
+Command::DM {
+ ACL "dm";
+ AdminKeyrings {
+ "/srv/keyring.debian.org/keyrings/debian-keyring.gpg";
+ };
+ Keyrings {
+ "/srv/keyring.debian.org/keyrings/debian-maintainers.gpg";
+ };
+};
function dm() {
log "Updating DM html page"
$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
+ dak acl export-per-source dm >$exportdir/dm.txt
}
function bts() {
session.commit()
+def acl_export_per_source(acl_name):
+ session = DBConn().session()
+ acl = session.query(ACL).filter_by(name=acl_name).one()
+
+ query = """
+ SELECT
+ f.fingerprint,
+ (SELECT COALESCE(u.name, '') || ' <' || u.uid || '>'
+ FROM uid u
+ JOIN fingerprint f2 ON u.id = f2.uid
+ WHERE f2.id = f.id) AS name,
+ STRING_AGG(a.source, ' ' ORDER BY a.source)
+ FROM acl_per_source a
+ JOIN fingerprint f ON a.fingerprint_id = f.id
+ LEFT JOIN uid u ON f.uid = u.id
+ WHERE a.acl_id = :acl_id
+ GROUP BY f.id, f.fingerprint
+ ORDER BY name
+ """
+
+ for row in session.execute(query, {'acl_id': acl.id}):
+ print "Fingerprint:", row[0]
+ print "Uid:", row[1]
+ print "Allow:", row[2]
+ print
+
+ session.rollback()
+ session.close()
+
def main(argv=None):
if argv is None:
argv = sys.argv
- if len(argv) != 3 or argv[1] != 'set-fingerprints':
+ if len(argv) != 3:
usage()
sys.exit(1)
- acl_set_fingerprints(argv[2], sys.stdin)
+ if argv[1] == 'set-fingerprints':
+ acl_set_fingerprints(argv[2], sys.stdin)
+ elif argv[1] == 'export-per-source':
+ acl_export_per_source(argv[2])
+ else:
+ usage()
+ sys.exit(1)
"Process NEW and BYHAND packages"),
("process-upload",
"Process packages in queue/unchecked"),
+ ("process-commands",
+ "Process command files (*.dak-commands)"),
("process-policy",
"Process packages in policy queues from COMMENTS files"),
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+add table to keep track of seen signatures
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2012 Ansgar Burchardt <ansgar@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+import psycopg2
+from daklib.dak_exceptions import DBUpdateError
+from daklib.config import Config
+
+################################################################################
+def do_update(self):
+ print __doc__
+ try:
+ cnf = Config()
+
+ c = self.db.cursor()
+
+ c.execute("""CREATE TABLE signature_history (
+ fingerprint TEXT NOT NULL,
+ signature_timestamp TIMESTAMP NOT NULL,
+ contents_sha1 TEXT NOT NULL,
+ seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ PRIMARY KEY (signature_timestamp, fingerprint, contents_sha1)
+ )""")
+
+ c.execute("UPDATE config SET value = '89' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError as msg:
+ self.db.rollback()
+ raise DBUpdateError('Unable to apply sick update 89, rollback issued. Error message: {0}'.format(msg))
--- /dev/null
+#! /usr/bin/env python
+#
+# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import apt_pkg
+import datetime
+import os
+import sys
+
+from daklib.config import Config
+from daklib.command import CommandError, CommandFile
+from daklib.daklog import Logger
+from daklib.fstransactions import FilesystemTransaction
+from daklib.utils import find_next_free
+
+def usage():
+ print """Usage: dak command <command-file>...
+
+process command files
+"""
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv
+
+ arguments = [('h', 'help', 'Process-Commands::Options::Help'),
+ ('d', 'directory', 'Process-Commands::Options::Directory', 'HasArg')]
+
+ cnf = Config()
+ cnf['Command::Options::Dummy'] = ''
+ filenames = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
+ options = cnf.subtree('Command::Options')
+
+ if 'Help' in options or (len(filenames) == 0 and 'Directory' not in options):
+ usage()
+ sys.exit(0)
+
+ log = Logger('command')
+
+ now = datetime.datetime.now()
+ donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
+ rejectdir = cnf['Dir::Reject']
+
+ if len(filenames) == 0:
+ filenames = [ fn for fn in os.listdir(options['Directory']) if fn.endswith('.dak-commands') ]
+
+ for fn in filenames:
+ basename = os.path.basename(fn)
+ if not fn.endswith('.dak-commands'):
+ log.log(['unexpected filename', basename])
+ continue
+
+ command = CommandFile(fn, log)
+ if command.evaluate():
+ log.log(['moving to done', basename])
+ dst = find_next_free(os.path.join(donedir, basename))
+ else:
+ log.log(['moving to reject', basename])
+ dst = find_next_free(os.path.join(rejectdir, basename))
+
+ with FilesystemTransaction() as fs:
+ fs.move(fn, dst, mode=0o644)
+ fs.commit()
+
+ log.close()
+
+if __name__ == '__main__':
+ main()
pu.program = "process-upload"
+ pu.warnings = upload.warnings
+
return pu
@try_or_reject
cnf = Config()
Logger.log(['ACCEPT', upload.changes.filename])
+ print "ACCEPT"
upload.install()
cnf = Config()
Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
+ print "ACCEPT-TO-NEW"
upload.install_to_new()
# TODO: tag bugs pending
cnf = Config()
Logger.log(['REJECT', upload.changes.filename])
+ print "REJECT"
fs = upload.transaction.fs
rejectdir = cnf['Dir::Reject']
################################################################################
Cnf = None
-required_database_schema = 88
+required_database_schema = 89
################################################################################
# program
program = "unknown-program"
+ warnings = []
+
def _subst_for_upload(upload):
cnf = Config()
'__SOURCE__': upload.source,
'__VERSION__': upload.version,
'__ARCHITECTURE__': upload.architecture,
+ '__WARNINGS__': '\n'.join(upload.warnings),
}
override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
self._new = self._new_queue.suite
+ def warn(self, message):
+ """add a warning message
+
+ Adds a warning message that can later be seen in C{self.warnings}
+
+ @type message: string
+ @param message: warning message
+ """
+ self.warnings.append(message)
+
def prepare(self):
"""prepare upload for further processing
# XXX: Drop DMUA part here and switch to new implementation.
# XXX: Send warning mail once users can set the new DMUA flag
dmua_status, dmua_reason = self._check_dmua(upload)
- if not dmua_status:
- return False, dmua_reason
+ if acl_per_source is None:
+ if not dmua_status:
+ return False, dmua_reason
+ else:
+ upload.warn('DM flag not set, but accepted as DMUA was set.')
#if acl_per_source is None:
# return False, "not allowed to upload source package '{0}'".format(source_name)
if acl.deny_per_source and acl_per_source is not None:
--- /dev/null
+"""module to handle command files
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2012, Ansgar Burchardt <ansgar@debian.org>
+@license: GPL-2+
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import apt_pkg
+import os
+import re
+import tempfile
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib.gpg import SignedFile
+from daklib.regexes import re_field_package
+from daklib.textutils import fix_maintainer
+from daklib.utils import gpg_get_key_addresses, send_mail, TemplateSubst
+
+class CommandError(Exception):
+ pass
+
+class CommandFile(object):
+ def __init__(self, path, log=None):
+ if log is None:
+ from daklib.daklog import Logger
+ log = Logger()
+ self.cc = []
+ self.result = []
+ self.log = log
+ self.path = path
+ self.filename = os.path.basename(path)
+
+ def _check_replay(self, signed_file, session):
+ """check for replays
+
+ @note: Will commit changes to the database.
+
+ @type signed_file: L{daklib.gpg.SignedFile}
+
+ @param session: database session
+ """
+ # Mark commands file as seen to prevent replays.
+ signature_history = SignatureHistory.from_signed_file(signed_file)
+ session.add(signature_history)
+ session.commit()
+
+ def _evaluate_sections(self, sections, session):
+ session.rollback()
+ try:
+ sections.next()
+ section = sections.section
+
+ action = section.get('Action', None)
+ if action is None:
+ raise CommandError('Encountered section without Action field')
+ self.result.append('Action: {0}'.format(action))
+
+ if action == 'dm':
+ self.action_dm(self.fingerprint, section, session)
+ else:
+ raise CommandError('Unknown action: {0}'.format(action))
+ except StopIteration:
+ pass
+ finally:
+ session.rollback()
+
+ def _notify_uploader(self):
+ cnf = Config()
+
+ bcc = 'X-DAK: dak process-command'
+ if 'Dinstall::Bcc' in cnf:
+ bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
+
+ cc = set(fix_maintainer(address)[1] for address in self.cc)
+
+ subst = {
+ '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
+ '__MAINTAINER_TO__': fix_maintainer(self.uploader)[1],
+ '__CC__': ", ".join(cc),
+ '__BCC__': bcc,
+ '__RESULTS__': "\n".join(self.result),
+ '__FILENAME__': self.filename,
+ }
+
+ message = TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-command.processed'))
+
+ send_mail(message)
+
+ def evaluate(self):
+ """evaluate commands file
+
+ @rtype: bool
+ @returns: C{True} if the file was processed sucessfully,
+ C{False} otherwise
+ """
+ result = True
+
+ session = DBConn().session()
+
+ keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
+ keyring_files = [ k.keyring_name for k in keyrings ]
+
+ raw_contents = open(self.path, 'r').read()
+ signed_file = SignedFile(raw_contents, keyring_files)
+ if not signed_file.valid:
+ self.log.log(['invalid signature', self.filename])
+ return False
+
+ self.fingerprint = session.query(Fingerprint).filter_by(fingerprint=signed_file.primary_fingerprint).one()
+ if self.fingerprint.keyring is None:
+ self.log.log(['singed by key in unknown keyring', self.filename])
+ return False
+ assert self.fingerprint.keyring.active
+
+ self.log.log(['processing', self.filename, 'signed-by={0}'.format(self.fingerprint.fingerprint)])
+
+ with tempfile.TemporaryFile() as fh:
+ fh.write(signed_file.contents)
+ fh.seek(0)
+ sections = apt_pkg.TagFile(fh)
+
+ self.uploader = None
+ addresses = gpg_get_key_addresses(self.fingerprint.fingerprint)
+ if len(addresses) > 0:
+ self.uploader = addresses[0]
+
+ try:
+ sections.next()
+ section = sections.section
+ if 'Uploader' in section:
+ self.uploader = section['Uploader']
+ # TODO: Verify first section has valid Archive field
+ if 'Archive' not in section:
+ raise CommandError('No Archive field in first section.')
+
+ # TODO: send mail when we detected a replay.
+ self._check_replay(signed_file, session)
+
+ self._evaluate_sections(sections, session)
+ self.result.append('')
+ except Exception as e:
+ self.log.log(['ERROR', e])
+ self.result.append("There was an error processing this section:\n{0}".format(e))
+ result = False
+
+ self._notify_uploader()
+
+ session.close()
+ self.log.log(['done', self.filename])
+
+ return result
+
+ def _split_packages(self, value):
+ names = value.split()
+ for name in names:
+ if not re_field_package.match(name):
+ raise CommandError('Invalid package name "{0}"'.format(name))
+ return names
+
+ def action_dm(self, fingerprint, section, session):
+ cnf = Config()
+
+ if 'Command::DM::AdminKeyrings' not in cnf \
+ or 'Command::DM::ACL' not in cnf \
+ or 'Command::DM::Keyrings' not in cnf:
+ raise CommandError('DM command is not configured for this archive.')
+
+ allowed_keyrings = cnf.value_list('Command::DM::AdminKeyrings')
+ if fingerprint.keyring.keyring_name not in allowed_keyrings:
+ raise CommandError('Key {0} is not allowed to set DM'.format(fingerprint.fingerprint))
+
+ acl_name = cnf.get('Command::DM::ACL', 'dm')
+ acl = session.query(ACL).filter_by(name=acl_name).one()
+
+ fpr = session.query(Fingerprint).filter_by(fingerprint=section['Fingerprint']).one()
+ if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list('Command::DM::Keyrings'):
+ raise CommandError('Key {0} is not in DM keyring.'.format(fpr.fingerprint))
+ addresses = gpg_get_key_addresses(fpr.fingerprint)
+ if len(addresses) > 0:
+ self.cc.append(addresses[0])
+
+ self.log.log(['dm', 'fingerprint', fpr.fingerprint])
+ self.result.append('Fingerprint: {0}'.format(fpr.fingerprint))
+ if len(addresses) > 0:
+ self.log.log(['dm', 'uid', addresses[0]])
+ self.result.append('Uid: {0}'.format(addresses[0]))
+
+ for source in self._split_packages(section.get('Allow', '')):
+ if session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).first() is None:
+ aps = ACLPerSource()
+ aps.acl = acl
+ aps.fingerprint = fpr
+ aps.source = source
+ session.add(aps)
+ self.log.log(['dm', 'allow', fpr.fingerprint, source])
+ self.result.append('Allowed: {0}'.format(source))
+ else:
+ self.result.append('Already-Allowed: {0}'.format(source))
+
+ session.flush()
+
+ for source in self._split_packages(section.get('Deny', '')):
+ session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).delete()
+ self.log.log(['dm', 'deny', fpr.fingerprint, source])
+ self.result.append('Denied: {0}'.format(source))
+
+ session.commit()
################################################################################
+class SignatureHistory(ORMObject):
+ @classmethod
+ def from_signed_file(cls, signed_file):
+ """signature history entry from signed file
+
+ @type signed_file: L{daklib.gpg.SignedFile}
+ @param signed_file: signed file
+
+ @rtype: L{SignatureHistory}
+ """
+ self = cls()
+ self.fingerprint = signed_file.primary_fingerprint
+ self.signature_timestamp = signed_file.signature_timestamp
+ self.contents_sha1 = signed_file.contents_sha1()
+ return self
+
+__all__.append('SignatureHistory')
+
+################################################################################
+
class SrcContents(ORMObject):
def __init__(self, file = None, source = None):
self.file = file
'policy_queue_byhand_file',
'priority',
'section',
+ 'signature_history',
'source',
'source_metadata',
'src_associations',
properties = dict(section_id = self.tbl_section.c.id,
section=self.tbl_section.c.section))
+ mapper(SignatureHistory, self.tbl_signature_history)
+
mapper(DBSource, self.tbl_source,
properties = dict(source_id = self.tbl_source.c.id,
version = self.tbl_source.c.version,
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+import apt_pkg
+import datetime
import errno
import fcntl
import os
return dict( (fd, "".join(read_lines[fd])) for fd in read_lines.keys() )
+ def _parse_date(self, value):
+ """parse date string in YYYY-MM-DD format
+
+ @rtype: L{datetime.datetime}
+ @returns: datetime objects for 0:00 on the given day
+ """
+ year, month, day = value.split('-')
+ date = datetime.date(int(year), int(month), int(day))
+ time = datetime.time(0, 0)
+ return datetime.datetime.combine(date, time)
+
def _parse_status(self, line):
fields = line.split()
if fields[0] != "[GNUPG:]":
self.valid = True
self.fingerprint = fields[2]
self.primary_fingerprint = fields[11]
+ self.signature_timestamp = self._parse_date(fields[3])
if fields[1] == "BADARMOR":
raise GpgException("Bad armor.")
finally:
os._exit(1)
+ def contents_sha1(self):
+ return apt_pkg.sha1sum(self.contents)
+
# vim: set sw=4 et:
--- /dev/null
+From: __DAK_ADDRESS__
+To: __MAINTAINER_TO__
+Cc: __CC__
+__BCC__
+X-Debian: DAK
+Precedence: bulk
+MIME-Version: 1.0
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 8bit
+Subject: Results of processing __FILENAME__
+
+__RESULTS__
Content-Transfer-Encoding: 8bit
Subject: __CHANGES_FILENAME__ ACCEPTED into __SUITE__
+__WARNINGS__
+
Accepted:
+
__FILE_CONTENTS__
Thank you for your contribution to __DISTRO__.
# ping target only if there is the possibility that we'll contact it (but
# also don't wait too long).
- my @have_changes = <*.changes *.commands>;
+ my @have_changes = <*.changes *.commands *.dak-commands>;
for ( my $delayed_dirs = 0 ;
$delayed_dirs <= $conf::max_delayed ;
$delayed_dirs++ )
return
);
- # look for *.commands files but not in delayed queues
+ # look for *.commands and *.dak-commands files but not in delayed queues
if ( $adelay == -1 ) {
foreach $file (<*.commands>) {
init_mail($file);
write_status_file() if $conf::statusdelay;
finish_mail();
} ## end foreach $file (<*.commands>)
+ foreach $file (<*.dak-commands>) {
+ init_mail($file);
+ block_signals();
+ process_dak_commands($file);
+ unblock_signals();
+ $main::dstat = "c";
+ write_status_file() if $conf::statusdelay;
+ finish_mail();
+ }
} ## end if ( $adelay == -1 )
opendir( INC, "." )
or (
#}
} ## end sub process_changes($\@)
+#
+# process one .dak-commands file
+#
+sub process_dak_commands {
+ my $commands = shift;
+
+ # TODO: get mail address from signed contents
+ # and NOT implement a third parser for armored PGP...
+ $main::mail_addr = undef;
+
+ # check signature
+ my $signator = pgp_check($commands);
+ if (!$signator) {
+ msg("log,mail",
+ "$main::current_incoming_short/$commands has bad PGP/GnuPG signature!\n");
+ msg("log,mail",
+ "Removing $main::current_incoming_short/$commands\n");
+ rm($commands);
+ return;
+ }
+ elsif ($signator eq 'LOCAL ERROR') {
+ debug("Can't check signature for $main::current_incoming_short/$commands -- don't process it for now");
+ return;
+ }
+ msg("log,mail", "(PGP/GnuPG signature by $signator)\n");
+
+ # check target
+ my @filenames = ($commands);
+ if (my $ls_l = is_on_target($commands, @filenames)) {
+ msg("log,mail", "$main::current_incoming_short/$commands is already present on target host:\n");
+ msg("log,mail", "$ls_l\n");
+ msg("log,mail", "Job $commands removed.\n");
+ rm($commands);
+ return;
+ }
+
+ if (!copy_to_target($commands)) {
+ msg("log,mail", "$commands couldn't be uploaded to target.\n");
+ msg("log,mail", "Giving up and removing it.\n");
+ rm($commands);
+ return;
+ }
+
+ rm($commands);
+ msg("mail", "$commands uploaded successfully to $conf::target\n");
+}
+
#
# process one .commands file
#
projects / dak.git / commitdiff