From: Joerg Jaspert Date: Mon, 17 Sep 2012 13:12:24 +0000 (+0200) Subject: Merge remote-tracking branch 'algernon/f/external_files' into merge X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=12e10a29eed2843efda0dfa13483a81877216cec;hp=b11a2110eda1996ab0ea43f95d285ff1014b65b8;p=dak.git Merge remote-tracking branch 'algernon/f/external_files' into merge * algernon/f/external_files: update87: Insert a default use_extfiles value to the config table too debian/cron.hourly: Dump & push the external_files table to security Introduce the external_files table Signed-off-by: Joerg Jaspert --- diff --git a/config/debian/cron.hourly b/config/debian/cron.hourly index fa565c4e..e1036550 100755 --- a/config/debian/cron.hourly +++ b/config/debian/cron.hourly @@ -46,6 +46,12 @@ ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1 $scriptsdir/generate-di +# Push files over to security +#pg_dump -a -F p -t files | sed -e "s,^COPY files (,DELETE FROM external_files; COPY external_files (," | xz -3 | \ +# ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/push-external_files dak@wherever sync +# +# The key should run the following command: +# 'xzcat | pg_restore -1 -a' # do the buildd key updates BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX) diff --git a/dak/dakdb/update87.py b/dak/dakdb/update87.py new file mode 100644 index 00000000..18e2509a --- /dev/null +++ b/dak/dakdb/update87.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +add external_files table for security + +@contact: Debian FTP Master +@copyright: 2012 Gergely Nagy +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError +from daklib.config import Config + +statements = [ +""" +CREATE TABLE external_files ( + id integer, + filename text NOT NULL, + size bigint NOT NULL, + md5sum text NOT NULL, + last_used timestamp with time zone, + sha1sum text, + sha256sum text, + created timestamp with time zone DEFAULT now() NOT NULL, + modified timestamp with time zone DEFAULT now() NOT NULL +); +""", +""" +INSERT INTO config(name, value) VALUES ('use_extfiles', 0); +""" +] + +################################################################################ +def do_update(self): + print __doc__ + try: + cnf = Config() + + c = self.db.cursor() + + for stmt in statements: + c.execute(stmt) + + c.execute("UPDATE config SET value = '87' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError as msg: + self.db.rollback() + raise DBUpdateError('Unable to apply sick update 87, rollback issued. Error message: {0}'.format(msg)) diff --git a/dak/update_db.py b/dak/update_db.py index cf327b0b..9dc613ae 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -46,7 +46,7 @@ from daklib.daklog import Logger ################################################################################ Cnf = None -required_database_schema = 86 +required_database_schema = 87 ################################################################################ diff --git a/daklib/archive.py b/daklib/archive.py index 78df632e..577601c4 100644 --- a/daklib/archive.py +++ b/daklib/archive.py @@ -861,6 +861,7 @@ class ArchiveUpload(object): checks.SignatureCheck, checks.ChangesCheck, checks.HashesCheck, + checks.ExternalHashesCheck, checks.SourceCheck, checks.BinaryCheck, checks.BinaryTimestampCheck, diff --git a/daklib/checks.py b/daklib/checks.py index 38975e1e..793bc681 100644 --- a/daklib/checks.py +++ b/daklib/checks.py @@ -45,6 +45,12 @@ class Reject(Exception): """exception raised by failing checks""" pass +class RejectStupidMaintainerException(Exception): + """exception raised by failing the external hashes check""" + + def __str__(self): + return "'%s' has mismatching %s from the external files db ('%s' [current] vs '%s' [external])" % self.args[:4] + class Check(object): """base class for checks @@ -167,6 +173,43 @@ class HashesCheck(Check): for f in source.files.itervalues(): f.check(upload.directory) +class ExternalHashesCheck(Check): + """Checks hashes in .changes and .dsc against an external database.""" + def check_single(self, session, f): + q = session.execute("SELECT size, md5sum, sha1sum, sha256sum FROM external_files WHERE filename LIKE '%%/%s'" % f.filename) + (ext_size, ext_md5sum, ext_sha1sum, ext_sha256sum) = q.fetchone() or (None, None, None, None) + + if not ext_size: + return + + if ext_size != f.size: + raise RejectStupidMaintainerException(f.filename, 'size', f.size, ext_size)) + + if ext_md5sum != f.md5sum: + raise RejectStupidMaintainerException(f.filename, 'md5sum', f.md5sum, ext_md5sum) + + if ext_sha1sum != f.sha1sum: + raise RejectStupidMaintainerException(f.filename, 'sha1sum', f.sha1sum, ext_sha1sum) + + if ext_sha256sum != f.sha256sum: + raise RejectStupidMaintainerException(f.filename, 'sha256sum', f.sha256sum, ext_sha256sum) + + def check(self, upload): + cnf = Config() + + if not cnf.use_extfiles: + return + + session = upload.session + changes = upload.changes + + for f in changes.files.itervalues(): + self.check_single(session, f) + source = changes.source + if source is not None: + for f in source.files.itervalues(): + self.check_single(session, f) + class BinaryCheck(Check): """Check binary packages for syntax errors.""" def check(self, upload): diff --git a/daklib/config.py b/daklib/config.py index c79582c2..65874250 100755 --- a/daklib/config.py +++ b/daklib/config.py @@ -120,7 +120,8 @@ class Config(object): for field in [('db_revision', None, int), ('defaultsuitename', 'unstable', str), ('exportpath', '', str), - ('unprivgroup', None, str) + ('unprivgroup', None, str), + ('use_extfiles', None, int) ]: setattr(self, 'get_%s' % field[0], lambda s=None, x=field[0], y=field[1], z=field[2]: self.get_db_value(x, y, z)) setattr(Config, '%s' % field[0], property(fget=getattr(self, 'get_%s' % field[0])))