From: Ansgar Burchardt Date: Mon, 17 Sep 2012 14:27:33 +0000 (+0200) Subject: Merge branch 'master' into pu/backports-merge X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=1617994b76d0cd90ce89a7e6d9f7bb886439f010;hp=e627bb90af0d9c396b94c78708ef9a86ddf8b770;p=dak.git Merge branch 'master' into pu/backports-merge --- diff --git a/config/debian/cron.hourly b/config/debian/cron.hourly index fa565c4e..e1036550 100755 --- a/config/debian/cron.hourly +++ b/config/debian/cron.hourly @@ -46,6 +46,12 @@ ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1 $scriptsdir/generate-di +# Push files over to security +#pg_dump -a -F p -t files | sed -e "s,^COPY files (,DELETE FROM external_files; COPY external_files (," | xz -3 | \ +# ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/push-external_files dak@wherever sync +# +# The key should run the following command: +# 'xzcat | pg_restore -1 -a' # do the buildd key updates BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX) diff --git a/config/homedir/ssh/ftpmaster-authorized_keys b/config/homedir/ssh/ftpmaster-authorized_keys new file mode 100644 index 00000000..c04539f7 --- /dev/null +++ b/config/homedir/ssh/ftpmaster-authorized_keys @@ -0,0 +1,15 @@ +# projectb to flotow +command="rsync --server --sender -vlogDtprz --delete . /srv/ftp-master.debian.org/backup/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="67.192.254.200" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAn4p26q4kcoqfHjbR4CXMOOppS9zR3RtCAroL1fVeWvE8U7CVowC1MP/0pq8UAaTfdflVfIYpKbl0xMxBspyxG/fOa90TaPDT9JJrZbkQj0tMfTWFVCMg5mScT0T9wPnTkXvANU28QwDSfudvwokqA0aF2jIsBZakqtULmx6r3BED02iBNZQbbc2Sf/MvfHnpgz7yGfU/NCZzdQU0/mTbL1DqVSgbmebt6MvRfYhnxm/Tw+gfLTpG0PTKTDU5NnJBG5tPgHC2vf2jqHDn1cMu9siNjPB52sG/n+KO3Deq3dXMKMjt+9VxXC2gfND6RVnZRCfwm9QByMw5eqVejEW7iw== dak@flotow - projectb 2008-11-29 +# non-s3kr1t stuff to flotow +command="rsync --server --sender -vlogDtprz --delete . /srv/ftp-master.debian.org/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="67.192.254.200" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1viRQWCL/LNE+7Kfc/Ao+COZM4x/7TzpXtVTRgzBtaPUf6xPiec3ieYHriLNpxblNRlRrzgSu978jmNAs9yWnmaG/QvV9CMTGMyt3ZC+z7HFX6YwSz+hJOMl55myVNsWbwOCfnTmem5YFG8yJZcTREWtMW31GfSfiv64p4ths5fJCyNBGh6E3TDg1Z9PafshETXogZjn7Ff+OXvGPo/oDW+0gEGzaNK1gvIdJNkrDAzb3UGmIZ4qcKVMtJ/Oc+R0G3NBDJUlxe48ocuzu3YUernTiZgvGAmE0vNlLAeJaXvh4YRV1mxirNvPtmKX+HZfJbrq9Pmsawdt/Yl2yR9K8w== ftpmaster test machine sync + +# syncing ries +command="rsync --server --sender -vlHogDtpre.Lsf --timeout=3600 . /srv/ftp-master.debian.org/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="128.148.34.103" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPmVHveaikQYpiufc+fkZgD+r3HJ8wCr9MCwk4kkpA5VZ5cO89+yYupZcKtyrsz34W2IB5igo/RbXrEaonAKQTVbaJPE4RAWpLlpXx6PqSsbGd1QonXXXbk3HQSjesqOEiw7KLU/m1grk5Ad9xdhY5mA2dzgZBD76JzUo8FISO4Mb4CGcWxj2n+lw0mhOftXP5WSRt28F7UFTbY+ogC8RgvXAudTC5zhZm4APcqob+kgVjneMy2xJKF+1KOW+bAtEKlKe+yMDU1zDC3etgzKYR70oiOpKIkjjYCWE17lFiVEOlXwW3rzg6U2oZF8U3NE8sTcCe7XzpCOI+bVSLm5jL ftpmaster dd access sync ries 2011-03-21 +command="rsync --server --sender -vlHogDtpre.Lsf --timeout=3600 . /srv/ftp.debian.org/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="128.148.34.103" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDF612GP1J9UZVH0G9EnLTgTfzmlNYvaCdYt9OUqzMnqFZhNjfZpaAM7m2n+f0j55j8ktS6jgmC8bpKXdwFsWjZVi4AJ3toHyfvFtOv0ec0j1p+5RqqxdvsbhAYDwGGUk0Wldc8d6g/uIy//gKuqyWuo5tOmuxXIYpG1SR7MzQNdgLRhcJK/ZsR302geQ4kbjyk8DnbVZUhWxQFELZ4cKLFETxXitr45TiUGon006MLmxWribwZVwYl7ZcTJlefK7Z4VOA39YEgacFUt9LtRmV4dipPli/I6z4DTrjaMPH8VgkMCZtSM/igoXod3/ExS2yrzZHJ/NDMLiR6hha7GuiV ftpmaster dd access sync public ries 2011-03-21 + +# whenever we have to read a new dataset for testing/squeeze-updates this runs +command="/srv/ftp-master.debian.org/dak/scripts/debian/import_dataset.sh testing",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="128.148.34.3,franck.debian.org" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAqvcRf4LLH9WLz3YGg/vj62I6aMihd9eF8tEYIMvRUNIqcI95YQP6nPpnIovom30RI9l5vJP+xpd0ABoiVxGDr0fw4hfp137BxpOL2WDHoqYX0KWP5mdWpA2PV2HVOJ4xp0q18pZ0DIdhxAGDd1QRrkR2yD9CH4dhRNcYRN8TA970y5Tweesh19Ba583f25NrSv0+A1200qiSdMbn9KIQYwC0Gc9xcKS1/Tygf2Sz3ekVrODog/nACPLbHRxO+mPcHJVBb9Sf8l393l5eln7ZfmSD0wZD6X/2M9+rRoXtVycLbmISxJV8zdady/3HCX33fcWCI7xCfOsikcVWDzygtQ== release@ries +command="/srv/ftp-master.debian.org/dak/scripts/debian/import_dataset.sh squeeze-updates",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="128.148.34.3,franck.debian.org" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAs0CFETy4E2rR7sH5kl5tgPVltcimtdmkpWSYLO+AJrrTvN447KjL0GhAc9raWv/wp6UeGw9zhOOxH6UGGD2DKI+lIZKW2PraLnQMs9g67B7Q/7MH7rHIzKue1niOANgPZppQ18rdiexagWyj+E8z/A1cFqpfaIIupi543eXZ4yZV3fjrHIE6zTvIzoTzlAZ5IaCOYyFT8wx6Ql53aEZfMk6S1FvXou24wFBD08CArTjRMf2eYo/aPqWbJs955eZwNqp1kS4jtJKwc7DCKpY7elHCyIqfR7YZxTUOBEGpoaAIfjIitgEedZnuMmBl8IUi1jQ0HvM7HDb4n4NVR/hbew== release@franck + +# release team tpu removals +command="/srv/ftp-master.debian.org/dak/scripts/debian/release_team_removals.sh",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="128.148.34.3,franck.debian.org" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCw6DLpbCsiadOqxenRfW5In7UFG5HoIDt0xV/dRDbqNUUihNcDi6SqlREuSBCA75lOqbhL1w2tWsdsTIMnJeq3Fdr3LdFjIKlG6QQZVThaD3SI76EkGtjt0XQDoN2d4hi0Xn2LOPKz8hxaY4jKYzSUN0TVue3C1EHTJD0S8Grkd5tPaDgXt4pJzHmNwT4r2dH5OT3Y3vJL2UGhbY6Y+rNFfmnKzDcBtNdUTLTtrAfCCMkPITTYrMvZevA9u/SzNenN9qwEQicc06FrycSCi6+XSA+t4k1YNf1NTHhTQEncEX4/FRf+jgbkt1lkchiu+eShx3bUZCsKPuoNEsuWUU5v release@franck diff --git a/config/homedir/ssh/ftpmaster-config b/config/homedir/ssh/ftpmaster-config new file mode 100644 index 00000000..045ae29a --- /dev/null +++ b/config/homedir/ssh/ftpmaster-config @@ -0,0 +1,33 @@ +Protocol 2 +ConnectTimeout 30 +ServerAliveInterval 30 +ForwardX11 no +ForwardAgent no +StrictHostKeyChecking yes +PasswordAuthentication no +BatchMode yes + +Host bugs-sync + Hostname bugs-master.debian.org + User debbugs + IdentityFile /srv/ftp-master.debian.org/s3kr1t/id_debbugs-vt + +Host ddtp-sync + Hostname i18n.debian.net + User ddtp-dak + IdentityFile /srv/ftp-master.debian.org/s3kr1t/ddtp-dak.rsa + +Host morgue-sync + Hostname stabile.debian.org + User dak + IdentityFile /srv/ftp-master.debian.org/s3kr1t/push_morgue + +Host ries-sync + Hostname ries.debian.org + User dak + IdentityFile /srv/ftp-master.debian.org/scripts/s3kr1t/ssh/id_franck + +Host external-security + Hostname chopin.debian.org + User dak + IdentityFile /srv/ftp-master.debian.org/scripts/s3kr1t/ssh/push_external_files diff --git a/dak/dakdb/update87.py b/dak/dakdb/update87.py new file mode 100644 index 00000000..18e2509a --- /dev/null +++ b/dak/dakdb/update87.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +add external_files table for security + +@contact: Debian FTP Master +@copyright: 2012 Gergely Nagy +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError +from daklib.config import Config + +statements = [ +""" +CREATE TABLE external_files ( + id integer, + filename text NOT NULL, + size bigint NOT NULL, + md5sum text NOT NULL, + last_used timestamp with time zone, + sha1sum text, + sha256sum text, + created timestamp with time zone DEFAULT now() NOT NULL, + modified timestamp with time zone DEFAULT now() NOT NULL +); +""", +""" +INSERT INTO config(name, value) VALUES ('use_extfiles', 0); +""" +] + +################################################################################ +def do_update(self): + print __doc__ + try: + cnf = Config() + + c = self.db.cursor() + + for stmt in statements: + c.execute(stmt) + + c.execute("UPDATE config SET value = '87' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError as msg: + self.db.rollback() + raise DBUpdateError('Unable to apply sick update 87, rollback issued. Error message: {0}'.format(msg)) diff --git a/dak/update_db.py b/dak/update_db.py index cf327b0b..9dc613ae 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -46,7 +46,7 @@ from daklib.daklog import Logger ################################################################################ Cnf = None -required_database_schema = 86 +required_database_schema = 87 ################################################################################ diff --git a/daklib/archive.py b/daklib/archive.py index 78df632e..577601c4 100644 --- a/daklib/archive.py +++ b/daklib/archive.py @@ -861,6 +861,7 @@ class ArchiveUpload(object): checks.SignatureCheck, checks.ChangesCheck, checks.HashesCheck, + checks.ExternalHashesCheck, checks.SourceCheck, checks.BinaryCheck, checks.BinaryTimestampCheck, diff --git a/daklib/checks.py b/daklib/checks.py index 8111ef75..2e76e783 100644 --- a/daklib/checks.py +++ b/daklib/checks.py @@ -45,6 +45,12 @@ class Reject(Exception): """exception raised by failing checks""" pass +class RejectStupidMaintainerException(Exception): + """exception raised by failing the external hashes check""" + + def __str__(self): + return "'%s' has mismatching %s from the external files db ('%s' [current] vs '%s' [external])" % self.args[:4] + class Check(object): """base class for checks @@ -162,11 +168,48 @@ class HashesCheck(Check): changes = upload.changes for f in changes.files.itervalues(): f.check(upload.directory) - source = changes.source + source = changes.source if source is not None: for f in source.files.itervalues(): f.check(upload.directory) +class ExternalHashesCheck(Check): + """Checks hashes in .changes and .dsc against an external database.""" + def check_single(self, session, f): + q = session.execute("SELECT size, md5sum, sha1sum, sha256sum FROM external_files WHERE filename LIKE '%%/%s'" % f.filename) + (ext_size, ext_md5sum, ext_sha1sum, ext_sha256sum) = q.fetchone() or (None, None, None, None) + + if not ext_size: + return + + if ext_size != f.size: + raise RejectStupidMaintainerException(f.filename, 'size', f.size, ext_size) + + if ext_md5sum != f.md5sum: + raise RejectStupidMaintainerException(f.filename, 'md5sum', f.md5sum, ext_md5sum) + + if ext_sha1sum != f.sha1sum: + raise RejectStupidMaintainerException(f.filename, 'sha1sum', f.sha1sum, ext_sha1sum) + + if ext_sha256sum != f.sha256sum: + raise RejectStupidMaintainerException(f.filename, 'sha256sum', f.sha256sum, ext_sha256sum) + + def check(self, upload): + cnf = Config() + + if not cnf.use_extfiles: + return + + session = upload.session + changes = upload.changes + + for f in changes.files.itervalues(): + self.check_single(session, f) + source = changes.source + if source is not None: + for f in source.files.itervalues(): + self.check_single(session, f) + class BinaryCheck(Check): """Check binary packages for syntax errors.""" def check(self, upload): diff --git a/daklib/config.py b/daklib/config.py index 99b84dd6..339604a1 100755 --- a/daklib/config.py +++ b/daklib/config.py @@ -134,7 +134,8 @@ class Config(object): for field in [('db_revision', None, int), ('defaultsuitename', 'unstable', str), ('exportpath', '', str), - ('unprivgroup', None, str) + ('unprivgroup', None, str), + ('use_extfiles', None, int) ]: setattr(self, 'get_%s' % field[0], lambda s=None, x=field[0], y=field[1], z=field[2]: self.get_db_value(x, y, z)) setattr(Config, '%s' % field[0], property(fget=getattr(self, 'get_%s' % field[0]))) diff --git a/daklib/fstransactions.py b/daklib/fstransactions.py index 33f59c8c..eb4874a1 100644 --- a/daklib/fstransactions.py +++ b/daklib/fstransactions.py @@ -122,7 +122,7 @@ class FilesystemTransaction(object): def __init__(self): self.actions = [] - def copy(self, source, destination, link=True, symlink=False, mode=None): + def copy(self, source, destination, link=False, symlink=False, mode=None): """copy C{source} to C{destination} @type source: str diff --git a/daklib/utils.py b/daklib/utils.py index ec929652..1034628e 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -1513,7 +1513,8 @@ def temp_dirname(parent=None, prefix="dak", suffix="", mode=None, group=None): if mode: os.chmod(tfname, mode) if group: - os.chown(tfname, -1, group) + gid = grp.getgrnam(group).gr_gid + os.chown(tfname, -1, gid) return tfname ################################################################################