#
set -e
+set -o pipefail
set -u
export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars
# exit on errors
set -e
+set -o pipefail
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
# Executed hourly via cron, out of dak's crontab.
set -e
+set -o pipefail
set -u
export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars
#
set -e
+set -o pipefail
set -u
export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars
# exit on errors
set -e
+set -o pipefail
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
# exit on errors
set -e
+set -o pipefail
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
#
set -e
+set -o pipefail
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the functions is
ARCHLIST=$(tempfile)
log "Querying postgres"
- echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
+ local query='SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)'
+ psql -At -c "$query" | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
includedirs () {
perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
log "Generating suite lists"
suite_list () {
- printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
+ local query
+ query="$(printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)' $1 $1)"
+ psql -F' ' -A -t -c "$query"
- printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
+ query="$(printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d' $1)"
+ psql -F' ' -A -t -c "$query"
}
- printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
+ psql -F' ' -At -c 'SELECT id, suite_name FROM suite' |
while read id suite; do
[ -e $base/ftp/dists/$suite ] || continue
(
# Executed after cron.unchecked
set -e
+set -o pipefail
set -u
export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars
# Executed daily via cron, out of dak's crontab.
set -e
+set -o pipefail
export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars
. $SCRIPTVARS
# Executed hourly via cron, out of dak's crontab.
set -e
+set -o pipefail
set -u
export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars
#! /bin/bash
set -e
+set -o pipefail
set -u
export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars
# Executed weekly via cron, out of dak's crontab.
set -e
+set -o pipefail
export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars
. $SCRIPTVARS
# Run daily via cron, out of dak's crontab.
set -e
+set -o pipefail
set -u
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS
# exit on errors
set -e
+set -o pipefail
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
# Executed hourly via cron, out of dak's crontab.
set -e
+set -o pipefail
set -u
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
# Run at the beginning of the month via cron, out of dak's crontab.
set -e
+set -o pipefail
set -u
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS
# exit on errors
set -e
+set -o pipefail
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
# exit on errors
set -e
+set -o pipefail
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
# Run once a week via cron, out of dak's crontab.
set -e
+set -o pipefail
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the functions is
ARCHLIST=$(tempfile)
log "Querying postgres"
- echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
+ local query='SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)'
+ psql -At -c "$query" | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
includedirs () {
perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
log "Generating suite lists"
suite_list () {
- printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
+ local query
+ query="$(printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)' $1 $1)"
+ psql -F' ' -A -t -c "$query"
- printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
+ query="$(printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d' $1)"
+ psql -F' ' -A -t -c "$query"
}
- printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
+ psql -F' ' -At -c "SELECT id, suite_name FROM suite" |
while read id suite; do
[ -e $base/ftp/dists/$suite ] || continue
(
("dominate",
"Remove obsolete source and binary associations from suites"),
+ ("export",
+ "Export uploads from policy queues"),
("make-pkg-file-mapping",
"Generate package <-> file mapping"),
("generate-filelist",
--- /dev/null
+#! /usr/bin/env python
+#
+# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import apt_pkg
+import sys
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib.policy import UploadCopy
+
+def usage():
+ print """Usage: dak export -q <queue> [options] -a|--all|<source...>
+
+Export uploads from policy queues, that is the changes files for the given
+source package and all other files associated with that.
+
+ -a --all export all uploads
+ -c --copy copy files instead of symlinking them
+ -d <directory> target directory to export packages to
+ default: current directory
+ -q <queue> queue to grab uploads from
+ <source> source package name to export
+"""
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv
+
+ arguments = [('h', 'help', 'Export::Options::Help'),
+ ('a', 'all', 'Export::Options::All'),
+ ('c', 'copy', 'Export::Options::Copy'),
+ ('d', 'directory', 'Export::Options::Directory', 'HasArg'),
+ ('q', 'queue', 'Export::Options::Queue', 'HasArg')]
+
+ cnf = Config()
+ source_names = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
+ options = cnf.subtree('Export::Options')
+
+ if 'Help' in options or 'Queue' not in options:
+ usage()
+ sys.exit(0)
+
+ session = DBConn().session()
+
+ queue = session.query(PolicyQueue).filter_by(queue_name=options['Queue']).first()
+ if queue is None:
+ print "Unknown queue '{0}'".format(options['Queue'])
+ sys.exit(1)
+ uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=queue)
+ if 'All' not in options:
+ uploads = uploads.filter(DBChange.source.in_(source_names))
+ directory = options.get('Directory', '.')
+ symlink = 'Copy' not in options
+
+ for u in uploads:
+ print "Processing {0}...".format(u.changes.changesname)
+ UploadCopy(u).export(directory, symlink=symlink)
+
+if __name__ == '__main__':
+ main()
source = source_query.filter(DBSource.suites.contains(suite)).first()
if source is None:
if source_suites != True:
- source_query = source_query.filter(DBSource.suites.any(source_suites))
+ source_query = source_query.join(DBSource.suites) \
+ .filter(Suite.suite_id == source_suites.c.id)
source = source_query.first()
if source is None:
raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
return suites
+ def _mapped_component(self, component_name):
+ """get component after mappings
+
+ Evaluate component mappings from ComponentMappings in dak.conf for the
+ given component name.
+
+ NOTE: ansgar wants to get rid of this. It's currently only used for
+ the security archive
+
+ Args:
+ component_name (str): component name
+
+ Returns:
+ `daklib.dbconn.Component` object
+ """
+ cnf = Config()
+ for m in cnf.value_list("ComponentMappings"):
+ (src, dst) = m.split()
+ if component_name == src:
+ component_name = dst
+ component = self.session.query(Component).filter_by(component_name=component_name).one()
+ return component
+
def _check_new(self, suite):
"""Check if upload is NEW
daklib.dbconn.Override or None
"""
if suite.overridesuite is not None:
- suite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
+ suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
.join(Component).filter(Component.component_name == binary.component) \
daklib.dbconn.Override or None
"""
if suite.overridesuite is not None:
- suite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
+ suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
# XXX: component for source?
query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
except NoResultFound:
return None
+ def _binary_component(self, suite, binary, only_overrides=True):
+ """get component for a binary
+
+ By default this will only look at overrides to get the right component;
+ if `only_overrides` is False this method will also look at the Section field.
+
+ Args:
+ suite (daklib.dbconn.Suite)
+ binary (daklib.upload.Binary)
+
+ Kwargs:
+ only_overrides (bool): only use overrides to get the right component.
+ defaults to True.
+
+ Returns:
+ `daklib.dbconn.Component` object or None
+ """
+ override = self._binary_override(suite, binary)
+ if override is not None:
+ return override.component
+ if only_overrides:
+ return None
+ return self._mapped_component(binary.component)
+
def check(self, force=False):
"""run checks against the upload
checks.HashesCheck,
checks.SourceCheck,
checks.BinaryCheck,
+ checks.BinaryTimestampCheck,
checks.ACLCheck,
checks.SingleDistributionCheck,
checks.NoSourceOnlyCheck,
changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
if source_suites is None:
- source_suites = self.session.query(Suite).join(VersionCheck, VersionCheck.reference_id == Suite.suite_id).filter(VersionCheck.suite == suite).subquery()
+ source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
source = self.changes.source
if source is not None:
remaining = []
for f in byhand:
- package, version, archext = f.filename.split('_', 2)
+ parts = f.filename.split('_', 2)
+ if len(parts) != 3:
+ print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
+ remaining.append(f)
+ continue
+
+ package, version, archext = parts
arch, ext = archext.split('.', 1)
rule = automatic_byhand_packages.get(package)
redirected_suite = suite.policy_queue.suite
source_component_func = lambda source: self._source_override(overridesuite, source).component
- binary_component_func = lambda binary: self._binary_override(overridesuite, binary).component
+ binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
(db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
suite = suites[0]
def binary_component_func(binary):
- override = self._binary_override(suite, binary)
- if override is not None:
- return override.component
- component_name = binary.component
- component = self.session.query(Component).filter_by(component_name=component_name).one()
- return component
+ return self._binary_component(suite, binary, only_overrides=False)
# guess source component
# XXX: should be moved into an extra method
component = binary_component_func(binary)
binary_component_names.add(component.component_name)
source_component_name = None
- for guess in ('main', 'contrib', 'non-free'):
+ for c in self.session.query(Component).order_by(Component.component_id):
+ guess = c.component_name
if guess in binary_component_names:
source_component_name = guess
break
import daklib.lintian as lintian
import daklib.utils as utils
+import apt_inst
import apt_pkg
from apt_pkg import version_compare
import os
+import time
import yaml
# TODO: replace by subprocess
except:
raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
+class BinaryTimestampCheck(Check):
+ """check timestamps of files in binary packages
+
+ Files in the near future cause ugly warnings and extreme time travel
+ can cause errors on extraction.
+ """
+ def check(self, upload):
+ cnf = Config()
+ future_cutoff = time.time() + cnf.find_i('Dinstall::FutureTimeTravelGrace', 24*3600)
+ past_cutoff = time.mktime(time.strptime(cnf.find('Dinstall::PastCutoffYear', '1984'), '%Y'))
+
+ class TarTime(object):
+ def __init__(self):
+ self.future_files = dict()
+ self.past_files = dict()
+ def callback(self, member, data):
+ if member.mtime > future_cutoff:
+ future_files[member.name] = member.mtime
+ elif member.mtime < past_cutoff:
+ past_files[member.name] = member.mtime
+
+ def format_reason(filename, direction, files):
+ reason = "{0}: has {1} file(s) with a timestamp too far in the {2}:\n".format(filename, len(files), direction)
+ for fn, ts in files.iteritems():
+ reason += " {0} ({1})".format(fn, time.ctime(ts))
+ return reason
+
+ for binary in upload.changes.binaries:
+ filename = binary.hashed_file.filename
+ path = os.path.join(upload.directory, filename)
+ deb = apt_inst.DebFile(path)
+ tar = TarTime()
+ deb.control.go(tar.callback)
+ if tar.future_files:
+ raise Reject(format_reason(filename, 'future', tar.future_files))
+ if tar.past_files:
+ raise Reject(format_reason(filename, 'past', tar.past_files))
+
class SourceCheck(Check):
"""Check source package for syntax errors."""
def check_filename(self, control, filename, regex):
if 'source' not in upload.changes.architectures:
raise Reject('DM uploads must include source')
- distributions = upload.changes.distributions
- for dist in distributions:
- if dist not in ('unstable', 'experimental', 'squeeze-backports'):
- raise Reject("Uploading to {0} is not allowed for DMs.".format(dist))
for f in upload.changes.files.itervalues():
if f.section == 'byhand' or f.section[:4] == "raw-":
raise Reject("Uploading byhand packages is not allowed for DMs.")
# Reject NEW packages
+ distributions = upload.changes.distributions
assert len(distributions) == 1
suite = session.query(Suite).filter_by(suite_name=distributions[0]).one()
overridesuite = suite
def _highest_binary_version(self, session, binary_name, suite, architecture):
db_binary = session.query(DBBinary).filter_by(package=binary_name) \
.filter(DBBinary.suites.contains(suite)) \
+ .join(DBBinary.architecture) \
.filter(Architecture.arch_string.in_(['all', architecture])) \
.order_by(DBBinary.version.desc()).first()
if db_binary is None:
else:
return db_binary.version
- def _version_checks(self, upload, suite, expected_result):
+ def _version_checks(self, upload, suite, op):
session = upload.session
if upload.changes.source is not None:
source_name = upload.changes.source.dsc['Source']
source_version = upload.changes.source.dsc['Version']
v = self._highest_source_version(session, source_name, suite)
- if v is not None and version_compare(source_version, v) != expected_result:
- raise Reject('Version check failed (source={0}, version={1}, suite={2})'.format(source_name, source_version, suite.suite_name))
+ if v is not None and not op(version_compare(source_version, v)):
+ raise Reject('Version check failed (source={0}, version={1}, other-version={2}, suite={3})'.format(source_name, source_version, v, suite.suite_name))
for binary in upload.changes.binaries:
binary_name = binary.control['Package']
binary_version = binary.control['Version']
architecture = binary.control['Architecture']
v = self._highest_binary_version(session, binary_name, suite, architecture)
- if v is not None and version_compare(binary_version, v) != expected_result:
- raise Reject('Version check failed (binary={0}, version={1}, suite={2})'.format(binary_name, binary_version, suite.suite_name))
+ if v is not None and not op(version_compare(binary_version, v)):
+ raise Reject('Version check failed (binary={0}, version={1}, other-version={2}, suite={3})'.format(binary_name, binary_version, v, suite.suite_name))
def per_suite_check(self, upload, suite):
session = upload.session
must_be_newer_than.append(suite)
for s in must_be_newer_than:
- self._version_checks(upload, s, 1)
+ self._version_checks(upload, s, lambda result: result > 0)
vc_older = session.query(dbconn.VersionCheck).filter_by(suite=suite, check='MustBeOlderThan')
must_be_older_than = [ vc.reference for vc in vc_older ]
for s in must_be_older_than:
- self._version_checks(upload, s, -1)
+ self._version_checks(upload, s, lambda result: result < 0)
return True
self.value_list = self.Cnf.value_list
self.find = self.Cnf.find
self.find_b = self.Cnf.find_b
+ self.find_i = self.Cnf.find_i
def has_key(self, name):
return name in self.Cnf
overridetype.overridetype, component.component_name]
contents_writer = BinaryContentsWriter(suite, architecture, overridetype, component)
contents_writer.write_file()
+ session.close()
return log_message
def source_helper(suite_id, component_id):
log_message = [suite.suite_name, 'source', component.component_name]
contents_writer = SourceContentsWriter(suite, component)
contents_writer.write_file()
+ session.close()
return log_message
class ContentsWriter(object):
def fullpath(self):
return os.path.join(self.location.path, self.filename)
+ @property
+ def basename(self):
+ return os.path.basename(self.filename)
+
def is_valid(self, filesize = -1, md5sum = None):
return self.filesize == long(filesize) and self.md5sum == md5sum
else:
return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one()
+ @property
+ def path(self):
+ return os.path.join(self.archive.path, 'dists', self.suite_name)
+
__all__.append('Suite')
@session_wrapper
# Match binary packages
# Groups: package, version, architecture, type
-re_file_binary = re.compile(_re_file_prefix + r'_(?P<architecture>[a-z0-9]+)\.(?P<type>u?deb)$')
+re_file_binary = re.compile(_re_file_prefix + r'_(?P<architecture>[a-z0-9-]+)\.(?P<type>u?deb)$')
# Match changes files
# Groups: package, version, suffix
# Match source field
# Groups: package, version
-re_field_source = re.compile(r'^(?P<package>[a-z0-9][a-z0-9.+-]+)(:?\s*\((?P<version>[A-Za-z0-9.:~+-])\))?')
+re_field_source = re.compile(r'^(?P<package>[a-z0-9][a-z0-9.+-]+)(?:\s*\((?P<version>[A-Za-z0-9.:~+-]+)\))?$')
projects / dak.git / commitdiff