From: Joerg Jaspert Date: Wed, 28 Oct 2009 16:56:00 +0000 (+0100) Subject: Merge commit 'lamby/master' into merge X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=d0779ef69d429a210405002f252067806ddfa6f2;hp=a8a6edfad74150868322cdc57e18b7454a584fcf;p=dak.git Merge commit 'lamby/master' into merge * commit 'lamby/master': Drop the SourceFormat-specific validate_format classmethods. Don't validate a .changes Format as a source package version format (!) Add validate_changes_format utility. Move parse_format tests to test_formats.py Move parse_format into daklib.formats Remove X-Katie: headers Updated dinstall Background half of dinstall Dont do work without a tagfile Signed-off-by: Joerg Jaspert --- diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall index 5a8f1073..1c9fa5af 100755 --- a/config/debian/cron.dinstall +++ b/config/debian/cron.dinstall @@ -355,9 +355,12 @@ function merkel3() { ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1 } -function runparts() { - log "Using run-parts to run scripts in $base/scripts/distmnt" - run-parts --report $base/scripts/distmnt +function mirrorpush() { + log "Starting the mirrorpush" + date -u > /srv/ftp.debian.org/web/mirrorstart + echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart + echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart + sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 & } function i18n2() { @@ -638,7 +641,7 @@ GO=( ARGS="" ERR="false" ) -stage $GO +stage $GO & GO=( FUNC="punew" @@ -662,7 +665,7 @@ GO=( ARGS="" ERR="false" ) -stage $GO +stage $GO & lockfile "$LOCK_ACCEPTED" lockfile "$LOCK_NEW" @@ -717,7 +720,7 @@ GO=( ARGS="" ERR="false" ) -stage $GO +stage $GO & GO=( FUNC="overrides" @@ -797,7 +800,7 @@ GO=( ARGS="" ERR="" ) -stage $GO +stage $GO & rm -f "${NOTICE}" rm -f "${LOCK_DAILY}" @@ -810,7 +813,7 @@ GO=( ARGS="" ERR="" ) -stage $GO +stage $GO & GO=( FUNC="expire" @@ -818,7 +821,7 @@ GO=( ARGS="" ERR="" ) -stage $GO +stage $GO & GO=( FUNC="transitionsclean" @@ -826,7 +829,7 @@ GO=( ARGS="" ERR="" ) -stage $GO +stage $GO & GO=( FUNC="reports" @@ -834,7 +837,7 @@ GO=( ARGS="" ERR="" ) -stage $GO +stage $GO & GO=( FUNC="dm" @@ -842,7 +845,7 @@ GO=( ARGS="" ERR="" ) -stage $GO +stage $GO & GO=( FUNC="bts" @@ -850,7 +853,7 @@ GO=( ARGS="" ERR="false" ) -stage $GO +stage $GO & GO=( FUNC="merkel2" @@ -858,11 +861,11 @@ GO=( ARGS="" ERR="false" ) -stage $GO +stage $GO & GO=( - FUNC="runparts" - TIME="run-parts" + FUNC="mirrorpush" + TIME="mirrorpush" ARGS="" ERR="false" ) @@ -882,7 +885,7 @@ GO=( ARGS="" ERR="false" ) -stage $GO +stage $GO & GO=( FUNC="testingsourcelist" @@ -908,7 +911,7 @@ GO=( ARGS="" ERR="false" ) -stage $GO +stage $GO & GO=( FUNC="merkel3" @@ -916,7 +919,7 @@ GO=( ARGS="" ERR="false" ) -stage $GO +stage $GO & GO=( FUNC="compress" diff --git a/dak/override.py b/dak/override.py index 413c3447..e2539677 100755 --- a/dak/override.py +++ b/dak/override.py @@ -240,7 +240,7 @@ def main (): Subst["__BCC__"] = "Bcc: " + ", ".join(bcc) else: Subst["__BCC__"] = "X-Filler: 42" - Subst["__CC__"] = "Cc: " + package + "@" + Cnf["Dinstall::PackagesServer"] + "\nX-DAK: dak override\nX-Katie: alicia" + Subst["__CC__"] = "Cc: " + package + "@" + Cnf["Dinstall::PackagesServer"] + "\nX-DAK: dak override" Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"] Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"] Subst["__WHOAMI__"] = utils.whoami() diff --git a/dak/process_accepted.py b/dak/process_accepted.py index b18346c8..b203f498 100755 --- a/dak/process_accepted.py +++ b/dak/process_accepted.py @@ -582,7 +582,7 @@ def stable_install(u, session, summary, short_summary, fromsuite_name="proposed- if not Options["No-Mail"] and u.pkg.changes["architecture"].has_key("source"): u.Subst["__SUITE__"] = " into %s" % (tosuite) u.Subst["__SUMMARY__"] = summary - u.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $" + u.Subst["__BCC__"] = "X-DAK: dak process-accepted" if cnf.has_key("Dinstall::Bcc"): u.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"]) diff --git a/dak/process_new.py b/dak/process_new.py index 1423d705..9a6c8e33 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -975,7 +975,7 @@ def do_pkg(changes_file, session): u.update_subst() cnf = Config() - bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $" + bcc = "X-DAK: dak process-new" if cnf.has_key("Dinstall::Bcc"): u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) else: diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py index f7caf303..5463f1a6 100755 --- a/dak/process_unchecked.py +++ b/dak/process_unchecked.py @@ -455,7 +455,7 @@ def process_it(changes_file): u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"] # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header - bcc = "X-DAK: dak process-unchecked\nX-Katie: $Revision: 1.65 $" + bcc = "X-DAK: dak process-unchecked" if cnf.has_key("Dinstall::Bcc"): u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) else: diff --git a/dak/rm.py b/dak/rm.py index 51400194..be3e1676 100755 --- a/dak/rm.py +++ b/dak/rm.py @@ -550,7 +550,7 @@ def main (): Subst["__BCC__"] = "Bcc: " + ", ".join(bcc) else: Subst["__BCC__"] = "X-Filler: 42" - Subst["__CC__"] = "X-DAK: dak rm\nX-Katie: melanie" + Subst["__CC__"] = "X-DAK: dak rm" if carbon_copy: Subst["__CC__"] += "\nCc: " + ", ".join(carbon_copy) Subst["__SUITE_LIST__"] = suites_list diff --git a/daklib/formats.py b/daklib/formats.py new file mode 100644 index 00000000..aaad2715 --- /dev/null +++ b/daklib/formats.py @@ -0,0 +1,45 @@ +from regexes import re_verwithext +from dak_exceptions import UnknownFormatError + +def parse_format(txt): + """ + Parse a .changes Format string into a tuple representation for easy + comparison. + + >>> parse_format('1.0') + (1, 0) + >>> parse_format('8.4 (hardy)') + (8, 4, 'hardy') + + If the format doesn't match these forms, raises UnknownFormatError. + """ + + format = re_verwithext.search(txt) + + if format is None: + raise UnknownFormatError, txt + + format = format.groups() + + if format[1] is None: + format = int(float(format[0])), 0, format[2] + else: + format = int(format[0]), int(format[1]), format[2] + + if format[2] is None: + format = format[:2] + + return format + +def validate_changes_format(format, field): + """ + Validate a tuple-representation of a .changes Format: field. Raises + UnknownFormatError if the field is invalid, otherwise return type is + undefined. + """ + + if (format < (1, 5) or format > (1, 8)): + raise UnknownFormatError, repr(format) + + if field != 'files' and format < (1, 8): + raise UnknownFormatError, repr(format) diff --git a/daklib/queue.py b/daklib/queue.py index 0c79a260..f7d999a3 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -1225,7 +1225,10 @@ class Upload(object): self.ensure_all_source_exists() cnf = Config() - tagfile = cnf["Dinstall::LintianTags"] + tagfile = cnf.get("Dinstall::LintianTags") + if tagfile is None: + # We don't have a tagfile, so just don't do anything. + return # Parse the yaml file sourcefile = file(tagfile, 'r') sourcecontent = sourcefile.read() @@ -1932,7 +1935,7 @@ distribution.""" if not manual: self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] self.Subst["__MANUAL_REJECT_MESSAGE__"] = "" - self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)" + self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)" os.write(reason_fd, reject_message) reject_mail_message = utils.TemplateSubst(self.Subst, rej_template) else: @@ -2350,7 +2353,7 @@ distribution.""" self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] self.Subst["__REJECT_MESSAGE__"] = self.package_info() self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"] - self.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $" + self.Subst["__BCC__"] = "X-DAK: dak process-accepted" if cnf.has_key("Dinstall::Bcc"): self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"]) diff --git a/daklib/srcformats.py b/daklib/srcformats.py index ade3c453..7d7dd940 100644 --- a/daklib/srcformats.py +++ b/daklib/srcformats.py @@ -1,6 +1,5 @@ import re -from regexes import re_verwithext from dak_exceptions import UnknownFormatError srcformats = [] @@ -18,36 +17,6 @@ def get_format_from_string(txt): raise UnknownFormatError, "Unknown format %r" % txt -def parse_format(txt): - """ - Parse a .changes Format string into a tuple representation for easy - comparison. - - >>> parse_format('1.0') - (1, 0) - >>> parse_format('8.4 (hardy)') - (8, 4, 'hardy') - - If the format doesn't match these forms, raises UnknownFormatError. - """ - - format = re_verwithext.search(txt) - - if format is None: - raise UnknownFormatError, txt - - format = format.groups() - - if format[1] is None: - format = int(float(format[0])), 0, format[2] - else: - format = int(format[0]), int(format[1]), format[2] - - if format[2] is None: - format = format[:2] - - return format - class SourceFormat(type): def __new__(cls, name, bases, attrs): klass = super(SourceFormat, cls).__new__(cls, name, bases, attrs) @@ -70,15 +39,6 @@ class SourceFormat(type): if has[key]: yield "contains source files not allowed in format %s" % cls.name - @classmethod - def validate_format(cls, format, is_a_dsc=False, field='files'): - """ - Raises UnknownFormatError if the specified format tuple is not valid for - this format (for example, the format (1, 0) is not valid for the - "3.0 (quilt)" format). Return value is undefined in all other cases. - """ - pass - class FormatOne(SourceFormat): __metaclass__ = SourceFormat @@ -101,19 +61,6 @@ class FormatOne(SourceFormat): for msg in super(FormatOne, cls).reject_msgs(has): yield msg - @classmethod - def validate_format(cls, format, is_a_dsc=False, field='files'): - msg = "Invalid format %s definition: %r" % (cls.name, format) - - if is_a_dsc: - if format != (1, 0): - raise UnknownFormatError, msg - else: - if (format < (1,5) or format > (1,8)): - raise UnknownFormatError, msg - if field != "files" and format < (1,8): - raise UnknownFormatError, msg - class FormatThree(SourceFormat): __metaclass__ = SourceFormat @@ -123,12 +70,6 @@ class FormatThree(SourceFormat): requires = ('native_tar',) disallowed = ('orig_tar', 'debian_diff', 'debian_tar', 'more_orig_tar') - @classmethod - def validate_format(cls, format, **kwargs): - if format != (3, 0, 'native'): - raise UnknownFormatError, "Invalid format %s definition: %r" % \ - (cls.name, format) - class FormatThreeQuilt(SourceFormat): __metaclass__ = SourceFormat @@ -137,9 +78,3 @@ class FormatThreeQuilt(SourceFormat): requires = ('orig_tar', 'debian_tar') disallowed = ('debian_diff', 'native_tar') - - @classmethod - def validate_format(cls, format, **kwargs): - if format != (3, 0, 'quilt'): - raise UnknownFormatError, "Invalid format %s definition: %r" % \ - (cls.name, format) diff --git a/daklib/utils.py b/daklib/utils.py index 3305f695..b0b71c01 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -48,6 +48,7 @@ from regexes import re_html_escaping, html_escaping, re_single_line_field, \ re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource, \ re_is_orig_source +from formats import parse_format, validate_changes_format from srcformats import get_format_from_string from collections import defaultdict @@ -527,9 +528,8 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): if not changes.has_key(field): raise NoFilesFieldError - # Get SourceFormat object for this Format and validate it - format = get_format_from_string(changes['format']) - format.validate_format(is_a_dsc=is_a_dsc, field=field) + # Validate .changes Format: field + validate_changes_format(parse_format(changes['format']), field) includes_section = (not is_a_dsc) and field == "files" diff --git a/docs/TODO b/docs/TODO index 5a2b885f..8edd4e4e 100644 --- a/docs/TODO +++ b/docs/TODO @@ -4,16 +4,7 @@ Various ------- -* Lintian based automated rejects - - Have a set of lintian tags each package *must* not have. If it does - -> reject. - - If a tag is overriden by the maintainer, do not reject, but put it - into NEW. If the override is ok note that in a table and dont act on - it for any future uploads of this package anymore. - - possibly have two classes of tags. one for "shouldnt happen by - accident" and one "shouldnt happen". the first gets ignored from us - if overwritten in the package, the second only us can overwrite. - - its a suite option in dak, not active for all at once. +* Implement autosigning, see ftpmaster_autosigning on ftp-master host in text/. * Throw away all DD uploaded .debs. (Depend on "Lintian based automated rejects") @@ -27,8 +18,6 @@ Various - its a suite option, not active for all at once. - should have all buildd machines under dsa control -* Implement autosigning, see ftpmaster_autosigning on ftp-master host in text/. - * Check TODO.old and move still-valid/useful entries over here. * need a testsuite _badly_ @@ -47,10 +36,6 @@ Various - needs updateX.py written and then the rest of the code changed to deal with it. -* Checkout SQL Alchemy and probably use that for our database layer. - -* reject on > or < in a version constraint - * use pythonX.Y-tarfile to check orig.tar.gz timestamps too. * the .dak stuff is fundamentally braindamaged for various reasons, it diff --git a/tests/test_formats.py b/tests/test_formats.py new file mode 100755 index 00000000..1ae6860a --- /dev/null +++ b/tests/test_formats.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +import unittest + +import os, sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from daklib.formats import parse_format, validate_changes_format +from daklib.dak_exceptions import UnknownFormatError + +class ParseFormatTestCase(unittest.TestCase): + def assertParse(self, format, expected): + self.assertEqual(parse_format(format), expected) + + def assertParseFail(self, format): + self.assertRaises( + UnknownFormatError, + lambda: parse_format(format) + ) + + def testParse(self): + self.assertParse('1.0', (1, 0)) + + def testEmpty(self): + self.assertParseFail('') + self.assertParseFail(' ') + self.assertParseFail(' ') + + def textText(self): + self.assertParse('1.2 (three)', (1, 2, 'three')) + self.assertParseFail('0.0 ()') + +class ValidateChangesFormat(unittest.TestCase): + def assertValid(self, changes, field='files'): + validate_changes_format(changes, field) + + def assertInvalid(self, *args, **kwargs): + self.assertRaises( + UnknownFormatError, + lambda: self.assertValid(*args, **kwargs) + ) + + ## + + def testBinary(self): + self.assertValid((1, 5)) + self.assertValid((1, 8)) + self.assertInvalid((1, 0)) + + def testRange(self): + self.assertInvalid((1, 3)) + self.assertValid((1, 5)) + self.assertValid((1, 8)) + self.assertInvalid((1, 9)) + + def testFilesField(self): + self.assertInvalid((1, 7), field='notfiles') + self.assertValid((1, 8), field='notfiles') diff --git a/tests/test_srcformats.py b/tests/test_srcformats.py index f6d7215f..4ecaf8b7 100755 --- a/tests/test_srcformats.py +++ b/tests/test_srcformats.py @@ -8,6 +8,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from collections import defaultdict from daklib import srcformats +from daklib.formats import parse_format from daklib.dak_exceptions import UnknownFormatError class SourceFormatTestCase(unittest.TestCase): @@ -103,89 +104,6 @@ class FormatTreeQuiltTestCase(SourceFormatTestCase): 'native_tar': 1, }) -## - -class ParseFormatTestCase(unittest.TestCase): - def assertParse(self, format, expected): - self.assertEqual(srcformats.parse_format(format), expected) - - def assertParseFail(self, format): - self.assertRaises( - UnknownFormatError, - lambda: srcformats.parse_format(format) - ) - - def testParse(self): - self.assertParse('1.0', (1, 0)) - - def testEmpty(self): - self.assertParseFail('') - self.assertParseFail(' ') - self.assertParseFail(' ') - - def textText(self): - self.assertParse('1.2 (three)', (1, 2, 'three')) - self.assertParseFail('0.0 ()') - -class ValidateFormatTestCase(unittest.TestCase): - def assertValid(self, format, **kwargs): - kwargs['is_a_dsc'] = kwargs.get('is_a_dsc', True) - self.fmt.validate_format(format, **kwargs) - - def assertInvalid(self, *args, **kwargs): - self.assertRaises( - UnknownFormatError, - lambda: self.assertValid(*args, **kwargs), - ) - -class ValidateFormatOneTestCase(ValidateFormatTestCase): - fmt = srcformats.FormatOne - - def testValid(self): - self.assertValid((1, 0)) - - def testInvalid(self): - self.assertInvalid((0, 1)) - self.assertInvalid((3, 0, 'quilt')) - - ## - - def testBinary(self): - self.assertValid((1, 5), is_a_dsc=False) - self.assertInvalid((1, 0), is_a_dsc=False) - - def testRange(self): - self.assertInvalid((1, 3), is_a_dsc=False) - self.assertValid((1, 5), is_a_dsc=False) - self.assertValid((1, 8), is_a_dsc=False) - self.assertInvalid((1, 9), is_a_dsc=False) - - def testFilesField(self): - self.assertInvalid((1, 7), is_a_dsc=False, field='notfiles') - self.assertValid((1, 8), is_a_dsc=False, field='notfiles') - -class ValidateFormatThreeTestCase(ValidateFormatTestCase): - fmt = srcformats.FormatThree - - def testValid(self): - self.assertValid((3, 0, 'native')) - - def testInvalid(self): - self.assertInvalid((1, 0)) - self.assertInvalid((0, 0)) - self.assertInvalid((3, 0, 'quilt')) - -class ValidateFormatThreeQuiltTestCase(ValidateFormatTestCase): - fmt = srcformats.FormatThreeQuilt - - def testValid(self): - self.assertValid((3, 0, 'quilt')) - - def testInvalid(self): - self.assertInvalid((1, 0)) - self.assertInvalid((0, 0)) - self.assertInvalid((3, 0, 'native')) - class FormatFromStringTestCase(unittest.TestCase): def assertFormat(self, txt, klass): self.assertEqual(srcformats.get_format_from_string(txt), klass)