function cruft() {
log "Checking for cruft in overrides"
dak check-overrides
-
- log "Fixing symlinks in $ftpdir"
- symlinks -d -r $ftpdir
}
function msfl() {
function packages() {
log "Generating Packages and Sources files"
cd $configdir
+ GZIP='--rsyncable' ; export GZIP
apt-ftparchive generate apt.conf
}
)
stage $GO &
-GO=(
- FUNC="aptftpcleanup"
- TIME="apt-ftparchive cleanup"
- ARGS=""
- ERR="false"
-)
-stage $GO &
-
GO=(
FUNC="merkel3"
TIME="merkel ddaccessible sync"
ARGS=""
ERR=""
)
+stage $GO &
+
+GO=(
+ FUNC="aptftpcleanup"
+ TIME="apt-ftparchive cleanup"
+ ARGS=""
+ ERR="false"
+)
stage $GO
log "Daily cron scripts successful, all done"
echo "Update wanna-build database dump"
/org/ftp.debian.org/scripts/nfu/get-w-b-db
+echo "Fixing symlinks in $ftpdir"
+symlinks -d -r $ftpdir
+
echo "Finally, all is done, compressing logfile"
exec > /dev/null 2>&1
################################################################################
import sys
+import traceback
import daklib.utils
+from daklib.daklog import Logger
+from daklib.config import Config
+
################################################################################
def init():
def main():
"""Launch dak functionality."""
+ logger = Logger(Config(), 'dak top-level', print_starting=False)
+
functionality = init()
modules = [ command for (command, _) in functionality ]
# Invoke the module
module = __import__(cmdname.replace("-","_"))
- module.main()
+ try:
+ module.main()
+ except KeyboardInterrupt:
+ msg = 'KeyboardInterrupt caught; exiting'
+ print msg
+ logger.log([msg])
+ sys.exit(1)
+ except SystemExit:
+ pass
+ except:
+ for line in traceback.format_exc().split('\n')[:-1]:
+ logger.log(['exception', line])
+ raise
################################################################################
if not os.path.isdir(outdir):
os.mkdir(outdir)
- w = os.popen("diff --ed - %s | gzip -c -9 > %s.gz" %
+ w = os.popen("diff --ed - %s | gzip --rsyncable -c -9 > %s.gz" %
(newfile, difffile), "w")
pipe_file(oldf, w)
oldf.close()
u.check_distributions()
u.check_files(not Options["No-Action"])
valid_dsc_p = u.check_dsc(not Options["No-Action"])
- if valid_dsc_p:
+ if valid_dsc_p and not Options["No-Action"]:
u.check_source()
u.check_lintian()
u.check_hashes()
action(u)
- except SystemExit:
+ except (SystemExit, KeyboardInterrupt):
raise
except:
logfile = None
program = None
- def __init__ (self, Cnf, program, debug=0):
+ def __init__ (self, Cnf, program, debug=0, print_starting=True):
"Initialize a new Logger object"
self.Cnf = Cnf
self.program = program
logfile = utils.open_file(logfilename, 'a')
os.umask(umask)
self.logfile = logfile
- self.log(["program start"])
+ if print_starting:
+ self.log(["program start"])
def log (self, details):
"Log an event"
--- /dev/null
+from regexes import re_verwithext
+from dak_exceptions import UnknownFormatError
+
+def parse_format(txt):
+ """
+ Parse a .changes Format string into a tuple representation for easy
+ comparison.
+
+ >>> parse_format('1.0')
+ (1, 0)
+ >>> parse_format('8.4 (hardy)')
+ (8, 4, 'hardy')
+
+ If the format doesn't match these forms, raises UnknownFormatError.
+ """
+
+ format = re_verwithext.search(txt)
+
+ if format is None:
+ raise UnknownFormatError, txt
+
+ format = format.groups()
+
+ if format[1] is None:
+ format = int(float(format[0])), 0, format[2]
+ else:
+ format = int(format[0]), int(format[1]), format[2]
+
+ if format[2] is None:
+ format = format[:2]
+
+ return format
+
+def validate_changes_format(format, field):
+ """
+ Validate a tuple-representation of a .changes Format: field. Raises
+ UnknownFormatError if the field is invalid, otherwise return type is
+ undefined.
+ """
+
+ if (format < (1, 5) or format > (1, 8)):
+ raise UnknownFormatError, repr(format)
+
+ if field != 'files' and format < (1, 8):
+ raise UnknownFormatError, repr(format)
if not os.path.exists(src):
return
ftype = m.group(3)
- if re_is_orig_source.match(f) and pkg.orig_files.has_key(f) and \
- pkg.orig_files[f].has_key("path"):
+ if re_is_orig_source.match(f) and self.pkg.orig_files.has_key(f) and \
+ self.pkg.orig_files[f].has_key("path"):
continue
dest = os.path.join(os.getcwd(), f)
os.symlink(src, dest)
###########################################################################
def check_lintian(self):
+ cnf = Config()
+
# Only check some distributions
valid_dist = False
for dist in ('unstable', 'experimental'):
if not valid_dist:
return
- cnf = Config()
tagfile = cnf.get("Dinstall::LintianTags")
if tagfile is None:
# We don't have a tagfile, so just don't do anything.
return
+
# Parse the yaml file
sourcefile = file(tagfile, 'r')
sourcecontent = sourcefile.read()
utils.fubar("Can not read the lintian tags file %s, YAML error: %s." % (tagfile, msg))
return
+ # Try and find all orig mentioned in the .dsc
+ target_dir = '.'
+ symlinked = []
+ for filename, entry in self.pkg.dsc_files.iteritems():
+ if not re_is_orig_source.match(filename):
+ # File is not an orig; ignore
+ continue
+
+ if os.path.exists(filename):
+ # File exists, no need to continue
+ continue
+
+ def symlink_if_valid(path):
+ f = utils.open_file(path)
+ md5sum = apt_pkg.md5sum(f)
+ f.close()
+
+ fingerprint = (os.stat(path)[stat.ST_SIZE], md5sum)
+ expected = (int(entry['size']), entry['md5sum'])
+
+ if fingerprint != expected:
+ return False
+
+ dest = os.path.join(target_dir, filename)
+
+ os.symlink(path, dest)
+ symlinked.append(dest)
+
+ return True
+
+ session = DBConn().session()
+ found = False
+
+ # Look in the pool
+ for poolfile in get_poolfile_like_name('/%s' % filename, session):
+ poolfile_path = os.path.join(
+ poolfile.location.path, poolfile.filename
+ )
+
+ if symlink_if_valid(poolfile_path):
+ found = True
+ break
+
+ session.close()
+
+ if found:
+ continue
+
+ # Look in some other queues for the file
+ queues = ('Accepted', 'New', 'Byhand', 'ProposedUpdates',
+ 'OldProposedUpdates', 'Embargoed', 'Unembargoed')
+
+ for queue in queues:
+ if 'Dir::Queue::%s' % directory not in cnf:
+ continue
+
+ queuefile_path = os.path.join(
+ cnf['Dir::Queue::%s' % directory], filename
+ )
+
+ if not os.path.exists(queuefile_path):
+ # Does not exist in this queue
+ continue
+
+ if symlink_if_valid(queuefile_path):
+ break
+
# Now setup the input file for lintian. lintian wants "one tag per line" only,
# so put it together like it. We put all types of tags in one file and then sort
# through lintians output later to see if its a fatal tag we detected, or not.
# to then parse it.
command = "lintian --show-overrides --tags-from-file %s %s" % (temp_filename, self.pkg.changes_file)
(result, output) = commands.getstatusoutput(command)
- # We are done with lintian, remove our tempfile
+
+ # We are done with lintian, remove our tempfile and any symlinks we created
os.unlink(temp_filename)
+ for symlink in symlinked:
+ os.unlink(symlink)
+
if (result == 2):
utils.warn("lintian failed for %s [return code: %s]." % (self.pkg.changes_file, result))
utils.warn(utils.prefix_multi_line_string(output, " [possible output:] "))
if len(output) == 0:
return
+ def log(*txt):
+ if self.logger:
+ self.logger.log([self.pkg.changes_file, "check_lintian"] + list(txt))
+
# We have output of lintian, this package isn't clean. Lets parse it and see if we
# are having a victim for a reject.
# W: tzdata: binary-without-manpage usr/sbin/tzconfig
elif etag in lintiantags['error']:
# The tag is overriden - but is not allowed to be
self.rejects.append("%s: Overriden tag %s found, but this tag may not be overwritten." % (epackage, etag))
+ log("overidden tag is overridden", etag)
else:
# Tag is known, it is not overriden, direct reject.
self.rejects.append("%s: Found lintian output: '%s %s', automatically rejected package." % (epackage, etag, etext))
+ log("auto rejecting", etag)
# Now tell if they *might* override it.
if etag in lintiantags['warning']:
self.rejects.append("%s: If you have a good reason, you may override this lintian tag." % (epackage))
import re
-from regexes import re_verwithext
from dak_exceptions import UnknownFormatError
srcformats = []
raise UnknownFormatError, "Unknown format %r" % txt
-def parse_format(txt):
- """
- Parse a .changes Format string into a tuple representation for easy
- comparison.
-
- >>> parse_format('1.0')
- (1, 0)
- >>> parse_format('8.4 (hardy)')
- (8, 4, 'hardy')
-
- If the format doesn't match these forms, raises UnknownFormatError.
- """
-
- format = re_verwithext.search(txt)
-
- if format is None:
- raise UnknownFormatError, txt
-
- format = format.groups()
-
- if format[1] is None:
- format = int(float(format[0])), 0, format[2]
- else:
- format = int(format[0]), int(format[1]), format[2]
-
- if format[2] is None:
- format = format[:2]
-
- return format
-
class SourceFormat(type):
def __new__(cls, name, bases, attrs):
klass = super(SourceFormat, cls).__new__(cls, name, bases, attrs)
if has[key]:
yield "contains source files not allowed in format %s" % cls.name
- @classmethod
- def validate_format(cls, format, is_a_dsc=False, field='files'):
- """
- Raises UnknownFormatError if the specified format tuple is not valid for
- this format (for example, the format (1, 0) is not valid for the
- "3.0 (quilt)" format). Return value is undefined in all other cases.
- """
- pass
-
class FormatOne(SourceFormat):
__metaclass__ = SourceFormat
for msg in super(FormatOne, cls).reject_msgs(has):
yield msg
- @classmethod
- def validate_format(cls, format, is_a_dsc=False, field='files'):
- msg = "Invalid format %s definition: %r" % (cls.name, format)
-
- if is_a_dsc:
- if format != (1, 0):
- raise UnknownFormatError, msg
- else:
- if (format < (1,5) or format > (1,8)):
- raise UnknownFormatError, msg
- if field != "files" and format < (1,8):
- raise UnknownFormatError, msg
-
class FormatThree(SourceFormat):
__metaclass__ = SourceFormat
requires = ('native_tar',)
disallowed = ('orig_tar', 'debian_diff', 'debian_tar', 'more_orig_tar')
- @classmethod
- def validate_format(cls, format, **kwargs):
- if format != (3, 0, 'native'):
- raise UnknownFormatError, "Invalid format %s definition: %r" % \
- (cls.name, format)
-
class FormatThreeQuilt(SourceFormat):
__metaclass__ = SourceFormat
requires = ('orig_tar', 'debian_tar')
disallowed = ('debian_diff', 'native_tar')
-
- @classmethod
- def validate_format(cls, format, **kwargs):
- if format != (3, 0, 'quilt'):
- raise UnknownFormatError, "Invalid format %s definition: %r" % \
- (cls.name, format)
re_multi_line_field, re_srchasver, re_taint_free, \
re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource
+from formats import parse_format, validate_changes_format
from srcformats import get_format_from_string
from collections import defaultdict
if not changes.has_key(field):
raise NoFilesFieldError
- # Get SourceFormat object for this Format and validate it
- format = get_format_from_string(changes.get['format'])
- format.validate_format(is_a_dsc=is_a_dsc, field=field)
+ # Validate .changes Format: field
+ if not is_a_dsc:
+ validate_changes_format(parse_format(changes['format']), field)
includes_section = (not is_a_dsc) and field == "files"
if which_conf_file() != default_config:
apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
-
-###############################################################################
<dak> mhy: Error: "!!!11111iiiiiioneoneoneone" is not a valid command.
<mhy> dak: oh shut up
<dak> mhy: Error: "oh" is not a valid command.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+<sgran> hey, I think something's wrong with your git repo
+<sgran> when I git pulled this last time, I got something that looked almost like python instead of dak
+<mhy> sgran: slander
+<sgran> sorry, I take it back, I've had a better look now
find "$TARGET/$VERSION" -type d -exec chmod 755 {} +
find "$TARGET/$VERSION" -type f -exec chmod 644 {} +
+# Make sure nothing symlinks outside of the ftpdir
+# Shouldnt happen, but better be sure.
+symlinks -d -r /srv/ftp.debian.org/ftp
+
trap - EXIT
cleanup
cd $base/ftp
find ./dists -maxdepth 1 \! -type d
find ./dists \! -type d | grep "/source/"
-) | sort -u | gzip -9 > source.list.gz
+) | sort -u | gzip --rsyncable -9 > source.list.gz
echo "Generating arch lists..."
cd $base/ftp
find ./dists -maxdepth 1 \! -type d
find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
- ) | sort -u | gzip -9 > arch-$a.list.gz
+ ) | sort -u | gzip --rsyncable -9 > arch-$a.list.gz
done
echo "Generating suite lists..."
done
)
suite_list $id | tr -d ' ' | sed 's,^/srv/ftp.debian.org/ftp,.,'
- ) | sort -u | gzip -9 > suite-${suite}.list.gz
+ ) | sort -u | gzip --rsyncable -9 > suite-${suite}.list.gz
done
echo "Finding everything on the ftp site to generate sundries $(date +"%X")..."
(cd $base/ftp/
for dist in sid squeeze; do
- find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
+ find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip --rsyncable -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
done
)
mv -f ${filename}.gz $filename.old.gz
mv -f .$filename.new $filename
rm -f $filename.patch.gz
- zcat $filename.old.gz | diff -u - $filename | gzip -9cfn - >$filename.patch.gz
+ zcat $filename.old.gz | diff -u - $filename | gzip --rsyncable -9cfn - >$filename.patch.gz
rm -f $filename.old.gz
else
mv -f .$filename.new $filename
fi
-gzip -9cfN $filename >$filename.gz
+gzip --rsyncable -9cfN $filename >$filename.gz
rm -f $filename
if [ $rc = 1 ] || [ ! -f Maintainers ] ; then
echo -n "installing Maintainers ... "
mv -f .new-maintainers Maintainers
- gzip -9v <Maintainers >.new-maintainers.gz
+ gzip --rsyncable -9v <Maintainers >.new-maintainers.gz
mv -f .new-maintainers.gz Maintainers.gz
elif [ $rc = 0 ] ; then
echo '(same as before)'
--- /dev/null
+#!/usr/bin/env python
+
+import unittest
+
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from daklib.formats import parse_format, validate_changes_format
+from daklib.dak_exceptions import UnknownFormatError
+
+class ParseFormatTestCase(unittest.TestCase):
+ def assertParse(self, format, expected):
+ self.assertEqual(parse_format(format), expected)
+
+ def assertParseFail(self, format):
+ self.assertRaises(
+ UnknownFormatError,
+ lambda: parse_format(format)
+ )
+
+ def testParse(self):
+ self.assertParse('1.0', (1, 0))
+
+ def testEmpty(self):
+ self.assertParseFail('')
+ self.assertParseFail(' ')
+ self.assertParseFail(' ')
+
+ def textText(self):
+ self.assertParse('1.2 (three)', (1, 2, 'three'))
+ self.assertParseFail('0.0 ()')
+
+class ValidateChangesFormat(unittest.TestCase):
+ def assertValid(self, changes, field='files'):
+ validate_changes_format(changes, field)
+
+ def assertInvalid(self, *args, **kwargs):
+ self.assertRaises(
+ UnknownFormatError,
+ lambda: self.assertValid(*args, **kwargs)
+ )
+
+ ##
+
+ def testBinary(self):
+ self.assertValid((1, 5))
+ self.assertValid((1, 8))
+ self.assertInvalid((1, 0))
+
+ def testRange(self):
+ self.assertInvalid((1, 3))
+ self.assertValid((1, 5))
+ self.assertValid((1, 8))
+ self.assertInvalid((1, 9))
+
+ def testFilesField(self):
+ self.assertInvalid((1, 7), field='notfiles')
+ self.assertValid((1, 8), field='notfiles')
from collections import defaultdict
from daklib import srcformats
+from daklib.formats import parse_format
from daklib.dak_exceptions import UnknownFormatError
class SourceFormatTestCase(unittest.TestCase):
'native_tar': 1,
})
-##
-
-class ParseFormatTestCase(unittest.TestCase):
- def assertParse(self, format, expected):
- self.assertEqual(srcformats.parse_format(format), expected)
-
- def assertParseFail(self, format):
- self.assertRaises(
- UnknownFormatError,
- lambda: srcformats.parse_format(format)
- )
-
- def testParse(self):
- self.assertParse('1.0', (1, 0))
-
- def testEmpty(self):
- self.assertParseFail('')
- self.assertParseFail(' ')
- self.assertParseFail(' ')
-
- def textText(self):
- self.assertParse('1.2 (three)', (1, 2, 'three'))
- self.assertParseFail('0.0 ()')
-
-class ValidateFormatTestCase(unittest.TestCase):
- def assertValid(self, format, **kwargs):
- kwargs['is_a_dsc'] = kwargs.get('is_a_dsc', True)
- self.fmt.validate_format(format, **kwargs)
-
- def assertInvalid(self, *args, **kwargs):
- self.assertRaises(
- UnknownFormatError,
- lambda: self.assertValid(*args, **kwargs),
- )
-
-class ValidateFormatOneTestCase(ValidateFormatTestCase):
- fmt = srcformats.FormatOne
-
- def testValid(self):
- self.assertValid((1, 0))
-
- def testInvalid(self):
- self.assertInvalid((0, 1))
- self.assertInvalid((3, 0, 'quilt'))
-
- ##
-
- def testBinary(self):
- self.assertValid((1, 5), is_a_dsc=False)
- self.assertInvalid((1, 0), is_a_dsc=False)
-
- def testRange(self):
- self.assertInvalid((1, 3), is_a_dsc=False)
- self.assertValid((1, 5), is_a_dsc=False)
- self.assertValid((1, 8), is_a_dsc=False)
- self.assertInvalid((1, 9), is_a_dsc=False)
-
- def testFilesField(self):
- self.assertInvalid((1, 7), is_a_dsc=False, field='notfiles')
- self.assertValid((1, 8), is_a_dsc=False, field='notfiles')
-
-class ValidateFormatThreeTestCase(ValidateFormatTestCase):
- fmt = srcformats.FormatThree
-
- def testValid(self):
- self.assertValid((3, 0, 'native'))
-
- def testInvalid(self):
- self.assertInvalid((1, 0))
- self.assertInvalid((0, 0))
- self.assertInvalid((3, 0, 'quilt'))
-
-class ValidateFormatThreeQuiltTestCase(ValidateFormatTestCase):
- fmt = srcformats.FormatThreeQuilt
-
- def testValid(self):
- self.assertValid((3, 0, 'quilt'))
-
- def testInvalid(self):
- self.assertInvalid((1, 0))
- self.assertInvalid((0, 0))
- self.assertInvalid((3, 0, 'native'))
-
class FormatFromStringTestCase(unittest.TestCase):
def assertFormat(self, txt, klass):
self.assertEqual(srcformats.get_format_from_string(txt), klass)
projects / dak.git / commitdiff