From e2593d978b225881993196434b8a0bbe4802837b Mon Sep 17 00:00:00 2001 From: Matthias Klumpp Date: Sat, 5 Dec 2015 23:09:46 +0100 Subject: [PATCH] Include AppStream metadata from the generator server This patch pulls AppStream/DEP-11 metadata from the server where it was generated, then validates it for format issues and adds it to the archive. --- config/debian/cron.dinstall | 8 + config/debian/dinstall.functions | 26 ++++ config/homedir/ssh/ftpmaster-config | 5 + daklib/regexes.py | 2 +- scripts/debian/dep11-basic-validate.py | 199 +++++++++++++++++++++++++ 5 files changed, 239 insertions(+), 1 deletion(-) create mode 100755 scripts/debian/dep11-basic-validate.py diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall index 4e8c1f28..8aaaa648 100755 --- a/config/debian/cron.dinstall +++ b/config/debian/cron.dinstall @@ -210,6 +210,14 @@ GO=( ) stage $GO +GO=( + FUNC="dep11" + TIME="dep11 1" + ARGS="" + ERR="false" +) +stage $GO + lockfile "$LOCK_ACCEPTED" trap remove_all_locks EXIT TERM HUP INT QUIT diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index ae621185..f43a0aa8 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -97,6 +97,32 @@ function i18n1() { fi } +# Syncing AppStream/DEP-11 data +function dep11() { + log "Synchronizing AppStream metadata" + # First sync their newest data + mkdir -p ${scriptdir}/dep11 + cd ${scriptdir}/dep11 + rsync -aq --delete --delete-after dep11-sync:/does/not/matter . || true + + # Lets check! + if ${scriptsdir}/dep11-basic-validate.py . ${scriptdir}/dep11/; then + # Yay, worked, lets copy around + for dir in stretch sid; do + if [ -d ${dir}/ ]; then + for comp in main contrib non-free; do + cd dists/${dir}/${comp}/dep11 + rsync -aq --delete --delete-after --exclude *.tmp . ${ftpdir}/dists/${dir}/${comp}/dep11/. + cd ${scriptdir}/dep11 + fi + fi + done + else + echo "ARRRR, bad guys, wrong files, ARRR" + echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters " mak@debian.org + fi +} + function cruft() { log "Checking for cruft in overrides" dak check-overrides diff --git a/config/homedir/ssh/ftpmaster-config b/config/homedir/ssh/ftpmaster-config index 222338a5..083f81f6 100644 --- a/config/homedir/ssh/ftpmaster-config +++ b/config/homedir/ssh/ftpmaster-config @@ -17,6 +17,11 @@ Host ddtp-sync User ddtp-dak IdentityFile /srv/ftp-master.debian.org/s3kr1t/ddtp-dak.rsa +Host dep11-sync + Hostname mekeel.debian.org + User appstream + IdentityFile /srv/ftp-master.debian.org/s3kr1t/appstream.rsa + Host morgue-sync Hostname stabile.debian.org User dak diff --git a/daklib/regexes.py b/daklib/regexes.py index 21defe93..0a2045d0 100644 --- a/daklib/regexes.py +++ b/daklib/regexes.py @@ -102,7 +102,7 @@ re_parse_lintian = re.compile(r"^(?PW|E|O): (?P.*?): (?P[^ # in generate-releases re_gensubrelease = re.compile (r".*/(binary-[0-9a-z-]+|source)$") -re_includeinrelease = re.compile (r"(Translation-[a-zA-Z_]+\.(?:bz2|xz)|Contents-[0-9a-z-]+.gz|Index|Packages(.gz|.bz2|.xz)?|Sources(.gz|.bz2|.xz)?|MD5SUMS|SHA256SUMS|Release)$") +re_includeinrelease = re.compile (r"(Translation-[a-zA-Z_]+\.(?:bz2|xz)|Contents-[0-9a-z-]+.gz|Index|Packages(.gz|.bz2|.xz)?|Sources(.gz|.bz2|.xz)?|Components-[0-9a-z-]+(.gz|.xz)|icons-[0-9x-]+(.gz|.xz)|MD5SUMS|SHA256SUMS|Release)$") # in generate_index_diffs re_includeinpdiff = re.compile(r"(Translation-[a-zA-Z_]+\.(?:bz2|xz))") diff --git a/scripts/debian/dep11-basic-validate.py b/scripts/debian/dep11-basic-validate.py new file mode 100755 index 00000000..6f46e14b --- /dev/null +++ b/scripts/debian/dep11-basic-validate.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2015 Matthias Klumpp +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program. + +import os +import sys +import yaml +import gzip +from voluptuous import Schema, Required, All, Any, Length, Range, Match, Url +from optparse import OptionParser + +schema_header = Schema({ + Required('File'): All(str, 'DEP-11', msg="Must be \"DEP-11\""), + Required('Origin'): All(str, Length(min=1)), + Required('Version'): All(str, Match(r'(\d+\.?)+$'), msg="Must be a valid version number"), + Required('MediaBaseUrl'): All(str, Url()), + 'Time': All(str, str), + 'Priority': All(str, int), +}) + +schema_translated = Schema({ + Required('C'): All(str, Length(min=1), msg="Must have an unlocalized 'C' key"), + dict: All(str, Length(min=1)), +}, extra = True) + +schema_component = Schema({ + Required('Type'): All(str, Length(min=1)), + Required('ID'): All(str, Length(min=1)), + Required('Name'): All(dict, Length(min=1), schema_translated), + Required('Package'): All(str, Length(min=1)), +}, extra = True) + +def add_issue(msg): + print(msg) + +def test_custom_objects(lines): + ret = True + for i in range(0, len(lines)): + if "!!python/" in lines[i]: + add_issue("Python object encoded in line %i." % (i)) + ret = False + return ret + +def is_quoted(s): + return (s.startswith("\"") and s.endswith("\"")) or (s.startswith("\'") and s.endswith("\'")) + +def test_localized_dict(doc, ldict, id_string): + ret = True + for lang, value in ldict.items(): + if lang == 'x-test': + add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "Found cruft locale: x-test")) + if lang == 'xx': + add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "Found cruft locale: xx")) + if lang.endswith('.UTF-8'): + add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "AppStream locale names should not specify encoding (ends with .UTF-8)")) + if is_quoted(value): + add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "String is quoted: '%s' @ %s" % (value, lang))) + if " " in lang: + add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "Locale name contains space: '%s'" % (lang))) + # this - as opposed to the other issues - is an error + ret = False + return ret + +def test_localized(doc, key): + ldict = doc.get(key, None) + if not ldict: + return True + + return test_localized_dict(doc, ldict, key) + +def validate_data(data): + ret = True + lines = data.split("\n") + + # see if there are any Python-specific objects encoded + ret = test_custom_objects(lines) + + try: + docs = yaml.safe_load_all(data) + header = next(docs) + except Exception as e: + add_issue("Could not parse file: %s" % (str(e))) + return False + + try: + schema_header(header) + except Exception as e: + add_issue("Invalid DEP-11 header: %s" % (str(e))) + ret = False + + for doc in docs: + docid = doc.get('ID') + pkgname = doc.get('Package') + if not pkgname: + pkgname = "?unknown?" + if not doc: + add_issue("FATAL: Empty document found.") + ret = False + continue + if not docid: + add_issue("FATAL: Component without ID found.") + ret = False + continue + + try: + schema_component(doc) + except Exception as e: + add_issue("[%s]: %s" % (docid, str(e))) + ret = False + continue + + # more tests for the icon key + icon = doc.get('Icon') + if (doc['Type'] == "desktop-app") or (doc['Type'] == "web-app"): + if not doc.get('Icon'): + add_issue("[%s]: %s" % (docid, "Components containing an application must have an 'Icon' key.")) + ret = False + if icon: + if (not icon.get('stock')) and (not icon.get('cached')) and (not icon.get('local')): + add_issue("[%s]: %s" % (docid, "A 'stock', 'cached' or 'local' icon must at least be provided. @ data['Icon']")) + ret = False + + if not test_localized(doc, 'Name'): + ret = False + if not test_localized(doc, 'Summary'): + ret = False + if not test_localized(doc, 'Description'): + ret = False + if not test_localized(doc, 'DeveloperName'): + ret = False + + for shot in doc.get('Screenshots', list()): + caption = shot.get('caption') + if caption: + if not test_localized_dict(doc, caption, "Screenshots.x.caption"): + ret = False + + return ret + +def validate_file(fname): + f = None + if fname.endswith(".gz"): + f = gzip.open(fname, 'r') + else: + f = open(fname, 'r') + + data = str(f.read(), 'utf-8') + f.close() + + return validate_data(data) + +def validate_dir(dirname): + ret = True + for root, subfolders, files in os.walk(dirname): + for fname in files: + if fname.endswith(".yml.gz"): + if not validate_file(os.path.join(root, fname)): + ret = False + + return ret + +def main(): + parser = OptionParser() + + (options, args) = parser.parse_args() + + if len(args) < 1: + print("You need to specify a file to validate!") + sys.exit(4) + fname = args[0] + + if os.path.isdir(fname): + ret = validate_dir(fname) + else: + ret = validate_file(fname) + if ret: + msg = "DEP-11 basic validation successful." + else: + msg = "DEP-11 validation failed!" + print(msg) + + if not ret: + sys.exit(1) + +if __name__ == "__main__": + main() -- 2.39.2