scripts/debian/ddtp-i18n-check.sh

   1 #!/bin/bash
   2 #
   3 # $Id: ddtp_i18n_check.sh 1670 2009-03-31 20:57:49Z nekral-guest $
   4 #
   5 # Copyright (C) 2008, Felipe Augusto van de Wiel <faw@funlabs.org>
   6 # Copyright (C) 2008, 2009 Nicolas François <nicolas.francois@centraliens.net>
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # On Debian systems, you can find the full text of the license in
  14 # /usr/share/common-licenses/GPL-2
  15
  16 set -eu
  17 export LC_ALL=C
  18
  19 # This must be defined to either 0 or 1
  20 # When DEBUG=0, fail after the first error.
  21 # Otherwise, list all the errors.
  22 DEBUG=0
  23
  24 # When DRY_RUN=0, generate the compressed version of the Translation-*
  25 # files.
  26 DRY_RUN=0
  27
  28 dists_parent_dir=""
  29 # If no argument indicates the PACKAGES_LISTS_DIR then use '.'
  30 PACKAGES_LISTS_DIR=""
  31
  32 usage () {
  33         echo "Usage: $0 [options] <dists_parent_dir> [<packages_lists_directory>]" >&2
  34         echo "" >&2
  35         echo "    --debug      Debug mode: do not stop after the first error" >&2
  36         echo "    --dry-run    Do not generate the compressed version of the " >&2
  37         echo "                 Translation files">&2
  38         exit 1
  39 }
  40
  41 # Parse options
  42 for opt; do
  43         case "$opt" in
  44                 "--debug")
  45                         DEBUG=1
  46                         ;;
  47                 "--dry-run")
  48                         DRY_RUN=1
  49                         ;;
  50                 "-*")
  51                         usage
  52                         ;;
  53                 "")
  54                         echo "Empty parameter" >&2
  55                         echo "" >&2
  56                         usage
  57                         ;;
  58                 *)
  59                         if [ -z "$dists_parent_dir" ]; then
  60                                 # Removing trailing /
  61                                 dists_parent_dir=${opt%/}
  62                         elif [ -z "$PACKAGES_LISTS_DIR" ]; then
  63                                 PACKAGES_LISTS_DIR=$opt
  64                         else
  65                                 echo "$0: Invalid option: $opt" >&2
  66                                 usage
  67                         fi
  68                         ;;
  69         esac
  70 done
  71 PACKAGES_LISTS_DIR=${opt:-.}
  72
  73 if [ ! -d "$dists_parent_dir" ]; then
  74         echo "missing dists_parent_dir, or not a directory" >&2
  75         echo "" >&2
  76         usage
  77 elif [ ! -d "$PACKAGES_LISTS_DIR" ]; then
  78         echo "missing packages_lists_directory, or not a directory" >&2
  79         echo "" >&2
  80         usage
  81 fi
  82
  83 #STABLE="lenny"
  84 TESTING="wheezy"
  85 UNSTABLE="sid"
  86
  87 # Original SHA256SUMS, generated by i18n.debian.net
  88 SHA256SUMS="SHA256SUMS"
  89
  90 # DAK Timestamp
  91 TIMESTAMP="timestamp"
  92
  93 # These special files must exist on the top of dists_parent_dir
  94 SPECIAL_FILES="$SHA256SUMS $TIMESTAMP $TIMESTAMP.gpg"
  95
  96 # Temporary working directory. We need a full path to reduce the
  97 # complexity of checking SHA256SUMS and cleaning/removing TMPDIR
  98 TEMP_WORK_DIR=$(mktemp -d -t ddtp_dinstall_tmpdir.XXXXXX)
  99 cd "$TEMP_WORK_DIR"
 100 TMP_WORK_DIR=$(pwd)
 101 cd "$OLDPWD"
 102 unset TEMP_WORK_DIR
 103
 104 # If it's trapped, something bad happened.
 105 trap_exit () {
 106         rm -rf "$TMP_WORK_DIR"
 107         rm -f "$dists_parent_dir"/dists/*/main/i18n/Translation-*.bz2
 108         exit 1
 109 }
 110 trap trap_exit EXIT HUP INT QUIT TERM
 111
 112 is_filename_okay () {
 113         ifo_file="$1"
 114
 115         # Check that the file in on an "i18n" directory
 116         # This ensures that the Translation-$lang files are not e.g. in
 117         # dists/etch/ or dists/etch/main/
 118         ifo_d=$(basename $(dirname "$ifo_file"))
 119         if [ "x$ifo_d" = "xi18n" ]; then
 120
 121                 # Check that the file is named Translation-$lang
 122                 ifo_f=$(basename "$ifo_file")
 123                 case "$ifo_f" in
 124                         Translation-[a-z][a-z][a-z]_[A-Z][A-Z]) return 0;;
 125                         Translation-[a-z][a-z]_[A-Z][A-Z])      return 0;;
 126                         Translation-[a-z][a-z][a-z])            return 0;;
 127                         Translation-[a-z][a-z])                 return 0;;
 128                 esac
 129         fi
 130
 131         return 1
 132 }
 133
 134 # Check a directory name against a directory whitelist
 135 is_dirname_okay () {
 136         ido_dir="$1"
 137
 138         case "$ido_dir" in
 139                 "$dists_parent_dir")                               return 0;;
 140                 "$dists_parent_dir/dists")                         return 0;;
 141 # TODO/FIXME: It is undecided how to update at stable/point-releases, so we
 142 #             don't allow files to $STABLE.
 143 #               "$dists_parent_dir/dists/$STABLE")                 return 0;;
 144 #               "$dists_parent_dir/dists/$STABLE/main")            return 0;;
 145 #               "$dists_parent_dir/dists/$STABLE/main/i18n")       return 0;;
 146 #               "$dists_parent_dir/dists/$STABLE/contrib")         return 0;;
 147 #               "$dists_parent_dir/dists/$STABLE/contrib/i18n")    return 0;;
 148 #               "$dists_parent_dir/dists/$STABLE/non-free")        return 0;;
 149 #               "$dists_parent_dir/dists/$STABLE/non-free/i18n")   return 0;;
 150                 "$dists_parent_dir/dists/$TESTING")                return 0;;
 151                 "$dists_parent_dir/dists/$TESTING/main")           return 0;;
 152                 "$dists_parent_dir/dists/$TESTING/main/i18n")      return 0;;
 153                 "$dists_parent_dir/dists/$TESTING/contrib")        return 0;;
 154                 "$dists_parent_dir/dists/$TESTING/contrib/i18n")   return 0;;
 155                 "$dists_parent_dir/dists/$TESTING/non-free")       return 0;;
 156                 "$dists_parent_dir/dists/$TESTING/non-free/i18n")  return 0;;
 157                 "$dists_parent_dir/dists/$UNSTABLE")               return 0;;
 158                 "$dists_parent_dir/dists/$UNSTABLE/main")          return 0;;
 159                 "$dists_parent_dir/dists/$UNSTABLE/main/i18n")     return 0;;
 160                 "$dists_parent_dir/dists/$UNSTABLE/contrib")       return 0;;
 161                 "$dists_parent_dir/dists/$UNSTABLE/contrib/i18n")  return 0;;
 162                 "$dists_parent_dir/dists/$UNSTABLE/non-free")      return 0;;
 163                 "$dists_parent_dir/dists/$UNSTABLE/non-free/i18n") return 0;;
 164         esac
 165
 166         return 1
 167 }
 168
 169 has_valid_fields () {
 170         hvf_file="$1"
 171         hvf_lang=${hvf_file/*-}
 172
 173 awk "
 174 function print_status () {
 175         printf (\"p: %d, m: %d, s: %d, l: %d\n\", package, md5, s_description, l_description)
 176 }
 177 BEGIN {
 178         package       = 0 # Indicates if a Package field was found
 179         md5           = 0 # Indicates if a Description-md5 field was found
 180         s_description = 0 # Indicates if a short description was found
 181         l_description = 0 # Indicates if a long description was found
 182
 183         failures      = 0 # Number of failures (debug only)
 184         failed        = 0 # Failure already reported for the block
 185 }
 186
 187 /^Package: / {
 188         if (0 == failed) {
 189                 if (   (0 != package)       \
 190                     || (0 != md5)           \
 191                     || (0 != s_description) \
 192                     || (0 != l_description)) {
 193                         printf (\"Package field unexpected in $hvf_file (line %d)\n\", NR)
 194                         print_status()
 195                         failed = 1
 196                         if ($DEBUG) { failures++ } else { exit 1 }
 197                 }
 198                 package++
 199         }
 200         # Next input line
 201         next
 202 }
 203
 204 /^Description-md5: / {
 205         if (0 == failed) {
 206                 if (   (1 != package)       \
 207                     || (0 != md5)           \
 208                     || (0 != s_description) \
 209                     || (0 != l_description)) {
 210                         printf (\"Description-md5 field unexpected in $hvf_file (line %d)\n\", NR)
 211                         print_status()
 212                         failed = 1
 213                         if ($DEBUG) { failures++ } else { exit 1 }
 214                 }
 215                 md5++
 216         }
 217         # Next input line
 218         next
 219 }
 220
 221 /^Description-$hvf_lang: / {
 222         if (0 == failed) {
 223                 if (   (1 != package)       \
 224                     || (1 != md5)           \
 225                     || (0 != s_description) \
 226                     || (0 != l_description)) {
 227                         printf (\"Description-$hvf_lang field unexpected in $hvf_file (line %d)\n\", NR)
 228                         print_status()
 229                         failed = 1
 230                         if ($DEBUG) { failures++ } else { exit 1 }
 231                 }
 232                 s_description++
 233         }
 234         # Next input line
 235         next
 236 }
 237
 238 /^ / {
 239         if (0 == failed) {
 240                 if (   (1 != package)       \
 241                     || (1 != md5)           \
 242                     || (1 != s_description)) {
 243                         printf (\"Long description unexpected in $hvf_file (line %d)\n\", NR)
 244                         print_status()
 245                         failed = 1
 246                         if ($DEBUG) { failures++ } else { exit 1 }
 247                 }
 248                 l_description = 1 # There can be any number of long description
 249                                   # lines. Do not count.
 250         }
 251         # Next line
 252         next
 253 }
 254
 255 /^$/ {
 256         if (0 == failed) {
 257                 if (   (1 != package)       \
 258                     || (1 != md5)           \
 259                     || (1 != s_description) \
 260                     || (1 != l_description)) {
 261                         printf (\"End of block unexpected in $hvf_file (line %d)\n\", NR)
 262                         print_status()
 263                         failed = 1
 264                         if ($DEBUG) { failures++ } else { exit 1 }
 265                 }
 266         }
 267
 268         # Next package
 269         package = 0; md5 = 0; s_description = 0; l_description = 0
 270         failed = 0
 271
 272         # Next input line
 273         next
 274 }
 275
 276 # Anything else: fail
 277 {
 278         printf (\"Unexpected line '\$0' in $hvf_file (line %d)\n\", NR)
 279         print_status()
 280         failed = 1
 281         if ($DEBUG) { failures++ } else { exit 1 }
 282 }
 283
 284 END {
 285         if (0 == failed) {
 286                 # They must be all set to 0 or all set to 1
 287                 if (   (   (0 == package)        \
 288                         || (0 == md5)            \
 289                         || (0 == s_description)  \
 290                         || (0 == l_description)) \
 291                     && (   (0 != package)        \
 292                         || (0 != md5)            \
 293                         || (0 != s_description)  \
 294                         || (0 != l_description))) {
 295                         printf (\"End of file unexpected in $hvf_file (line %d)\n\", NR)
 296                         print_status()
 297                         exit 1
 298                 }
 299         }
 300
 301         if (failures > 0) {
 302                 exit 1
 303         }
 304 }
 305 " "$hvf_file" || return 1
 306
 307         return 0
 308 }
 309
 310 # $SPECIAL_FILES must exist
 311 for sf in $SPECIAL_FILES; do
 312         if [ ! -f "$dists_parent_dir/$sf" ]; then
 313                 echo "Special file ($sf) doesn't exist"
 314                 exit 1;
 315         fi
 316 done
 317
 318 # Comparing SHA256SUMS
 319 # We don use -c because a file could exist in the directory tree and not in
 320 # the SHA256SUMS, so we sort the existing SHA256SUMS and we create a new one
 321 # already sorted, if cmp fails then files are different and we don't want to
 322 # continue.
 323 cd "$dists_parent_dir"
 324 find dists -type f -print0 |xargs --null sha256sum > "$TMP_WORK_DIR/$SHA256SUMS.new"
 325 sort "$SHA256SUMS" > "$TMP_WORK_DIR/$SHA256SUMS.sorted"
 326 sort "$TMP_WORK_DIR/$SHA256SUMS.new" > "$TMP_WORK_DIR/$SHA256SUMS.new.sorted"
 327 if ! cmp --quiet "$TMP_WORK_DIR/$SHA256SUMS.sorted" "$TMP_WORK_DIR/$SHA256SUMS.new.sorted"; then
 328         echo "Failed to compare the SHA256SUMS, they are not identical!" >&2
 329         diff -au "$TMP_WORK_DIR/$SHA256SUMS.sorted" "$TMP_WORK_DIR/$SHA256SUMS.new.sorted" >&2
 330         exit 1
 331 fi
 332 cd "$OLDPWD"
 333
 334 # Get the list of valid packages (sorted, uniq)
 335 for t in "$TESTING" "$UNSTABLE"; do
 336         if [ ! -f "$PACKAGES_LISTS_DIR/$t" ]; then
 337                 echo "Missing $PACKAGES_LISTS_DIR/$t" >&2
 338                 exit 1
 339         fi
 340         cut -d' ' -f 1 "$PACKAGES_LISTS_DIR/$t" | sort -u > "$TMP_WORK_DIR/$t.pkgs"
 341 done
 342
 343 /usr/bin/find "$dists_parent_dir" |
 344 while read f; do
 345         if   [ -d "$f" ]; then
 346                 if ! is_dirname_okay "$f"; then
 347                         echo "Wrong directory name: $f" >&2
 348                         exit 1
 349                 fi
 350         elif [ -f "$f" ]; then
 351                 # If $f is in $SPECIAL_FILES, we skip to the next loop because
 352                 # we won't check it for format, fields and encoding.
 353                 for sf in $SPECIAL_FILES; do
 354                         if [ "$f" = "$dists_parent_dir/$sf" ]; then
 355                                 continue 2
 356                         fi
 357                 done
 358
 359                 if ! is_filename_okay "$f"; then
 360                         echo "Wrong file: $f" >&2
 361                         exit 1
 362                 fi
 363
 364                 # Check that all entries contains the right fields
 365                 if ! has_valid_fields "$f"; then
 366                         echo "File $f has an invalid format" >&2
 367                         exit 1
 368                 fi
 369
 370                 # Check that every packages in Translation-$lang exists
 371                 TPKGS=$(basename "$f").pkgs
 372                 grep "^Package: " "$f" | cut -d' ' -f 2 | sort -u > "$TMP_WORK_DIR/$TPKGS"
 373                 case "$f" in
 374                         */$TESTING/*)  t="$TESTING";;
 375                         */$UNSTABLE/*) t="$UNSTABLE";;
 376                 esac
 377                 if diff "$TMP_WORK_DIR/$t.pkgs" "$TMP_WORK_DIR/$TPKGS" | grep -q "^>"; then
 378                         diff -au "$TMP_WORK_DIR/$t.pkgs" "$TMP_WORK_DIR/$TPKGS" |grep "^+"
 379                         echo "$f contains packages which are not in $t" >&2
 380                         exit 1
 381                 fi
 382
 383                 # Check encoding
 384                 iconv -f utf-8 -t utf-8 < "$f" > /dev/null 2>&1 || {
 385                         echo "$f is not an UTF-8 file" >&2
 386                         exit 1
 387                 }
 388
 389                 # We do not check if the md5 in Translation-$lang are
 390                 # correct.
 391
 392                 if [ "$DRY_RUN" = "0" ]; then
 393                         # Now generate the compressed files
 394                         bzip2 "$f"
 395                 fi
 396         else
 397                 echo "Neither a file or directory: $f" >&2
 398                 exit 1
 399         fi
 400 done || false
 401 # The while will just fail if an internal check "exit 1", but the script
 402 # is not exited. "|| false" makes the script fail (and exit) in that case.
 403
 404 echo "$dists_parent_dir structure validated successfully ($(date +%c))"
 405
 406 # If we reach this point, everything went fine.
 407 trap - EXIT
 408 rm -rf "$TMP_WORK_DIR"
 409