scripts/debian/ddtp-i18n-check.sh

   1 #!/bin/bash
   2 #
   3 # $Id: ddtp_i18n_check.sh 1186 2008-08-12 18:31:25Z faw $
   4 #
   5 # Copyright (C) 2008, Felipe Augusto van de Wiel <faw@funlabs.org>
   6 # Copyright (C) 2008, Nicolas François <nicolas.francois@centraliens.net>
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # On Debian systems, you can find the full text of the license in
  14 # /usr/share/common-licenses/GPL-2
  15
  16 set -eu
  17 export LC_ALL=C
  18
  19 # This must be defined to either 0 or 1
  20 # When DEBUG=0, fail after the first error.
  21 # Otherwise, list all the errors.
  22 DEBUG=0
  23
  24 #STABLE="lenny"
  25 TESTING="squeeze"
  26 UNSTABLE="sid"
  27
  28 # Original SHA256SUMS, generated by i18n.debian.net
  29 SHA256SUMS="SHA256SUMS"
  30
  31 # DAK Timestamp
  32 TIMESTAMP="timestamp"
  33
  34 # These special files must exist on the top of dists_parent_dir
  35 SPECIAL_FILES="$SHA256SUMS $TIMESTAMP $TIMESTAMP.gpg"
  36
  37 usage () {
  38         echo "Usage: $0 <dists_parent_dir> [<packages_lists_directory>]" >&2
  39         exit 1
  40 }
  41
  42 if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ ! -d $1 ]
  43 then
  44         usage
  45 fi
  46
  47 # Temporary working directory. We need a full path to reduce the
  48 # complexity of checking SHA256SUMS and cleaning/removing TMPDIR
  49 TEMP_WORK_DIR=$(mktemp -d -t ddtp_dinstall_tmpdir.XXXXXX)
  50 cd "$TEMP_WORK_DIR"
  51 TMP_WORK_DIR=$(pwd)
  52 cd "$OLDPWD"
  53 unset TEMP_WORK_DIR
  54
  55 # If it's traped, something bad happened.
  56 trap_exit () {
  57         rm -rf "$TMP_WORK_DIR"
  58         rm -f "$dists_parent_dir"/dists/*/main/i18n/Translation-*.{bz2,gz}
  59         exit 1
  60 }
  61 trap trap_exit EXIT HUP INT QUIT TERM
  62
  63 # If no argument indicates the PACKAGES_LISTS_DIR then use '.'
  64 PACKAGES_LISTS_DIR=${2:-.}
  65
  66 if [ ! -d "$PACKAGES_LISTS_DIR" ]
  67 then
  68         usage
  69 fi
  70
  71 # Removing trailing /
  72 dists_parent_dir=${1%/}
  73
  74 is_filename_okay () {
  75         ifo_file="$1"
  76
  77         # Check that the file in on an "i18n" directory
  78         # This ensures that the Translation-$lang files are not e.g. in
  79         # dists/etch/ or dists/etch/main/
  80         ifo_d=$(basename $(dirname "$ifo_file"))
  81         if [ "x$ifo_d" = "xi18n" ]; then
  82
  83                 # Check that the file is named Translation-$lang
  84                 ifo_f=$(basename "$ifo_file")
  85                 case "$ifo_f" in
  86                         Translation-[a-z][a-z][a-z]_[A-Z][A-Z]) return 0;;
  87                         Translation-[a-z][a-z]_[A-Z][A-Z])      return 0;;
  88                         Translation-[a-z][a-z][a-z])            return 0;;
  89                         Translation-[a-z][a-z])                 return 0;;
  90                 esac
  91         fi
  92
  93         return 1
  94 }
  95
  96 # Check a directory name against a directory whitelist
  97 is_dirname_okay () {
  98         ido_dir="$1"
  99
 100         case "$ido_dir" in
 101                 "$dists_parent_dir")                               return 0;;
 102                 "$dists_parent_dir/dists")                         return 0;;
 103 # TODO/FIXME: It is undecided how to update at stable/point-releases, so we
 104 #             don't allow files to $STABLE.
 105 #               "$dists_parent_dir/dists/$STABLE")                 return 0;;
 106 #               "$dists_parent_dir/dists/$STABLE/main")            return 0;;
 107 #               "$dists_parent_dir/dists/$STABLE/main/i18n")       return 0;;
 108 #               "$dists_parent_dir/dists/$STABLE/contrib")         return 0;;
 109 #               "$dists_parent_dir/dists/$STABLE/contrib/i18n")    return 0;;
 110 #               "$dists_parent_dir/dists/$STABLE/non-free")        return 0;;
 111 #               "$dists_parent_dir/dists/$STABLE/non-free/i18n")   return 0;;
 112                 "$dists_parent_dir/dists/$TESTING")                return 0;;
 113                 "$dists_parent_dir/dists/$TESTING/main")           return 0;;
 114                 "$dists_parent_dir/dists/$TESTING/main/i18n")      return 0;;
 115                 "$dists_parent_dir/dists/$TESTING/contrib")        return 0;;
 116                 "$dists_parent_dir/dists/$TESTING/contrib/i18n")   return 0;;
 117                 "$dists_parent_dir/dists/$TESTING/non-free")       return 0;;
 118                 "$dists_parent_dir/dists/$TESTING/non-free/i18n")  return 0;;
 119                 "$dists_parent_dir/dists/$UNSTABLE")               return 0;;
 120                 "$dists_parent_dir/dists/$UNSTABLE/main")          return 0;;
 121                 "$dists_parent_dir/dists/$UNSTABLE/main/i18n")     return 0;;
 122                 "$dists_parent_dir/dists/$UNSTABLE/contrib")       return 0;;
 123                 "$dists_parent_dir/dists/$UNSTABLE/contrib/i18n")  return 0;;
 124                 "$dists_parent_dir/dists/$UNSTABLE/non-free")      return 0;;
 125                 "$dists_parent_dir/dists/$UNSTABLE/non-free/i18n") return 0;;
 126         esac
 127
 128         return 1
 129 }
 130
 131 has_valid_fields () {
 132         hvf_file="$1"
 133         hvf_lang=${hvf_file/*-}
 134
 135 awk "
 136 function print_status () {
 137         printf (\"p: %d, m: %d, s: %d, l: %d\n\", package, md5, s_description, l_description)
 138 }
 139 BEGIN {
 140         package       = 0 # Indicates if a Package field was found
 141         md5           = 0 # Indicates if a Description-md5 field was found
 142         s_description = 0 # Indicates if a short description was found
 143         l_description = 0 # Indicates if a long description was found
 144
 145         failures      = 0 # Number of failures (debug only)
 146         failed        = 0 # Failure already reported for the block
 147 }
 148
 149 /^Package: / {
 150         if (0 == failed) {
 151                 if (   (0 != package)       \
 152                     || (0 != md5)           \
 153                     || (0 != s_description) \
 154                     || (0 != l_description)) {
 155                         printf (\"Package field unexpected in $hvf_file (line %d)\n\", NR)
 156                         print_status()
 157                         failed = 1
 158                         if ($DEBUG) { failures++ } else { exit 1 }
 159                 }
 160                 package++
 161         }
 162         # Next input line
 163         next
 164 }
 165
 166 /^Description-md5: / {
 167         if (0 == failed) {
 168                 if (   (1 != package)       \
 169                     || (0 != md5)           \
 170                     || (0 != s_description) \
 171                     || (0 != l_description)) {
 172                         printf (\"Description-md5 field unexpected in $hvf_file (line %d)\n\", NR)
 173                         print_status()
 174                         failed = 1
 175                         if ($DEBUG) { failures++ } else { exit 1 }
 176                 }
 177                 md5++
 178         }
 179         # Next input line
 180         next
 181 }
 182
 183 /^Description-$hvf_lang: / {
 184         if (0 == failed) {
 185                 if (   (1 != package)       \
 186                     || (1 != md5)           \
 187                     || (0 != s_description) \
 188                     || (0 != l_description)) {
 189                         printf (\"Description-$hvf_lang field unexpected in $hvf_file (line %d)\n\", NR)
 190                         print_status()
 191                         failed = 1
 192                         if ($DEBUG) { failures++ } else { exit 1 }
 193                 }
 194                 s_description++
 195         }
 196         # Next input line
 197         next
 198 }
 199
 200 /^ / {
 201         if (0 == failed) {
 202                 if (   (1 != package)       \
 203                     || (1 != md5)           \
 204                     || (1 != s_description)) {
 205                         printf (\"Long description unexpected in $hvf_file (line %d)\n\", NR)
 206                         print_status()
 207                         failed = 1
 208                         if ($DEBUG) { failures++ } else { exit 1 }
 209                 }
 210                 l_description = 1 # There can be any number of long description
 211                                   # lines. Do not count.
 212         }
 213         # Next line
 214         next
 215 }
 216
 217 /^$/ {
 218         if (0 == failed) {
 219                 if (   (1 != package)       \
 220                     || (1 != md5)           \
 221                     || (1 != s_description) \
 222                     || (1 != l_description)) {
 223                         printf (\"End of block unexpected in $hvf_file (line %d)\n\", NR)
 224                         print_status()
 225                         failed = 1
 226                         if ($DEBUG) { failures++ } else { exit 1 }
 227                 }
 228         }
 229
 230         # Next package
 231         package = 0; md5 = 0; s_description = 0; l_description = 0
 232         failed = 0
 233
 234         # Next input line
 235         next
 236 }
 237
 238 # Anything else: fail
 239 {
 240         printf (\"Unexpected line '\$0' in $hvf_file (line %d)\n\", NR)
 241         print_status()
 242         failed = 1
 243         if ($DEBUG) { failures++ } else { exit 1 }
 244 }
 245
 246 END {
 247         if (0 == failed) {
 248                 # They must be all set to 0 or all set to 1
 249                 if (   (   (0 == package)        \
 250                         || (0 == md5)            \
 251                         || (0 == s_description)  \
 252                         || (0 == l_description)) \
 253                     && (   (0 != package)        \
 254                         || (0 != md5)            \
 255                         || (0 != s_description)  \
 256                         || (0 != l_description))) {
 257                         printf (\"End of file unexpected in $hvf_file (line %d)\n\", NR)
 258                         print_status()
 259                         exit 1
 260                 }
 261         }
 262
 263         if (failures > 0) {
 264                 exit 1
 265         }
 266 }
 267 " "$hvf_file" || return 1
 268
 269         return 0
 270 }
 271
 272 # $SPECIAL_FILES must exist
 273 for sf in $SPECIAL_FILES; do
 274         if [ ! -f "$dists_parent_dir/$sf" ]; then
 275                 echo "Special file ($sf) doesn't exist"
 276                 exit 1;
 277         fi
 278 done
 279
 280 # Comparing SHA256SUMS
 281 # We don use -c because a file could exist in the directory tree and not in
 282 # the SHA256SUMS, so we sort the existing SHA256SUMS and we create a new one
 283 # already sorted, if cmp fails then files are different and we don't want to
 284 # continue.
 285 cd "$dists_parent_dir"
 286 find dists -type f -print0 |xargs --null sha256sum > "$TMP_WORK_DIR/$SHA256SUMS.new"
 287 sort "$SHA256SUMS" > "$TMP_WORK_DIR/$SHA256SUMS.sorted"
 288 sort "$TMP_WORK_DIR/$SHA256SUMS.new" > "$TMP_WORK_DIR/$SHA256SUMS.new.sorted"
 289 if ! cmp --quiet "$TMP_WORK_DIR/$SHA256SUMS.sorted" "$TMP_WORK_DIR/$SHA256SUMS.new.sorted"; then
 290         echo "Failed to compare the SHA256SUMS, they are not identical!" >&2
 291         diff -au "$TMP_WORK_DIR/$SHA256SUMS.sorted" "$TMP_WORK_DIR/$SHA256SUMS.new.sorted" >&2
 292         exit 1
 293 fi
 294 cd "$OLDPWD"
 295
 296 # Get the list of valid packages (sorted, uniq)
 297 for t in "$TESTING" "$UNSTABLE"; do
 298         if [ ! -f "$PACKAGES_LISTS_DIR/$t" ]; then
 299                 echo "Missing $PACKAGES_LISTS_DIR/$t" >&2
 300                 exit 1
 301         fi
 302         cut -d' ' -f 1 "$PACKAGES_LISTS_DIR/$t" | sort -u > "$TMP_WORK_DIR/$t.pkgs"
 303 done
 304
 305 /usr/bin/find "$dists_parent_dir" |
 306 while read f; do
 307         if   [ -d "$f" ]; then
 308                 if ! is_dirname_okay "$f"; then
 309                         echo "Wrong directory name: $f" >&2
 310                         exit 1
 311                 fi
 312         elif [ -f "$f" ]; then
 313                 # If $f is in $SPECIAL_FILES, we skip to the next loop because
 314                 # we won't check it for format, fields and encoding.
 315                 for sf in $SPECIAL_FILES; do
 316                         if [ "$f" = "$dists_parent_dir/$sf" ]; then
 317                                 continue 2
 318                         fi
 319                 done
 320
 321                 if ! is_filename_okay "$f"; then
 322                         echo "Wrong file: $f" >&2
 323                         exit 1
 324                 fi
 325
 326                 # Check that all entries contains the right fields
 327                 if ! has_valid_fields "$f"; then
 328                         echo "File $f has an invalid format" >&2
 329                         exit 1
 330                 fi
 331
 332                 # Check that every packages in Translation-$lang exists
 333                 TPKGS=$(basename "$f").pkgs
 334                 grep "^Package: " "$f" | cut -d' ' -f 2 | sort -u > "$TMP_WORK_DIR/$TPKGS"
 335                 case "$f" in
 336                         */$TESTING/*)  t="$TESTING";;
 337                         */$UNSTABLE/*) t="$UNSTABLE";;
 338                 esac
 339                 if diff "$TMP_WORK_DIR/$t.pkgs" "$TMP_WORK_DIR/$TPKGS" | grep -q "^>"; then
 340                         diff -au "$TMP_WORK_DIR/$t.pkgs" "$TMP_WORK_DIR/$TPKGS" |grep "^+"
 341                         echo "$f contains packages which are not in $t" >&2
 342                         exit 1
 343                 fi
 344
 345                 # Check encoding
 346                 iconv -f utf-8 -t utf-8 < "$f" > /dev/null 2>&1 || {
 347                         echo "$f is not an UTF-8 file" >&2
 348                         exit 1
 349                 }
 350
 351                 # We do not check if the md5 in Translation-$lang are
 352                 # correct.
 353
 354                 # Now generate files
 355                 #   Compress the file
 356                 bzip2 -c "$f" > "$f.bz2"
 357                 gzip  -c "$f" > "$f.gz"
 358         else
 359                 echo "Neither a file or directory: $f" >&2
 360                 exit 1
 361         fi
 362 done || false
 363 # The while will just fail if an internal check "exit 1", but the script
 364 # is not exited. "|| false" makes the script fail (and exit) in that case.
 365
 366 echo "$dists_parent_dir structure validated successfully ($(date +%c))"
 367
 368 # If we reach this point, everything went fine.
 369 trap - EXIT
 370 rm -rf "$TMP_WORK_DIR"
 371