]> git.decadent.org.uk Git - dak.git/blob - scripts/debian/ddtp-i18n-check.sh
not everything can be done in parallel...
[dak.git] / scripts / debian / ddtp-i18n-check.sh
1 #!/bin/bash
2 #
3 # $Id: ddtp_i18n_check.sh 1670 2009-03-31 20:57:49Z nekral-guest $
4
5 # Copyright (C) 2008, Felipe Augusto van de Wiel <faw@funlabs.org>
6 # Copyright (C) 2008, 2009 Nicolas François <nicolas.francois@centraliens.net>
7 #
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
12 #
13 # On Debian systems, you can find the full text of the license in
14 # /usr/share/common-licenses/GPL-2
15
16 set -eu
17 export LC_ALL=C
18
19 # This must be defined to either 0 or 1
20 # When DEBUG=0, fail after the first error.
21 # Otherwise, list all the errors.
22 DEBUG=0
23
24 # When DRY_RUN=0, generate the compressed version of the Translation-*
25 # files.
26 DRY_RUN=0
27
28 dists_parent_dir=""
29 # If no argument indicates the PACKAGES_LISTS_DIR then use '.'
30 PACKAGES_LISTS_DIR=""
31
32 usage () {
33         echo "Usage: $0 [options] <dists_parent_dir> [<packages_lists_directory>]" >&2
34         echo "" >&2
35         echo "    --debug      Debug mode: do not stop after the first error" >&2
36         echo "    --dry-run    Do not generate the compressed version of the " >&2
37         echo "                 Translation files">&2
38         exit 1
39 }
40
41 # Parse options
42 for opt; do
43         case "$opt" in
44                 "--debug")
45                         DEBUG=1
46                         ;;
47                 "--dry-run")
48                         DRY_RUN=1
49                         ;;
50                 "-*")
51                         usage
52                         ;;
53                 "")
54                         echo "Empty parameter" >&2
55                         echo "" >&2
56                         usage
57                         ;;
58                 *)
59                         if [ -z "$dists_parent_dir" ]; then
60                                 # Removing trailing /
61                                 dists_parent_dir=${opt%/}
62                         elif [ -z "$PACKAGES_LISTS_DIR" ]; then
63                                 PACKAGES_LISTS_DIR=$opt
64                         else
65                                 echo "$0: Invalid option: $opt" >&2
66                                 usage
67                         fi
68                         ;;
69         esac
70 done
71 PACKAGES_LISTS_DIR=${opt:-.}
72
73 if [ ! -d "$dists_parent_dir" ]; then
74         echo "missing dists_parent_dir, or not a directory" >&2
75         echo "" >&2
76         usage
77 elif [ ! -d "$PACKAGES_LISTS_DIR" ]; then
78         echo "missing packages_lists_directory, or not a directory" >&2
79         echo "" >&2
80         usage
81 fi
82
83 #STABLE="lenny"
84 TESTING="squeeze"
85 UNSTABLE="sid"
86
87 # Original SHA256SUMS, generated by i18n.debian.net
88 SHA256SUMS="SHA256SUMS"
89
90 # DAK Timestamp
91 TIMESTAMP="timestamp"
92
93 # These special files must exist on the top of dists_parent_dir
94 SPECIAL_FILES="$SHA256SUMS $TIMESTAMP $TIMESTAMP.gpg"
95
96 # Temporary working directory. We need a full path to reduce the
97 # complexity of checking SHA256SUMS and cleaning/removing TMPDIR
98 TEMP_WORK_DIR=$(mktemp -d -t ddtp_dinstall_tmpdir.XXXXXX)
99 cd "$TEMP_WORK_DIR"
100 TMP_WORK_DIR=$(pwd)
101 cd "$OLDPWD"
102 unset TEMP_WORK_DIR
103
104 # If it's trapped, something bad happened.
105 trap_exit () {
106         rm -rf "$TMP_WORK_DIR"
107         rm -f "$dists_parent_dir"/dists/*/main/i18n/Translation-*.bz2
108         exit 1
109 }
110 trap trap_exit EXIT HUP INT QUIT TERM
111
112 is_filename_okay () {
113         ifo_file="$1"
114
115         # Check that the file in on an "i18n" directory
116         # This ensures that the Translation-$lang files are not e.g. in
117         # dists/etch/ or dists/etch/main/
118         ifo_d=$(basename $(dirname "$ifo_file"))
119         if [ "x$ifo_d" = "xi18n" ]; then
120
121                 # Check that the file is named Translation-$lang
122                 ifo_f=$(basename "$ifo_file")
123                 case "$ifo_f" in
124                         Translation-[a-z][a-z][a-z]_[A-Z][A-Z]) return 0;;
125                         Translation-[a-z][a-z]_[A-Z][A-Z])      return 0;;
126                         Translation-[a-z][a-z][a-z])            return 0;;
127                         Translation-[a-z][a-z])                 return 0;;
128                 esac
129         fi
130
131         return 1
132 }
133
134 # Check a directory name against a directory whitelist 
135 is_dirname_okay () {
136         ido_dir="$1"
137
138         case "$ido_dir" in
139                 "$dists_parent_dir")                               return 0;;
140                 "$dists_parent_dir/dists")                         return 0;;
141 # TODO/FIXME: It is undecided how to update at stable/point-releases, so we
142 #             don't allow files to $STABLE.
143 #               "$dists_parent_dir/dists/$STABLE")                 return 0;;
144 #               "$dists_parent_dir/dists/$STABLE/main")            return 0;;
145 #               "$dists_parent_dir/dists/$STABLE/main/i18n")       return 0;;
146 #               "$dists_parent_dir/dists/$STABLE/contrib")         return 0;;
147 #               "$dists_parent_dir/dists/$STABLE/contrib/i18n")    return 0;;
148 #               "$dists_parent_dir/dists/$STABLE/non-free")        return 0;;
149 #               "$dists_parent_dir/dists/$STABLE/non-free/i18n")   return 0;;
150                 "$dists_parent_dir/dists/$TESTING")                return 0;;
151                 "$dists_parent_dir/dists/$TESTING/main")           return 0;;
152                 "$dists_parent_dir/dists/$TESTING/main/i18n")      return 0;;
153                 "$dists_parent_dir/dists/$TESTING/contrib")        return 0;;
154                 "$dists_parent_dir/dists/$TESTING/contrib/i18n")   return 0;;
155                 "$dists_parent_dir/dists/$TESTING/non-free")       return 0;;
156                 "$dists_parent_dir/dists/$TESTING/non-free/i18n")  return 0;;
157                 "$dists_parent_dir/dists/$UNSTABLE")               return 0;;
158                 "$dists_parent_dir/dists/$UNSTABLE/main")          return 0;;
159                 "$dists_parent_dir/dists/$UNSTABLE/main/i18n")     return 0;;
160                 "$dists_parent_dir/dists/$UNSTABLE/contrib")       return 0;;
161                 "$dists_parent_dir/dists/$UNSTABLE/contrib/i18n")  return 0;;
162                 "$dists_parent_dir/dists/$UNSTABLE/non-free")      return 0;;
163                 "$dists_parent_dir/dists/$UNSTABLE/non-free/i18n") return 0;;
164         esac
165
166         return 1
167 }
168
169 has_valid_fields () {
170         hvf_file="$1"
171         hvf_lang=${hvf_file/*-}
172
173 awk "
174 function print_status () {
175         printf (\"p: %d, m: %d, s: %d, l: %d\n\", package, md5, s_description, l_description)
176 }
177 BEGIN {
178         package       = 0 # Indicates if a Package field was found
179         md5           = 0 # Indicates if a Description-md5 field was found
180         s_description = 0 # Indicates if a short description was found
181         l_description = 0 # Indicates if a long description was found
182
183         failures      = 0 # Number of failures (debug only)
184         failed        = 0 # Failure already reported for the block
185 }
186
187 /^Package: / {
188         if (0 == failed) {
189                 if (   (0 != package)       \
190                     || (0 != md5)           \
191                     || (0 != s_description) \
192                     || (0 != l_description)) {
193                         printf (\"Package field unexpected in $hvf_file (line %d)\n\", NR)
194                         print_status()
195                         failed = 1
196                         if ($DEBUG) { failures++ } else { exit 1 }
197                 }
198                 package++
199         }
200         # Next input line
201         next
202 }
203
204 /^Description-md5: / {
205         if (0 == failed) {
206                 if (   (1 != package)       \
207                     || (0 != md5)           \
208                     || (0 != s_description) \
209                     || (0 != l_description)) {
210                         printf (\"Description-md5 field unexpected in $hvf_file (line %d)\n\", NR)
211                         print_status()
212                         failed = 1
213                         if ($DEBUG) { failures++ } else { exit 1 }
214                 }
215                 md5++
216         }
217         # Next input line
218         next
219 }
220
221 /^Description-$hvf_lang: / {
222         if (0 == failed) {
223                 if (   (1 != package)       \
224                     || (1 != md5)           \
225                     || (0 != s_description) \
226                     || (0 != l_description)) {
227                         printf (\"Description-$hvf_lang field unexpected in $hvf_file (line %d)\n\", NR)
228                         print_status()
229                         failed = 1
230                         if ($DEBUG) { failures++ } else { exit 1 }
231                 }
232                 s_description++
233         }
234         # Next input line
235         next
236 }
237
238 /^ / {
239         if (0 == failed) {
240                 if (   (1 != package)       \
241                     || (1 != md5)           \
242                     || (1 != s_description)) {
243                         printf (\"Long description unexpected in $hvf_file (line %d)\n\", NR)
244                         print_status()
245                         failed = 1
246                         if ($DEBUG) { failures++ } else { exit 1 }
247                 }
248                 l_description = 1 # There can be any number of long description
249                                   # lines. Do not count.
250         }
251         # Next line
252         next
253 }
254
255 /^$/ {
256         if (0 == failed) {
257                 if (   (1 != package)       \
258                     || (1 != md5)           \
259                     || (1 != s_description) \
260                     || (1 != l_description)) {
261                         printf (\"End of block unexpected in $hvf_file (line %d)\n\", NR)
262                         print_status()
263                         failed = 1
264                         if ($DEBUG) { failures++ } else { exit 1 }
265                 }
266         }
267
268         # Next package
269         package = 0; md5 = 0; s_description = 0; l_description = 0
270         failed = 0
271
272         # Next input line
273         next
274 }
275
276 # Anything else: fail
277 {
278         printf (\"Unexpected line '\$0' in $hvf_file (line %d)\n\", NR)
279         print_status()
280         failed = 1
281         if ($DEBUG) { failures++ } else { exit 1 }
282 }
283
284 END {
285         if (0 == failed) {
286                 # They must be all set to 0 or all set to 1
287                 if (   (   (0 == package)        \
288                         || (0 == md5)            \
289                         || (0 == s_description)  \
290                         || (0 == l_description)) \
291                     && (   (0 != package)        \
292                         || (0 != md5)            \
293                         || (0 != s_description)  \
294                         || (0 != l_description))) {
295                         printf (\"End of file unexpected in $hvf_file (line %d)\n\", NR)
296                         print_status()
297                         exit 1
298                 }
299         }
300
301         if (failures > 0) {
302                 exit 1
303         }
304 }
305 " "$hvf_file" || return 1
306
307         return 0
308 }
309
310 # $SPECIAL_FILES must exist
311 for sf in $SPECIAL_FILES; do
312         if [ ! -f "$dists_parent_dir/$sf" ]; then
313                 echo "Special file ($sf) doesn't exist"
314                 exit 1;
315         fi
316 done
317
318 # Comparing SHA256SUMS
319 # We don use -c because a file could exist in the directory tree and not in
320 # the SHA256SUMS, so we sort the existing SHA256SUMS and we create a new one
321 # already sorted, if cmp fails then files are different and we don't want to
322 # continue.
323 cd "$dists_parent_dir"
324 find dists -type f -print0 |xargs --null sha256sum > "$TMP_WORK_DIR/$SHA256SUMS.new"
325 sort "$SHA256SUMS" > "$TMP_WORK_DIR/$SHA256SUMS.sorted"
326 sort "$TMP_WORK_DIR/$SHA256SUMS.new" > "$TMP_WORK_DIR/$SHA256SUMS.new.sorted"
327 if ! cmp --quiet "$TMP_WORK_DIR/$SHA256SUMS.sorted" "$TMP_WORK_DIR/$SHA256SUMS.new.sorted"; then
328         echo "Failed to compare the SHA256SUMS, they are not identical!" >&2
329         diff -au "$TMP_WORK_DIR/$SHA256SUMS.sorted" "$TMP_WORK_DIR/$SHA256SUMS.new.sorted" >&2
330         exit 1
331 fi
332 cd "$OLDPWD"
333
334 # Get the list of valid packages (sorted, uniq)
335 for t in "$TESTING" "$UNSTABLE"; do
336         if [ ! -f "$PACKAGES_LISTS_DIR/$t" ]; then
337                 echo "Missing $PACKAGES_LISTS_DIR/$t" >&2
338                 exit 1
339         fi
340         cut -d' ' -f 1 "$PACKAGES_LISTS_DIR/$t" | sort -u > "$TMP_WORK_DIR/$t.pkgs"
341 done
342
343 /usr/bin/find "$dists_parent_dir" |
344 while read f; do
345         if   [ -d "$f" ]; then
346                 if ! is_dirname_okay "$f"; then
347                         echo "Wrong directory name: $f" >&2
348                         exit 1
349                 fi
350         elif [ -f "$f" ]; then
351                 # If $f is in $SPECIAL_FILES, we skip to the next loop because
352                 # we won't check it for format, fields and encoding.
353                 for sf in $SPECIAL_FILES; do
354                         if [ "$f" = "$dists_parent_dir/$sf" ]; then
355                                 continue 2
356                         fi
357                 done
358
359                 if ! is_filename_okay "$f"; then
360                         echo "Wrong file: $f" >&2
361                         exit 1
362                 fi
363
364                 # Check that all entries contains the right fields
365                 if ! has_valid_fields "$f"; then
366                         echo "File $f has an invalid format" >&2
367                         exit 1
368                 fi
369
370                 # Check that every packages in Translation-$lang exists
371                 TPKGS=$(basename "$f").pkgs
372                 grep "^Package: " "$f" | cut -d' ' -f 2 | sort -u > "$TMP_WORK_DIR/$TPKGS"
373                 case "$f" in
374                         */$TESTING/*)  t="$TESTING";;
375                         */$UNSTABLE/*) t="$UNSTABLE";;
376                 esac
377                 if diff "$TMP_WORK_DIR/$t.pkgs" "$TMP_WORK_DIR/$TPKGS" | grep -q "^>"; then
378                         diff -au "$TMP_WORK_DIR/$t.pkgs" "$TMP_WORK_DIR/$TPKGS" |grep "^+"
379                         echo "$f contains packages which are not in $t" >&2
380                         exit 1
381                 fi
382
383                 # Check encoding
384                 iconv -f utf-8 -t utf-8 < "$f" > /dev/null 2>&1 || {
385                         echo "$f is not an UTF-8 file" >&2
386                         exit 1
387                 }
388
389                 # We do not check if the md5 in Translation-$lang are
390                 # correct.
391
392                 if [ "$DRY_RUN" = "0" ]; then
393                         # Now generate the compressed files
394                         bzip2 "$f"
395                 fi
396         else
397                 echo "Neither a file or directory: $f" >&2
398                 exit 1
399         fi
400 done || false
401 # The while will just fail if an internal check "exit 1", but the script
402 # is not exited. "|| false" makes the script fail (and exit) in that case.
403
404 echo "$dists_parent_dir structure validated successfully ($(date +%c))"
405
406 # If we reach this point, everything went fine.
407 trap - EXIT
408 rm -rf "$TMP_WORK_DIR"
409