]> git.decadent.org.uk Git - dak.git/blob - scripts/debian/ddtp-i18n-check.sh
Fixed 822 queue report to actually print out the timestamp sanely
[dak.git] / scripts / debian / ddtp-i18n-check.sh
1 #!/bin/bash
2 #
3 # $Id: ddtp_i18n_check.sh 1186 2008-08-12 18:31:25Z faw $
4
5 # Copyright (C) 2008, Felipe Augusto van de Wiel <faw@funlabs.org>
6 # Copyright (C) 2008, Nicolas François <nicolas.francois@centraliens.net>
7 #
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
12 #
13 # On Debian systems, you can find the full text of the license in
14 # /usr/share/common-licenses/GPL-2
15
16 set -eu
17 export LC_ALL=C
18
19 # This must be defined to either 0 or 1
20 # When DEBUG=0, fail after the first error.
21 # Otherwise, list all the errors.
22 DEBUG=0
23
24 #STABLE="etch"
25 TESTING="lenny"
26 UNSTABLE="sid"
27
28 # Original SHA256SUMS, generated by i18n.debian.net
29 SHA256SUMS="SHA256SUMS"
30
31 # DAK Timestamp
32 TIMESTAMP="timestamp"
33
34 # These special files must exist on the top of dists_parent_dir
35 SPECIAL_FILES="$SHA256SUMS $TIMESTAMP $TIMESTAMP.gpg"
36
37 usage () {
38         echo "Usage: $0 <dists_parent_dir> [<packages_lists_directory>]" >&2
39         exit 1
40 }
41
42 if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ ! -d $1 ]
43 then
44         usage
45 fi
46
47 # Temporary working directory. We need a full path to reduce the
48 # complexity of checking SHA256SUMS and cleaning/removing TMPDIR
49 TEMP_WORK_DIR=$(mktemp -d -t ddtp_dinstall_tmpdir.XXXXXX)
50 cd "$TEMP_WORK_DIR"
51 TMP_WORK_DIR=$(pwd)
52 cd "$OLDPWD"
53 unset TEMP_WORK_DIR
54
55 # If it's traped, something bad happened.
56 trap_exit () {
57         rm -rf "$TMP_WORK_DIR"
58         rm -f "$dists_parent_dir"/dists/*/main/i18n/Translation-*.{bz2,gz}
59         exit 1
60 }
61 trap trap_exit EXIT HUP INT QUIT TERM
62
63 # If no argument indicates the PACKAGES_LISTS_DIR then use '.'
64 PACKAGES_LISTS_DIR=${2:-.}
65
66 if [ ! -d "$PACKAGES_LISTS_DIR" ]
67 then
68         usage
69 fi
70
71 # Removing trailing /
72 dists_parent_dir=${1%/}
73
74 is_filename_okay () {
75         ifo_file="$1"
76
77         # Check that the file in on an "i18n" directory
78         # This ensures that the Translation-$lang files are not e.g. in
79         # dists/etch/ or dists/etch/main/
80         ifo_d=$(basename $(dirname "$ifo_file"))
81         if [ "x$ifo_d" = "xi18n" ]; then
82
83                 # Check that the file is named Translation-$lang
84                 ifo_f=$(basename "$ifo_file")
85                 case "$ifo_f" in
86                         Translation-[a-z][a-z][a-z]_[A-Z][A-Z]) return 0;;
87                         Translation-[a-z][a-z]_[A-Z][A-Z])      return 0;;
88                         Translation-[a-z][a-z][a-z])            return 0;;
89                         Translation-[a-z][a-z])                 return 0;;
90                 esac
91         fi
92
93         return 1
94 }
95
96 # Check a directory name against a directory whitelist 
97 is_dirname_okay () {
98         ido_dir="$1"
99
100         case "$ido_dir" in
101                 "$dists_parent_dir")                               return 0;;
102                 "$dists_parent_dir/dists")                         return 0;;
103 # TODO/FIXME: It is undecided how to update at stable/point-releases, so we
104 #             don't allow files to $STABLE.
105 #               "$dists_parent_dir/dists/$STABLE")                 return 0;;
106 #               "$dists_parent_dir/dists/$STABLE/main")            return 0;;
107 #               "$dists_parent_dir/dists/$STABLE/main/i18n")       return 0;;
108 #               "$dists_parent_dir/dists/$STABLE/contrib")         return 0;;
109 #               "$dists_parent_dir/dists/$STABLE/contrib/i18n")    return 0;;
110 #               "$dists_parent_dir/dists/$STABLE/non-free")        return 0;;
111 #               "$dists_parent_dir/dists/$STABLE/non-free/i18n")   return 0;;
112                 "$dists_parent_dir/dists/$TESTING")                return 0;;
113                 "$dists_parent_dir/dists/$TESTING/main")           return 0;;
114                 "$dists_parent_dir/dists/$TESTING/main/i18n")      return 0;;
115                 "$dists_parent_dir/dists/$TESTING/contrib")        return 0;;
116                 "$dists_parent_dir/dists/$TESTING/contrib/i18n")   return 0;;
117                 "$dists_parent_dir/dists/$TESTING/non-free")       return 0;;
118                 "$dists_parent_dir/dists/$TESTING/non-free/i18n")  return 0;;
119                 "$dists_parent_dir/dists/$UNSTABLE")               return 0;;
120                 "$dists_parent_dir/dists/$UNSTABLE/main")          return 0;;
121                 "$dists_parent_dir/dists/$UNSTABLE/main/i18n")     return 0;;
122                 "$dists_parent_dir/dists/$UNSTABLE/contrib")       return 0;;
123                 "$dists_parent_dir/dists/$UNSTABLE/contrib/i18n")  return 0;;
124                 "$dists_parent_dir/dists/$UNSTABLE/non-free")      return 0;;
125                 "$dists_parent_dir/dists/$UNSTABLE/non-free/i18n") return 0;;
126         esac
127
128         return 1
129 }
130
131 has_valid_fields () {
132         hvf_file="$1"
133         hvf_lang=${hvf_file/*-}
134
135 awk "
136 function print_status () {
137         printf (\"p: %d, m: %d, s: %d, l: %d\n\", package, md5, s_description, l_description)
138 }
139 BEGIN {
140         package       = 0 # Indicates if a Package field was found
141         md5           = 0 # Indicates if a Description-md5 field was found
142         s_description = 0 # Indicates if a short description was found
143         l_description = 0 # Indicates if a long description was found
144
145         failures      = 0 # Number of failures (debug only)
146         failed        = 0 # Failure already reported for the block
147 }
148
149 /^Package: / {
150         if (0 == failed) {
151                 if (   (0 != package)       \
152                     || (0 != md5)           \
153                     || (0 != s_description) \
154                     || (0 != l_description)) {
155                         printf (\"Package field unexpected in $hvf_file (line %d)\n\", NR)
156                         print_status()
157                         failed = 1
158                         if ($DEBUG) { failures++ } else { exit 1 }
159                 }
160                 package++
161         }
162         # Next input line
163         next
164 }
165
166 /^Description-md5: / {
167         if (0 == failed) {
168                 if (   (1 != package)       \
169                     || (0 != md5)           \
170                     || (0 != s_description) \
171                     || (0 != l_description)) {
172                         printf (\"Description-md5 field unexpected in $hvf_file (line %d)\n\", NR)
173                         print_status()
174                         failed = 1
175                         if ($DEBUG) { failures++ } else { exit 1 }
176                 }
177                 md5++
178         }
179         # Next input line
180         next
181 }
182
183 /^Description-$hvf_lang: / {
184         if (0 == failed) {
185                 if (   (1 != package)       \
186                     || (1 != md5)           \
187                     || (0 != s_description) \
188                     || (0 != l_description)) {
189                         printf (\"Description-$hvf_lang field unexpected in $hvf_file (line %d)\n\", NR)
190                         print_status()
191                         failed = 1
192                         if ($DEBUG) { failures++ } else { exit 1 }
193                 }
194                 s_description++
195         }
196         # Next input line
197         next
198 }
199
200 /^ / {
201         if (0 == failed) {
202                 if (   (1 != package)       \
203                     || (1 != md5)           \
204                     || (1 != s_description)) {
205                         printf (\"Long description unexpected in $hvf_file (line %d)\n\", NR)
206                         print_status()
207                         failed = 1
208                         if ($DEBUG) { failures++ } else { exit 1 }
209                 }
210                 l_description = 1 # There can be any number of long description
211                                   # lines. Do not count.
212         }
213         # Next line
214         next
215 }
216
217 /^$/ {
218         if (0 == failed) {
219                 if (   (1 != package)       \
220                     || (1 != md5)           \
221                     || (1 != s_description) \
222                     || (1 != l_description)) {
223                         printf (\"End of block unexpected in $hvf_file (line %d)\n\", NR)
224                         print_status()
225                         failed = 1
226                         if ($DEBUG) { failures++ } else { exit 1 }
227                 }
228         }
229
230         # Next package
231         package = 0; md5 = 0; s_description = 0; l_description = 0
232         failed = 0
233
234         # Next input line
235         next
236 }
237
238 # Anything else: fail
239 {
240         printf (\"Unexpected line '\$0' in $hvf_file (line %d)\n\", NR)
241         print_status()
242         failed = 1
243         if ($DEBUG) { failures++ } else { exit 1 }
244 }
245
246 END {
247         if (0 == failed) {
248                 # They must be all set to 0 or all set to 1
249                 if (   (   (0 == package)        \
250                         || (0 == md5)            \
251                         || (0 == s_description)  \
252                         || (0 == l_description)) \
253                     && (   (0 != package)        \
254                         || (0 != md5)            \
255                         || (0 != s_description)  \
256                         || (0 != l_description))) {
257                         printf (\"End of file unexpected in $hvf_file (line %d)\n\", NR)
258                         print_status()
259                         exit 1
260                 }
261         }
262
263         if (failures > 0) {
264                 exit 1
265         }
266 }
267 " "$hvf_file" || return 1
268
269         return 0
270 }
271
272 # $SPECIAL_FILES must exist
273 for sf in $SPECIAL_FILES; do
274         if [ ! -f "$dists_parent_dir/$sf" ]; then
275                 echo "Special file ($sf) doesn't exist"
276                 exit 1;
277         fi
278 done
279
280 # Comparing SHA256SUMS
281 # We don use -c because a file could exist in the directory tree and not in
282 # the SHA256SUMS, so we sort the existing SHA256SUMS and we create a new one
283 # already sorted, if cmp fails then files are different and we don't want to
284 # continue.
285 cd "$dists_parent_dir"
286 find dists -type f -print0 |xargs --null sha256sum > "$TMP_WORK_DIR/$SHA256SUMS.new"
287 sort "$SHA256SUMS" > "$TMP_WORK_DIR/$SHA256SUMS.sorted"
288 sort "$TMP_WORK_DIR/$SHA256SUMS.new" > "$TMP_WORK_DIR/$SHA256SUMS.new.sorted"
289 if ! cmp --quiet "$TMP_WORK_DIR/$SHA256SUMS.sorted" "$TMP_WORK_DIR/$SHA256SUMS.new.sorted"; then
290         echo "Failed to compare the SHA256SUMS, they are not identical!" >&2
291         diff -au "$TMP_WORK_DIR/$SHA256SUMS.sorted" "$TMP_WORK_DIR/$SHA256SUMS.new.sorted" >&2
292         exit 1
293 fi
294 cd "$OLDPWD"
295
296 # Get the list of valid packages (sorted, uniq)
297 for t in "$TESTING" "$UNSTABLE"; do
298         if [ ! -f "$PACKAGES_LISTS_DIR/$t" ]; then
299                 echo "Missing $PACKAGES_LISTS_DIR/$t" >&2
300                 exit 1
301         fi
302         cut -d' ' -f 1 "$PACKAGES_LISTS_DIR/$t" | sort -u > "$TMP_WORK_DIR/$t.pkgs"
303 done
304
305 /usr/bin/find "$dists_parent_dir" |
306 while read f; do
307         if   [ -d "$f" ]; then
308                 if ! is_dirname_okay "$f"; then
309                         echo "Wrong directory name: $f" >&2
310                         exit 1
311                 fi
312         elif [ -f "$f" ]; then
313                 # If $f is in $SPECIAL_FILES, we skip to the next loop because
314                 # we won't check it for format, fields and encoding.
315                 for sf in $SPECIAL_FILES; do
316                         if [ "$f" = "$dists_parent_dir/$sf" ]; then
317                                 continue 2
318                         fi
319                 done
320
321                 if ! is_filename_okay "$f"; then
322                         echo "Wrong file: $f" >&2
323                         exit 1
324                 fi
325
326                 # Check that all entries contains the right fields
327                 if ! has_valid_fields "$f"; then
328                         echo "File $f has an invalid format" >&2
329                         exit 1
330                 fi
331
332                 # Check that every packages in Translation-$lang exists
333                 TPKGS=$(basename "$f").pkgs
334                 grep "^Package: " "$f" | cut -d' ' -f 2 | sort -u > "$TMP_WORK_DIR/$TPKGS"
335                 case "$f" in
336                         */$TESTING/*)  t="$TESTING";;
337                         */$UNSTABLE/*) t="$UNSTABLE";;
338                 esac
339                 if diff "$TMP_WORK_DIR/$t.pkgs" "$TMP_WORK_DIR/$TPKGS" | grep -q "^>"; then
340                         diff -au "$TMP_WORK_DIR/$t.pkgs" "$TMP_WORK_DIR/$TPKGS" |grep "^+"
341                         echo "$f contains packages which are not in $t" >&2
342                         exit 1
343                 fi
344
345                 # Check encoding
346                 iconv -f utf-8 -t utf-8 < "$f" > /dev/null 2>&1 || {
347                         echo "$f is not an UTF-8 file" >&2
348                         exit 1
349                 }
350
351                 # We do not check if the md5 in Translation-$lang are
352                 # correct.
353
354                 # Now generate files
355                 #   Compress the file
356                 bzip2 -c "$f" > "$f.bz2"
357                 gzip  -c "$f" > "$f.gz"
358         else
359                 echo "Neither a file or directory: $f" >&2
360                 exit 1
361         fi
362 done || false
363 # The while will just fail if an internal check "exit 1", but the script
364 # is not exited. "|| false" makes the script fail (and exit) in that case.
365
366 echo "$dists_parent_dir structure validated successfully ($(date +%c))"
367
368 # If we reach this point, everything went fine.
369 trap - EXIT
370 rm -rf "$TMP_WORK_DIR"
371