1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5import 'dart:convert';
6import 'dart:io';
7
8import 'package:args/args.dart' as argslib;
9import 'package:meta/meta.dart';
10
11import 'language_subtag_registry.dart';
12
13typedef HeaderGenerator = String Function(String regenerateInstructions);
14typedef ConstructorGenerator = String Function(LocaleInfo locale);
15
16int sortFilesByPath(FileSystemEntity a, FileSystemEntity b) {
17 return a.path.compareTo(b.path);
18}
19
20/// Simple data class to hold parsed locale. Does not promise validity of any data.
21@immutable
22class LocaleInfo implements Comparable<LocaleInfo> {
23 const LocaleInfo({
24 required this.languageCode,
25 this.scriptCode,
26 this.countryCode,
27 required this.length,
28 required this.originalString,
29 });
30
31 /// Simple parser. Expects the locale string to be in the form of 'language_script_COUNTRY'
32 /// where the language is 2 characters, script is 4 characters with the first uppercase,
33 /// and country is 2-3 characters and all uppercase.
34 ///
35 /// 'language_COUNTRY' or 'language_script' are also valid. Missing fields will be null.
36 ///
37 /// When `deriveScriptCode` is true, if [scriptCode] was unspecified, it will
38 /// be derived from the [languageCode] and [countryCode] if possible.
39 factory LocaleInfo.fromString(String locale, {bool deriveScriptCode = false}) {
40 final List<String> codes = locale.split('_'); // [language, script, country]
41 assert(codes.isNotEmpty && codes.length < 4);
42 final String languageCode = codes[0];
43 String? scriptCode;
44 String? countryCode;
45 int length = codes.length;
46 String originalString = locale;
47 if (codes.length == 2) {
48 scriptCode = codes[1].length >= 4 ? codes[1] : null;
49 countryCode = codes[1].length < 4 ? codes[1] : null;
50 } else if (codes.length == 3) {
51 scriptCode = codes[1].length > codes[2].length ? codes[1] : codes[2];
52 countryCode = codes[1].length < codes[2].length ? codes[1] : codes[2];
53 }
54 assert(codes[0].isNotEmpty);
55 assert(countryCode == null || countryCode.isNotEmpty);
56 assert(scriptCode == null || scriptCode.isNotEmpty);
57
58 /// Adds scriptCodes to locales where we are able to assume it to provide
59 /// finer granularity when resolving locales.
60 ///
61 /// The basis of the assumptions here are based off of known usage of scripts
62 /// across various countries. For example, we know Taiwan uses traditional (Hant)
63 /// script, so it is safe to apply (Hant) to Taiwanese languages.
64 if (deriveScriptCode && scriptCode == null) {
65 scriptCode = switch ((languageCode, countryCode)) {
66 ('zh', 'CN' || 'SG' || null) => 'Hans',
67 ('zh', 'TW' || 'HK' || 'MO') => 'Hant',
68 ('sr', null) => 'Cyrl',
69 _ => null,
70 };
71 // Increment length if we were able to assume a scriptCode.
72 if (scriptCode != null) {
73 length += 1;
74 }
75 // Update the base string to reflect assumed scriptCodes.
76 originalString = languageCode;
77 if (scriptCode != null) {
78 originalString += '_$scriptCode';
79 }
80 if (countryCode != null) {
81 originalString += '_$countryCode';
82 }
83 }
84
85 return LocaleInfo(
86 languageCode: languageCode,
87 scriptCode: scriptCode,
88 countryCode: countryCode,
89 length: length,
90 originalString: originalString,
91 );
92 }
93
94 final String languageCode;
95 final String? scriptCode;
96 final String? countryCode;
97 final int length; // The number of fields. Ranges from 1-3.
98 final String originalString; // Original un-parsed locale string.
99
100 String camelCase() {
101 return originalString
102 .split('_')
103 .map<String>(
104 (String part) => part.substring(0, 1).toUpperCase() + part.substring(1).toLowerCase(),
105 )
106 .join();
107 }
108
109 @override
110 bool operator ==(Object other) {
111 return other is LocaleInfo && other.originalString == originalString;
112 }
113
114 @override
115 int get hashCode => originalString.hashCode;
116
117 @override
118 String toString() {
119 return originalString;
120 }
121
122 @override
123 int compareTo(LocaleInfo other) {
124 return originalString.compareTo(other.originalString);
125 }
126}
127
128/// Parse the data for a locale from a file, and store it in the [attributes]
129/// and [resources] keys.
130void loadMatchingArbsIntoBundleMaps({
131 required Directory directory,
132 required RegExp filenamePattern,
133 required Map<LocaleInfo, Map<String, String>> localeToResources,
134 required Map<LocaleInfo, Map<String, dynamic>> localeToResourceAttributes,
135}) {
136 /// Set that holds the locales that were assumed from the existing locales.
137 ///
138 /// For example, when the data lacks data for zh_Hant, we will use the data of
139 /// the first Hant Chinese locale as a default by repeating the data. If an
140 /// explicit match is later found, we can reference this set to see if we should
141 /// overwrite the existing assumed data.
142 final Set<LocaleInfo> assumedLocales = <LocaleInfo>{};
143
144 for (final FileSystemEntity entity in directory.listSync().toList()..sort(sortFilesByPath)) {
145 final String entityPath = entity.path;
146 if (FileSystemEntity.isFileSync(entityPath) && filenamePattern.hasMatch(entityPath)) {
147 final String localeString = filenamePattern.firstMatch(entityPath)![1]!;
148 final File arbFile = File(entityPath);
149
150 // Helper method to fill the maps with the correct data from file.
151 void populateResources(LocaleInfo locale, File file) {
152 final Map<String, String> resources = localeToResources[locale]!;
153 final Map<String, dynamic> attributes = localeToResourceAttributes[locale]!;
154 final Map<String, dynamic> bundle =
155 json.decode(file.readAsStringSync()) as Map<String, dynamic>;
156 for (final String key in bundle.keys) {
157 // The ARB file resource "attributes" for foo are called @foo.
158 if (key.startsWith('@')) {
159 attributes[key.substring(1)] = bundle[key];
160 } else {
161 resources[key] = bundle[key] as String;
162 }
163 }
164 }
165
166 // Only pre-assume scriptCode if there is a country or script code to assume off of.
167 // When we assume scriptCode based on languageCode-only, we want this initial pass
168 // to use the un-assumed version as a base class.
169 LocaleInfo locale = LocaleInfo.fromString(
170 localeString,
171 deriveScriptCode: localeString.split('_').length > 1,
172 );
173 // Allow overwrite if the existing data is assumed.
174 if (assumedLocales.contains(locale)) {
175 localeToResources[locale] = <String, String>{};
176 localeToResourceAttributes[locale] = <String, dynamic>{};
177 assumedLocales.remove(locale);
178 } else {
179 localeToResources[locale] ??= <String, String>{};
180 localeToResourceAttributes[locale] ??= <String, dynamic>{};
181 }
182 populateResources(locale, arbFile);
183 // Add an assumed locale to default to when there is no info on scriptOnly locales.
184 locale = LocaleInfo.fromString(localeString, deriveScriptCode: true);
185 if (locale.scriptCode != null) {
186 final LocaleInfo scriptLocale = LocaleInfo.fromString(
187 '${locale.languageCode}_${locale.scriptCode}',
188 );
189 if (!localeToResources.containsKey(scriptLocale)) {
190 assumedLocales.add(scriptLocale);
191 localeToResources[scriptLocale] ??= <String, String>{};
192 localeToResourceAttributes[scriptLocale] ??= <String, dynamic>{};
193 populateResources(scriptLocale, arbFile);
194 }
195 }
196 }
197 }
198}
199
200void exitWithError(String errorMessage) {
201 stderr.writeln('fatal: $errorMessage');
202 exit(1);
203}
204
205void checkCwdIsRepoRoot(String commandName) {
206 final bool isRepoRoot = Directory('.git').existsSync();
207
208 if (!isRepoRoot) {
209 exitWithError(
210 '$commandName must be run from the root of the Flutter repository. The '
211 'current working directory is: ${Directory.current.path}',
212 );
213 }
214}
215
216GeneratorOptions parseArgs(List<String> rawArgs) {
217 final argslib.ArgParser argParser = argslib.ArgParser()
218 ..addFlag('help', abbr: 'h', help: 'Print the usage message for this command')
219 ..addFlag('overwrite', abbr: 'w', help: 'Overwrite existing localizations')
220 ..addFlag(
221 'remove-undefined',
222 help: 'Remove any localizations that are not defined in the canonical locale.',
223 )
224 ..addFlag(
225 'widgets',
226 help:
227 'Whether to print the generated classes for the Widgets package only. Ignored when --overwrite is passed.',
228 )
229 ..addFlag(
230 'material',
231 help:
232 'Whether to print the generated classes for the Material package only. Ignored when --overwrite is passed.',
233 )
234 ..addFlag(
235 'cupertino',
236 help:
237 'Whether to print the generated classes for the Cupertino package only. Ignored when --overwrite is passed.',
238 );
239 final argslib.ArgResults args = argParser.parse(rawArgs);
240 if (args.wasParsed('help') && args['help'] == true) {
241 stderr.writeln(argParser.usage);
242 exit(0);
243 }
244 final bool writeToFile = args['overwrite'] as bool;
245 final bool removeUndefined = args['remove-undefined'] as bool;
246 final bool widgetsOnly = args['widgets'] as bool;
247 final bool materialOnly = args['material'] as bool;
248 final bool cupertinoOnly = args['cupertino'] as bool;
249
250 return GeneratorOptions(
251 writeToFile: writeToFile,
252 materialOnly: materialOnly,
253 cupertinoOnly: cupertinoOnly,
254 widgetsOnly: widgetsOnly,
255 removeUndefined: removeUndefined,
256 );
257}
258
259class GeneratorOptions {
260 GeneratorOptions({
261 required this.writeToFile,
262 required this.removeUndefined,
263 required this.materialOnly,
264 required this.cupertinoOnly,
265 required this.widgetsOnly,
266 });
267
268 final bool writeToFile;
269 final bool removeUndefined;
270 final bool materialOnly;
271 final bool cupertinoOnly;
272 final bool widgetsOnly;
273}
274
275// See also //master/tools/gen_locale.dart in the engine repo.
276Map<String, List<String>> _parseSection(String section) {
277 final Map<String, List<String>> result = <String, List<String>>{};
278 late List<String> lastHeading;
279 for (final String line in section.split('\n')) {
280 if (line == '') {
281 continue;
282 }
283 if (line.startsWith(' ')) {
284 lastHeading[lastHeading.length - 1] = '${lastHeading.last}${line.substring(1)}';
285 continue;
286 }
287 final int colon = line.indexOf(':');
288 if (colon <= 0) {
289 throw 'not sure how to deal with "$line"';
290 }
291 final String name = line.substring(0, colon);
292 final String value = line.substring(colon + 2);
293 lastHeading = result.putIfAbsent(name, () => <String>[]);
294 result[name]!.add(value);
295 }
296 return result;
297}
298
299final Map<String, String> _languages = <String, String>{};
300final Map<String, String> _regions = <String, String>{};
301final Map<String, String> _scripts = <String, String>{};
302const String kProvincePrefix = ', Province of ';
303const String kParentheticalPrefix = ' (';
304
305/// Prepares the data for the [describeLocale] method below.
306///
307/// The data is obtained from the official IANA registry.
308void precacheLanguageAndRegionTags() {
309 final List<Map<String, List<String>>> sections = languageSubtagRegistry
310 .split('%%')
311 .skip(1)
312 .map<Map<String, List<String>>>(_parseSection)
313 .toList();
314 for (final Map<String, List<String>> section in sections) {
315 assert(section.containsKey('Type'), section.toString());
316 final String type = section['Type']!.single;
317 if (type == 'language' || type == 'region' || type == 'script') {
318 assert(
319 section.containsKey('Subtag') && section.containsKey('Description'),
320 section.toString(),
321 );
322 final String subtag = section['Subtag']!.single;
323 String description = section['Description']!.join(' ');
324 if (description.startsWith('United ')) {
325 description = 'the $description';
326 }
327 if (description.contains(kParentheticalPrefix)) {
328 description = description.substring(0, description.indexOf(kParentheticalPrefix));
329 }
330 if (description.contains(kProvincePrefix)) {
331 description = description.substring(0, description.indexOf(kProvincePrefix));
332 }
333 if (description.endsWith(' Republic')) {
334 description = 'the $description';
335 }
336 switch (type) {
337 case 'language':
338 _languages[subtag] = description;
339 case 'region':
340 _regions[subtag] = description;
341 case 'script':
342 _scripts[subtag] = description;
343 }
344 }
345 }
346}
347
348String describeLocale(String tag) {
349 final List<String> subtags = tag.split('_');
350 assert(subtags.isNotEmpty);
351 assert(_languages.containsKey(subtags[0]));
352 final String language = _languages[subtags[0]]!;
353 String output = language;
354 String? region;
355 String? script;
356 if (subtags.length == 2) {
357 region = _regions[subtags[1]];
358 script = _scripts[subtags[1]];
359 assert(region != null || script != null);
360 } else if (subtags.length >= 3) {
361 region = _regions[subtags[2]];
362 script = _scripts[subtags[1]];
363 assert(region != null && script != null);
364 }
365 if (region != null) {
366 output += ', as used in $region';
367 }
368 if (script != null) {
369 output += ', using the $script script';
370 }
371 return output;
372}
373
374/// Writes the header of each class which corresponds to a locale.
375String generateClassDeclaration(LocaleInfo locale, String classNamePrefix, String superClass) {
376 final String camelCaseName = locale.camelCase();
377 return '''
378
379/// The translations for ${describeLocale(locale.originalString)} (`${locale.originalString}`).
380class $classNamePrefix$camelCaseName extends $superClass {''';
381}
382
383/// Return the input string as a Dart-parseable string.
384///
385/// ```none
386/// foo => 'foo'
387/// foo "bar" => 'foo "bar"'
388/// foo 'bar' => "foo 'bar'"
389/// foo 'bar' "baz" => '''foo 'bar' "baz"'''
390/// ```
391///
392/// This function is used by tools that take in a JSON-formatted file to
393/// generate Dart code. For this reason, characters with special meaning
394/// in JSON files are escaped. For example, the backspace character (\b)
395/// has to be properly escaped by this function so that the generated
396/// Dart code correctly represents this character:
397/// ```none
398/// foo\bar => 'foo\\bar'
399/// foo\nbar => 'foo\\nbar'
400/// foo\\nbar => 'foo\\\\nbar'
401/// foo\\bar => 'foo\\\\bar'
402/// foo\ bar => 'foo\\ bar'
403/// foo$bar = 'foo\$bar'
404/// ```
405String generateString(String value) {
406 if (<String>['\n', '\f', '\t', '\r', '\b'].every((String pattern) => !value.contains(pattern))) {
407 final bool hasDollar = value.contains(r'$');
408 final bool hasBackslash = value.contains(r'\');
409 final bool hasQuote = value.contains("'");
410 final bool hasDoubleQuote = value.contains('"');
411 if (!hasQuote) {
412 return hasBackslash || hasDollar ? "r'$value'" : "'$value'";
413 }
414 if (!hasDoubleQuote) {
415 return hasBackslash || hasDollar ? 'r"$value"' : '"$value"';
416 }
417 }
418
419 const String backslash = '__BACKSLASH__';
420 assert(
421 !value.contains(backslash),
422 'Input string cannot contain the sequence: '
423 '"__BACKSLASH__", as it is used as part of '
424 'backslash character processing.',
425 );
426
427 value = value
428 // Replace backslashes with a placeholder for now to properly parse
429 // other special characters.
430 .replaceAll(r'\', backslash)
431 .replaceAll(r'$', r'\$')
432 .replaceAll("'", r"\'")
433 .replaceAll('"', r'\"')
434 .replaceAll('\n', r'\n')
435 .replaceAll('\f', r'\f')
436 .replaceAll('\t', r'\t')
437 .replaceAll('\r', r'\r')
438 .replaceAll('\b', r'\b')
439 // Reintroduce escaped backslashes into generated Dart string.
440 .replaceAll(backslash, r'\\');
441
442 return "'$value'";
443}
444
445/// Only used to generate localization strings for the Kannada locale ('kn') because
446/// some of the localized strings contain characters that can crash Emacs on Linux.
447/// See packages/flutter_localizations/lib/src/l10n/README for more information.
448String generateEncodedString(String? locale, String value) {
449 if (locale != 'kn' || value.runes.every((int code) => code <= 0xFF)) {
450 return generateString(value);
451 }
452
453 final String unicodeEscapes = value.runes
454 .map((int code) => '\\u{${code.toRadixString(16)}}')
455 .join();
456 return "'$unicodeEscapes'";
457}
458