Skip to content

Commit

Permalink
CLDR-15830 Extensive refactoring of NameGetter; new enum NameType (#4246
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@btangmu
btangmu authored Dec 19, 2024
1 parent 162bff6 commit 72b7e8a
Show file tree
Hide file tree
Showing 123 changed files with 1,233 additions and 1,135 deletions.
3 changes: 2 additions & 1 deletion tools/cldr-apps/src/main/java/org/unicode/cldr/web/LocaleTree.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.util.TreeMap;
import org.unicode.cldr.util.CLDRLocale;
import org.unicode.cldr.util.CLDRLocale.CLDRFormatter;
import org.unicode.cldr.util.NameGetter;

public class LocaleTree {
CLDRFormatter displayLocale;
Expand Down Expand Up @@ -97,7 +98,7 @@ public CLDRLocale getLocaleCode(String localeName) {
}

public String getLocaleDisplayName(CLDRLocale locale) {
return displayLocale.getDisplayName(locale, true, null);
return displayLocale.getDisplayName(locale, NameGetter.NameOpt.COMPOUND_ONLY, null);
}

public Map<String, CLDRLocale> getMap() {
Expand Down
8 changes: 5 additions & 3 deletions tools/cldr-code/src/main/java/org/unicode/cldr/draft/GenerateLanguageData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.unicode.cldr.util.Factory;
import org.unicode.cldr.util.LanguageTagParser;
import org.unicode.cldr.util.NameGetter;
import org.unicode.cldr.util.NameType;
import org.unicode.cldr.util.StandardCodes;
import org.unicode.cldr.util.SupplementalDataInfo;
import org.unicode.cldr.util.SupplementalDataInfo.OfficialStatus;
Expand Down Expand Up @@ -47,7 +48,8 @@ private void run() throws IOException {
Map<String, String> languageNameToCode = new TreeMap<>();
NameGetter nameGetter = english.nameGetter();
for (String languageCode : info.getLanguages()) {
languageNameToCode.put(nameGetter.getNameFromBCP47(languageCode), languageCode);
languageNameToCode.put(
nameGetter.getNameFromIdentifier(languageCode), languageCode);
}
out.println("\n@sheet:CLDR County Data");
out.println("code\tgdp\tlit-pop\tpopulation\tliteracy");
Expand All @@ -73,7 +75,7 @@ private void run() throws IOException {
Map<String, Counter2<String>> langToCountriesOfficial = new TreeMap<>();

for (String languageCode : info.getLanguages()) {
String languageName = nameGetter.getNameFromBCP47(languageCode);
String languageName = nameGetter.getNameFromIdentifier(languageCode);

String baseLanguage = languageCode;
ltp.set(languageCode);
Expand All @@ -93,7 +95,7 @@ private void run() throws IOException {
for (String territory : territories) {
PopulationData terrData = info.getPopulationDataForTerritory(territory);
String territoryName =
nameGetter.getNameFromTypenumCode(CLDRFile.TERRITORY_NAME, territory);
nameGetter.getNameFromTypeEnumCode(NameType.TERRITORY, territory);

PopulationData data =
info.getLanguageAndTerritoryPopulationData(languageCode, territory);
Expand Down
10 changes: 7 additions & 3 deletions tools/cldr-code/src/main/java/org/unicode/cldr/draft/Misc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,11 @@ private static void showDefaultContent(String... strings) {
}
}
System.out.println(
string + "\t" + defCon + "\t" + english.nameGetter().getNameFromBCP47(defCon));
string
+ "\t"
+ defCon
+ "\t"
+ english.nameGetter().getNameFromIdentifier(defCon));
}
}

Expand Down Expand Up @@ -329,7 +333,7 @@ private static void showExemplarSize() {
if (temp != null) {
baseLanguage = temp.get0().get(0);
}
String englishName = english.nameGetter().getNameFromBCP47(baseLanguage);
String englishName = english.nameGetter().getNameFromIdentifier(baseLanguage);
CLDRFile cldrFile = factory.make(baseLanguage, false);
UnicodeSet set = cldrFile.getExemplarSet("", WinningChoice.WINNING);
int script = -1;
Expand All @@ -340,7 +344,7 @@ private static void showExemplarSize() {
break;
}
}
String nativeName = cldrFile.nameGetter().getNameFromBCP47(baseLanguage);
String nativeName = cldrFile.nameGetter().getNameFromIdentifier(baseLanguage);
nameAndInfo.add(
englishName
+ "\t"
Expand Down
7 changes: 4 additions & 3 deletions tools/cldr-code/src/main/java/org/unicode/cldr/draft/XLocaleDistance.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.unicode.cldr.draft.XLocaleDistance.RegionMapper.Builder;
import org.unicode.cldr.util.CLDRConfig;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.NameType;
import org.unicode.cldr.util.SupplementalDataInfo;

public class XLocaleDistance {
Expand Down Expand Up @@ -971,7 +972,7 @@ private static String fixedName(List<String> match) {
} else {
result.append(
english.nameGetter()
.getNameFromTypenumCode(CLDRFile.TERRITORY_NAME, region));
.getNameFromTypeEnumCode(NameType.TERRITORY, region));
}
case 2:
String script = alt.get(1);
Expand All @@ -981,7 +982,7 @@ private static String fixedName(List<String> match) {
result.insert(
0,
english.nameGetter()
.getNameFromTypenumCode(CLDRFile.TERRITORY_NAME, script));
.getNameFromTypeEnumCode(NameType.TERRITORY, script));
}
case 1:
String language = alt.get(0);
Expand All @@ -991,7 +992,7 @@ private static String fixedName(List<String> match) {
result.insert(
0,
english.nameGetter()
.getNameFromTypenumCode(CLDRFile.TERRITORY_NAME, language));
.getNameFromTypeEnumCode(NameType.TERRITORY, language));
}
}
return Joiner.on("; ").join(alt);
Expand Down
146 changes: 91 additions & 55 deletions tools/cldr-code/src/main/java/org/unicode/cldr/test/CLDRTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.security.InvalidParameterException;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
Expand Down Expand Up @@ -77,8 +79,6 @@ public class CLDRTest extends TestFmwk {
private CLDRFile resolvedEnglish;
private final UnicodeSet commonAndInherited =
new UnicodeSet("[[:script=common:][:script=inherited:][:alphabetic=false:]]");
private static final String[] WIDTHS = {"narrow", "wide", "abbreviated", "short"};
private static final String[] MONTHORDAYS = {"day", "month"};
private Map<String, String> localeNameCache = new HashMap<>();
private CLDRFile english = null;

Expand Down Expand Up @@ -413,7 +413,7 @@ public String getLocaleName(String locale) {
String name = localeNameCache.get(locale);
if (name != null) return name;
if (english == null) english = cldrFactory.make("en", true);
String result = english.nameGetter().getNameFromBCP47(locale);
String result = english.nameGetter().getNameFromIdentifier(locale);
/*
* Collection c = Utility.splitList(locale, '_', false, null);
* String[] pieces = new String[c.size()];
Expand Down Expand Up @@ -494,9 +494,14 @@ public void TestForIllegalAttributeValues() {
public void TestDisplayNameCollisions() {
if (disableUntilLater("TestDisplayNameCollisions")) return;

Map<String, String>[] maps = new HashMap[CLDRFile.LIMIT_TYPES];
for (int i = 0; i < maps.length; ++i) {
maps[i] = new HashMap<>();
Set<NameType> nameTypeSet = EnumSet.allOf(NameType.class);
Map<String, String>[] maps = new HashMap[nameTypeSet.size()];
Map<NameType, Integer> nameTypeIntegerMap = new HashMap();
int j = 0;
for (NameType nameType : NameType.values()) {
maps[j] = new HashMap<>();
nameTypeIntegerMap.put(nameType, j);
++j;
}
Set<String> collisions = new TreeSet<>();
for (Iterator<String> it = locales.iterator(); it.hasNext(); ) {
Expand All @@ -509,22 +514,17 @@ public void TestDisplayNameCollisions() {

for (Iterator<String> it2 = item.iterator(); it2.hasNext(); ) {
String xpath = it2.next();
int nameType = CLDRFile.getNameType(xpath);
if (nameType < 0) continue;
NameType nameType = NameType.fromPath(xpath);
if (nameType == NameType.NONE) continue;
String value = item.getStringValue(xpath);
String xpath2 = maps[nameType].get(value);
int nameTypeIndex = nameTypeIntegerMap.get(nameType);
String xpath2 = maps[nameTypeIndex].get(value);
if (xpath2 == null) {
maps[nameType].put(value, xpath);
maps[nameTypeIndex].put(value, xpath);
continue;
}
collisions.add(
CLDRFile.getNameTypeName(nameType)
+ "\t"
+ value
+ "\t"
+ xpath
+ "\t"
+ xpath2);
String theName = nameType.getNameTypeName();
collisions.add(theName + "\t" + value + "\t" + xpath + "\t" + xpath2);
surveyInfo.add(
locale
+ "\t"
Expand Down Expand Up @@ -843,6 +843,12 @@ void getSupplementalData(
}
}

/** Settings for checkForItems behavior */
private enum KeyOpt {
DEFAULT,
DATE
};

/** Verify that the minimal localizations are present. */
public void TestMinimalLocalization() throws IOException {
if (disableUntilLater("TestMinimalLocalization")) return;
Expand Down Expand Up @@ -876,10 +882,7 @@ public void TestMinimalLocalization() throws IOException {
// languages
Set<String> languages = new TreeSet<>(CldrUtility.MINIMUM_LANGUAGES);
languages.add(language);
// LANGUAGE_NAME = 0, SCRIPT_NAME = 1, TERRITORY_NAME = 2, VARIANT_NAME = 3,
// CURRENCY_NAME = 4, CURRENCY_SYMBOL = 5, TZID = 6

checkForItems(item, languages, CLDRFile.LANGUAGE_NAME, missing, failureCount, null);
checkForItems(item, languages, NameType.LANGUAGE, missing, failureCount);

/*
* checkTranslatedCode(cldrfile, codes, "currency", "//ldml/numbers/currencies/currency");
Expand All @@ -891,12 +894,12 @@ public void TestMinimalLocalization() throws IOException {
scripts.add("Latn");
Set<String> others = language_scripts.get(language);
if (others != null) scripts.addAll(others);
checkForItems(item, scripts, CLDRFile.SCRIPT_NAME, missing, failureCount, null);
checkForItems(item, scripts, NameType.SCRIPT, missing, failureCount);

Set<String> countries = new TreeSet<>(CldrUtility.MINIMUM_TERRITORIES);
others = language_territories.get(language);
if (others != null) countries.addAll(others);
checkForItems(item, countries, CLDRFile.TERRITORY_NAME, missing, failureCount, null);
checkForItems(item, countries, NameType.TERRITORY, missing, failureCount);

Set<String> currencies = new TreeSet<>();
StandardCodes sc = StandardCodes.make();
Expand All @@ -909,9 +912,9 @@ public void TestMinimalLocalization() throws IOException {
currencies.addAll(countryCurrencies);
}
}
checkForItems(item, currencies, CLDRFile.CURRENCY_NAME, missing, failureCount, null);
checkForItems(item, currencies, NameType.CURRENCY, missing, failureCount);
checkForItems(
item, currencies, CLDRFile.CURRENCY_SYMBOL, missing, failureCount, exemplars);
item, currencies, NameType.CURRENCY_SYMBOL, missing, failureCount, exemplars);

// context=format and width=wide; context=stand-alone & width=abbreviated
Set<String> months = new TreeSet<>();
Expand All @@ -922,10 +925,13 @@ public void TestMinimalLocalization() throws IOException {
new String[] {
"sun", "mon", "tue", "wed", "thu", "fri", "sat"
}));
for (int i = -7; i < 0; ++i) {
checkForItems(item, (i < -4 ? months : days), i, missing, failureCount, null);
}

checkForItemsDate(item, months, NameType.TZ_EXEMPLAR, missing, failureCount);
checkForItemsDate(item, months, NameType.CURRENCY_SYMBOL, missing, failureCount);
checkForItemsDate(item, months, NameType.CURRENCY, missing, failureCount);
checkForItemsDate(item, days, NameType.VARIANT, missing, failureCount);
checkForItemsDate(item, days, NameType.TERRITORY, missing, failureCount);
checkForItemsDate(item, days, NameType.SCRIPT, missing, failureCount);
checkForItemsDate(item, days, NameType.LANGUAGE, missing, failureCount);
String filename = "missing_" + locale + ".xml";
if (failureCount[0] > 0 || warningCount[0] > 0) {
PrintWriter out =
Expand Down Expand Up @@ -964,38 +970,70 @@ private String getDateKey(String monthOrDay, String width, String code) {
}

/** Internal */
private String getDateKey(int type, String code) {
// type is 6..4 for months abbrev..narrow, 3..0 for days short..narrow
int monthOrDayType = 0, widthType = type;
if (type >= 4) {
monthOrDayType = 1;
widthType -= 4;
private String getDateKey(NameType type, String code) {
switch (type) {
case TZ_EXEMPLAR:
return getDateKey("month", "abbreviated", code);
case CURRENCY_SYMBOL:
return getDateKey("month", "wide", code);
case CURRENCY:
return getDateKey("month", "narrow", code);
case VARIANT:
return getDateKey("day", "short", code);
case TERRITORY:
return getDateKey("day", "abbreviated", code);
case SCRIPT:
return getDateKey("day", "wide", code);
case LANGUAGE:
return getDateKey("day", "narrow", code);
default:
throw new InvalidParameterException("getDateKey: " + type);
}
return getDateKey(MONTHORDAYS[monthOrDayType], WIDTHS[widthType], code);
}

/**
* @param item
* @param codes
* @param missing
* @param exemplarTest TODO TODO
*/
private void checkForItems(
CLDRFile item,
Set<String> codes,
int type,
NameType nameType,
CLDRFile missing,
int[] failureCount) {
checkForItems(item, codes, nameType, KeyOpt.DEFAULT, missing, failureCount, null);
}

private void checkForItems(
CLDRFile item,
Set<String> codes,
NameType nameType,
CLDRFile missing,
int failureCount[],
int[] failureCount,
UnicodeSet exemplarTest) {
checkForItems(item, codes, nameType, KeyOpt.DEFAULT, missing, failureCount, exemplarTest);
}

private void checkForItemsDate(
CLDRFile item,
Set<String> codes,
NameType nameType,
CLDRFile missing,
int[] failureCount) {
checkForItems(item, codes, nameType, KeyOpt.DATE, missing, failureCount, null);
}

private void checkForItems(
CLDRFile item,
Set<String> codes,
NameType nameType,
KeyOpt keyOpt,
CLDRFile missing,
int[] failureCount,
UnicodeSet exemplarTest) {
// check codes
for (Iterator<String> it2 = codes.iterator(); it2.hasNext(); ) {
String code = it2.next();
String key;
if (type >= 0) {
key = CLDRFile.getKey(type, code);
} else {
key = getDateKey(-type - 1, code);
}
String key =
(keyOpt == KeyOpt.DATE)
? getDateKey(nameType, code)
: nameType.getKeyPath(code);
String v = item.getStringValue(key);
String rootValue = resolvedRoot.getStringValue(key);
if (v == null
Expand Down Expand Up @@ -1048,8 +1086,7 @@ public Object transform(Object source) {
public Object transform(Object source) {
if (english == null) english = cldrFactory.make("en", true);
return english.nameGetter()
.getNameFromTypenumCode(
CLDRFile.CURRENCY_NAME, source.toString())
.getNameFromTypeEnumCode(NameType.CURRENCY, source.toString())
+ " ("
+ source
+ ")";
Expand Down Expand Up @@ -1195,8 +1232,7 @@ public void TestSupplementalData() {
NameGetter englishNameGetter = english.nameGetter();
for (Iterator<String> it = legalCurrencies.iterator(); it.hasNext(); ) {
String currency = it.next();
String name =
englishNameGetter.getNameFromTypenumCode(CLDRFile.CURRENCY_NAME, currency);
String name = englishNameGetter.getNameFromTypeEnumCode(NameType.CURRENCY, currency);
if (name == null) {
String standardName = sc.getFullData("currency", currency).get(0);
logln("\t\t\t<currency type=\"" + currency + "\">");
Expand Down
3 changes: 2 additions & 1 deletion tools/cldr-code/src/main/java/org/unicode/cldr/test/CheckNumbers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.unicode.cldr.util.Factory;
import org.unicode.cldr.util.ICUServiceBuilder;
import org.unicode.cldr.util.LocaleIDParser;
import org.unicode.cldr.util.NameType;
import org.unicode.cldr.util.PathHeader;
import org.unicode.cldr.util.PatternCache;
import org.unicode.cldr.util.PluralRulesUtil;
Expand Down Expand Up @@ -577,7 +578,7 @@ private void checkPattern(
private void checkCurrencyFormats(
String path, String fullPath, String value, List result, boolean generateExamples)
throws ParseException {
DecimalFormat x = icuServiceBuilder.getCurrencyFormat(CLDRFile.getCode(path));
DecimalFormat x = icuServiceBuilder.getCurrencyFormat(NameType.getCode(path));
addOrTestSamples(x, x.toPattern(), value, result, generateExamples);
}

Expand Down
Loading

0 comments on commit 72b7e8a

Please sign in to comment.