Datagen: Cache supported_locales impls (#4470)

unicode-org · Dec 19, 2023 · bd36f1e · bd36f1e
1 parent ae4c162
commit bd36f1e
Show file tree

Hide file tree

Showing 24 changed files with 118 additions and 66 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/provider/core/src/datagen/iter.rs b/provider/core/src/datagen/iter.rs
@@ -22,6 +22,10 @@ pub trait IterableDynamicDataProvider<M: DataMarker>: DynamicDataProvider<M> {
 pub trait IterableDataProvider<M: KeyedDataMarker>: DataProvider<M> {
     /// Returns a list of [`DataLocale`].
     fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError>;
+    /// Returns whether a [`DataLocale`] is in the supported locales list.
+    fn supports_locale(&self, locale: &DataLocale) -> Result<bool, DataError> {
+        self.supported_locales().map(|v| v.contains(locale))
+    }
 }
 
 impl<M, P> IterableDynamicDataProvider<M> for Box<P>

diff --git a/provider/datagen/Cargo.toml b/provider/datagen/Cargo.toml
@@ -73,7 +73,7 @@ proc-macro2 = {version = "1", optional = true }
 
 # Other external dependencies
 displaydoc = { version = "0.2.3", default-features = false }
-elsa = "1.9"
+elsa = "1.10"
 itertools = "0.10"
 log = "0.4"
 memchr = "2.5.0"

diff --git a/provider/datagen/src/provider.rs b/provider/datagen/src/provider.rs
@@ -7,7 +7,10 @@
 use crate::source::*;
 use crate::transform::cldr::source::CldrCache;
 use crate::{CollationHanDatabase, CoverageLevel};
+use elsa::sync::FrozenMap;
+use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
+use std::collections::HashSet;
 use std::fmt::Debug;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -78,6 +81,7 @@ impl DatagenProvider {
                 icuexport_dictionary_fallback: None,
                 #[cfg(feature = "legacy_api")]
                 collations: Default::default(),
+                supported_locales_cache: Default::default(),
             },
         }
     }
@@ -258,6 +262,24 @@ impl DatagenProvider {
     ) -> Result<impl IntoIterator<Item = icu_locid::LanguageIdentifier>, DataError> {
         self.cldr()?.locales(levels)
     }
+
+    pub(crate) fn supported_locales_set<M>(&self) -> Result<&HashSet<DataLocale>, DataError>
+    where
+        M: KeyedDataMarker,
+        Self: IterableDataProviderInternal<M>,
+    {
+        #[allow(deprecated)] // SourceData
+        self.source
+            .supported_locales_cache
+            .insert_with(M::KEY, || {
+                Box::new(
+                    self.supported_locales_impl()
+                        .map(|v| v.into_iter().collect()),
+                )
+            })
+            .as_ref()
+            .map_err(|e| *e)
+    }
 }
 
 /// Specifies the trie type to use.
@@ -306,6 +328,9 @@ pub struct SourceData {
     pub(crate) icuexport_dictionary_fallback: Option<Arc<SerdeCache>>,
     #[cfg(feature = "legacy_api")]
     pub(crate) collations: Vec<String>,
+    #[allow(clippy::type_complexity)] // not as complex as it appears
+    pub(crate) supported_locales_cache:
+        Arc<FrozenMap<DataKey, Box<Result<HashSet<DataLocale>, DataError>>>>,
 }
 
 #[cfg(feature = "legacy_api")]
@@ -485,3 +510,21 @@ impl SourceData {
             .locales(levels.iter().copied())
     }
 }
+
+pub(crate) trait IterableDataProviderInternal<M: KeyedDataMarker>: DataProvider<M> {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError>;
+}
+
+impl<M: KeyedDataMarker> IterableDataProvider<M> for DatagenProvider
+where
+    DatagenProvider: IterableDataProviderInternal<M>,
+{
+    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+        self.supported_locales_set()
+            .map(|v| v.iter().cloned().collect())
+    }
+
+    fn supports_locale(&self, locale: &DataLocale) -> Result<bool, DataError> {
+        self.supported_locales_set().map(|v| v.contains(locale))
+    }
+}
diff --git a/provider/datagen/src/transform/cldr/characters/mod.rs b/provider/datagen/src/transform/cldr/characters/mod.rs
@@ -5,10 +5,10 @@
 use std::collections::HashSet;
 use std::marker::PhantomData;
 
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde;
 use icu_collections::codepointinvliststringlist::CodePointInversionListAndStringList;
 use icu_properties::provider::*;
-use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
 use itertools::Itertools;
 
@@ -46,8 +46,8 @@ macro_rules! exemplar_chars_impls {
             }
         }
 
-        impl IterableDataProvider<$data_marker_name> for crate::DatagenProvider {
-            fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+        impl IterableDataProviderInternal<$data_marker_name> for crate::DatagenProvider {
+            fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
                 Ok(self
                     .cldr()?
                     .misc()

diff --git a/provider/datagen/src/transform/cldr/currency/mod.rs b/provider/datagen/src/transform/cldr/currency/mod.rs
@@ -10,10 +10,10 @@ use tinystr::UnvalidatedTinyAsciiStr;
 use zerovec::VarZeroVec;
 use zerovec::ZeroMap;
 
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde;
 use crate::DatagenProvider;
 use icu_dimension::provider::*;
-use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
@@ -95,8 +95,8 @@ impl DataProvider<CurrencyEssentialsV1Marker> for crate::DatagenProvider {
     }
 }
 
-impl IterableDataProvider<CurrencyEssentialsV1Marker> for crate::DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<CurrencyEssentialsV1Marker> for crate::DatagenProvider {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         Ok(self
             .cldr()?
             .numbers()

diff --git a/provider/datagen/src/transform/cldr/datetime/mod.rs b/provider/datagen/src/transform/cldr/datetime/mod.rs
@@ -2,13 +2,13 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde;
 use icu_datetime::provider::calendar::*;
 use icu_locid::{
     extensions::unicode::{key, value},
     Locale,
 };
-use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
 use once_cell::sync::OnceCell;
 use std::collections::HashMap;
@@ -207,8 +207,8 @@ macro_rules! impl_data_provider {
             }
         }
 
-        impl IterableDataProvider<$marker> for crate::DatagenProvider {
-            fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+        impl IterableDataProviderInternal<$marker> for crate::DatagenProvider {
+            fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
                 let mut r = Vec::new();
                 if DateSkeletonPatternsV1Marker::KEY == $marker::KEY {
                     for (cal_value, cldr_cal) in supported_cals() {

diff --git a/provider/datagen/src/transform/cldr/datetime/neo.rs b/provider/datagen/src/transform/cldr/datetime/neo.rs
@@ -3,6 +3,7 @@
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 use super::supported_cals;
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde::ca;
 use crate::DatagenProvider;
 use icu_datetime::pattern::{self, CoarseHourCycle};
@@ -681,8 +682,8 @@ impl DataProvider<TimePatternV1Marker> for DatagenProvider {
 // subtag actually should be produced (by returning a special error), then this code is no longer necessary
 // and we can use a union of the H12/H24 key lengths arrays, instead checking for preferred hc
 // in timepattern_convert
-impl IterableDataProvider<TimePatternV1Marker> for DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<TimePatternV1Marker> for DatagenProvider {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         let calendar = value!("gregory");
         let mut r = Vec::new();
 
@@ -727,8 +728,8 @@ macro_rules! impl_symbols_datagen {
             }
         }
 
-        impl IterableDataProvider<$marker> for DatagenProvider {
-            fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+        impl IterableDataProviderInternal<$marker> for DatagenProvider {
+            fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
                 self.supported_locales_neo(value!($calendar), $lengths)
             }
         }
@@ -743,8 +744,8 @@ macro_rules! impl_pattern_datagen {
             }
         }
 
-        impl IterableDataProvider<$marker> for DatagenProvider {
-            fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+        impl IterableDataProviderInternal<$marker> for DatagenProvider {
+            fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
                 self.supported_locales_neo(value!($calendar), $lengths)
             }
         }

diff --git a/provider/datagen/src/transform/cldr/datetime/week_data.rs b/provider/datagen/src/transform/cldr/datetime/week_data.rs
@@ -2,18 +2,18 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde::{
     self,
     week_data::{Territory, DEFAULT_TERRITORY},
 };
 use icu_calendar::provider::{WeekDataV1, WeekDataV1Marker};
 use icu_locid::LanguageIdentifier;
-use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
 use std::collections::HashSet;
 
-impl IterableDataProvider<WeekDataV1Marker> for crate::DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<WeekDataV1Marker> for crate::DatagenProvider {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         let week_data: &cldr_serde::week_data::Resource = self
             .cldr()?
             .core()

diff --git a/provider/datagen/src/transform/cldr/decimal/compact.rs b/provider/datagen/src/transform/cldr/decimal/compact.rs
@@ -2,10 +2,10 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde;
 use icu_compactdecimal::provider::*;
 use icu_locid::extensions::unicode::key;
-use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
 use std::convert::TryFrom;
 
@@ -105,14 +105,16 @@ impl DataProvider<LongCompactDecimalFormatDataV1Marker> for crate::DatagenProvid
     }
 }
 
-impl IterableDataProvider<ShortCompactDecimalFormatDataV1Marker> for crate::DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<ShortCompactDecimalFormatDataV1Marker>
+    for crate::DatagenProvider
+{
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         self.supported_locales()
     }
 }
 
-impl IterableDataProvider<LongCompactDecimalFormatDataV1Marker> for crate::DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<LongCompactDecimalFormatDataV1Marker> for crate::DatagenProvider {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         self.supported_locales()
     }
 }

diff --git a/provider/datagen/src/transform/cldr/decimal/symbols.rs b/provider/datagen/src/transform/cldr/decimal/symbols.rs
@@ -2,10 +2,10 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde;
 use icu_decimal::provider::*;
 use icu_locid::extensions::unicode::key;
-use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
 use std::borrow::Cow;
 use std::convert::TryFrom;
@@ -47,8 +47,8 @@ impl DataProvider<DecimalSymbolsV1Marker> for crate::DatagenProvider {
     }
 }
 
-impl IterableDataProvider<DecimalSymbolsV1Marker> for crate::DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<DecimalSymbolsV1Marker> for crate::DatagenProvider {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         self.supported_locales()
     }
 }

diff --git a/provider/datagen/src/transform/cldr/displaynames/language.rs b/provider/datagen/src/transform/cldr/displaynames/language.rs
@@ -2,11 +2,11 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde;
 use core::convert::TryFrom;
 use icu_displaynames::provider::*;
 use icu_locid::subtags::Language;
-use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
 use std::collections::BTreeMap;
 use zerovec::ule::UnvalidatedStr;
@@ -58,8 +58,8 @@ impl DataProvider<LocaleDisplayNamesV1Marker> for crate::DatagenProvider {
     }
 }
 
-impl IterableDataProvider<LanguageDisplayNamesV1Marker> for crate::DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<LanguageDisplayNamesV1Marker> for crate::DatagenProvider {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         Ok(self
             .cldr()?
             .displaynames()
@@ -77,8 +77,8 @@ impl IterableDataProvider<LanguageDisplayNamesV1Marker> for crate::DatagenProvid
     }
 }
 
-impl IterableDataProvider<LocaleDisplayNamesV1Marker> for crate::DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<LocaleDisplayNamesV1Marker> for crate::DatagenProvider {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         Ok(self
             .cldr()?
             .displaynames()

diff --git a/provider/datagen/src/transform/cldr/displaynames/region.rs b/provider/datagen/src/transform/cldr/displaynames/region.rs
@@ -2,11 +2,11 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde;
 use core::convert::TryFrom;
 use icu_displaynames::provider::*;
 use icu_locid::subtags::Region;
-use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
 use std::collections::BTreeMap;
 use std::str::FromStr;
@@ -35,8 +35,8 @@ impl DataProvider<RegionDisplayNamesV1Marker> for crate::DatagenProvider {
     }
 }
 
-impl IterableDataProvider<RegionDisplayNamesV1Marker> for crate::DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<RegionDisplayNamesV1Marker> for crate::DatagenProvider {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         Ok(self
             .cldr()?
             .displaynames()

diff --git a/provider/datagen/src/transform/cldr/displaynames/script.rs b/provider/datagen/src/transform/cldr/displaynames/script.rs
@@ -2,11 +2,11 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
+use crate::provider::IterableDataProviderInternal;
 use crate::transform::cldr::cldr_serde;
 use core::convert::TryFrom;
 use icu_displaynames::provider::*;
 use icu_locid::{subtags::Script, ParserError};
-use icu_provider::datagen::IterableDataProvider;
 use icu_provider::prelude::*;
 use std::collections::BTreeMap;
 use std::str::FromStr;
@@ -35,8 +35,8 @@ impl DataProvider<ScriptDisplayNamesV1Marker> for crate::DatagenProvider {
     }
 }
 
-impl IterableDataProvider<ScriptDisplayNamesV1Marker> for crate::DatagenProvider {
-    fn supported_locales(&self) -> Result<Vec<DataLocale>, DataError> {
+impl IterableDataProviderInternal<ScriptDisplayNamesV1Marker> for crate::DatagenProvider {
+    fn supported_locales_impl(&self) -> Result<Vec<DataLocale>, DataError> {
         Ok(self
             .cldr()?
             .displaynames()