Group loading of translations for integrations to reduce executor jobs at startup (#110674)

homeassistant/bootstrap.py

@@ -710,6 +710,21 @@ async def _async_resolve_domains_to_setup(
         requirements.async_load_installed_versions(hass, needed_requirements),
         "check installed requirements",
     )
+    # Start loading translations for all integrations we are going to set up
+    # in the background so they are ready when we need them. This avoids a
+    # lot of waiting for the translation load lock and a thundering herd of
+    # tasks trying to load the same translations at the same time as each
+    # integration is loaded.
+    #
+    # We do not wait for this since as soon as the task runs it will
+    # hold the translation load lock and if anything is fast enough to
+    # wait for the translation load lock, loading will be done by the
+    # time it gets to it.
+    hass.async_create_background_task(
+        translation.async_load_integrations(hass, {*BASE_PLATFORMS, *domains_to_setup}),
+        "load translations",
+    )
+
     return domains_to_setup, integration_cache
 
 

homeassistant/helpers/translation.py

@@ -158,7 +158,7 @@ async def _async_get_component_strings(
     """Load translations."""
     translations: dict[str, Any] = {}
     # Determine paths of missing components/platforms
-    files_to_load = {}
+    files_to_load: dict[str, str] = {}
     for loaded in components:
         domain = loaded.partition(".")[0]
         if not (integration := integrations.get(domain)):
@@ -264,13 +264,13 @@ class _TranslationCache:
         _LOGGER.debug(
             "Cache miss for %s: %s",
             language,
-            ", ".join(components),
+            components,
         )
         # Fetch the English resources, as a fallback for missing keys
         languages = [LOCALE_EN] if language == LOCALE_EN else [LOCALE_EN, language]
 
         integrations: dict[str, Integration] = {}
-        domains = list({loaded.partition(".")[0] for loaded in components})
+        domains = {loaded.partition(".")[0] for loaded in components}
         ints_or_excs = await async_get_integrations(self.hass, domains)
         for domain, int_or_exc in ints_or_excs.items():
             if isinstance(int_or_exc, Exception):
@@ -392,31 +392,15 @@ async def async_get_translations(
     """
     if integrations is None and config_flow:
         components = (await async_get_config_flows(hass)) - hass.config.components
+    elif integrations is not None:
+        components = set(integrations)
     else:
-        components = _async_get_components(hass, category, integrations)
+        components = _async_get_components(hass, category)
 
-    cache: _TranslationCache = hass.data[TRANSLATION_FLATTEN_CACHE]
+    return await _async_get_translations_cache(hass).async_fetch(
-
+        language, category, components
-    return await cache.async_fetch(language, category, components)
-
-
-async def _async_load_translations(
-    hass: HomeAssistant,
-    language: str,
-    category: str,
-    integration: str | None,
-) -> None:
-    """Prime backend translation cache.
-
-    If integration is not specified, translation cache is primed for all loaded integrations.
-    """
-    components = _async_get_components(
-        hass, category, [integration] if integration is not None else None
     )
 
-    cache = hass.data[TRANSLATION_FLATTEN_CACHE]
-    await cache.async_load(language, components)
-
 
 @callback
 def async_get_cached_translations(
@@ -430,42 +414,36 @@ def async_get_cached_translations(
     If integration is specified, return translations for it.
     Otherwise, default to all loaded integrations.
     """
-    components = _async_get_components(
+    if integration is not None:
-        hass, category, [integration] if integration is not None else None
+        components = {integration}
+    else:
+        components = _async_get_components(hass, category)
+
+    return _async_get_translations_cache(hass).get_cached(
+        language, category, components
     )
 
+
+@callback
+def _async_get_translations_cache(hass: HomeAssistant) -> _TranslationCache:
+    """Return the translation cache."""
     cache: _TranslationCache = hass.data[TRANSLATION_FLATTEN_CACHE]
-    return cache.get_cached(language, category, components)
+    return cache
+
+
+_DIRECT_MAPPED_CATEGORIES = {"state", "entity_component", "services"}
 
 
 @callback
 def _async_get_components(
     hass: HomeAssistant,
     category: str,
-    integrations: Iterable[str] | None = None,
 ) -> set[str]:
     """Return a set of components for which translations should be loaded."""
-    if integrations is not None:
+    if category in _DIRECT_MAPPED_CATEGORIES:
-        components = set(integrations)
+        return hass.config.components
-    elif category in ("state", "entity_component", "services"):
+    # Only 'state' supports merging, so remove platforms from selection
-        components = hass.config.components
+    return {component for component in hass.config.components if "." not in component}
-    else:
-        # Only 'state' supports merging, so remove platforms from selection
-        components = {
-            component for component in hass.config.components if "." not in component
-        }
-    return components
-
-
-async def _async_load_state_translations_to_cache(
-    hass: HomeAssistant,
-    language: str,
-    integration: str | None,
-) -> None:
-    """Load state translations to cache."""
-    await _async_load_translations(hass, language, "entity", integration)
-    await _async_load_translations(hass, language, "state", integration)
-    await _async_load_translations(hass, language, "entity_component", integration)
 
 
 @callback
@@ -492,7 +470,7 @@ def async_setup(hass: HomeAssistant) -> None:
     async def _async_load_translations(event: Event) -> None:
         new_language = event.data["language"]
         _LOGGER.debug("Loading translations for language: %s", new_language)
-        await _async_load_state_translations_to_cache(hass, new_language, None)
+        await cache.async_load(new_language, hass.config.components)
 
     @callback
     def _async_load_translations_for_component_filter(event: Event) -> bool:
@@ -506,16 +484,17 @@ def async_setup(hass: HomeAssistant) -> None:
         )
 
     async def _async_load_translations_for_component(event: Event) -> None:
+        """Load translations for a component."""
         component: str | None = event.data.get("component")
         if TYPE_CHECKING:
             assert component is not None
         language = hass.config.language
         _LOGGER.debug(
             "Loading translations for language: %s and component: %s",
-            hass.config.language,
+            language,
             component,
         )
-        await _async_load_state_translations_to_cache(hass, language, component)
+        await cache.async_load(language, {component})
 
     hass.bus.async_listen(
         EVENT_COMPONENT_LOADED,
@@ -529,6 +508,13 @@ def async_setup(hass: HomeAssistant) -> None:
     )
 
 
+async def async_load_integrations(hass: HomeAssistant, integrations: set[str]) -> None:
+    """Load translations for integrations."""
+    await _async_get_translations_cache(hass).async_load(
+        hass.config.language, integrations
+    )
+
+
 @callback
 def async_translate_state(
     hass: HomeAssistant,

tests/common.py

@@ -1377,7 +1377,7 @@ def mock_integration(
         f"{loader.PACKAGE_BUILTIN}.{module.DOMAIN}"
         if built_in
         else f"{loader.PACKAGE_CUSTOM_COMPONENTS}.{module.DOMAIN}",
-        None,
+        pathlib.Path(""),
         module.mock_manifest(),
     )
 

tests/helpers/test_template.py

@@ -1972,7 +1972,7 @@ async def test_state_translated(
         },
     )
     await hass.async_block_till_done()
-    await translation._async_load_state_translations_to_cache(hass, "en", None)
+    await translation._async_get_translations_cache(hass).async_load("en", set())
 
     hass.states.async_set("switch.without_translations", "on", attributes={})
     hass.states.async_set("binary_sensor.without_device_class", "on", attributes={})

tests/helpers/test_translation.py

@@ -544,38 +544,6 @@ async def test_custom_component_translations(
     assert await translation.async_get_translations(hass, "en", "state") == {}
 
 
-async def test_load_state_translations_to_cache(
-    hass: HomeAssistant, mock_config_flows, enable_custom_integrations: None
-):
-    """Test the load state translations to cache helper."""
-
-    with patch(
-        "homeassistant.helpers.translation._async_load_translations",
-    ) as mock:
-        await translation._async_load_state_translations_to_cache(hass, "en", None)
-        mock.assert_has_calls(
-            [
-                call(hass, "en", "entity", None),
-                call(hass, "en", "state", None),
-                call(hass, "en", "entity_component", None),
-            ]
-        )
-
-    with patch(
-        "homeassistant.helpers.translation._async_load_translations",
-    ) as mock:
-        await translation._async_load_state_translations_to_cache(
-            hass, "en", "some_integration"
-        )
-        mock.assert_has_calls(
-            [
-                call(hass, "en", "entity", "some_integration"),
-                call(hass, "en", "state", "some_integration"),
-                call(hass, "en", "entity_component", "some_integration"),
-            ]
-        )
-
-
 async def test_get_cached_translations(
     hass: HomeAssistant, mock_config_flows, enable_custom_integrations: None
 ):
@@ -586,27 +554,33 @@ async def test_get_cached_translations(
     assert await async_setup_component(hass, "switch", {"switch": {"platform": "test"}})
     await hass.async_block_till_done()
 
-    await translation._async_load_state_translations_to_cache(hass, "en", None)
+    await translation._async_get_translations_cache(hass).async_load(
+        "en", hass.config.components
+    )
     translations = translation.async_get_cached_translations(hass, "en", "state")
 
     assert translations["component.switch.state.string1"] == "Value 1"
     assert translations["component.switch.state.string2"] == "Value 2"
 
-    await translation._async_load_state_translations_to_cache(hass, "de", None)
+    await translation._async_get_translations_cache(hass).async_load(
+        "de", hass.config.components
+    )
     translations = translation.async_get_cached_translations(hass, "de", "state")
     assert "component.switch.something" not in translations
     assert translations["component.switch.state.string1"] == "German Value 1"
     assert translations["component.switch.state.string2"] == "German Value 2"
 
     # Test a partial translation
-    await translation._async_load_state_translations_to_cache(hass, "es", None)
+    await translation._async_get_translations_cache(hass).async_load(
+        "es", hass.config.components
+    )
     translations = translation.async_get_cached_translations(hass, "es", "state")
     assert translations["component.switch.state.string1"] == "Spanish Value 1"
     assert translations["component.switch.state.string2"] == "Value 2"
 
     # Test that an untranslated language falls back to English.
-    await translation._async_load_state_translations_to_cache(
+    await translation._async_get_translations_cache(hass).async_load(
-        hass, "invalid-language", None
+        "invalid-language", hass.config.components
     )
     translations = translation.async_get_cached_translations(
         hass, "invalid-language", "state"
@@ -620,14 +594,14 @@ async def test_setup(hass: HomeAssistant):
     translation.async_setup(hass)
 
     with patch(
-        "homeassistant.helpers.translation._async_load_state_translations_to_cache",
+        "homeassistant.helpers.translation._TranslationCache.async_load",
     ) as mock:
         hass.bus.async_fire(EVENT_COMPONENT_LOADED, {"component": "loaded_component"})
         await hass.async_block_till_done()
-        mock.assert_called_once_with(hass, hass.config.language, "loaded_component")
+        mock.assert_called_once_with(hass.config.language, {"loaded_component"})
 
     with patch(
-        "homeassistant.helpers.translation._async_load_state_translations_to_cache",
+        "homeassistant.helpers.translation._TranslationCache.async_load",
     ) as mock:
         hass.bus.async_fire(EVENT_COMPONENT_LOADED, {"component": "config.component"})
         await hass.async_block_till_done()
@@ -635,7 +609,7 @@ async def test_setup(hass: HomeAssistant):
 
     # Should not be called if the language is the current language
     with patch(
-        "homeassistant.helpers.translation._async_load_state_translations_to_cache",
+        "homeassistant.helpers.translation._TranslationCache.async_load",
     ) as mock:
         hass.bus.async_fire(EVENT_CORE_CONFIG_UPDATE, {"language": "en"})
         await hass.async_block_till_done()
@@ -643,14 +617,14 @@ async def test_setup(hass: HomeAssistant):
 
     # Should be called if the language is different
     with patch(
-        "homeassistant.helpers.translation._async_load_state_translations_to_cache",
+        "homeassistant.helpers.translation._TranslationCache.async_load",
     ) as mock:
         hass.bus.async_fire(EVENT_CORE_CONFIG_UPDATE, {"language": "es"})
         await hass.async_block_till_done()
-        mock.assert_called_once_with(hass, "es", None)
+        mock.assert_called_once_with("es", set())
 
     with patch(
-        "homeassistant.helpers.translation._async_load_state_translations_to_cache",
+        "homeassistant.helpers.translation._TranslationCache.async_load",
     ) as mock:
         hass.bus.async_fire(EVENT_CORE_CONFIG_UPDATE, {})
         await hass.async_block_till_done()