Translation fixes and tweaks (#34489)

Co-Authored-By: Martin Hjelmare <marhje52@gmail.com>

homeassistant/components/.translations/history_graph.no.json

@@ -1,3 +0,0 @@
-{
-    "title": "Historisk graf"
-}

homeassistant/components/.translations/weblink.no.json

@@ -1,3 +0,0 @@
-{
-    "title": "Weblink"
-}

homeassistant/components/binary_sensor/strings.json

@@ -140,8 +140,8 @@
       "on": "[%key:common::state::open%]"
     },
     "presence": {
-      "off": "[%key:component::device_tracker::state::not_home%]",
+      "off": "[%key:component::device_tracker::state::_::not_home%]",
-      "on": "[%key:component::device_tracker::state::home%]"
+      "on": "[%key:component::device_tracker::state::_::home%]"
     },
     "problem": {
       "off": "OK",

homeassistant/components/group/strings.json

@@ -4,8 +4,8 @@
     "_": {
       "off": "[%key:common::state::off%]",
       "on": "[%key:common::state::on%]",
-      "home": "[%key:component::device_tracker::state::home%]",
+      "home": "[%key:component::device_tracker::state::_::home%]",
-      "not_home": "[%key:component::device_tracker::state::not_home%]",
+      "not_home": "[%key:component::device_tracker::state::_::not_home%]",
       "open": "[%key:common::state::open%]",
       "closed": "[%key:common::state::closed%]",
       "locked": "[%key:common::state::locked%]",

homeassistant/helpers/translation.py

@@ -3,7 +3,8 @@ import asyncio
 import logging
 from typing import Any, Dict, Optional, Set
 
-from homeassistant.core import callback
+from homeassistant.const import EVENT_COMPONENT_LOADED
+from homeassistant.core import Event, callback
 from homeassistant.loader import (
     Integration,
     async_get_config_flows,
@@ -17,7 +18,7 @@ from .typing import HomeAssistantType
 _LOGGER = logging.getLogger(__name__)
 
 TRANSLATION_LOAD_LOCK = "translation_load_lock"
-TRANSLATION_STRING_CACHE = "translation_string_cache"
+TRANSLATION_FLATTEN_CACHE = "translation_flatten_cache"
 
 MOVED_TRANSLATIONS_DIRECTORY_MSG = (
     "%s: the '.translations' directory has been moved, the new name is 'translations', "
@@ -87,16 +88,24 @@ def load_translations_files(
     loaded = {}
     for component, translation_file in translation_files.items():
         loaded_json = load_json(translation_file)
-        assert isinstance(loaded_json, dict)
+
+        if not isinstance(loaded_json, dict):
+            _LOGGER.warning(
+                "Translation file is unexpected type %s. Expected dict for %s",
+                type(loaded_json),
+                translation_file,
+            )
+            continue
+
         loaded[component] = loaded_json
 
     return loaded
 
 
-def build_resources(
+def merge_resources(
-    translation_cache: Dict[str, Dict[str, Any]], components: Set[str], category: str,
+    translation_strings: Dict[str, Dict[str, Any]], components: Set[str], category: str,
 ) -> Dict[str, Dict[str, Any]]:
-    """Build the resources response for the given components."""
+    """Build and merge the resources response for the given components and platforms."""
     # Build response
     resources: Dict[str, Dict[str, Any]] = {}
     for component in components:
@@ -115,7 +124,7 @@ def build_resources(
         # We are going to merge the translations for the custom device classes into
         # the translations of sensor.
 
-        new_value = translation_cache[component].get(category)
+        new_value = translation_strings[component].get(category)
 
         if new_value is None:
             continue
@@ -154,50 +163,56 @@ def build_resources(
     return {"component": resources}
 
 
-async def async_get_component_cache(
+def build_resources(
+    translation_strings: Dict[str, Dict[str, Any]], components: Set[str], category: str,
+) -> Dict[str, Dict[str, Any]]:
+    """Build the resources response for the given components."""
+    # Build response
+    resources: Dict[str, Dict[str, Any]] = {}
+    for component in components:
+        new_value = translation_strings[component].get(category)
+
+        if new_value is None:
+            continue
+
+        resources[component] = {category: new_value}
+
+    return {"component": resources}
+
+
+async def async_get_component_strings(
     hass: HomeAssistantType, language: str, components: Set[str]
 ) -> Dict[str, Any]:
-    """Return translation cache that includes all specified components."""
+    """Load translations."""
-    # Get cache for this language
+    domains = list({loaded.split(".")[-1] for loaded in components})
-    cache: Dict[str, Dict[str, Any]] = hass.data.setdefault(
+    integrations = dict(
-        TRANSLATION_STRING_CACHE, {}
-    )
-    translation_cache: Dict[str, Any] = cache.setdefault(language, {})
-
-    # Calculate the missing components and platforms
-    missing_loaded = components - set(translation_cache)
-
-    if not missing_loaded:
-        return translation_cache
-
-    missing_domains = list({loaded.split(".")[-1] for loaded in missing_loaded})
-    missing_integrations = dict(
         zip(
-            missing_domains,
+            domains,
             await asyncio.gather(
-                *[async_get_integration(hass, domain) for domain in missing_domains]
+                *[async_get_integration(hass, domain) for domain in domains]
             ),
         )
     )
 
+    translations: Dict[str, Any] = {}
+
     # Determine paths of missing components/platforms
-    missing_files = {}
+    files_to_load = {}
-    for loaded in missing_loaded:
+    for loaded in components:
         parts = loaded.split(".")
         domain = parts[-1]
-        integration = missing_integrations[domain]
+        integration = integrations[domain]
 
         path = component_translation_path(loaded, language, integration)
         # No translation available
         if path is None:
-            translation_cache[loaded] = {}
+            translations[loaded] = {}
         else:
-            missing_files[loaded] = path
+            files_to_load[loaded] = path
 
-    # Load missing files
+    # Load files
-    if missing_files:
+    load_translations_job = hass.async_add_executor_job(
-        load_translations_job = hass.async_add_job(
+        load_translations_files, files_to_load
-            load_translations_files, missing_files
     )
     assert load_translations_job is not None
     loaded_translations = await load_translations_job
@@ -208,12 +223,42 @@ async def async_get_component_cache(
             continue
 
         if "title" not in translations:
-                translations["title"] = missing_integrations[loaded].name
+            translations["title"] = integrations[loaded].name
 
-        # Update cache
+    translations.update(loaded_translations)
-        translation_cache.update(loaded_translations)
 
-    return translation_cache
+    return translations
+
+
+class FlatCache:
+    """Cache for flattened translations."""
+
+    def __init__(self, hass: HomeAssistantType) -> None:
+        """Initialize the cache."""
+        self.hass = hass
+        self.cache: Dict[str, Dict[str, Dict[str, str]]] = {}
+
+    @callback
+    def async_setup(self) -> None:
+        """Initialize the cache clear listeners."""
+        self.hass.bus.async_listen(EVENT_COMPONENT_LOADED, self._async_component_loaded)
+
+    @callback
+    def _async_component_loaded(self, event: Event) -> None:
+        """Clear cache when a new component is loaded."""
+        self.cache = {}
+
+    @callback
+    def async_get_cache(self, language: str, category: str) -> Optional[Dict[str, str]]:
+        """Get cache."""
+        return self.cache.setdefault(language, {}).get(category)
+
+    @callback
+    def async_set_cache(
+        self, language: str, category: str, data: Dict[str, str]
+    ) -> None:
+        """Set cache."""
+        self.cache.setdefault(language, {})[category] = data
 
 
 @bind_hass
@@ -230,30 +275,70 @@ async def async_get_translations(
     Otherwise default to loaded intgrations combined with config flow
     integrations if config_flow is true.
     """
-    if integration is not None:
-        components = {integration}
-    elif config_flow:
-        components = hass.config.components | await async_get_config_flows(hass)
-    else:
-        components = set(hass.config.components)
-
     lock = hass.data.get(TRANSLATION_LOAD_LOCK)
     if lock is None:
         lock = hass.data[TRANSLATION_LOAD_LOCK] = asyncio.Lock()
 
-    tasks = [async_get_component_cache(hass, language, components)]
+    if integration is not None:
+        components = {integration}
+    elif config_flow:
+        # When it's a config flow, we're going to merge the cached loaded component results
+        # with the integrations that have not been loaded yet. We merge this at the end.
+        # We can't cache with config flow, as we can't monitor it during runtime.
+        components = (await async_get_config_flows(hass)) - hass.config.components
+    else:
+        # Only 'state' supports merging, so remove platforms from selection
+        if category == "state":
+            components = set(hass.config.components)
+        else:
+            components = {
+                component
+                for component in hass.config.components
+                if "." not in component
+            }
+
+    async with lock:
+        if integration is None and not config_flow:
+            cache = hass.data.get(TRANSLATION_FLATTEN_CACHE)
+            if cache is None:
+                cache = hass.data[TRANSLATION_FLATTEN_CACHE] = FlatCache(hass)
+                cache.async_setup()
+
+            cached_translations = cache.async_get_cache(language, category)
+
+            if cached_translations is not None:
+                return cached_translations
+
+        tasks = [async_get_component_strings(hass, language, components)]
 
         # Fetch the English resources, as a fallback for missing keys
         if language != "en":
-        tasks.append(async_get_component_cache(hass, "en", components))
+            tasks.append(async_get_component_strings(hass, "en", components))
+
+        _LOGGER.debug(
+            "Cache miss for %s, %s: %s", language, category, ", ".join(components)
+        )
 
-    async with lock:
         results = await asyncio.gather(*tasks)
 
-    resources = flatten(build_resources(results[0], components, category))
+    if category == "state":
+        resource_func = merge_resources
+    else:
+        resource_func = build_resources
+
+    resources = flatten(resource_func(results[0], components, category))
 
     if language != "en":
-        base_resources = flatten(build_resources(results[1], components, category))
+        base_resources = flatten(resource_func(results[1], components, category))
         resources = {**base_resources, **resources}
 
+    if integration is not None:
+        pass
+    elif config_flow:
+        loaded_comp_resources = await async_get_translations(hass, language, category)
+        resources.update(loaded_comp_resources)
+    else:
+        assert cache is not None
+        cache.async_set_cache(language, category, resources)
+
     return resources

script/hassfest/translations.py

@@ -259,8 +259,7 @@ def validate_translation_file(config: Config, integration: Integration, all_stri
             search = search[key]
             key = parts.pop(0)
 
-        if parts:
+        if parts or key not in search:
-            print(key, list(search))
             integration.add_error(
                 "translations",
                 f"{reference['source']} contains invalid reference {reference['ref']}: Could not find {key}",

tests/helpers/test_translation.py

@@ -6,8 +6,9 @@ import pathlib
 from asynctest import Mock, patch
 import pytest
 
+from homeassistant.const import EVENT_COMPONENT_LOADED
 from homeassistant.generated import config_flows
-import homeassistant.helpers.translation as translation
+from homeassistant.helpers import translation
 from homeassistant.loader import async_get_integration
 from homeassistant.setup import async_setup_component
 
@@ -214,26 +215,53 @@ async def test_translation_merging(hass, caplog):
     assert "component.sensor.state.moon__phase.first_quarter" in translations
     assert "component.sensor.state.season__season.summer" in translations
 
-    # Merge in some bad translation data
+    # Clear cache
-    integration = Mock(file_path=pathlib.Path(__file__))
+    hass.bus.async_fire(EVENT_COMPONENT_LOADED)
-    hass.config.components.add("sensor.bad_translations")
+    await hass.async_block_till_done()
+
+    # Patch in some bad translation data
+
+    orig_load_translations = translation.load_translations_files
+
+    def mock_load_translations_files(files):
+        """Mock loading."""
+        result = orig_load_translations(files)
+        result["sensor.season"] = {"state": "bad data"}
+        return result
 
     with patch.object(
-        translation, "component_translation_path", return_value="bla.json"
-    ), patch.object(
         translation,
         "load_translations_files",
-        return_value={"sensor.bad_translations": {"state": "bad data"}},
+        side_effect=mock_load_translations_files,
-    ), patch(
-        "homeassistant.helpers.translation.async_get_integration",
-        return_value=integration,
     ):
         translations = await translation.async_get_translations(hass, "en", "state")
 
         assert "component.sensor.state.moon__phase.first_quarter" in translations
-        assert "component.sensor.state.season__season.summer" in translations
 
     assert (
         "An integration providing translations for sensor provided invalid data: bad data"
         in caplog.text
     )
+
+
+async def test_caching(hass):
+    """Test we cache data."""
+    hass.config.components.add("sensor")
+
+    # Patch with same method so we can count invocations
+    with patch(
+        "homeassistant.helpers.translation.merge_resources",
+        side_effect=translation.merge_resources,
+    ) as mock_merge:
+        await translation.async_get_translations(hass, "en", "state")
+        assert len(mock_merge.mock_calls) == 1
+
+        await translation.async_get_translations(hass, "en", "state")
+        assert len(mock_merge.mock_calls) == 1
+
+        # This event clears the cache so we should record another call
+        hass.bus.async_fire(EVENT_COMPONENT_LOADED)
+        await hass.async_block_till_done()
+
+        await translation.async_get_translations(hass, "en", "state")
+        assert len(mock_merge.mock_calls) == 2