From e9ff1940d6ce0eb7e384cdffa6df9a754a21f5aa Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Tue, 21 Apr 2020 17:57:21 -0700 Subject: [PATCH] Translation fixes and tweaks (#34489) Co-Authored-By: Martin Hjelmare --- .../.translations/history_graph.no.json | 3 - .../components/.translations/weblink.no.json | 3 - .../components/binary_sensor/strings.json | 4 +- homeassistant/components/group/strings.json | 4 +- homeassistant/helpers/translation.py | 205 +++++++++++++----- script/hassfest/translations.py | 3 +- tests/helpers/test_translation.py | 50 ++++- 7 files changed, 189 insertions(+), 83 deletions(-) delete mode 100644 homeassistant/components/.translations/history_graph.no.json delete mode 100644 homeassistant/components/.translations/weblink.no.json diff --git a/homeassistant/components/.translations/history_graph.no.json b/homeassistant/components/.translations/history_graph.no.json deleted file mode 100644 index 25a9cdab104..00000000000 --- a/homeassistant/components/.translations/history_graph.no.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "title": "Historisk graf" -} \ No newline at end of file diff --git a/homeassistant/components/.translations/weblink.no.json b/homeassistant/components/.translations/weblink.no.json deleted file mode 100644 index c9fbb307a4c..00000000000 --- a/homeassistant/components/.translations/weblink.no.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "title": "Weblink" -} \ No newline at end of file diff --git a/homeassistant/components/binary_sensor/strings.json b/homeassistant/components/binary_sensor/strings.json index 35922081e9b..045fcdae707 100644 --- a/homeassistant/components/binary_sensor/strings.json +++ b/homeassistant/components/binary_sensor/strings.json @@ -140,8 +140,8 @@ "on": "[%key:common::state::open%]" }, "presence": { - "off": "[%key:component::device_tracker::state::not_home%]", - "on": "[%key:component::device_tracker::state::home%]" + "off": "[%key:component::device_tracker::state::_::not_home%]", + "on": "[%key:component::device_tracker::state::_::home%]" }, "problem": { "off": "OK", diff --git a/homeassistant/components/group/strings.json b/homeassistant/components/group/strings.json index 3f015741230..e29407bf932 100644 --- a/homeassistant/components/group/strings.json +++ b/homeassistant/components/group/strings.json @@ -4,8 +4,8 @@ "_": { "off": "[%key:common::state::off%]", "on": "[%key:common::state::on%]", - "home": "[%key:component::device_tracker::state::home%]", - "not_home": "[%key:component::device_tracker::state::not_home%]", + "home": "[%key:component::device_tracker::state::_::home%]", + "not_home": "[%key:component::device_tracker::state::_::not_home%]", "open": "[%key:common::state::open%]", "closed": "[%key:common::state::closed%]", "locked": "[%key:common::state::locked%]", diff --git a/homeassistant/helpers/translation.py b/homeassistant/helpers/translation.py index ac871e89635..abf39972186 100644 --- a/homeassistant/helpers/translation.py +++ b/homeassistant/helpers/translation.py @@ -3,7 +3,8 @@ import asyncio import logging from typing import Any, Dict, Optional, Set -from homeassistant.core import callback +from homeassistant.const import EVENT_COMPONENT_LOADED +from homeassistant.core import Event, callback from homeassistant.loader import ( Integration, async_get_config_flows, @@ -17,7 +18,7 @@ from .typing import HomeAssistantType _LOGGER = logging.getLogger(__name__) TRANSLATION_LOAD_LOCK = "translation_load_lock" -TRANSLATION_STRING_CACHE = "translation_string_cache" +TRANSLATION_FLATTEN_CACHE = "translation_flatten_cache" MOVED_TRANSLATIONS_DIRECTORY_MSG = ( "%s: the '.translations' directory has been moved, the new name is 'translations', " @@ -87,16 +88,24 @@ def load_translations_files( loaded = {} for component, translation_file in translation_files.items(): loaded_json = load_json(translation_file) - assert isinstance(loaded_json, dict) + + if not isinstance(loaded_json, dict): + _LOGGER.warning( + "Translation file is unexpected type %s. Expected dict for %s", + type(loaded_json), + translation_file, + ) + continue + loaded[component] = loaded_json return loaded -def build_resources( - translation_cache: Dict[str, Dict[str, Any]], components: Set[str], category: str, +def merge_resources( + translation_strings: Dict[str, Dict[str, Any]], components: Set[str], category: str, ) -> Dict[str, Dict[str, Any]]: - """Build the resources response for the given components.""" + """Build and merge the resources response for the given components and platforms.""" # Build response resources: Dict[str, Dict[str, Any]] = {} for component in components: @@ -115,7 +124,7 @@ def build_resources( # We are going to merge the translations for the custom device classes into # the translations of sensor. - new_value = translation_cache[component].get(category) + new_value = translation_strings[component].get(category) if new_value is None: continue @@ -154,66 +163,102 @@ def build_resources( return {"component": resources} -async def async_get_component_cache( +def build_resources( + translation_strings: Dict[str, Dict[str, Any]], components: Set[str], category: str, +) -> Dict[str, Dict[str, Any]]: + """Build the resources response for the given components.""" + # Build response + resources: Dict[str, Dict[str, Any]] = {} + for component in components: + new_value = translation_strings[component].get(category) + + if new_value is None: + continue + + resources[component] = {category: new_value} + + return {"component": resources} + + +async def async_get_component_strings( hass: HomeAssistantType, language: str, components: Set[str] ) -> Dict[str, Any]: - """Return translation cache that includes all specified components.""" - # Get cache for this language - cache: Dict[str, Dict[str, Any]] = hass.data.setdefault( - TRANSLATION_STRING_CACHE, {} - ) - translation_cache: Dict[str, Any] = cache.setdefault(language, {}) - - # Calculate the missing components and platforms - missing_loaded = components - set(translation_cache) - - if not missing_loaded: - return translation_cache - - missing_domains = list({loaded.split(".")[-1] for loaded in missing_loaded}) - missing_integrations = dict( + """Load translations.""" + domains = list({loaded.split(".")[-1] for loaded in components}) + integrations = dict( zip( - missing_domains, + domains, await asyncio.gather( - *[async_get_integration(hass, domain) for domain in missing_domains] + *[async_get_integration(hass, domain) for domain in domains] ), ) ) + translations: Dict[str, Any] = {} + # Determine paths of missing components/platforms - missing_files = {} - for loaded in missing_loaded: + files_to_load = {} + for loaded in components: parts = loaded.split(".") domain = parts[-1] - integration = missing_integrations[domain] + integration = integrations[domain] path = component_translation_path(loaded, language, integration) # No translation available if path is None: - translation_cache[loaded] = {} + translations[loaded] = {} else: - missing_files[loaded] = path + files_to_load[loaded] = path - # Load missing files - if missing_files: - load_translations_job = hass.async_add_job( - load_translations_files, missing_files - ) - assert load_translations_job is not None - loaded_translations = await load_translations_job + # Load files + load_translations_job = hass.async_add_executor_job( + load_translations_files, files_to_load + ) + assert load_translations_job is not None + loaded_translations = await load_translations_job - # Translations that miss "title" will get integration put in. - for loaded, translations in loaded_translations.items(): - if "." in loaded: - continue + # Translations that miss "title" will get integration put in. + for loaded, translations in loaded_translations.items(): + if "." in loaded: + continue - if "title" not in translations: - translations["title"] = missing_integrations[loaded].name + if "title" not in translations: + translations["title"] = integrations[loaded].name - # Update cache - translation_cache.update(loaded_translations) + translations.update(loaded_translations) - return translation_cache + return translations + + +class FlatCache: + """Cache for flattened translations.""" + + def __init__(self, hass: HomeAssistantType) -> None: + """Initialize the cache.""" + self.hass = hass + self.cache: Dict[str, Dict[str, Dict[str, str]]] = {} + + @callback + def async_setup(self) -> None: + """Initialize the cache clear listeners.""" + self.hass.bus.async_listen(EVENT_COMPONENT_LOADED, self._async_component_loaded) + + @callback + def _async_component_loaded(self, event: Event) -> None: + """Clear cache when a new component is loaded.""" + self.cache = {} + + @callback + def async_get_cache(self, language: str, category: str) -> Optional[Dict[str, str]]: + """Get cache.""" + return self.cache.setdefault(language, {}).get(category) + + @callback + def async_set_cache( + self, language: str, category: str, data: Dict[str, str] + ) -> None: + """Set cache.""" + self.cache.setdefault(language, {})[category] = data @bind_hass @@ -230,30 +275,70 @@ async def async_get_translations( Otherwise default to loaded intgrations combined with config flow integrations if config_flow is true. """ - if integration is not None: - components = {integration} - elif config_flow: - components = hass.config.components | await async_get_config_flows(hass) - else: - components = set(hass.config.components) - lock = hass.data.get(TRANSLATION_LOAD_LOCK) if lock is None: lock = hass.data[TRANSLATION_LOAD_LOCK] = asyncio.Lock() - tasks = [async_get_component_cache(hass, language, components)] - - # Fetch the English resources, as a fallback for missing keys - if language != "en": - tasks.append(async_get_component_cache(hass, "en", components)) + if integration is not None: + components = {integration} + elif config_flow: + # When it's a config flow, we're going to merge the cached loaded component results + # with the integrations that have not been loaded yet. We merge this at the end. + # We can't cache with config flow, as we can't monitor it during runtime. + components = (await async_get_config_flows(hass)) - hass.config.components + else: + # Only 'state' supports merging, so remove platforms from selection + if category == "state": + components = set(hass.config.components) + else: + components = { + component + for component in hass.config.components + if "." not in component + } async with lock: + if integration is None and not config_flow: + cache = hass.data.get(TRANSLATION_FLATTEN_CACHE) + if cache is None: + cache = hass.data[TRANSLATION_FLATTEN_CACHE] = FlatCache(hass) + cache.async_setup() + + cached_translations = cache.async_get_cache(language, category) + + if cached_translations is not None: + return cached_translations + + tasks = [async_get_component_strings(hass, language, components)] + + # Fetch the English resources, as a fallback for missing keys + if language != "en": + tasks.append(async_get_component_strings(hass, "en", components)) + + _LOGGER.debug( + "Cache miss for %s, %s: %s", language, category, ", ".join(components) + ) + results = await asyncio.gather(*tasks) - resources = flatten(build_resources(results[0], components, category)) + if category == "state": + resource_func = merge_resources + else: + resource_func = build_resources + + resources = flatten(resource_func(results[0], components, category)) if language != "en": - base_resources = flatten(build_resources(results[1], components, category)) + base_resources = flatten(resource_func(results[1], components, category)) resources = {**base_resources, **resources} + if integration is not None: + pass + elif config_flow: + loaded_comp_resources = await async_get_translations(hass, language, category) + resources.update(loaded_comp_resources) + else: + assert cache is not None + cache.async_set_cache(language, category, resources) + return resources diff --git a/script/hassfest/translations.py b/script/hassfest/translations.py index 394247313ca..1e49530e275 100644 --- a/script/hassfest/translations.py +++ b/script/hassfest/translations.py @@ -259,8 +259,7 @@ def validate_translation_file(config: Config, integration: Integration, all_stri search = search[key] key = parts.pop(0) - if parts: - print(key, list(search)) + if parts or key not in search: integration.add_error( "translations", f"{reference['source']} contains invalid reference {reference['ref']}: Could not find {key}", diff --git a/tests/helpers/test_translation.py b/tests/helpers/test_translation.py index 4bbff89b25f..3957d2c19ee 100644 --- a/tests/helpers/test_translation.py +++ b/tests/helpers/test_translation.py @@ -6,8 +6,9 @@ import pathlib from asynctest import Mock, patch import pytest +from homeassistant.const import EVENT_COMPONENT_LOADED from homeassistant.generated import config_flows -import homeassistant.helpers.translation as translation +from homeassistant.helpers import translation from homeassistant.loader import async_get_integration from homeassistant.setup import async_setup_component @@ -214,26 +215,53 @@ async def test_translation_merging(hass, caplog): assert "component.sensor.state.moon__phase.first_quarter" in translations assert "component.sensor.state.season__season.summer" in translations - # Merge in some bad translation data - integration = Mock(file_path=pathlib.Path(__file__)) - hass.config.components.add("sensor.bad_translations") + # Clear cache + hass.bus.async_fire(EVENT_COMPONENT_LOADED) + await hass.async_block_till_done() + + # Patch in some bad translation data + + orig_load_translations = translation.load_translations_files + + def mock_load_translations_files(files): + """Mock loading.""" + result = orig_load_translations(files) + result["sensor.season"] = {"state": "bad data"} + return result with patch.object( - translation, "component_translation_path", return_value="bla.json" - ), patch.object( translation, "load_translations_files", - return_value={"sensor.bad_translations": {"state": "bad data"}}, - ), patch( - "homeassistant.helpers.translation.async_get_integration", - return_value=integration, + side_effect=mock_load_translations_files, ): translations = await translation.async_get_translations(hass, "en", "state") assert "component.sensor.state.moon__phase.first_quarter" in translations - assert "component.sensor.state.season__season.summer" in translations assert ( "An integration providing translations for sensor provided invalid data: bad data" in caplog.text ) + + +async def test_caching(hass): + """Test we cache data.""" + hass.config.components.add("sensor") + + # Patch with same method so we can count invocations + with patch( + "homeassistant.helpers.translation.merge_resources", + side_effect=translation.merge_resources, + ) as mock_merge: + await translation.async_get_translations(hass, "en", "state") + assert len(mock_merge.mock_calls) == 1 + + await translation.async_get_translations(hass, "en", "state") + assert len(mock_merge.mock_calls) == 1 + + # This event clears the cache so we should record another call + hass.bus.async_fire(EVENT_COMPONENT_LOADED) + await hass.async_block_till_done() + + await translation.async_get_translations(hass, "en", "state") + assert len(mock_merge.mock_calls) == 2