Gather loading platforms in async_process_component_config (#113573)

homeassistant/config.py

@@ -2,12 +2,13 @@
 
 from __future__ import annotations
 
+import asyncio
 from collections import OrderedDict
 from collections.abc import Callable, Hashable, Iterable, Sequence
 from contextlib import suppress
 from dataclasses import dataclass
 from enum import StrEnum
-from functools import reduce
+from functools import partial, reduce
 import logging
 import operator
 import os
@@ -65,6 +66,7 @@ from .helpers.entity_values import EntityValues
 from .helpers.typing import ConfigType
 from .loader import ComponentProtocol, Integration, IntegrationNotFound
 from .requirements import RequirementsNotFound, async_get_integration_with_requirements
+from .util.async_ import create_eager_task
 from .util.package import is_docker_env
 from .util.unit_system import get_unit_system, validate_unit_system
 from .util.yaml import SECRET_YAML, Secrets, YamlTypeError, load_yaml_dict
@@ -1434,6 +1436,67 @@ def extract_domain_configs(config: ConfigType, domain: str) -> Sequence[str]:
     return domain_configs
 
 
+@dataclass(slots=True)
+class _PlatformIntegration:
+    """Class to hold platform integration information."""
+
+    path: str  # integration.platform; ex: filter.sensor
+    name: str  # integration; ex: filter
+    integration: Integration  # <Integration filter>
+    config: ConfigType  # un-validated config
+    validated_config: ConfigType  # component validated config
+
+
+async def _async_load_and_validate_platform_integration(
+    domain: str,
+    integration_docs: str | None,
+    config_exceptions: list[ConfigExceptionInfo],
+    p_integration: _PlatformIntegration,
+) -> ConfigType | None:
+    """Load a platform integration and validate its config."""
+    try:
+        platform = await p_integration.integration.async_get_platform(domain)
+    except LOAD_EXCEPTIONS as exc:
+        exc_info = ConfigExceptionInfo(
+            exc,
+            ConfigErrorTranslationKey.PLATFORM_COMPONENT_LOAD_EXC,
+            p_integration.path,
+            p_integration.config,
+            integration_docs,
+        )
+        config_exceptions.append(exc_info)
+        return None
+
+    # If the platform does not have a config schema
+    # the top level component validated schema will be used
+    if not hasattr(platform, "PLATFORM_SCHEMA"):
+        return p_integration.validated_config
+
+    # Validate platform specific schema
+    try:
+        return platform.PLATFORM_SCHEMA(p_integration.config)  # type: ignore[no-any-return]
+    except vol.Invalid as exc:
+        exc_info = ConfigExceptionInfo(
+            exc,
+            ConfigErrorTranslationKey.PLATFORM_CONFIG_VALIDATION_ERR,
+            p_integration.path,
+            p_integration.config,
+            p_integration.integration.documentation,
+        )
+        config_exceptions.append(exc_info)
+    except Exception as exc:  # pylint: disable=broad-except
+        exc_info = ConfigExceptionInfo(
+            exc,
+            ConfigErrorTranslationKey.PLATFORM_SCHEMA_VALIDATOR_ERR,
+            p_integration.name,
+            p_integration.config,
+            p_integration.integration.documentation,
+        )
+        config_exceptions.append(exc_info)
+
+    return None
+
+
 async def async_process_component_config(
     hass: HomeAssistant,
     config: ConfigType,
@@ -1548,6 +1611,7 @@ async def async_process_component_config(
     if component_platform_schema is None:
         return IntegrationConfigInfo(config, [])
 
+    platform_integrations_to_load: list[_PlatformIntegration] = []
     platforms: list[ConfigType] = []
     for p_name, p_config in config_per_platform(config, domain):
         # Validate component specific platform schema
@@ -1595,45 +1659,44 @@ async def async_process_component_config(
             config_exceptions.append(exc_info)
             continue
 
-        try:
-            platform = await p_integration.async_get_platform(domain)
-        except LOAD_EXCEPTIONS as exc:
-            exc_info = ConfigExceptionInfo(
-                exc,
-                ConfigErrorTranslationKey.PLATFORM_COMPONENT_LOAD_EXC,
-                platform_path,
-                p_config,
+        platform_integration = _PlatformIntegration(
+            platform_path, p_name, p_integration, p_config, p_validated
+        )
+        platform_integrations_to_load.append(platform_integration)
+
+    #
+    # Since bootstrap will order base platform (ie sensor) integrations
+    # first, we eagerly gather importing the platforms that need to be
+    # validated for the base platform since everything that uses the
+    # base platform has to wait for it to finish.
+    #
+    # For example if `hue` where to load first and than called
+    # `async_forward_entry_setup` for the `sensor` platform it would have to
+    # wait for the sensor platform to finish loading before it could continue.
+    # Since the base `sensor` platform must also import all of its platform
+    # integrations to do validation before it can finish setup, its important
+    # that the platform integrations are imported first so we do not waste
+    # time importing `hue` first when we could have been importing the platforms
+    # that the base `sensor` platform need to load to do validation and allow
+    # all integrations that need the base `sensor` platform to proceed with setup.
+    #
+    if platform_integrations_to_load:
+        async_load_and_validate = partial(
+            _async_load_and_validate_platform_integration,
+            domain,
             integration_docs,
+            config_exceptions,
         )
-            config_exceptions.append(exc_info)
-            continue
-
-        # Validate platform specific schema
-        if hasattr(platform, "PLATFORM_SCHEMA"):
-            try:
-                p_validated = platform.PLATFORM_SCHEMA(p_config)
-            except vol.Invalid as exc:
-                exc_info = ConfigExceptionInfo(
-                    exc,
-                    ConfigErrorTranslationKey.PLATFORM_CONFIG_VALIDATION_ERR,
-                    platform_path,
-                    p_config,
-                    p_integration.documentation,
+        platforms.extend(
+            validated_config
+            for validated_config in await asyncio.gather(
+                *(
+                    create_eager_task(async_load_and_validate(p_integration))
+                    for p_integration in platform_integrations_to_load
                 )
-                config_exceptions.append(exc_info)
-                continue
-            except Exception as exc:  # pylint: disable=broad-except
-                exc_info = ConfigExceptionInfo(
-                    exc,
-                    ConfigErrorTranslationKey.PLATFORM_SCHEMA_VALIDATOR_ERR,
-                    p_name,
-                    p_config,
-                    p_integration.documentation,
             )
-                config_exceptions.append(exc_info)
-                continue
-
-        platforms.append(p_validated)
+            if validated_config is not None
+        )
 
     # Create a copy of the configuration with all config for current
     # component removed and add validated config back in.

tests/test_config.py

@@ -1,5 +1,6 @@
 """Test config utils."""
 
+import asyncio
 from collections import OrderedDict
 import contextlib
 import copy
@@ -15,6 +16,7 @@ import voluptuous as vol
 from voluptuous import Invalid, MultipleInvalid
 import yaml
 
+from homeassistant import config, loader
 import homeassistant.config as config_util
 from homeassistant.const import (
     ATTR_ASSUMED_STATE,
@@ -2372,3 +2374,80 @@ def test_extract_platform_integrations() -> None:
     ) == {"zone": {"hello 2", "hello"}, "notzone": {"nothello"}}
     assert config_util.extract_platform_integrations(config, {"zoneq"}) == {}
     assert config_util.extract_platform_integrations(config, {"zoneempty"}) == {}
+
+
+@pytest.mark.parametrize("load_registries", [False])
+async def test_loading_platforms_gathers(hass: HomeAssistant) -> None:
+    """Test loading platform integrations gathers."""
+
+    mock_integration(
+        hass,
+        MockModule(
+            domain="platform_int",
+        ),
+    )
+    mock_integration(
+        hass,
+        MockModule(
+            domain="platform_int2",
+        ),
+    )
+
+    # Its important that we do not mock the platforms with mock_platform
+    # as the loader is smart enough to know they are already loaded and
+    # will not create an executor job to load them. We are testing in
+    # what order the executor jobs happen here as we want to make
+    # sure the platform integrations are at the front of the line
+    light_integration = await loader.async_get_integration(hass, "light")
+    sensor_integration = await loader.async_get_integration(hass, "sensor")
+
+    order: list[tuple[str, str]] = []
+
+    def _load_platform(self, platform: str) -> MockModule:
+        order.append((self.domain, platform))
+        return MockModule()
+
+    # We need to patch what runs in the executor so we are counting
+    # the order that jobs are scheduled in th executor
+    with patch(
+        "homeassistant.loader.Integration._load_platform",
+        _load_platform,
+    ):
+        light_task = hass.async_create_task(
+            config.async_process_component_config(
+                hass,
+                {
+                    "light": [
+                        {"platform": "platform_int"},
+                        {"platform": "platform_int2"},
+                    ]
+                },
+                light_integration,
+            ),
+            eager_start=True,
+        )
+        sensor_task = hass.async_create_task(
+            config.async_process_component_config(
+                hass,
+                {
+                    "sensor": [
+                        {"platform": "platform_int"},
+                        {"platform": "platform_int2"},
+                    ]
+                },
+                sensor_integration,
+            ),
+            eager_start=True,
+        )
+
+        await asyncio.gather(light_task, sensor_task)
+
+    # Should be called in order so that
+    # all the light platforms are imported
+    # before the sensor platforms
+    assert order == [
+        ("platform_int", "light"),
+        ("platform_int2", "light"),
+        ("platform_int", "sensor"),
+        ("platform_int2", "sensor"),
+    ]