Fix initial fetch of Home Connect appliance data to handle API rate limit errors (#139379)

* Fix initial fetch of appliance data to handle API rate limit errors

* Apply comments

* Delete stale function

* Handle api rate limit error at options fetching

* Update appliances after stream non-breaking error

* Always initialize coordinator data

* Improve device update

* Update test description

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

---------

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

homeassistant/components/home_connect/__init__.py

@@ -629,14 +629,19 @@ async def async_setup_entry(hass: HomeAssistant, entry: HomeConnectConfigEntry)
     home_connect_client = HomeConnectClient(config_entry_auth)
 
     coordinator = HomeConnectCoordinator(hass, entry, home_connect_client)
-    await coordinator.async_config_entry_first_refresh()
+    await coordinator.async_setup()
-
     entry.runtime_data = coordinator
 
     await hass.config_entries.async_forward_entry_setups(entry, PLATFORMS)
 
     entry.runtime_data.start_event_listener()
 
+    entry.async_create_background_task(
+        hass,
+        coordinator.async_refresh(),
+        f"home_connect-initial-full-refresh-{entry.entry_id}",
+    )
+
     return True
 
 

homeassistant/components/home_connect/common.py

@@ -137,41 +137,6 @@ def setup_home_connect_entry(
         defaultdict(list)
     )
 
-    entities: list[HomeConnectEntity] = []
-    for appliance in entry.runtime_data.data.values():
-        entities_to_add = get_entities_for_appliance(entry, appliance)
-        if get_option_entities_for_appliance:
-            entities_to_add.extend(get_option_entities_for_appliance(entry, appliance))
-            for event_key in (
-                EventKey.BSH_COMMON_ROOT_ACTIVE_PROGRAM,
-                EventKey.BSH_COMMON_ROOT_SELECTED_PROGRAM,
-            ):
-                changed_options_listener_remove_callback = (
-                    entry.runtime_data.async_add_listener(
-                        partial(
-                            _create_option_entities,
-                            entry,
-                            appliance,
-                            known_entity_unique_ids,
-                            get_option_entities_for_appliance,
-                            async_add_entities,
-                        ),
-                        (appliance.info.ha_id, event_key),
-                    )
-                )
-                entry.async_on_unload(changed_options_listener_remove_callback)
-                changed_options_listener_remove_callbacks[appliance.info.ha_id].append(
-                    changed_options_listener_remove_callback
-                )
-        known_entity_unique_ids.update(
-            {
-                cast(str, entity.unique_id): appliance.info.ha_id
-                for entity in entities_to_add
-            }
-        )
-        entities.extend(entities_to_add)
-    async_add_entities(entities)
-
     entry.async_on_unload(
         entry.runtime_data.async_add_special_listener(
             partial(

homeassistant/components/home_connect/coordinator.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-import asyncio
+from asyncio import sleep as asyncio_sleep
 from collections import defaultdict
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -29,6 +29,7 @@ from aiohomeconnect.model.error import (
     HomeConnectApiError,
     HomeConnectError,
     HomeConnectRequestError,
+    TooManyRequestsError,
     UnauthorizedError,
 )
 from aiohomeconnect.model.program import EnumerateProgram, ProgramDefinitionOption
@@ -36,11 +37,11 @@ from propcache.api import cached_property
 
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.core import CALLBACK_TYPE, HomeAssistant, callback
-from homeassistant.exceptions import ConfigEntryAuthFailed
+from homeassistant.exceptions import ConfigEntryAuthFailed, ConfigEntryNotReady
 from homeassistant.helpers import device_registry as dr
 from homeassistant.helpers.update_coordinator import DataUpdateCoordinator, UpdateFailed
 
-from .const import APPLIANCES_WITH_PROGRAMS, DOMAIN
+from .const import API_DEFAULT_RETRY_AFTER, APPLIANCES_WITH_PROGRAMS, DOMAIN
 from .utils import get_dict_from_home_connect_error
 
 _LOGGER = logging.getLogger(__name__)
@@ -154,7 +155,7 @@ class HomeConnectCoordinator(
             f"home_connect-events_listener_task-{self.config_entry.entry_id}",
         )
 
-    async def _event_listener(self) -> None:
+    async def _event_listener(self) -> None:  # noqa: C901
         """Match event with listener for event type."""
         retry_time = 10
         while True:
@@ -269,7 +270,7 @@ class HomeConnectCoordinator(
                     error,
                     retry_time,
                 )
-                await asyncio.sleep(retry_time)
+                await asyncio_sleep(retry_time)
                 retry_time = min(retry_time * 2, 3600)
             except HomeConnectApiError as error:
                 _LOGGER.error("Error while listening for events: %s", error)
@@ -278,6 +279,13 @@ class HomeConnectCoordinator(
                 )
                 break
 
+            # Trigger to delete the possible depaired device entities
+            # from known_entities variable at common.py
+            for listener, context in self._special_listeners.values():
+                assert isinstance(context, tuple)
+                if EventKey.BSH_COMMON_APPLIANCE_DEPAIRED in context:
+                    listener()
+
     @callback
     def _call_event_listener(self, event_message: EventMessage) -> None:
         """Call listener for event."""
@@ -295,6 +303,42 @@ class HomeConnectCoordinator(
 
     async def _async_update_data(self) -> dict[str, HomeConnectApplianceData]:
         """Fetch data from Home Connect."""
+        await self._async_setup()
+
+        for appliance_data in self.data.values():
+            appliance = appliance_data.info
+            ha_id = appliance.ha_id
+            while True:
+                try:
+                    self.data[ha_id] = await self._get_appliance_data(
+                        appliance, self.data.get(ha_id)
+                    )
+                except TooManyRequestsError as err:
+                    _LOGGER.debug(
+                        "Rate limit exceeded on initial fetch: %s",
+                        err,
+                    )
+                    await asyncio_sleep(err.retry_after or API_DEFAULT_RETRY_AFTER)
+                else:
+                    break
+
+        for listener, context in self._special_listeners.values():
+            assert isinstance(context, tuple)
+            if EventKey.BSH_COMMON_APPLIANCE_PAIRED in context:
+                listener()
+
+        return self.data
+
+    async def async_setup(self) -> None:
+        """Set up the devices."""
+        try:
+            await self._async_setup()
+        except UpdateFailed as err:
+            raise ConfigEntryNotReady from err
+
+    async def _async_setup(self) -> None:
+        """Set up the devices."""
+        old_appliances = set(self.data.keys())
         try:
             appliances = await self.client.get_home_appliances()
         except UnauthorizedError as error:
@@ -312,12 +356,38 @@ class HomeConnectCoordinator(
                 translation_placeholders=get_dict_from_home_connect_error(error),
             ) from error
 
-        return {
+        for appliance in appliances.homeappliances:
-            appliance.ha_id: await self._get_appliance_data(
+            self.device_registry.async_get_or_create(
-                appliance, self.data.get(appliance.ha_id)
+                config_entry_id=self.config_entry.entry_id,
+                identifiers={(DOMAIN, appliance.ha_id)},
+                manufacturer=appliance.brand,
+                name=appliance.name,
+                model=appliance.vib,
             )
-            for appliance in appliances.homeappliances
+            if appliance.ha_id not in self.data:
-        }
+                self.data[appliance.ha_id] = HomeConnectApplianceData(
+                    commands=set(),
+                    events={},
+                    info=appliance,
+                    options={},
+                    programs=[],
+                    settings={},
+                    status={},
+                )
+            else:
+                self.data[appliance.ha_id].info.connected = appliance.connected
+                old_appliances.remove(appliance.ha_id)
+
+        for ha_id in old_appliances:
+            self.data.pop(ha_id, None)
+            device = self.device_registry.async_get_device(
+                identifiers={(DOMAIN, ha_id)}
+            )
+            if device:
+                self.device_registry.async_update_device(
+                    device_id=device.id,
+                    remove_config_entry_id=self.config_entry.entry_id,
+                )
 
     async def _get_appliance_data(
         self,
@@ -339,6 +409,8 @@ class HomeConnectCoordinator(
                     await self.client.get_settings(appliance.ha_id)
                 ).settings
             }
+        except TooManyRequestsError:
+            raise
         except HomeConnectError as error:
             _LOGGER.debug(
                 "Error fetching settings for %s: %s",
@@ -351,6 +423,8 @@ class HomeConnectCoordinator(
                 status.key: status
                 for status in (await self.client.get_status(appliance.ha_id)).status
             }
+        except TooManyRequestsError:
+            raise
         except HomeConnectError as error:
             _LOGGER.debug(
                 "Error fetching status for %s: %s",
@@ -365,6 +439,8 @@ class HomeConnectCoordinator(
         if appliance.type in APPLIANCES_WITH_PROGRAMS:
             try:
                 all_programs = await self.client.get_all_programs(appliance.ha_id)
+            except TooManyRequestsError:
+                raise
             except HomeConnectError as error:
                 _LOGGER.debug(
                     "Error fetching programs for %s: %s",
@@ -421,6 +497,8 @@ class HomeConnectCoordinator(
                     await self.client.get_available_commands(appliance.ha_id)
                 ).commands
             }
+        except TooManyRequestsError:
+            raise
         except HomeConnectError:
             commands = set()
 
@@ -455,6 +533,8 @@ class HomeConnectCoordinator(
                 ).options
                 or []
             }
+        except TooManyRequestsError:
+            raise
         except HomeConnectError as error:
             _LOGGER.debug(
                 "Error fetching options for %s: %s",

tests/components/home_connect/test_coordinator.py

@@ -29,6 +29,7 @@ from homeassistant.components.home_connect.const import (
     BSH_DOOR_STATE_OPEN,
     BSH_EVENT_PRESENT_STATE_PRESENT,
     BSH_POWER_OFF,
+    DOMAIN,
 )
 from homeassistant.config_entries import ConfigEntries, ConfigEntryState
 from homeassistant.const import EVENT_STATE_REPORTED, Platform
@@ -38,7 +39,7 @@ from homeassistant.core import (
     HomeAssistant,
     callback,
 )
-from homeassistant.helpers import entity_registry as er
+from homeassistant.helpers import device_registry as dr, entity_registry as er
 from homeassistant.setup import async_setup_component
 from homeassistant.util import dt as dt_util
 
@@ -499,3 +500,44 @@ async def test_event_listener_resilience(
     state = hass.states.get(entity_id)
     assert state
     assert state.state == after_event_expected_state
+
+
+async def test_devices_updated_on_refresh(
+    hass: HomeAssistant,
+    config_entry: MockConfigEntry,
+    integration_setup: Callable[[MagicMock], Awaitable[bool]],
+    setup_credentials: None,
+    client: MagicMock,
+    device_registry: dr.DeviceRegistry,
+) -> None:
+    """Test handling of devices added or deleted while event stream is down."""
+    appliances: list[HomeAppliance] = (
+        client.get_home_appliances.return_value.homeappliances
+    )
+    assert len(appliances) >= 3
+    client.get_home_appliances = AsyncMock(
+        return_value=ArrayOfHomeAppliances(appliances[:2]),
+    )
+
+    await async_setup_component(hass, "homeassistant", {})
+    assert config_entry.state == ConfigEntryState.NOT_LOADED
+    await integration_setup(client)
+    assert config_entry.state == ConfigEntryState.LOADED
+
+    for appliance in appliances[:2]:
+        assert device_registry.async_get_device({(DOMAIN, appliance.ha_id)})
+    assert not device_registry.async_get_device({(DOMAIN, appliances[2].ha_id)})
+
+    client.get_home_appliances = AsyncMock(
+        return_value=ArrayOfHomeAppliances(appliances[1:3]),
+    )
+    await hass.services.async_call(
+        "homeassistant",
+        "update_entity",
+        {"entity_id": "switch.dishwasher_power"},
+        blocking=True,
+    )
+
+    assert not device_registry.async_get_device({(DOMAIN, appliances[0].ha_id)})
+    for appliance in appliances[2:3]:
+        assert device_registry.async_get_device({(DOMAIN, appliance.ha_id)})

tests/components/home_connect/test_init.py

@@ -3,11 +3,15 @@
 from collections.abc import Awaitable, Callable
 from http import HTTPStatus
 from typing import Any
-from unittest.mock import MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 from aiohomeconnect.const import OAUTH2_TOKEN
 from aiohomeconnect.model import OptionKey, ProgramKey, SettingKey, StatusKey
-from aiohomeconnect.model.error import HomeConnectError, UnauthorizedError
+from aiohomeconnect.model.error import (
+    HomeConnectError,
+    TooManyRequestsError,
+    UnauthorizedError,
+)
 import aiohttp
 import pytest
 from syrupy.assertion import SnapshotAssertion
@@ -355,6 +359,48 @@ async def test_client_error(
     assert client_with_exception.get_home_appliances.call_count == 1
 
 
+@pytest.mark.parametrize(
+    "raising_exception_method",
+    [
+        "get_settings",
+        "get_status",
+        "get_all_programs",
+        "get_available_commands",
+        "get_available_program",
+    ],
+)
+async def test_client_rate_limit_error(
+    raising_exception_method: str,
+    hass: HomeAssistant,
+    config_entry: MockConfigEntry,
+    integration_setup: Callable[[MagicMock], Awaitable[bool]],
+    setup_credentials: None,
+    client: MagicMock,
+) -> None:
+    """Test client errors during setup integration."""
+    retry_after = 42
+
+    original_mock = getattr(client, raising_exception_method)
+    mock = AsyncMock()
+
+    async def side_effect(*args, **kwargs):
+        if mock.call_count <= 1:
+            raise TooManyRequestsError("error.key", retry_after=retry_after)
+        return await original_mock(*args, **kwargs)
+
+    mock.side_effect = side_effect
+    setattr(client, raising_exception_method, mock)
+
+    assert config_entry.state == ConfigEntryState.NOT_LOADED
+    with patch(
+        "homeassistant.components.home_connect.coordinator.asyncio_sleep",
+    ) as asyncio_sleep_mock:
+        assert await integration_setup(client)
+    assert config_entry.state == ConfigEntryState.LOADED
+    assert mock.call_count >= 2
+    asyncio_sleep_mock.assert_called_once_with(retry_after)
+
+
 @pytest.mark.parametrize(
     "service_call",
     SERVICE_KV_CALL_PARAMS + SERVICE_COMMAND_CALL_PARAMS + SERVICE_PROGRAM_CALL_PARAMS,