Core API check during startup can timeout (#4595)

* Core API check during startup can timeout

* Use a more specific exception so caller can differentiate

supervisor/exceptions.py

@@ -67,6 +67,10 @@ class HomeAssistantCrashError(HomeAssistantError):
     """Error on crash of a Home Assistant startup."""
 
 
+class HomeAssistantStartupTimeout(HomeAssistantCrashError):
+    """Timeout waiting for Home Assistant successful startup."""
+
+
 class HomeAssistantAPIError(HomeAssistantError):
     """Home Assistant API exception."""
 

supervisor/homeassistant/core.py

@@ -2,12 +2,14 @@
 import asyncio
 from collections.abc import Awaitable
 from contextlib import suppress
+from dataclasses import dataclass
+from datetime import datetime, timedelta
 import logging
 import re
 import secrets
 import shutil
+from typing import Final
 
-import attr
 from awesomeversion import AwesomeVersion
 
 from ..const import ATTR_HOMEASSISTANT, BusEvent
@@ -21,6 +23,7 @@ from ..exceptions import (
     HomeAssistantCrashError,
     HomeAssistantError,
     HomeAssistantJobError,
+    HomeAssistantStartupTimeout,
     HomeAssistantUpdateError,
     JobException,
 )
@@ -40,15 +43,17 @@ from .const import (
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 
+SECONDS_BETWEEN_API_CHECKS: Final[int] = 5
+STARTUP_API_CHECK_TIMEOUT: Final[timedelta] = timedelta(minutes=5)
 RE_YAML_ERROR = re.compile(r"homeassistant\.util\.yaml")
 
 
-@attr.s(frozen=True)
+@dataclass
 class ConfigResult:
     """Return object from config check."""
 
-    valid = attr.ib()
-    log = attr.ib()
+    valid: bool
+    log: str
 
 
 class HomeAssistantCore(JobGroup):
@@ -435,8 +440,9 @@ class HomeAssistantCore(JobGroup):
             return
         _LOGGER.info("Wait until Home Assistant is ready")
 
-        while True:
-            await asyncio.sleep(5)
+        start = datetime.now()
+        while not (timeout := datetime.now() >= start + STARTUP_API_CHECK_TIMEOUT):
+            await asyncio.sleep(SECONDS_BETWEEN_API_CHECKS)
 
             # 1: Check if Container is is_running
             if not await self.instance.is_running():
@@ -450,6 +456,11 @@ class HomeAssistantCore(JobGroup):
                 return
 
         self._error_state = True
+        if timeout:
+            raise HomeAssistantStartupTimeout(
+                "No API response in 5 minutes, assuming core has had a fatal startup error",
+                _LOGGER.error,
+            )
         raise HomeAssistantCrashError()
 
     @Job(

tests/homeassistant/test_core.py

@@ -1,9 +1,12 @@
 """Test Home Assistant core."""
+
+from datetime import datetime, timedelta
 from unittest.mock import MagicMock, Mock, PropertyMock, patch
 
 from awesomeversion import AwesomeVersion
 from docker.errors import DockerException, ImageNotFound, NotFound
 import pytest
+from time_machine import travel
 
 from supervisor.const import CpuArch
 from supervisor.coresys import CoreSys
@@ -14,6 +17,7 @@ from supervisor.exceptions import (
     AudioUpdateError,
     CodeNotaryError,
     DockerError,
+    HomeAssistantCrashError,
     HomeAssistantError,
     HomeAssistantJobError,
 )
@@ -263,3 +267,35 @@ async def test_stats_failures(
 
     with pytest.raises(HomeAssistantError):
         await coresys.homeassistant.core.stats()
+
+
+async def test_api_check_timeout(
+    coresys: CoreSys, container: MagicMock, caplog: pytest.LogCaptureFixture
+):
+    """Test attempts to contact the API timeout."""
+    container.status = "stopped"
+    coresys.homeassistant.version = AwesomeVersion("2023.9.0")
+    coresys.homeassistant.api.check_api_state.return_value = False
+
+    async def mock_instance_start(*_):
+        container.status = "running"
+
+    with patch.object(
+        DockerHomeAssistant, "start", new=mock_instance_start
+    ), patch.object(DockerAPI, "container_is_initialized", return_value=True), travel(
+        datetime(2023, 10, 2, 0, 0, 0), tick=False
+    ) as traveller:
+
+        async def mock_sleep(*args):
+            traveller.shift(timedelta(minutes=1))
+
+        with patch(
+            "supervisor.homeassistant.core.asyncio.sleep", new=mock_sleep
+        ), pytest.raises(HomeAssistantCrashError):
+            await coresys.homeassistant.core.start()
+
+    assert coresys.homeassistant.api.check_api_state.call_count == 5
+    assert (
+        "No API response in 5 minutes, assuming core has had a fatal startup error"
+        in caplog.text
+    )