Implement retry and backoff strategy for requirements install (#56580)

Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>

homeassistant/helpers/check_config.py

@@ -26,6 +26,7 @@ from homeassistant.exceptions import HomeAssistantError
 from homeassistant.helpers.typing import ConfigType
 from homeassistant.requirements import (
     RequirementsNotFound,
+    async_clear_install_history,
     async_get_integration_with_requirements,
 )
 import homeassistant.util.yaml.loader as yaml_loader
@@ -71,6 +72,7 @@ async def async_check_ha_config_file(  # noqa: C901
     This method is a coroutine.
     """
     result = HomeAssistantConfig()
+    async_clear_install_history(hass)
 
     def _pack_error(
         package: str, component: str, config: ConfigType, message: str

homeassistant/requirements.py

@@ -3,10 +3,11 @@ from __future__ import annotations
 
 import asyncio
 from collections.abc import Iterable
+import logging
 import os
 from typing import Any, cast
 
-from homeassistant.core import HomeAssistant
+from homeassistant.core import HomeAssistant, callback
 from homeassistant.exceptions import HomeAssistantError
 from homeassistant.helpers.typing import UNDEFINED, UndefinedType
 from homeassistant.loader import Integration, IntegrationNotFound, async_get_integration
@@ -15,9 +16,11 @@ import homeassistant.util.package as pkg_util
 # mypy: disallow-any-generics
 
 PIP_TIMEOUT = 60  # The default is too low when the internet connection is satellite or high latency
+MAX_INSTALL_FAILURES = 3
 DATA_PIP_LOCK = "pip_lock"
 DATA_PKG_CACHE = "pkg_cache"
 DATA_INTEGRATIONS_WITH_REQS = "integrations_with_reqs"
+DATA_INSTALL_FAILURE_HISTORY = "install_failure_history"
 CONSTRAINT_FILE = "package_constraints.txt"
 DISCOVERY_INTEGRATIONS: dict[str, Iterable[str]] = {
     "dhcp": ("dhcp",),
@@ -25,6 +28,7 @@ DISCOVERY_INTEGRATIONS: dict[str, Iterable[str]] = {
     "ssdp": ("ssdp",),
     "zeroconf": ("zeroconf", "homekit"),
 }
+_LOGGER = logging.getLogger(__name__)
 
 
 class RequirementsNotFound(HomeAssistantError):
@@ -135,6 +139,13 @@ async def _async_process_integration(
             raise result
 
 
+@callback
+def async_clear_install_history(hass: HomeAssistant) -> None:
+    """Forget the install history."""
+    if install_failure_history := hass.data.get(DATA_INSTALL_FAILURE_HISTORY):
+        install_failure_history.clear()
+
+
 async def async_process_requirements(
     hass: HomeAssistant, name: str, requirements: list[str]
 ) -> None:
@@ -146,22 +157,47 @@ async def async_process_requirements(
     pip_lock = hass.data.get(DATA_PIP_LOCK)
     if pip_lock is None:
         pip_lock = hass.data[DATA_PIP_LOCK] = asyncio.Lock()
+    install_failure_history = hass.data.get(DATA_INSTALL_FAILURE_HISTORY)
+    if install_failure_history is None:
+        install_failure_history = hass.data[DATA_INSTALL_FAILURE_HISTORY] = set()
 
     kwargs = pip_kwargs(hass.config.config_dir)
 
     async with pip_lock:
         for req in requirements:
-            if pkg_util.is_installed(req):
+            await _async_process_requirements(
-                continue
+                hass, name, req, install_failure_history, kwargs
+            )
 
-            def _install(req: str, kwargs: dict[str, Any]) -> bool:
-                """Install requirement."""
-                return pkg_util.install_package(req, **kwargs)
 
-            ret = await hass.async_add_executor_job(_install, req, kwargs)
+async def _async_process_requirements(
+    hass: HomeAssistant,
+    name: str,
+    req: str,
+    install_failure_history: set[str],
+    kwargs: Any,
+) -> None:
+    """Install a requirement and save failures."""
+    if pkg_util.is_installed(req):
+        return
 
-            if not ret:
+    if req in install_failure_history:
-                raise RequirementsNotFound(name, [req])
+        _LOGGER.info(
+            "Multiple attempts to install %s failed, install will be retried after next configuration check or restart",
+            req,
+        )
+        raise RequirementsNotFound(name, [req])
+
+    def _install(req: str, kwargs: dict[str, Any]) -> bool:
+        """Install requirement."""
+        return pkg_util.install_package(req, **kwargs)
+
+    for _ in range(MAX_INSTALL_FAILURES):
+        if await hass.async_add_executor_job(_install, req, kwargs):
+            return
+
+    install_failure_history.add(req)
+    raise RequirementsNotFound(name, [req])
 
 
 def pip_kwargs(config_dir: str | None) -> dict[str, Any]:

tests/test_requirements.py

@@ -8,6 +8,7 @@ from homeassistant import loader, setup
 from homeassistant.requirements import (
     CONSTRAINT_FILE,
     RequirementsNotFound,
+    async_clear_install_history,
     async_get_integration_with_requirements,
     async_process_requirements,
 )
@@ -89,7 +90,7 @@ async def test_install_missing_package(hass):
     ) as mock_inst, pytest.raises(RequirementsNotFound):
         await async_process_requirements(hass, "test_component", ["hello==1.0.0"])
 
-    assert len(mock_inst.mock_calls) == 1
+    assert len(mock_inst.mock_calls) == 3
 
 
 async def test_get_integration_with_requirements(hass):
@@ -188,9 +189,13 @@ async def test_get_integration_with_requirements_pip_install_fails_two_passes(ha
         "test-comp==1.0.0",
     ]
 
-    assert len(mock_inst.mock_calls) == 3
+    assert len(mock_inst.mock_calls) == 7
     assert sorted(mock_call[1][0] for mock_call in mock_inst.mock_calls) == [
         "test-comp-after-dep==1.0.0",
+        "test-comp-after-dep==1.0.0",
+        "test-comp-after-dep==1.0.0",
+        "test-comp-dep==1.0.0",
+        "test-comp-dep==1.0.0",
         "test-comp-dep==1.0.0",
         "test-comp==1.0.0",
     ]
@@ -215,6 +220,67 @@ async def test_get_integration_with_requirements_pip_install_fails_two_passes(ha
         "test-comp==1.0.0",
     ]
 
+    # On another attempt we remember failures and don't try again
+    assert len(mock_inst.mock_calls) == 1
+    assert sorted(mock_call[1][0] for mock_call in mock_inst.mock_calls) == [
+        "test-comp==1.0.0"
+    ]
+
+    # Now clear the history and so we try again
+    async_clear_install_history(hass)
+
+    with pytest.raises(RequirementsNotFound), patch(
+        "homeassistant.util.package.is_installed", return_value=False
+    ) as mock_is_installed, patch(
+        "homeassistant.util.package.install_package", side_effect=_mock_install_package
+    ) as mock_inst:
+
+        integration = await async_get_integration_with_requirements(
+            hass, "test_component"
+        )
+        assert integration
+        assert integration.domain == "test_component"
+
+    assert len(mock_is_installed.mock_calls) == 3
+    assert sorted(mock_call[1][0] for mock_call in mock_is_installed.mock_calls) == [
+        "test-comp-after-dep==1.0.0",
+        "test-comp-dep==1.0.0",
+        "test-comp==1.0.0",
+    ]
+
+    assert len(mock_inst.mock_calls) == 7
+    assert sorted(mock_call[1][0] for mock_call in mock_inst.mock_calls) == [
+        "test-comp-after-dep==1.0.0",
+        "test-comp-after-dep==1.0.0",
+        "test-comp-after-dep==1.0.0",
+        "test-comp-dep==1.0.0",
+        "test-comp-dep==1.0.0",
+        "test-comp-dep==1.0.0",
+        "test-comp==1.0.0",
+    ]
+
+    # Now clear the history and mock success
+    async_clear_install_history(hass)
+
+    with patch(
+        "homeassistant.util.package.is_installed", return_value=False
+    ) as mock_is_installed, patch(
+        "homeassistant.util.package.install_package", return_value=True
+    ) as mock_inst:
+
+        integration = await async_get_integration_with_requirements(
+            hass, "test_component"
+        )
+        assert integration
+        assert integration.domain == "test_component"
+
+    assert len(mock_is_installed.mock_calls) == 3
+    assert sorted(mock_call[1][0] for mock_call in mock_is_installed.mock_calls) == [
+        "test-comp-after-dep==1.0.0",
+        "test-comp-dep==1.0.0",
+        "test-comp==1.0.0",
+    ]
+
     assert len(mock_inst.mock_calls) == 3
     assert sorted(mock_call[1][0] for mock_call in mock_inst.mock_calls) == [
         "test-comp-after-dep==1.0.0",