diff --git a/supervisor/bus.py b/supervisor/bus.py index 4061cf37c..715ad70f2 100644 --- a/supervisor/bus.py +++ b/supervisor/bus.py @@ -2,6 +2,7 @@ from __future__ import annotations +from asyncio import Task from collections.abc import Callable, Coroutine import logging from typing import Any @@ -38,11 +39,13 @@ class Bus(CoreSysAttributes): self._listeners.setdefault(event, []).append(listener) return listener - def fire_event(self, event: BusEvent, reference: Any) -> None: + def fire_event(self, event: BusEvent, reference: Any) -> list[Task]: """Fire an event to the bus.""" _LOGGER.debug("Fire event '%s' with '%s'", event, reference) + tasks: list[Task] = [] for listener in self._listeners.get(event, []): - self.sys_create_task(listener.callback(reference)) + tasks.append(self.sys_create_task(listener.callback(reference))) + return tasks def remove_listener(self, listener: EventListener) -> None: """Unregister an listener.""" diff --git a/supervisor/docker/interface.py b/supervisor/docker/interface.py index f4acc62f9..16142557d 100644 --- a/supervisor/docker/interface.py +++ b/supervisor/docker/interface.py @@ -452,11 +452,23 @@ class DockerInterface(JobGroup, ABC): suggestions=[SuggestionType.REGISTRY_LOGIN], ) raise DockerHubRateLimitExceeded(_LOGGER.error) from err + await async_capture_exception(err) + raise DockerError( + f"Can't install {image}:{version!s}: {err}", _LOGGER.error + ) from err + except aiodocker.DockerError as err: + if err.status == HTTPStatus.TOO_MANY_REQUESTS: + self.sys_resolution.create_issue( + IssueType.DOCKER_RATELIMIT, + ContextType.SYSTEM, + suggestions=[SuggestionType.REGISTRY_LOGIN], + ) + raise DockerHubRateLimitExceeded(_LOGGER.error) from err + await async_capture_exception(err) raise DockerError( f"Can't install {image}:{version!s}: {err}", _LOGGER.error ) from err except ( - aiodocker.DockerError, docker.errors.DockerException, requests.RequestException, ) as err: diff --git a/supervisor/docker/manager.py b/supervisor/docker/manager.py index 75ca6f1ab..9b186ffa2 100644 --- a/supervisor/docker/manager.py +++ b/supervisor/docker/manager.py @@ -17,6 +17,7 @@ from typing import Any, Final, Self, cast import aiodocker from aiodocker.images import DockerImages +from aiohttp import ClientSession, ClientTimeout, UnixConnector import attr from awesomeversion import AwesomeVersion, AwesomeVersionCompareException from docker import errors as docker_errors @@ -211,7 +212,10 @@ class DockerAPI(CoreSysAttributes): # We keep both until we can fully refactor to aiodocker self._dockerpy: DockerClient | None = None self.docker: aiodocker.Docker = aiodocker.Docker( - url=f"unix:/{str(SOCKET_DOCKER)}", api_version="auto" + url="unix://localhost", # dummy hostname for URL composition + connector=(connector := UnixConnector(SOCKET_DOCKER.as_posix())), + session=ClientSession(connector=connector, timeout=ClientTimeout(900)), + api_version="auto", ) self._network: DockerNetwork | None = None @@ -221,11 +225,13 @@ class DockerAPI(CoreSysAttributes): async def post_init(self) -> Self: """Post init actions that must be done in event loop.""" + # Use /var/run/docker.sock for this one so aiodocker and dockerpy don't + # share the same handle. Temporary fix while refactoring this client out self._dockerpy = await asyncio.get_running_loop().run_in_executor( None, partial( DockerClient, - base_url=f"unix:/{str(SOCKET_DOCKER)}", + base_url=f"unix://var{SOCKET_DOCKER.as_posix()}", version="auto", timeout=900, ), @@ -433,20 +439,16 @@ class DockerAPI(CoreSysAttributes): raises only if the get fails afterwards. Additionally it fires progress reports for the pull on the bus so listeners can use that to update status for users. """ - - def api_pull(): - pull_log = self.dockerpy.api.pull( - repository, tag=tag, platform=platform, stream=True, decode=True + async for e in self.images.pull( + repository, tag=tag, platform=platform, stream=True + ): + entry = PullLogEntry.from_pull_log_dict(job_id, e) + if entry.error: + raise entry.exception + await asyncio.gather( + *self.sys_bus.fire_event(BusEvent.DOCKER_IMAGE_PULL_UPDATE, entry) ) - for e in pull_log: - entry = PullLogEntry.from_pull_log_dict(job_id, e) - if entry.error: - raise entry.exception - self.sys_loop.call_soon_threadsafe( - self.sys_bus.fire_event, BusEvent.DOCKER_IMAGE_PULL_UPDATE, entry - ) - await self.sys_run_in_executor(api_pull) sep = "@" if tag.startswith("sha256:") else ":" return await self.images.inspect(f"{repository}{sep}{tag}") diff --git a/tests/addons/test_addon.py b/tests/addons/test_addon.py index 35b9b0d17..2d26acabd 100644 --- a/tests/addons/test_addon.py +++ b/tests/addons/test_addon.py @@ -9,7 +9,7 @@ from unittest.mock import MagicMock, PropertyMock, call, patch import aiodocker from awesomeversion import AwesomeVersion -from docker.errors import DockerException, NotFound +from docker.errors import APIError, DockerException, NotFound import pytest from securetar import SecureTarFile @@ -931,7 +931,7 @@ async def test_addon_loads_missing_image( @pytest.mark.parametrize( "pull_image_exc", - [DockerException(), aiodocker.DockerError(400, {"message": "error"})], + [APIError("error"), aiodocker.DockerError(400, {"message": "error"})], ) @pytest.mark.usefixtures("container", "mock_amd64_arch_supported") async def test_addon_load_succeeds_with_docker_errors( @@ -973,7 +973,7 @@ async def test_addon_load_succeeds_with_docker_errors( caplog.clear() with patch.object(DockerAPI, "pull_image", side_effect=pull_image_exc): await install_addon_ssh.load() - assert "Unknown error with test/amd64-addon-ssh:9.2.1" in caplog.text + assert "Can't install test/amd64-addon-ssh:9.2.1:" in caplog.text async def test_addon_manual_only_boot(coresys: CoreSys, install_addon_example: Addon): diff --git a/tests/api/test_homeassistant.py b/tests/api/test_homeassistant.py index 40cc37fda..76e10aa16 100644 --- a/tests/api/test_homeassistant.py +++ b/tests/api/test_homeassistant.py @@ -19,7 +19,7 @@ from supervisor.homeassistant.core import HomeAssistantCore from supervisor.homeassistant.module import HomeAssistant from tests.api import common_test_api_advanced_logs -from tests.common import load_json_fixture +from tests.common import AsyncIterator, load_json_fixture @pytest.mark.parametrize("legacy_route", [True, False]) @@ -283,9 +283,9 @@ async def test_api_progress_updates_home_assistant_update( """Test progress updates sent to Home Assistant for updates.""" coresys.hardware.disk.get_disk_free_space = lambda x: 5000 coresys.core.set_state(CoreState.RUNNING) - coresys.docker.dockerpy.api.pull.return_value = load_json_fixture( - "docker_pull_image_log.json" - ) + + logs = load_json_fixture("docker_pull_image_log.json") + coresys.docker.images.pull.return_value = AsyncIterator(logs) coresys.homeassistant.version = AwesomeVersion("2025.8.0") with ( diff --git a/tests/api/test_store.py b/tests/api/test_store.py index bf81944a9..760f33e3e 100644 --- a/tests/api/test_store.py +++ b/tests/api/test_store.py @@ -24,7 +24,7 @@ from supervisor.homeassistant.module import HomeAssistant from supervisor.store.addon import AddonStore from supervisor.store.repository import Repository -from tests.common import load_json_fixture +from tests.common import AsyncIterator, load_json_fixture from tests.const import TEST_ADDON_SLUG REPO_URL = "https://github.com/awesome-developer/awesome-repo" @@ -732,9 +732,10 @@ async def test_api_progress_updates_addon_install_update( """Test progress updates sent to Home Assistant for installs/updates.""" coresys.hardware.disk.get_disk_free_space = lambda x: 5000 coresys.core.set_state(CoreState.RUNNING) - coresys.docker.dockerpy.api.pull.return_value = load_json_fixture( - "docker_pull_image_log.json" - ) + + logs = load_json_fixture("docker_pull_image_log.json") + coresys.docker.images.pull.return_value = AsyncIterator(logs) + coresys.arch._supported_arch = ["amd64"] # pylint: disable=protected-access install_addon_example.data_store["version"] = AwesomeVersion("2.0.0") diff --git a/tests/api/test_supervisor.py b/tests/api/test_supervisor.py index 9e353dd3e..cd9d93fe5 100644 --- a/tests/api/test_supervisor.py +++ b/tests/api/test_supervisor.py @@ -19,7 +19,7 @@ from supervisor.supervisor import Supervisor from supervisor.updater import Updater from tests.api import common_test_api_advanced_logs -from tests.common import load_json_fixture +from tests.common import AsyncIterator, load_json_fixture from tests.dbus_service_mocks.base import DBusServiceMock from tests.dbus_service_mocks.os_agent import OSAgent as OSAgentService @@ -332,9 +332,9 @@ async def test_api_progress_updates_supervisor_update( """Test progress updates sent to Home Assistant for updates.""" coresys.hardware.disk.get_disk_free_space = lambda x: 5000 coresys.core.set_state(CoreState.RUNNING) - coresys.docker.dockerpy.api.pull.return_value = load_json_fixture( - "docker_pull_image_log.json" - ) + + logs = load_json_fixture("docker_pull_image_log.json") + coresys.docker.images.pull.return_value = AsyncIterator(logs) with ( patch.object( diff --git a/tests/common.py b/tests/common.py index 944f4348c..08408979e 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,13 +1,14 @@ """Common test functions.""" import asyncio +from collections.abc import Sequence from datetime import datetime from functools import partial from importlib import import_module from inspect import getclosurevars import json from pathlib import Path -from typing import Any +from typing import Any, Self from dbus_fast.aio.message_bus import MessageBus @@ -145,3 +146,22 @@ class MockResponse: async def __aexit__(self, exc_type, exc, tb): """Exit the context manager.""" + + +class AsyncIterator: + """Make list/fixture into async iterator for test mocks.""" + + def __init__(self, seq: Sequence[Any]) -> None: + """Initialize with sequence.""" + self.iter = iter(seq) + + def __aiter__(self) -> Self: + """Implement aiter.""" + return self + + async def __anext__(self) -> Any: + """Return next in sequence.""" + try: + return next(self.iter) + except StopIteration: + raise StopAsyncIteration() from None diff --git a/tests/conftest.py b/tests/conftest.py index ebaac8b83..6936198b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,6 +56,7 @@ from supervisor.store.repository import Repository from supervisor.utils.dt import utcnow from .common import ( + AsyncIterator, MockResponse, load_binary_fixture, load_fixture, @@ -125,14 +126,8 @@ async def docker() -> DockerAPI: patch( "supervisor.docker.manager.DockerAPI.containers", return_value=MagicMock() ), - patch( - "supervisor.docker.manager.DockerAPI.api", - return_value=(api_mock := MagicMock()), - ), - patch( - "supervisor.docker.manager.DockerAPI.info", - return_value=MagicMock(), - ), + patch("supervisor.docker.manager.DockerAPI.api", return_value=MagicMock()), + patch("supervisor.docker.manager.DockerAPI.info", return_value=MagicMock()), patch("supervisor.docker.manager.DockerAPI.unload"), patch("supervisor.docker.manager.aiodocker.Docker", return_value=MagicMock()), patch( @@ -153,13 +148,12 @@ async def docker() -> DockerAPI: {"stream": "Loaded image: test:latest\n"} ] + docker_images.pull.return_value = AsyncIterator([{}]) + docker_obj.info.logging = "journald" docker_obj.info.storage = "overlay2" docker_obj.info.version = AwesomeVersion("1.0.0") - # Need an iterable for logs - api_mock.pull.return_value = [] - yield docker_obj diff --git a/tests/docker/test_interface.py b/tests/docker/test_interface.py index 524d602b4..adbf5a0c2 100644 --- a/tests/docker/test_interface.py +++ b/tests/docker/test_interface.py @@ -28,7 +28,7 @@ from supervisor.exceptions import ( ) from supervisor.jobs import JobSchedulerOptions, SupervisorJob -from tests.common import load_json_fixture +from tests.common import AsyncIterator, load_json_fixture @pytest.fixture(autouse=True) @@ -59,8 +59,8 @@ async def test_docker_image_platform( """Test platform set correctly from arch.""" coresys.docker.images.inspect.return_value = {"Id": "test:1.2.3"} await test_docker_interface.install(AwesomeVersion("1.2.3"), "test", arch=cpu_arch) - coresys.docker.dockerpy.api.pull.assert_called_once_with( - "test", tag="1.2.3", platform=platform, stream=True, decode=True + coresys.docker.images.pull.assert_called_once_with( + "test", tag="1.2.3", platform=platform, stream=True ) coresys.docker.images.inspect.assert_called_once_with("test:1.2.3") @@ -76,8 +76,8 @@ async def test_docker_image_default_platform( ), ): await test_docker_interface.install(AwesomeVersion("1.2.3"), "test") - coresys.docker.dockerpy.api.pull.assert_called_once_with( - "test", tag="1.2.3", platform="linux/386", stream=True, decode=True + coresys.docker.images.pull.assert_called_once_with( + "test", tag="1.2.3", platform="linux/386", stream=True ) coresys.docker.images.inspect.assert_called_once_with("test:1.2.3") @@ -276,8 +276,9 @@ async def test_install_fires_progress_events( coresys: CoreSys, test_docker_interface: DockerInterface ): """Test progress events are fired during an install for listeners.""" + # This is from a sample pull. Filtered log to just one per unique status for test - coresys.docker.dockerpy.api.pull.return_value = [ + logs = [ { "status": "Pulling from home-assistant/odroid-n2-homeassistant", "id": "2025.7.2", @@ -299,7 +300,11 @@ async def test_install_fires_progress_events( "id": "1578b14a573c", }, {"status": "Pull complete", "progressDetail": {}, "id": "1578b14a573c"}, - {"status": "Verifying Checksum", "progressDetail": {}, "id": "6a1e931d8f88"}, + { + "status": "Verifying Checksum", + "progressDetail": {}, + "id": "6a1e931d8f88", + }, { "status": "Digest: sha256:490080d7da0f385928022927990e04f604615f7b8c622ef3e58253d0f089881d" }, @@ -307,6 +312,7 @@ async def test_install_fires_progress_events( "status": "Status: Downloaded newer image for ghcr.io/home-assistant/odroid-n2-homeassistant:2025.7.2" }, ] + coresys.docker.images.pull.return_value = AsyncIterator(logs) events: list[PullLogEntry] = [] @@ -321,8 +327,8 @@ async def test_install_fires_progress_events( ), ): await test_docker_interface.install(AwesomeVersion("1.2.3"), "test") - coresys.docker.dockerpy.api.pull.assert_called_once_with( - "test", tag="1.2.3", platform="linux/386", stream=True, decode=True + coresys.docker.images.pull.assert_called_once_with( + "test", tag="1.2.3", platform="linux/386", stream=True ) coresys.docker.images.inspect.assert_called_once_with("test:1.2.3") @@ -402,10 +408,11 @@ async def test_install_progress_rounding_does_not_cause_misses( ): """Test extremely close progress events do not create rounding issues.""" coresys.core.set_state(CoreState.RUNNING) + # Current numbers chosen to create a rounding issue with original code # Where a progress update came in with a value between the actual previous # value and what it was rounded to. It should not raise an out of order exception - coresys.docker.dockerpy.api.pull.return_value = [ + logs = [ { "status": "Pulling from home-assistant/odroid-n2-homeassistant", "id": "2025.7.1", @@ -445,6 +452,7 @@ async def test_install_progress_rounding_does_not_cause_misses( "status": "Status: Downloaded newer image for ghcr.io/home-assistant/odroid-n2-homeassistant:2025.7.1" }, ] + coresys.docker.images.pull.return_value = AsyncIterator(logs) with ( patch.object( @@ -500,7 +508,8 @@ async def test_install_raises_on_pull_error( exc_msg: str, ): """Test exceptions raised from errors in pull log.""" - coresys.docker.dockerpy.api.pull.return_value = [ + + logs = [ { "status": "Pulling from home-assistant/odroid-n2-homeassistant", "id": "2025.7.2", @@ -513,6 +522,7 @@ async def test_install_raises_on_pull_error( }, error_log, ] + coresys.docker.images.pull.return_value = AsyncIterator(logs) with pytest.raises(exc_type, match=exc_msg): await test_docker_interface.install(AwesomeVersion("1.2.3"), "test") @@ -526,11 +536,11 @@ async def test_install_progress_handles_download_restart( ): """Test install handles docker progress events that include a download restart.""" coresys.core.set_state(CoreState.RUNNING) + # Fixture emulates a download restart as it docker logs it # A log out of order exception should not be raised - coresys.docker.dockerpy.api.pull.return_value = load_json_fixture( - "docker_pull_image_log_restart.json" - ) + logs = load_json_fixture("docker_pull_image_log_restart.json") + coresys.docker.images.pull.return_value = AsyncIterator(logs) with ( patch.object( diff --git a/tests/homeassistant/test_core.py b/tests/homeassistant/test_core.py index a151aaca0..069289f3d 100644 --- a/tests/homeassistant/test_core.py +++ b/tests/homeassistant/test_core.py @@ -1,12 +1,14 @@ """Test Home Assistant core.""" from datetime import datetime, timedelta +from http import HTTPStatus from unittest.mock import ANY, MagicMock, Mock, PropertyMock, call, patch import aiodocker from awesomeversion import AwesomeVersion from docker.errors import APIError, DockerException, NotFound import pytest +from requests import RequestException from time_machine import travel from supervisor.const import CpuArch @@ -24,8 +26,12 @@ from supervisor.exceptions import ( from supervisor.homeassistant.api import APIState from supervisor.homeassistant.core import HomeAssistantCore from supervisor.homeassistant.module import HomeAssistant +from supervisor.resolution.const import ContextType, IssueType +from supervisor.resolution.data import Issue from supervisor.updater import Updater +from tests.common import AsyncIterator + async def test_update_fails_if_out_of_date(coresys: CoreSys): """Test update of Home Assistant fails when supervisor or plugin is out of date.""" @@ -53,11 +59,23 @@ async def test_update_fails_if_out_of_date(coresys: CoreSys): await coresys.homeassistant.core.update() -async def test_install_landingpage_docker_error( - coresys: CoreSys, capture_exception: Mock, caplog: pytest.LogCaptureFixture +@pytest.mark.parametrize( + "err", + [ + aiodocker.DockerError(HTTPStatus.TOO_MANY_REQUESTS, {"message": "ratelimit"}), + APIError("ratelimit", MagicMock(status_code=HTTPStatus.TOO_MANY_REQUESTS)), + ], +) +async def test_install_landingpage_docker_ratelimit_error( + coresys: CoreSys, + capture_exception: Mock, + caplog: pytest.LogCaptureFixture, + err: Exception, ): - """Test install landing page fails due to docker error.""" + """Test install landing page fails due to docker ratelimit error.""" coresys.security.force = True + coresys.docker.images.pull.side_effect = [err, AsyncIterator([{}])] + with ( patch.object(DockerHomeAssistant, "attach", side_effect=DockerError), patch.object( @@ -70,19 +88,35 @@ async def test_install_landingpage_docker_error( ), patch("supervisor.homeassistant.core.asyncio.sleep") as sleep, ): - coresys.docker.dockerpy.api.pull.side_effect = [APIError("fail"), MagicMock()] await coresys.homeassistant.core.install_landingpage() sleep.assert_awaited_once_with(30) assert "Failed to install landingpage, retrying after 30sec" in caplog.text capture_exception.assert_not_called() + assert ( + Issue(IssueType.DOCKER_RATELIMIT, ContextType.SYSTEM) + in coresys.resolution.issues + ) +@pytest.mark.parametrize( + "err", + [ + aiodocker.DockerError(HTTPStatus.INTERNAL_SERVER_ERROR, {"message": "fail"}), + APIError("fail"), + DockerException(), + RequestException(), + OSError(), + ], +) async def test_install_landingpage_other_error( - coresys: CoreSys, capture_exception: Mock, caplog: pytest.LogCaptureFixture + coresys: CoreSys, + capture_exception: Mock, + caplog: pytest.LogCaptureFixture, + err: Exception, ): """Test install landing page fails due to other error.""" - coresys.docker.images.inspect.side_effect = [(err := OSError()), MagicMock()] + coresys.docker.images.inspect.side_effect = [err, MagicMock()] with ( patch.object(DockerHomeAssistant, "attach", side_effect=DockerError), @@ -103,11 +137,23 @@ async def test_install_landingpage_other_error( capture_exception.assert_called_once_with(err) -async def test_install_docker_error( - coresys: CoreSys, capture_exception: Mock, caplog: pytest.LogCaptureFixture +@pytest.mark.parametrize( + "err", + [ + aiodocker.DockerError(HTTPStatus.TOO_MANY_REQUESTS, {"message": "ratelimit"}), + APIError("ratelimit", MagicMock(status_code=HTTPStatus.TOO_MANY_REQUESTS)), + ], +) +async def test_install_docker_ratelimit_error( + coresys: CoreSys, + capture_exception: Mock, + caplog: pytest.LogCaptureFixture, + err: Exception, ): - """Test install fails due to docker error.""" + """Test install fails due to docker ratelimit error.""" coresys.security.force = True + coresys.docker.images.pull.side_effect = [err, AsyncIterator([{}])] + with ( patch.object(HomeAssistantCore, "start"), patch.object(DockerHomeAssistant, "cleanup"), @@ -124,19 +170,35 @@ async def test_install_docker_error( ), patch("supervisor.homeassistant.core.asyncio.sleep") as sleep, ): - coresys.docker.dockerpy.api.pull.side_effect = [APIError("fail"), MagicMock()] await coresys.homeassistant.core.install() sleep.assert_awaited_once_with(30) assert "Error on Home Assistant installation. Retrying in 30sec" in caplog.text capture_exception.assert_not_called() + assert ( + Issue(IssueType.DOCKER_RATELIMIT, ContextType.SYSTEM) + in coresys.resolution.issues + ) +@pytest.mark.parametrize( + "err", + [ + aiodocker.DockerError(HTTPStatus.INTERNAL_SERVER_ERROR, {"message": "fail"}), + APIError("fail"), + DockerException(), + RequestException(), + OSError(), + ], +) async def test_install_other_error( - coresys: CoreSys, capture_exception: Mock, caplog: pytest.LogCaptureFixture + coresys: CoreSys, + capture_exception: Mock, + caplog: pytest.LogCaptureFixture, + err: Exception, ): """Test install fails due to other error.""" - coresys.docker.images.inspect.side_effect = [(err := OSError()), MagicMock()] + coresys.docker.images.inspect.side_effect = [err, MagicMock()] with ( patch.object(HomeAssistantCore, "start"),