Addon startup waits for healthy (#4376)

* Addon startup waits for healthy

* fix import for pylint

* wait_for to 5 in tests

* Adjust tests to simplify async tasks

* Remove wait_boot time from addons.boot tests

* Eliminate async task race conditions in tests

supervisor/addons/__init__.py

@@ -1,5 +1,6 @@
 """Init file for Supervisor add-ons."""
 import asyncio
+from collections.abc import Awaitable
 from contextlib import suppress
 import logging
 import tarfile
@@ -104,9 +105,13 @@ class AddonManager(CoreSysAttributes):
 
         # Start Add-ons sequential
         # avoid issue on slow IO
+        # Config.wait_boot is deprecated. Until addons update with healthchecks,
+        # add a sleep task for it to keep the same minimum amount of wait time
+        wait_boot: list[Awaitable[None]] = [asyncio.sleep(self.sys_config.wait_boot)]
         for addon in tasks:
             try:
-                await addon.start()
+                if start_task := await addon.start():
+                    wait_boot.append(start_task)
             except AddonsError as err:
                 # Check if there is an system/user issue
                 if check_exception_chain(
@@ -121,7 +126,8 @@ class AddonManager(CoreSysAttributes):
 
             _LOGGER.warning("Can't start Add-on %s", addon.slug)
 
-        await asyncio.sleep(self.sys_config.wait_boot)
+        # Ignore exceptions from waiting for addon startup, addon errors handled elsewhere
+        await asyncio.gather(*wait_boot, return_exceptions=True)
 
     async def shutdown(self, stage: AddonStartup) -> None:
         """Shutdown addons."""
@@ -244,8 +250,14 @@ class AddonManager(CoreSysAttributes):
         conditions=ADDON_UPDATE_CONDITIONS,
         on_condition=AddonsJobError,
     )
-    async def update(self, slug: str, backup: bool | None = False) -> None:
+    async def update(
-        """Update add-on."""
+        self, slug: str, backup: bool | None = False
+    ) -> Awaitable[None] | None:
+        """Update add-on.
+
+        Returns a coroutine that completes when addon has state 'started' (see addon.start)
+        if addon is started after update. Else nothing is returned.
+        """
         if slug not in self.local:
             raise AddonsError(f"Add-on {slug} is not installed", _LOGGER.error)
         addon = self.local[slug]
@@ -288,8 +300,11 @@ class AddonManager(CoreSysAttributes):
         await addon.install_apparmor()
 
         # restore state
-        if last_state == AddonState.STARTED:
+        return (
             await addon.start()
+            if last_state in [AddonState.STARTED, AddonState.STARTUP]
+            else None
+        )
 
     @Job(
         conditions=[
@@ -299,8 +314,12 @@ class AddonManager(CoreSysAttributes):
         ],
         on_condition=AddonsJobError,
     )
-    async def rebuild(self, slug: str) -> None:
+    async def rebuild(self, slug: str) -> Awaitable[None] | None:
-        """Perform a rebuild of local build add-on."""
+        """Perform a rebuild of local build add-on.
+
+        Returns a coroutine that completes when addon has state 'started' (see addon.start)
+        if addon is started after rebuild. Else nothing is returned.
+        """
         if slug not in self.local:
             raise AddonsError(f"Add-on {slug} is not installed", _LOGGER.error)
         addon = self.local[slug]
@@ -333,8 +352,11 @@ class AddonManager(CoreSysAttributes):
         _LOGGER.info("Add-on '%s' successfully rebuilt", slug)
 
         # restore state
-        if last_state == AddonState.STARTED:
+        return (
             await addon.start()
+            if last_state in [AddonState.STARTED, AddonState.STARTUP]
+            else None
+        )
 
     @Job(
         conditions=[
@@ -344,8 +366,14 @@ class AddonManager(CoreSysAttributes):
         ],
         on_condition=AddonsJobError,
     )
-    async def restore(self, slug: str, tar_file: tarfile.TarFile) -> None:
+    async def restore(
-        """Restore state of an add-on."""
+        self, slug: str, tar_file: tarfile.TarFile
+    ) -> Awaitable[None] | None:
+        """Restore state of an add-on.
+
+        Returns a coroutine that completes when addon has state 'started' (see addon.start)
+        if addon is started after restore. Else nothing is returned.
+        """
         if slug not in self.local:
             _LOGGER.debug("Add-on %s is not local available for restore", slug)
             addon = Addon(self.coresys, slug)
@@ -353,7 +381,7 @@ class AddonManager(CoreSysAttributes):
             _LOGGER.debug("Add-on %s is local available for restore", slug)
             addon = self.local[slug]
 
-        await addon.restore(tar_file)
+        wait_for_start = await addon.restore(tar_file)
 
         # Check if new
         if slug not in self.local:
@@ -366,6 +394,8 @@ class AddonManager(CoreSysAttributes):
             with suppress(HomeAssistantAPIError):
                 await self.sys_ingress.update_hass_panel(addon)
 
+        return wait_for_start
+
     @Job(conditions=[JobCondition.FREE_SPACE, JobCondition.INTERNET_HOST])
     async def repair(self) -> None:
         """Repair local add-ons."""

supervisor/addons/addon.py

@@ -99,6 +99,7 @@ RE_WATCHDOG = re.compile(
 )
 
 WATCHDOG_TIMEOUT = aiohttp.ClientTimeout(total=10)
+STARTUP_TIMEOUT = 120
 
 _OPTIONS_MERGER: Final = Merger(
     type_strategies=[(dict, ["merge"])],
@@ -106,6 +107,14 @@ _OPTIONS_MERGER: Final = Merger(
     type_conflict_strategies=["override"],
 )
 
+# Backups just need to know if an addon was running or not
+# Map other addon states to those two
+_MAP_ADDON_STATE = {
+    AddonState.STARTUP: AddonState.STARTED,
+    AddonState.ERROR: AddonState.STOPPED,
+    AddonState.UNKNOWN: AddonState.STOPPED,
+}
+
 
 class Addon(AddonModel):
     """Hold data for add-on inside Supervisor."""
@@ -119,6 +128,7 @@ class Addon(AddonModel):
             self.sys_hardware.helper.last_boot != self.sys_config.last_boot
         )
         self._listeners: list[EventListener] = []
+        self._startup_event = asyncio.Event()
 
         @Job(
             name=f"addon_{slug}_restart_after_problem",
@@ -144,9 +154,9 @@ class Addon(AddonModel):
                                 with suppress(DockerError):
                                     await addon.instance.stop(remove_container=True)
 
-                            await addon.start()
+                            await (await addon.start())
                         else:
-                            await addon.restart()
+                            await (await addon.restart())
                     except AddonsError as err:
                         attempts = attempts + 1
                         _LOGGER.error(
@@ -182,7 +192,13 @@ class Addon(AddonModel):
         """Set the add-on into new state."""
         if self._state == new_state:
             return
+        old_state = self._state
         self._state = new_state
+
+        # Signal listeners about addon state change
+        if new_state == AddonState.STARTED or old_state == AddonState.STARTUP:
+            self._startup_event.set()
+
         self.sys_homeassistant.websocket.send_message(
             {
                 ATTR_TYPE: WSType.SUPERVISOR_EVENT,
@@ -680,8 +696,24 @@ class Addon(AddonModel):
             return False
         return True
 
-    async def start(self) -> None:
+    async def _wait_for_startup(self) -> None:
-        """Set options and start add-on."""
+        """Wait for startup event to be set with timeout."""
+        try:
+            await asyncio.wait_for(self._startup_event.wait(), STARTUP_TIMEOUT)
+        except asyncio.TimeoutError:
+            _LOGGER.warning(
+                "Timeout while waiting for addon %s to start, took more then %s seconds",
+                self.name,
+                STARTUP_TIMEOUT,
+            )
+
+    async def start(self) -> Awaitable[None]:
+        """Set options and start add-on.
+
+        Returns a coroutine that completes when addon has state 'started'.
+        For addons with a healthcheck, that is when they become healthy or unhealthy.
+        Addons without a healthcheck have state 'started' immediately.
+        """
         if await self.instance.is_running():
             _LOGGER.warning("%s is already running!", self.slug)
             return
@@ -698,12 +730,15 @@ class Addon(AddonModel):
             self.write_pulse()
 
         # Start Add-on
+        self._startup_event.clear()
         try:
             await self.instance.run()
         except DockerError as err:
             self.state = AddonState.ERROR
             raise AddonsError() from err
 
+        return self._wait_for_startup()
+
     async def stop(self) -> None:
         """Stop add-on."""
         self._manual_stop = True
@@ -713,11 +748,14 @@ class Addon(AddonModel):
             self.state = AddonState.ERROR
             raise AddonsError() from err
 
-    async def restart(self) -> None:
+    async def restart(self) -> Awaitable[None]:
-        """Restart add-on."""
+        """Restart add-on.
+
+        Returns a coroutine that completes when addon has state 'started' (see start).
+        """
         with suppress(AddonsError):
             await self.stop()
-        await self.start()
+        return await self.start()
 
     def logs(self) -> Awaitable[bytes]:
         """Return add-ons log output.
@@ -772,8 +810,13 @@ class Addon(AddonModel):
                 _LOGGER.error,
             ) from err
 
-    async def backup(self, tar_file: tarfile.TarFile) -> None:
+    async def backup(self, tar_file: tarfile.TarFile) -> Awaitable[None] | None:
-        """Backup state of an add-on."""
+        """Backup state of an add-on.
+
+        Returns a coroutine that completes when addon has state 'started' (see start)
+        for cold backup. Else nothing is returned.
+        """
+        wait_for_start: Awaitable[None] | None = None
         is_running = await self.is_running()
 
         with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
@@ -790,7 +833,7 @@ class Addon(AddonModel):
                 ATTR_USER: self.persist,
                 ATTR_SYSTEM: self.data,
                 ATTR_VERSION: self.version,
-                ATTR_STATE: self.state,
+                ATTR_STATE: _MAP_ADDON_STATE.get(self.state, self.state),
             }
 
             # Store local configs/state
@@ -852,12 +895,18 @@ class Addon(AddonModel):
                     await self._backup_command(self.backup_post)
                 elif is_running and self.backup_mode is AddonBackupMode.COLD:
                     _LOGGER.info("Starting add-on %s again", self.slug)
-                    await self.start()
+                    wait_for_start = await self.start()
 
         _LOGGER.info("Finish backup for addon %s", self.slug)
+        return wait_for_start
 
-    async def restore(self, tar_file: tarfile.TarFile) -> None:
+    async def restore(self, tar_file: tarfile.TarFile) -> Awaitable[None] | None:
-        """Restore state of an add-on."""
+        """Restore state of an add-on.
+
+        Returns a coroutine that completes when addon has state 'started' (see start)
+        if addon is started after restore. Else nothing is returned.
+        """
+        wait_for_start: Awaitable[None] | None = None
         with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
             # extract backup
             def _extract_tarfile():
@@ -958,9 +1007,10 @@ class Addon(AddonModel):
 
             # Run add-on
             if data[ATTR_STATE] == AddonState.STARTED:
-                return await self.start()
+                wait_for_start = await self.start()
 
         _LOGGER.info("Finished restore for add-on %s", self.slug)
+        return wait_for_start
 
     def check_trust(self) -> Awaitable[None]:
         """Calculate Addon docker content trust.
@@ -974,12 +1024,15 @@ class Addon(AddonModel):
         if event.name != self.instance.name:
             return
 
-        if event.state in [
+        if event.state == ContainerState.RUNNING:
-            ContainerState.RUNNING,
+            self._manual_stop = False
+            self.state = (
+                AddonState.STARTUP if self.instance.healthcheck else AddonState.STARTED
+            )
+        elif event.state in [
             ContainerState.HEALTHY,
             ContainerState.UNHEALTHY,
         ]:
-            self._manual_stop = False
             self.state = AddonState.STARTED
         elif event.state == ContainerState.STOPPED:
             self.state = AddonState.STOPPED

supervisor/addons/model.py

@@ -673,10 +673,10 @@ class AddonModel(CoreSysAttributes, ABC):
         """Uninstall this add-on."""
         return self.sys_addons.uninstall(self.slug)
 
-    def update(self, backup: bool | None = False) -> Awaitable[None]:
+    def update(self, backup: bool | None = False) -> Awaitable[Awaitable[None] | None]:
         """Update this add-on."""
         return self.sys_addons.update(self.slug, backup=backup)
 
-    def rebuild(self) -> Awaitable[None]:
+    def rebuild(self) -> Awaitable[Awaitable[None] | None]:
         """Rebuild this add-on."""
         return self.sys_addons.rebuild(self.slug)

supervisor/api/addons.py

@@ -391,10 +391,11 @@ class APIAddons(CoreSysAttributes):
         return asyncio.shield(addon.uninstall())
 
     @api_process
-    def start(self, request: web.Request) -> Awaitable[None]:
+    async def start(self, request: web.Request) -> None:
         """Start add-on."""
         addon = self._extract_addon(request)
-        return asyncio.shield(addon.start())
+        if start_task := await asyncio.shield(addon.start()):
+            await start_task
 
     @api_process
     def stop(self, request: web.Request) -> Awaitable[None]:
@@ -403,16 +404,18 @@ class APIAddons(CoreSysAttributes):
         return asyncio.shield(addon.stop())
 
     @api_process
-    def restart(self, request: web.Request) -> Awaitable[None]:
+    async def restart(self, request: web.Request) -> None:
         """Restart add-on."""
         addon: Addon = self._extract_addon(request)
-        return asyncio.shield(addon.restart())
+        if start_task := await asyncio.shield(addon.restart()):
+            await start_task
 
     @api_process
-    def rebuild(self, request: web.Request) -> Awaitable[None]:
+    async def rebuild(self, request: web.Request) -> None:
         """Rebuild local build add-on."""
         addon = self._extract_addon(request)
-        return asyncio.shield(addon.rebuild())
+        if start_task := await asyncio.shield(addon.rebuild()):
+            await start_task
 
     @api_process_raw(CONTENT_TYPE_BINARY)
     def logs(self, request: web.Request) -> Awaitable[bytes]:

supervisor/api/store.py

@@ -208,7 +208,10 @@ class APIStore(CoreSysAttributes):
 
         body = await api_validate(SCHEMA_UPDATE, request)
 
-        return await asyncio.shield(addon.update(backup=body.get(ATTR_BACKUP)))
+        if start_task := await asyncio.shield(
+            addon.update(backup=body.get(ATTR_BACKUP))
+        ):
+            await start_task
 
     @api_process
     async def addons_addon_info(self, request: web.Request) -> dict[str, Any]:

supervisor/api/supervisor.py

@@ -94,7 +94,6 @@ class APISupervisor(CoreSysAttributes):
             ATTR_SUPPORTED: self.sys_core.supported,
             ATTR_HEALTHY: self.sys_core.healthy,
             ATTR_IP_ADDRESS: str(self.sys_supervisor.ip_address),
-            ATTR_WAIT_BOOT: self.sys_config.wait_boot,
             ATTR_TIMEZONE: self.sys_config.timezone,
             ATTR_LOGGING: self.sys_config.logging,
             ATTR_DEBUG: self.sys_config.debug,
@@ -102,6 +101,7 @@ class APISupervisor(CoreSysAttributes):
             ATTR_DIAGNOSTICS: self.sys_config.diagnostics,
             ATTR_AUTO_UPDATE: self.sys_updater.auto_update,
             # Depricated
+            ATTR_WAIT_BOOT: self.sys_config.wait_boot,
             ATTR_ADDONS: [
                 {
                     ATTR_NAME: addon.name,
@@ -132,9 +132,6 @@ class APISupervisor(CoreSysAttributes):
         if ATTR_TIMEZONE in body:
             self.sys_config.timezone = body[ATTR_TIMEZONE]
 
-        if ATTR_WAIT_BOOT in body:
-            self.sys_config.wait_boot = body[ATTR_WAIT_BOOT]
-
         if ATTR_DEBUG in body:
             self.sys_config.debug = body[ATTR_DEBUG]
 
@@ -156,6 +153,10 @@ class APISupervisor(CoreSysAttributes):
         if ATTR_AUTO_UPDATE in body:
             self.sys_updater.auto_update = body[ATTR_AUTO_UPDATE]
 
+        # Deprecated
+        if ATTR_WAIT_BOOT in body:
+            self.sys_config.wait_boot = body[ATTR_WAIT_BOOT]
+
         # Save changes before processing addons in case of errors
         self.sys_updater.save_data()
         self.sys_config.save_data()

supervisor/backups/backup.py

@@ -332,10 +332,14 @@ class Backup(CoreSysAttributes):
         finally:
             self._tmp.cleanup()
 
-    async def store_addons(self, addon_list: list[str]):
+    async def store_addons(self, addon_list: list[str]) -> list[Awaitable[None]]:
-        """Add a list of add-ons into backup."""
+        """Add a list of add-ons into backup.
 
-        async def _addon_save(addon: Addon):
+        For each addon that needs to be started after backup, returns a task which
+        completes when that addon has state 'started' (see addon.start).
+        """
+
+        async def _addon_save(addon: Addon) -> Awaitable[None] | None:
             """Task to store an add-on into backup."""
             tar_name = f"{addon.slug}.tar{'.gz' if self.compressed else ''}"
             addon_file = SecureTarFile(
@@ -348,7 +352,7 @@ class Backup(CoreSysAttributes):
 
             # Take backup
             try:
-                await addon.backup(addon_file)
+                start_task = await addon.backup(addon_file)
             except AddonsError:
                 _LOGGER.error("Can't create backup for %s", addon.slug)
                 return
@@ -363,18 +367,24 @@ class Backup(CoreSysAttributes):
                 }
             )
 
+            return start_task
+
         # Save Add-ons sequential
         # avoid issue on slow IO
+        start_tasks: list[Awaitable[None]] = []
         for addon in addon_list:
             try:
-                await _addon_save(addon)
+                if start_task := await _addon_save(addon):
+                    start_tasks.append(start_task)
             except Exception as err:  # pylint: disable=broad-except
                 _LOGGER.warning("Can't save Add-on %s: %s", addon.slug, err)
 
-    async def restore_addons(self, addon_list: list[str]):
+        return start_tasks
+
+    async def restore_addons(self, addon_list: list[str]) -> list[Awaitable[None]]:
         """Restore a list add-on from backup."""
 
-        async def _addon_restore(addon_slug: str):
+        async def _addon_restore(addon_slug: str) -> Awaitable[None] | None:
             """Task to restore an add-on into backup."""
             tar_name = f"{addon_slug}.tar{'.gz' if self.compressed else ''}"
             addon_file = SecureTarFile(
@@ -392,18 +402,22 @@ class Backup(CoreSysAttributes):
 
             # Perform a restore
             try:
-                await self.sys_addons.restore(addon_slug, addon_file)
+                return await self.sys_addons.restore(addon_slug, addon_file)
             except AddonsError:
                 _LOGGER.error("Can't restore backup %s", addon_slug)
 
         # Save Add-ons sequential
         # avoid issue on slow IO
+        start_tasks: list[Awaitable[None]] = []
         for slug in addon_list:
             try:
-                await _addon_restore(slug)
+                if start_task := await _addon_restore(slug):
+                    start_tasks.append(start_task)
             except Exception as err:  # pylint: disable=broad-except
                 _LOGGER.warning("Can't restore Add-on %s: %s", slug, err)
 
+        return start_tasks
+
     async def store_folders(self, folder_list: list[str]):
         """Backup Supervisor data into backup."""
 

supervisor/backups/manager.py

@@ -2,7 +2,7 @@
 from __future__ import annotations
 
 import asyncio
-from collections.abc import Iterable
+from collections.abc import Awaitable, Iterable
 import logging
 from pathlib import Path
 
@@ -190,6 +190,7 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
         folder_list: list[str],
         homeassistant: bool,
     ):
+        addon_start_tasks: list[Awaitable[None]] | None = None
         try:
             self.sys_core.state = CoreState.FREEZE
 
@@ -197,7 +198,7 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
                 # Backup add-ons
                 if addon_list:
                     _LOGGER.info("Backing up %s store Add-ons", backup.slug)
-                    await backup.store_addons(addon_list)
+                    addon_start_tasks = await backup.store_addons(addon_list)
 
                 # HomeAssistant Folder is for v1
                 if homeassistant:
@@ -214,6 +215,11 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
             return None
         else:
             self._backups[backup.slug] = backup
+
+            if addon_start_tasks:
+                # Ignore exceptions from waiting for addon startup, addon errors handled elsewhere
+                await asyncio.gather(*addon_start_tasks, return_exceptions=True)
+
             return backup
         finally:
             self.sys_core.state = CoreState.RUNNING
@@ -300,6 +306,7 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
         homeassistant: bool,
         replace: bool,
     ):
+        addon_start_tasks: list[Awaitable[None]] | None = None
         try:
             task_hass: asyncio.Task | None = None
             async with backup:
@@ -336,7 +343,7 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
                     await backup.restore_repositories(replace)
 
                     _LOGGER.info("Restoring %s Add-ons", backup.slug)
-                    await backup.restore_addons(addon_list)
+                    addon_start_tasks = await backup.restore_addons(addon_list)
 
                 # Wait for Home Assistant Core update/downgrade
                 if task_hass:
@@ -348,6 +355,10 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
             capture_exception(err)
             return False
         else:
+            if addon_start_tasks:
+                # Ignore exceptions from waiting for addon startup, addon errors handled elsewhere
+                await asyncio.gather(*addon_start_tasks, return_exceptions=True)
+
             return True
         finally:
             # Do we need start Home Assistant Core?

supervisor/const.py

@@ -395,6 +395,7 @@ class AddonStage(str, Enum):
 class AddonState(str, Enum):
     """State of add-on."""
 
+    STARTUP = "startup"
     STARTED = "started"
     STOPPED = "stopped"
     UNKNOWN = "unknown"

supervisor/docker/interface.py

@@ -159,6 +159,11 @@ class DockerInterface(CoreSysAttributes):
         # causes problems on some types of host systems.
         return ["seccomp=unconfined"]
 
+    @property
+    def healthcheck(self) -> dict[str, Any] | None:
+        """Healthcheck of instance if it has one."""
+        return self.meta_config.get("Healthcheck")
+
     def _get_credentials(self, image: str) -> dict:
         """Return a dictionay with credentials for docker login."""
         registry = None

supervisor/docker/manager.py

@@ -185,7 +185,7 @@ class DockerAPI:
 
         # Create container
         try:
-            container = self.docker.containers.create(
+            container = self.containers.create(
                 f"{image}:{tag}", use_config_proxy=False, **kwargs
             )
         except docker_errors.NotFound as err:

supervisor/misc/tasks.py

@@ -1,4 +1,6 @@
 """A collection of tasks."""
+import asyncio
+from collections.abc import Awaitable
 import logging
 
 from ..addons.const import ADDON_UPDATE_CONDITIONS
@@ -84,6 +86,7 @@ class Tasks(CoreSysAttributes):
     @Job(conditions=ADDON_UPDATE_CONDITIONS + [JobCondition.RUNNING])
     async def _update_addons(self):
         """Check if an update is available for an Add-on and update it."""
+        start_tasks: list[Awaitable[None]] = []
         for addon in self.sys_addons.all:
             if not addon.is_installed or not addon.auto_update:
                 continue
@@ -101,10 +104,13 @@ class Tasks(CoreSysAttributes):
             # avoid issue on slow IO
             _LOGGER.info("Add-on auto update process %s", addon.slug)
             try:
-                await addon.update(backup=True)
+                if start_task := await addon.update(backup=True):
+                    start_tasks.append(start_task)
             except AddonsError:
                 _LOGGER.error("Can't auto update Add-on %s", addon.slug)
 
+        await asyncio.gather(*start_tasks)
+
     @Job(
         conditions=[
             JobCondition.AUTO_UPDATE,
@@ -265,7 +271,7 @@ class Tasks(CoreSysAttributes):
 
             _LOGGER.warning("Watchdog found a problem with %s application!", addon.slug)
             try:
-                await addon.restart()
+                await (await addon.restart())
             except AddonsError as err:
                 _LOGGER.error("%s watchdog reanimation failed with %s", addon.slug, err)
                 capture_exception(err)

supervisor/resolution/fixups/addon_execute_rebuild.py

@@ -41,7 +41,7 @@ class FixupAddonExecuteRebuild(FixupBase):
             )
             await addon.stop()
         else:
-            await addon.restart()
+            await (await addon.restart())
 
     @property
     def suggestion(self) -> SuggestionType:

tests/addons/test_addon.py

@@ -6,8 +6,12 @@ from unittest.mock import MagicMock, PropertyMock, patch
 
 from docker.errors import DockerException
 import pytest
+from securetar import SecureTarFile
 
 from supervisor.addons.addon import Addon
+from supervisor.addons.const import AddonBackupMode
+from supervisor.addons.model import AddonModel
+from supervisor.arch import CpuArch
 from supervisor.const import AddonState, BusEvent
 from supervisor.coresys import CoreSys
 from supervisor.docker.addon import DockerAddon
@@ -17,7 +21,8 @@ from supervisor.exceptions import AddonsJobError, AudioUpdateError
 from supervisor.store.repository import Repository
 from supervisor.utils.dt import utcnow
 
-from ..const import TEST_ADDON_SLUG
+from tests.common import get_fixture_path
+from tests.const import TEST_ADDON_SLUG
 
 
 def _fire_test_event(coresys: CoreSys, name: str, state: ContainerState):
@@ -131,6 +136,7 @@ async def test_addon_watchdog(coresys: CoreSys, install_addon_ssh: Addon) -> Non
         await install_addon_ssh.load()
 
     install_addon_ssh.watchdog = True
+    install_addon_ssh._manual_stop = False  # pylint: disable=protected-access
 
     with patch.object(Addon, "restart") as restart, patch.object(
         Addon, "start"
@@ -219,7 +225,7 @@ async def test_listener_attached_on_install(coresys: CoreSys, repository):
 
     _fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
     await asyncio.sleep(0)
-    assert coresys.addons.get(TEST_ADDON_SLUG).state == AddonState.STARTED
+    assert coresys.addons.get(TEST_ADDON_SLUG).state == AddonState.STARTUP
 
 
 @pytest.mark.parametrize(
@@ -304,3 +310,164 @@ async def test_listeners_removed_on_uninstall(
             listener
             not in coresys.bus._listeners[BusEvent.DOCKER_CONTAINER_STATE_CHANGE]
         )
+
+
+async def test_start(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container,
+    tmp_supervisor_data,
+    path_extern,
+) -> None:
+    """Test starting an addon without healthcheck."""
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STOPPED
+
+    start_task = await install_addon_ssh.start()
+    assert start_task
+
+    _fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
+    await start_task
+    assert install_addon_ssh.state == AddonState.STARTED
+
+
+@pytest.mark.parametrize("state", [ContainerState.HEALTHY, ContainerState.UNHEALTHY])
+async def test_start_wait_healthcheck(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    state: ContainerState,
+    tmp_supervisor_data,
+    path_extern,
+) -> None:
+    """Test starting an addon with a healthcheck waits for health status."""
+    install_addon_ssh.path_data.mkdir()
+    container.attrs["Config"] = {"Healthcheck": "exists"}
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STOPPED
+
+    start_task = asyncio.create_task(await install_addon_ssh.start())
+    assert start_task
+
+    _fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
+    await asyncio.sleep(0.01)
+
+    assert not start_task.done()
+    assert install_addon_ssh.state == AddonState.STARTUP
+
+    _fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", state)
+    await asyncio.sleep(0.01)
+
+    assert start_task.done()
+    assert install_addon_ssh.state == AddonState.STARTED
+
+
+async def test_start_timeout(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    caplog: pytest.LogCaptureFixture,
+    container,
+    tmp_supervisor_data,
+    path_extern,
+) -> None:
+    """Test starting an addon times out while waiting."""
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STOPPED
+
+    start_task = await install_addon_ssh.start()
+    assert start_task
+
+    caplog.clear()
+    with patch(
+        "supervisor.addons.addon.asyncio.wait_for", side_effect=asyncio.TimeoutError
+    ):
+        await start_task
+
+    assert "Timeout while waiting for addon Terminal & SSH to start" in caplog.text
+
+
+async def test_restart(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container,
+    tmp_supervisor_data,
+    path_extern,
+) -> None:
+    """Test restarting an addon."""
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STOPPED
+
+    start_task = await install_addon_ssh.restart()
+    assert start_task
+
+    _fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
+    await start_task
+    assert install_addon_ssh.state == AddonState.STARTED
+
+
+@pytest.mark.parametrize("status", ["running", "stopped"])
+async def test_backup(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    status: str,
+    tmp_supervisor_data,
+    path_extern,
+) -> None:
+    """Test backing up an addon."""
+    container.status = status
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+
+    tarfile = SecureTarFile(coresys.config.path_tmp / "test.tar.gz", "w")
+    assert await install_addon_ssh.backup(tarfile) is None
+
+
+@pytest.mark.parametrize("status", ["running", "stopped"])
+async def test_backup_cold_mode(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    status: str,
+    tmp_supervisor_data,
+    path_extern,
+) -> None:
+    """Test backing up an addon in cold mode."""
+    container.status = status
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+
+    tarfile = SecureTarFile(coresys.config.path_tmp / "test.tar.gz", "w")
+    with patch.object(
+        AddonModel, "backup_mode", new=PropertyMock(return_value=AddonBackupMode.COLD)
+    ), patch.object(
+        DockerAddon, "_is_running", side_effect=[status == "running", False, False]
+    ):
+        start_task = await install_addon_ssh.backup(tarfile)
+
+    assert bool(start_task) is (status == "running")
+
+
+@pytest.mark.parametrize("status", ["running", "stopped"])
+async def test_restore(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    status: str,
+    tmp_supervisor_data,
+    path_extern,
+) -> None:
+    """Test restoring an addon."""
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+
+    tarfile = SecureTarFile(get_fixture_path(f"backup_local_ssh_{status}.tar.gz"), "r")
+    with patch.object(DockerAddon, "_is_running", return_value=False), patch.object(
+        CpuArch, "supported", new=PropertyMock(return_value=["aarch64"])
+    ):
+        start_task = await coresys.addons.restore(TEST_ADDON_SLUG, tarfile)
+
+    assert bool(start_task) is (status == "running")

tests/addons/test_manager.py

@@ -8,10 +8,12 @@ import pytest
 
 from supervisor.addons.addon import Addon
 from supervisor.arch import CpuArch
-from supervisor.const import AddonBoot, AddonStartup, AddonState
+from supervisor.const import AddonBoot, AddonStartup, AddonState, BusEvent
 from supervisor.coresys import CoreSys
 from supervisor.docker.addon import DockerAddon
+from supervisor.docker.const import ContainerState
 from supervisor.docker.interface import DockerInterface
+from supervisor.docker.monitor import DockerContainerStateEvent
 from supervisor.exceptions import (
     AddonConfigurationError,
     AddonsError,
@@ -34,6 +36,12 @@ async def fixture_mock_arch_disk() -> None:
         yield
 
 
+@pytest.fixture(autouse=True)
+async def fixture_remove_wait_boot(coresys: CoreSys) -> None:
+    """Remove default wait boot time for tests."""
+    coresys.config.wait_boot = 0
+
+
 async def test_image_added_removed_on_update(
     coresys: CoreSys, install_addon_ssh: Addon
 ):
@@ -182,3 +190,89 @@ async def test_load(
         write_hosts.assert_called_once()
 
     assert "Found 1 installed add-ons" in caplog.text
+
+
+async def test_boot_waits_for_addons(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test addon manager boot waits for addons."""
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STOPPED
+
+    addon_state: AddonState | None = None
+
+    async def fire_container_event(*args, **kwargs):
+        nonlocal addon_state
+
+        addon_state = install_addon_ssh.state
+        coresys.bus.fire_event(
+            BusEvent.DOCKER_CONTAINER_STATE_CHANGE,
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.RUNNING,
+                id="abc123",
+                time=1,
+            ),
+        )
+
+    with patch.object(DockerAddon, "run", new=fire_container_event):
+        await coresys.addons.boot(AddonStartup.APPLICATION)
+
+    assert addon_state == AddonState.STOPPED
+    assert install_addon_ssh.state == AddonState.STARTED
+
+
+@pytest.mark.parametrize("status", ["running", "stopped"])
+async def test_update(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    status: str,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test addon update."""
+    container.status = status
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+    with patch(
+        "supervisor.store.data.read_json_or_yaml_file",
+        return_value=load_json_fixture("addon-config-add-image.json"),
+    ):
+        await coresys.store.data.update()
+
+    assert install_addon_ssh.need_update is True
+
+    with patch.object(DockerInterface, "_install"), patch.object(
+        DockerAddon, "_is_running", return_value=False
+    ):
+        start_task = await coresys.addons.update(TEST_ADDON_SLUG)
+
+    assert bool(start_task) is (status == "running")
+
+
+@pytest.mark.parametrize("status", ["running", "stopped"])
+async def test_rebuild(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    status: str,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test addon rebuild."""
+    container.status = status
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+
+    with patch.object(DockerAddon, "_build"), patch.object(
+        DockerAddon, "_is_running", return_value=False
+    ), patch.object(Addon, "need_build", new=PropertyMock(return_value=True)):
+        start_task = await coresys.addons.rebuild(TEST_ADDON_SLUG)
+
+    assert bool(start_task) is (status == "running")

tests/api/test_addons.py

@@ -1,17 +1,33 @@
 """Test addons api."""
 
-from unittest.mock import MagicMock
+import asyncio
+from unittest.mock import MagicMock, PropertyMock, patch
 
 from aiohttp.test_utils import TestClient
 
 from supervisor.addons.addon import Addon
+from supervisor.addons.build import AddonBuild
+from supervisor.arch import CpuArch
 from supervisor.const import AddonState
 from supervisor.coresys import CoreSys
+from supervisor.docker.addon import DockerAddon
+from supervisor.docker.const import ContainerState
+from supervisor.docker.monitor import DockerContainerStateEvent
 from supervisor.store.repository import Repository
 
 from ..const import TEST_ADDON_SLUG
 
 
+def _create_test_event(name: str, state: ContainerState) -> DockerContainerStateEvent:
+    """Create a container state event."""
+    return DockerContainerStateEvent(
+        name=name,
+        state=state,
+        id="abc123",
+        time=1,
+    )
+
+
 async def test_addons_info(
     api_client: TestClient, coresys: CoreSys, install_addon_ssh: Addon
 ):
@@ -62,3 +78,137 @@ async def test_api_addon_logs(
         "\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os\x1b[0m",
         "\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/AppArmor\x1b[0m",
     ]
+
+
+async def test_api_addon_start_healthcheck(
+    api_client: TestClient,
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test starting an addon waits for healthy."""
+    install_addon_ssh.path_data.mkdir()
+    container.attrs["Config"] = {"Healthcheck": "exists"}
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STOPPED
+
+    state_changes: list[AddonState] = []
+
+    async def container_events():
+        nonlocal state_changes
+        await asyncio.sleep(0.01)
+
+        await install_addon_ssh.container_state_changed(
+            _create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
+        )
+        state_changes.append(install_addon_ssh.state)
+
+        await install_addon_ssh.container_state_changed(
+            _create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.HEALTHY)
+        )
+
+    async def container_events_task(*args, **kwargs):
+        asyncio.create_task(container_events())
+
+    with patch.object(DockerAddon, "run", new=container_events_task):
+        resp = await api_client.post("/addons/local_ssh/start")
+
+    assert state_changes == [AddonState.STARTUP]
+    assert install_addon_ssh.state == AddonState.STARTED
+    assert resp.status == 200
+
+
+async def test_api_addon_restart_healthcheck(
+    api_client: TestClient,
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test restarting an addon waits for healthy."""
+    install_addon_ssh.path_data.mkdir()
+    container.attrs["Config"] = {"Healthcheck": "exists"}
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STOPPED
+
+    state_changes: list[AddonState] = []
+
+    async def container_events():
+        nonlocal state_changes
+        await asyncio.sleep(0.01)
+
+        await install_addon_ssh.container_state_changed(
+            _create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
+        )
+        state_changes.append(install_addon_ssh.state)
+
+        await install_addon_ssh.container_state_changed(
+            _create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.HEALTHY)
+        )
+
+    async def container_events_task(*args, **kwargs):
+        asyncio.create_task(container_events())
+
+    with patch.object(DockerAddon, "run", new=container_events_task):
+        resp = await api_client.post("/addons/local_ssh/restart")
+
+    assert state_changes == [AddonState.STARTUP]
+    assert install_addon_ssh.state == AddonState.STARTED
+    assert resp.status == 200
+
+
+async def test_api_addon_rebuild_healthcheck(
+    api_client: TestClient,
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test rebuilding an addon waits for healthy."""
+    container.status = "running"
+    install_addon_ssh.path_data.mkdir()
+    container.attrs["Config"] = {"Healthcheck": "exists"}
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STARTUP
+
+    state_changes: list[AddonState] = []
+
+    async def container_events():
+        nonlocal state_changes
+
+        await install_addon_ssh.container_state_changed(
+            _create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.STOPPED)
+        )
+        state_changes.append(install_addon_ssh.state)
+
+        await install_addon_ssh.container_state_changed(
+            _create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
+        )
+        state_changes.append(install_addon_ssh.state)
+        await asyncio.sleep(0)
+
+        await install_addon_ssh.container_state_changed(
+            _create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.HEALTHY)
+        )
+
+    async def container_events_task(*args, **kwargs):
+        asyncio.create_task(container_events())
+
+    with patch.object(
+        AddonBuild, "is_valid", new=PropertyMock(return_value=True)
+    ), patch.object(DockerAddon, "_is_running", return_value=False), patch.object(
+        Addon, "need_build", new=PropertyMock(return_value=True)
+    ), patch.object(
+        CpuArch, "supported", new=PropertyMock(return_value=["amd64"])
+    ), patch.object(
+        DockerAddon, "run", new=container_events_task
+    ):
+        resp = await api_client.post("/addons/local_ssh/rebuild")
+
+    assert state_changes == [AddonState.STOPPED, AddonState.STARTUP]
+    assert install_addon_ssh.state == AddonState.STARTED
+    assert resp.status == 200

tests/api/test_store.py

@@ -1,13 +1,25 @@
 """Test Store API."""
-from unittest.mock import patch
+
+import asyncio
+from unittest.mock import MagicMock, PropertyMock, patch
 
 from aiohttp.test_utils import TestClient
 import pytest
 
+from supervisor.addons.addon import Addon
+from supervisor.arch import CpuArch
+from supervisor.const import AddonState
 from supervisor.coresys import CoreSys
+from supervisor.docker.addon import DockerAddon
+from supervisor.docker.const import ContainerState
+from supervisor.docker.interface import DockerInterface
+from supervisor.docker.monitor import DockerContainerStateEvent
 from supervisor.store.addon import AddonStore
 from supervisor.store.repository import Repository
 
+from tests.common import load_json_fixture
+from tests.const import TEST_ADDON_SLUG
+
 REPO_URL = "https://github.com/awesome-developer/awesome-repo"
 
 
@@ -102,3 +114,74 @@ async def test_api_store_remove_repository(
     assert response.status == 200
     assert repository.source not in coresys.store.repository_urls
     assert repository.slug not in coresys.store.repositories
+
+
+async def test_api_store_update_healthcheck(
+    api_client: TestClient,
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test updating an addon with healthcheck waits for health status."""
+    container.status = "running"
+    container.attrs["Config"] = {"Healthcheck": "exists"}
+    install_addon_ssh.path_data.mkdir()
+    await install_addon_ssh.load()
+    with patch(
+        "supervisor.store.data.read_json_or_yaml_file",
+        return_value=load_json_fixture("addon-config-add-image.json"),
+    ):
+        await coresys.store.data.update()
+
+    assert install_addon_ssh.need_update is True
+
+    state_changes: list[AddonState] = []
+
+    async def container_events():
+        nonlocal state_changes
+        await asyncio.sleep(0.01)
+
+        await install_addon_ssh.container_state_changed(
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.STOPPED,
+                id="abc123",
+                time=1,
+            )
+        )
+
+        state_changes.append(install_addon_ssh.state)
+        await install_addon_ssh.container_state_changed(
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.RUNNING,
+                id="abc123",
+                time=1,
+            )
+        )
+
+        state_changes.append(install_addon_ssh.state)
+        await install_addon_ssh.container_state_changed(
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.HEALTHY,
+                id="abc123",
+                time=1,
+            )
+        )
+
+    async def container_events_task(*args, **kwargs):
+        asyncio.create_task(container_events())
+
+    with patch.object(DockerAddon, "run", new=container_events_task), patch.object(
+        DockerInterface, "_install"
+    ), patch.object(DockerAddon, "_is_running", return_value=False), patch.object(
+        CpuArch, "supported", new=PropertyMock(return_value=["amd64"])
+    ):
+        resp = await api_client.post(f"/store/addons/{TEST_ADDON_SLUG}/update")
+
+    assert state_changes == [AddonState.STOPPED, AddonState.STARTUP]
+    assert install_addon_ssh.state == AddonState.STARTED
+    assert resp.status == 200

tests/backups/test_manager.py

@@ -1,5 +1,6 @@
 """Test BackupManager class."""
 
+import asyncio
 from shutil import rmtree
 from unittest.mock import AsyncMock, MagicMock, Mock, PropertyMock, patch
 
@@ -8,11 +9,16 @@ from dbus_fast import DBusError
 import pytest
 
 from supervisor.addons.addon import Addon
+from supervisor.addons.const import AddonBackupMode
+from supervisor.addons.model import AddonModel
 from supervisor.backups.backup import Backup
 from supervisor.backups.const import BackupType
 from supervisor.backups.manager import BackupManager
-from supervisor.const import FOLDER_HOMEASSISTANT, FOLDER_SHARE, CoreState
+from supervisor.const import FOLDER_HOMEASSISTANT, FOLDER_SHARE, AddonState, CoreState
 from supervisor.coresys import CoreSys
+from supervisor.docker.addon import DockerAddon
+from supervisor.docker.const import ContainerState
+from supervisor.docker.monitor import DockerContainerStateEvent
 from supervisor.exceptions import AddonsError, DockerError
 from supervisor.homeassistant.core import HomeAssistantCore
 from supervisor.homeassistant.module import HomeAssistant
@@ -696,3 +702,138 @@ async def test_load_network_error(
         await coresys.backups.load()
 
     assert "Could not list backups from /data/backup_test" in caplog.text
+
+
+async def test_backup_with_healthcheck(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test backup of addon with healthcheck in cold mode."""
+    container.status = "running"
+    container.attrs["Config"] = {"Healthcheck": "exists"}
+    install_addon_ssh.path_data.mkdir()
+    coresys.core.state = CoreState.RUNNING
+    coresys.hardware.disk.get_disk_free_space = lambda x: 5000
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STARTUP
+
+    state_changes: list[AddonState] = []
+
+    async def container_events():
+        nonlocal state_changes
+        await asyncio.sleep(0.01)
+
+        await install_addon_ssh.container_state_changed(
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.STOPPED,
+                id="abc123",
+                time=1,
+            )
+        )
+
+        state_changes.append(install_addon_ssh.state)
+        await install_addon_ssh.container_state_changed(
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.RUNNING,
+                id="abc123",
+                time=1,
+            )
+        )
+
+        state_changes.append(install_addon_ssh.state)
+        await install_addon_ssh.container_state_changed(
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.HEALTHY,
+                id="abc123",
+                time=1,
+            )
+        )
+
+    async def container_events_task(*args, **kwargs):
+        asyncio.create_task(container_events())
+
+    with patch.object(DockerAddon, "run", new=container_events_task), patch.object(
+        AddonModel, "backup_mode", new=PropertyMock(return_value=AddonBackupMode.COLD)
+    ), patch.object(DockerAddon, "_is_running", side_effect=[True, False, False]):
+        backup = await coresys.backups.do_backup_partial(
+            homeassistant=False, addons=["local_ssh"]
+        )
+
+    assert backup
+    assert state_changes == [AddonState.STOPPED, AddonState.STARTUP]
+    assert install_addon_ssh.state == AddonState.STARTED
+    assert coresys.core.state == CoreState.RUNNING
+
+
+async def test_restore_with_healthcheck(
+    coresys: CoreSys,
+    install_addon_ssh: Addon,
+    container: MagicMock,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test backup of addon with healthcheck in cold mode."""
+    container.status = "running"
+    container.attrs["Config"] = {"Healthcheck": "exists"}
+    install_addon_ssh.path_data.mkdir()
+    coresys.core.state = CoreState.RUNNING
+    coresys.hardware.disk.get_disk_free_space = lambda x: 5000
+    await install_addon_ssh.load()
+    assert install_addon_ssh.state == AddonState.STARTUP
+
+    backup = await coresys.backups.do_backup_partial(
+        homeassistant=False, addons=["local_ssh"]
+    )
+    state_changes: list[AddonState] = []
+
+    async def container_events():
+        nonlocal state_changes
+
+        await install_addon_ssh.container_state_changed(
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.STOPPED,
+                id="abc123",
+                time=1,
+            )
+        )
+
+        state_changes.append(install_addon_ssh.state)
+        await install_addon_ssh.container_state_changed(
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.RUNNING,
+                id="abc123",
+                time=1,
+            )
+        )
+
+        state_changes.append(install_addon_ssh.state)
+        await install_addon_ssh.container_state_changed(
+            DockerContainerStateEvent(
+                name=f"addon_{TEST_ADDON_SLUG}",
+                state=ContainerState.HEALTHY,
+                id="abc123",
+                time=1,
+            )
+        )
+
+    async def container_events_task(*args, **kwargs):
+        asyncio.create_task(container_events())
+
+    with patch.object(DockerAddon, "run", new=container_events_task), patch.object(
+        DockerAddon, "_is_running", return_value=False
+    ), patch.object(AddonModel, "_validate_availability"), patch.object(
+        Addon, "with_ingress", new=PropertyMock(return_value=False)
+    ):
+        await coresys.backups.do_restore_partial(backup, addons=["local_ssh"])
+
+    assert state_changes == [AddonState.STOPPED, AddonState.STARTUP]
+    assert install_addon_ssh.state == AddonState.STARTED
+    assert coresys.core.state == CoreState.RUNNING

tests/conftest.py

@@ -377,6 +377,7 @@ async def tmp_supervisor_data(coresys: CoreSys, tmp_path: Path) -> Path:
         coresys.config.path_audio.mkdir()
         coresys.config.path_dns.mkdir()
         coresys.config.path_share.mkdir()
+        coresys.config.path_addons_data.mkdir(parents=True)
         yield tmp_path
 
 
@@ -641,3 +642,15 @@ async def mount_propagation(docker: DockerAPI, coresys: CoreSys) -> None:
     }
     await coresys.supervisor.load()
     yield
+
+
+@pytest.fixture
+async def container(docker: DockerAPI) -> MagicMock:
+    """Mock attrs and status for container on attach."""
+    docker.containers.get.return_value = addon = MagicMock()
+    docker.containers.create.return_value = addon
+    docker.images.pull.return_value = addon
+    docker.images.build.return_value = (addon, "")
+    addon.status = "stopped"
+    addon.attrs = {"State": {"ExitCode": 0}}
+    yield addon

tests/docker/test_addon.py

@@ -189,7 +189,7 @@ async def test_addon_run_docker_error(
 ):
     """Test docker error when addon is run."""
     await coresys.dbus.timedate.connect(coresys.dbus.bus)
-    coresys.docker.docker.containers.create.side_effect = NotFound("Missing")
+    coresys.docker.containers.create.side_effect = NotFound("Missing")
     docker_addon = get_docker_addon(
         coresys, addonsdata_system, "basic-addon-config.json"
     )

tests/resolution/fixup/test_addon_execute_rebuild.py

@@ -25,6 +25,10 @@ def make_mock_container_get(status: str):
     return mock_container_get
 
 
+async def _mock_wait_for_container() -> None:
+    """Mock of wait for container."""
+
+
 async def test_fixup(docker: DockerAPI, coresys: CoreSys, install_addon_ssh: Addon):
     """Test fixup rebuilds addon's container."""
     docker.containers.get = make_mock_container_get("running")
@@ -39,7 +43,9 @@ async def test_fixup(docker: DockerAPI, coresys: CoreSys, install_addon_ssh: Add
         reference="local_ssh",
         suggestions=[SuggestionType.EXECUTE_REBUILD],
     )
-    with patch.object(Addon, "restart") as restart:
+    with patch.object(
+        Addon, "restart", return_value=_mock_wait_for_container()
+    ) as restart:
         await addon_execute_rebuild()
         restart.assert_called_once()