Watchdog shouldn't rebuild addons (#3766)

* Watchdog shouldn't rebuild addons

* pylint fix
This commit is contained in:
Mike Degatano 2022-08-03 06:06:59 -04:00 committed by GitHub
parent 5d6e2eeaac
commit 2b4f46f6b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 26 additions and 54 deletions

View File

@ -914,23 +914,17 @@ class Addon(AddonModel):
await asyncio.sleep(WATCHDOG_RETRY_SECONDS) await asyncio.sleep(WATCHDOG_RETRY_SECONDS)
elif event.state == ContainerState.FAILED: elif event.state == ContainerState.FAILED:
rebuild = False # Ensure failed container is removed before attempting reanimation
with suppress(DockerError):
await self.instance.stop(remove_container=True)
while await self.instance.current_state() == event.state: while await self.instance.current_state() == event.state:
if not self.in_progress: if not self.in_progress:
_LOGGER.warning( _LOGGER.warning(
"Watchdog found addon %s failed, restarting...", self.name "Watchdog found addon %s failed, restarting...", self.name
) )
if not rebuild:
try: try:
await self.start() await self.start()
except AddonsError as err:
self.sys_capture_exception(err)
rebuild = True
else:
break
try:
await self.rebuild()
except AddonsError as err: except AddonsError as err:
_LOGGER.error( _LOGGER.error(
"Watchdog reanimation of addon %s failed!", self.name "Watchdog reanimation of addon %s failed!", self.name

View File

@ -10,7 +10,6 @@ from supervisor.const import AddonState, BusEvent
from supervisor.coresys import CoreSys from supervisor.coresys import CoreSys
from supervisor.docker.const import ContainerState from supervisor.docker.const import ContainerState
from supervisor.docker.monitor import DockerContainerStateEvent from supervisor.docker.monitor import DockerContainerStateEvent
from supervisor.exceptions import AddonsError
from ..const import TEST_ADDON_SLUG from ..const import TEST_ADDON_SLUG
@ -131,6 +130,10 @@ async def mock_current_state(state: ContainerState) -> ContainerState:
return state return state
async def mock_stop() -> None:
"""Mock for stop method."""
async def test_addon_watchdog(coresys: CoreSys, install_addon_ssh: Addon) -> None: async def test_addon_watchdog(coresys: CoreSys, install_addon_ssh: Addon) -> None:
"""Test addon watchdog works correctly.""" """Test addon watchdog works correctly."""
with patch.object(type(install_addon_ssh.instance), "attach"): with patch.object(type(install_addon_ssh.instance), "attach"):
@ -159,6 +162,10 @@ async def test_addon_watchdog(coresys: CoreSys, install_addon_ssh: Addon) -> Non
restart.reset_mock() restart.reset_mock()
current_state.return_value = mock_current_state(ContainerState.FAILED) current_state.return_value = mock_current_state(ContainerState.FAILED)
with patch.object(
type(install_addon_ssh.instance), "stop", return_value=mock_stop()
) as stop:
coresys.bus.fire_event( coresys.bus.fire_event(
BusEvent.DOCKER_CONTAINER_STATE_CHANGE, BusEvent.DOCKER_CONTAINER_STATE_CHANGE,
DockerContainerStateEvent( DockerContainerStateEvent(
@ -169,6 +176,7 @@ async def test_addon_watchdog(coresys: CoreSys, install_addon_ssh: Addon) -> Non
), ),
) )
await asyncio.sleep(0) await asyncio.sleep(0)
stop.assert_called_once_with(remove_container=True)
restart.assert_not_called() restart.assert_not_called()
start.assert_called_once() start.assert_called_once()
@ -217,36 +225,6 @@ async def test_addon_watchdog(coresys: CoreSys, install_addon_ssh: Addon) -> Non
start.assert_not_called() start.assert_not_called()
async def test_addon_watchdog_rebuild_on_failure(
coresys: CoreSys, install_addon_ssh: Addon
) -> None:
"""Test addon watchdog rebuilds if start fails."""
with patch.object(type(install_addon_ssh.instance), "attach"):
await install_addon_ssh.load()
install_addon_ssh.watchdog = True
with patch.object(Addon, "start", side_effect=AddonsError()) as start, patch.object(
Addon, "rebuild"
) as rebuild, patch.object(
type(install_addon_ssh.instance),
"current_state",
return_value=mock_current_state(ContainerState.FAILED),
):
coresys.bus.fire_event(
BusEvent.DOCKER_CONTAINER_STATE_CHANGE,
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.FAILED,
id="abc123",
time=1,
),
)
await asyncio.sleep(0)
start.assert_called_once()
rebuild.assert_called_once()
async def test_listener_attached_on_install(coresys: CoreSys, repository): async def test_listener_attached_on_install(coresys: CoreSys, repository):
"""Test events listener attached on addon install.""" """Test events listener attached on addon install."""
container_collection = MagicMock() container_collection = MagicMock()