Observer: rebuild container if the application don't response (#2121)

* Observer: rebuild container if the application don't response

* add network mask

* Fix version

supervisor/docker/observer.py

@@ -1,13 +1,14 @@
 """Observer docker object."""
 import logging
 
-from ..const import ENV_TIME, ENV_TOKEN
+from ..const import DOCKER_NETWORK_MASK, ENV_TIME, ENV_TOKEN
 from ..coresys import CoreSysAttributes
 from .interface import DockerInterface
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 
 OBSERVER_DOCKER_NAME: str = "hassio_observer"
+ENV_NETWORK_MASK: str = "NETWORK_MASK"
 
 
 class DockerObserver(DockerInterface, CoreSysAttributes):
@@ -48,6 +49,7 @@ class DockerObserver(DockerInterface, CoreSysAttributes):
             environment={
                 ENV_TIME: self.sys_config.timezone,
                 ENV_TOKEN: self.sys_plugins.observer.supervisor_token,
+                ENV_NETWORK_MASK: DOCKER_NETWORK_MASK,
             },
             volumes={"/run/docker.sock": {"bind": "/run/docker.sock", "mode": "ro"}},
             ports={"80/tcp": 4357},

supervisor/misc/tasks.py

@@ -42,6 +42,7 @@ RUN_WATCHDOG_MULTICAST_DOCKER = 60
 
 RUN_WATCHDOG_ADDON_DOCKER = 30
 RUN_WATCHDOG_ADDON_APPLICATON = 120
+RUN_WATCHDOG_OBSERVER_APPLICATION = 180
 
 RUN_REFRESH_ADDON = 15
 
@@ -93,6 +94,9 @@ class Tasks(CoreSysAttributes):
         self.sys_scheduler.register_task(
             self._watchdog_observer_docker, RUN_WATCHDOG_OBSERVER_DOCKER
         )
+        self.sys_scheduler.register_task(
+            self._watchdog_observer_application, RUN_WATCHDOG_OBSERVER_APPLICATION
+        )
         self.sys_scheduler.register_task(
             self._watchdog_multicast_docker, RUN_WATCHDOG_MULTICAST_DOCKER
         )
@@ -301,13 +305,28 @@ class Tasks(CoreSysAttributes):
             or self.sys_plugins.observer.in_progress
         ):
             return
-        _LOGGER.warning("Watchdog found a problem with observer plugin!")
+        _LOGGER.warning("Watchdog/Docker found a problem with observer plugin!")
 
         try:
             await self.sys_plugins.observer.start()
         except ObserverError:
             _LOGGER.error("Watchdog observer reanimation failed!")
 
+    async def _watchdog_observer_application(self):
+        """Check running state of application and rebuild if they is not response."""
+        # if observer plugin is active
+        if (
+            self.sys_plugins.observer.in_progress
+            or await self.sys_plugins.observer.check_system_runtime()
+        ):
+            return
+        _LOGGER.warning("Watchdog/Application found a problem with observer plugin!")
+
+        try:
+            await self.sys_plugins.observer.rebuild()
+        except ObserverError:
+            _LOGGER.error("Watchdog observer reanimation failed!")
+
     async def _watchdog_multicast_docker(self):
         """Check running state of Docker and start if they is close."""
         # if multicast plugin is active

supervisor/plugins/__init__.py

@@ -21,7 +21,7 @@ class PluginManager(CoreSysAttributes):
     required_cli: LegacyVersion = pkg_parse("26")
     required_dns: LegacyVersion = pkg_parse("9")
     required_audio: LegacyVersion = pkg_parse("17")
-    required_observer: LegacyVersion = pkg_parse("2")
+    required_observer: LegacyVersion = pkg_parse("2020.10.1")
     required_multicast: LegacyVersion = pkg_parse("3")
 
     def __init__(self, coresys: CoreSys):

supervisor/plugins/observer.py

@@ -8,6 +8,8 @@ import logging
 import secrets
 from typing import Awaitable, Optional
 
+import aiohttp
+
 from ..const import ATTR_ACCESS_TOKEN, ATTR_IMAGE, ATTR_VERSION
 from ..coresys import CoreSys, CoreSysAttributes
 from ..docker.observer import DockerObserver
@@ -175,6 +177,26 @@ class Observer(CoreSysAttributes, JsonConfig):
         """
         return self.instance.is_running()
 
+    async def rebuild(self) -> None:
+        """Rebuild Observer Docker container."""
+        with suppress(DockerError):
+            await self.instance.stop()
+        await self.start()
+
+    async def check_system_runtime(self) -> bool:
+        """Check if the observer is running."""
+        try:
+            timeout = aiohttp.ClientTimeout(total=5)
+            async with self.sys_websession.get(
+                f"http://{self.sys_docker.network.observer!s}/ping", timeout=timeout
+            ) as request:
+                if request.status == 200:
+                    return True
+        except (aiohttp.ClientError, asyncio.TimeoutError):
+            pass
+
+        return False
+
     async def repair(self) -> None:
         """Repair observer container."""
         if await self.instance.exists():