Rollback homeassistant on failover (#549)

* Rollback homeassistant on failover

* Check running system

hassio/homeassistant.py

@@ -37,6 +37,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
         self.coresys = coresys
         self.instance = DockerHomeAssistant(coresys)
         self.lock = asyncio.Lock(loop=coresys.loop)
+        self._error_state = False
 
     async def load(self):
         """Prepare HomeAssistant object."""
@@ -51,6 +52,11 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
         """Return System Machines."""
         return self.instance.machine
 
+    @property
+    def error_state(self):
+        """Return True if system is in error."""
+        return self._error_state
+
     @property
     def api_ip(self):
         """Return IP of HomeAssistant instance."""
@@ -207,6 +213,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
     async def update(self, version=None):
         """Update HomeAssistant version."""
         version = version or self.last_version
+        rollback = self.version
         running = await self.instance.is_running()
         exists = await self.instance.exists()
 
@@ -214,12 +221,25 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
             _LOGGER.warning("Version %s is already installed", version)
             return False
 
+        # process a update
+        async def _update(to_version):
+            """Run Home Assistant update."""
             try:
-            return await self.instance.update(version)
+                return await self.instance.update(to_version)
             finally:
                 if running:
                     await self._start()
 
+        # Update Home Assistant
+        ret = await _update(version)
+
+        # Update going wrong, revert it
+        if self.error_state and rollback:
+            _LOGGER.fatal("Home Assistant update fails -> rollback!")
+            ret = await _update(rollback)
+
+        return ret
+
     async def _start(self):
         """Start HomeAssistant docker & wait."""
         if not await self.instance.run():
@@ -361,10 +381,20 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
                 pass
 
         while time.monotonic() - start_time < self.wait_boot:
+            # Check if API response
             if await self.sys_run_in_executor(check_port):
                 _LOGGER.info("Detect a running Home-Assistant instance")
+                self._error_state = False
                 return True
+
+            # Check if Container is is_running
+            if not await self.instance.is_running():
+                _LOGGER.error("Home Assistant is crashed!")
+                break
+
+            # wait and don't hit the system
             await asyncio.sleep(10)
 
         _LOGGER.warning("Don't wait anymore of Home-Assistant startup!")
+        self._error_state = True
         return False

hassio/tasks.py

@@ -6,49 +6,51 @@ from .coresys import CoreSysAttributes
 
 _LOGGER = logging.getLogger(__name__)
 
+HASS_WATCHDOG_API = 'HASS_WATCHDOG_API'
+
+RUN_UPDATE_SUPERVISOR = 29100
+RUN_UPDATE_ADDONS = 57600
+
+RUN_RELOAD_ADDONS = 21600
+RUN_RELOAD_SNAPSHOTS = 72000
+RUN_RELOAD_HOST = 72000
+RUN_RELOAD_UPDATER = 21600
+
+RUN_WATCHDOG_HOMEASSISTANT_DOCKER = 15
+RUN_WATCHDOG_HOMEASSISTANT_API = 300
+
 
 class Tasks(CoreSysAttributes):
     """Handle Tasks inside HassIO."""
 
-    RUN_UPDATE_SUPERVISOR = 29100
-    RUN_UPDATE_ADDONS = 57600
-
-    RUN_RELOAD_ADDONS = 21600
-    RUN_RELOAD_SNAPSHOTS = 72000
-    RUN_RELOAD_HOST = 72000
-    RUN_RELOAD_UPDATER = 21600
-
-    RUN_WATCHDOG_HOMEASSISTANT_DOCKER = 15
-    RUN_WATCHDOG_HOMEASSISTANT_API = 300
-
     def __init__(self, coresys):
         """Initialize Tasks."""
         self.coresys = coresys
         self.jobs = set()
-        self._data = {}
+        self._cache = {}
 
     async def load(self):
         """Add Tasks to scheduler."""
         self.jobs.add(self.sys_scheduler.register_task(
-            self._update_addons, self.RUN_UPDATE_ADDONS))
+            self._update_addons, RUN_UPDATE_ADDONS))
         self.jobs.add(self.sys_scheduler.register_task(
-            self._update_supervisor, self.RUN_UPDATE_SUPERVISOR))
+            self._update_supervisor, RUN_UPDATE_SUPERVISOR))
 
         self.jobs.add(self.sys_scheduler.register_task(
-            self.sys_addons.reload, self.RUN_RELOAD_ADDONS))
+            self.sys_addons.reload, RUN_RELOAD_ADDONS))
         self.jobs.add(self.sys_scheduler.register_task(
-            self.sys_updater.reload, self.RUN_RELOAD_UPDATER))
+            self.sys_updater.reload, RUN_RELOAD_UPDATER))
         self.jobs.add(self.sys_scheduler.register_task(
-            self.sys_snapshots.reload, self.RUN_RELOAD_SNAPSHOTS))
+            self.sys_snapshots.reload, RUN_RELOAD_SNAPSHOTS))
         self.jobs.add(self.sys_scheduler.register_task(
-            self.sys_host.reload, self.RUN_RELOAD_HOST))
+            self.sys_host.reload, RUN_RELOAD_HOST))
 
         self.jobs.add(self.sys_scheduler.register_task(
             self._watchdog_homeassistant_docker,
-            self.RUN_WATCHDOG_HOMEASSISTANT_DOCKER))
+            RUN_WATCHDOG_HOMEASSISTANT_DOCKER))
         self.jobs.add(self.sys_scheduler.register_task(
             self._watchdog_homeassistant_api,
-            self.RUN_WATCHDOG_HOMEASSISTANT_API))
+            RUN_WATCHDOG_HOMEASSISTANT_API))
 
         _LOGGER.info("All core tasks are scheduled")
 
@@ -89,7 +91,8 @@ class Tasks(CoreSysAttributes):
         """Check running state of docker and start if they is close."""
         # if Home-Assistant is active
         if not await self.sys_homeassistant.is_initialize() or \
-                not self.sys_homeassistant.watchdog:
+                not self.sys_homeassistant.watchdog or \
+                self.sys_homeassistant.error_state:
             return
 
         # if Home-Assistant is running
@@ -106,13 +109,15 @@ class Tasks(CoreSysAttributes):
         Try 2 times to call API before we restart Home-Assistant. Maybe we had
         a delay in our system.
         """
-        retry_scan = self._data.get('HASS_WATCHDOG_API', 0)
-
         # If Home-Assistant is active
         if not await self.sys_homeassistant.is_initialize() or \
-                not self.sys_homeassistant.watchdog:
+                not self.sys_homeassistant.watchdog or \
+                self.sys_homeassistant.error_state:
             return
 
+        # Init cache data
+        retry_scan = self._cache.get(HASS_WATCHDOG_API, 0)
+
         # If Home-Assistant API is up
         if self.sys_homeassistant.in_progress or \
                 await self.sys_homeassistant.check_api_state():
@@ -121,10 +126,10 @@ class Tasks(CoreSysAttributes):
         # Look like we run into a problem
         retry_scan += 1
         if retry_scan == 1:
-            self._data['HASS_WATCHDOG_API'] = retry_scan
+            self._cache[HASS_WATCHDOG_API] = retry_scan
             _LOGGER.warning("Watchdog miss API response from Home-Assistant")
             return
 
         _LOGGER.error("Watchdog found a problem with Home-Assistant API!")
         await self.sys_homeassistant.restart()
-        self._data['HASS_WATCHDOG_API'] = 0
+        self._cache[HASS_WATCHDOG_API] = 0