From 70dd6593e4f40108223b5e02c3e95a00fb2798b7 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 3 Jul 2018 22:41:50 +0200 Subject: [PATCH] Rollback homeassistant on failover (#549) * Rollback homeassistant on failover * Check running system --- hassio/homeassistant.py | 40 +++++++++++++++++++++++++---- hassio/tasks.py | 57 ++++++++++++++++++++++------------------- 2 files changed, 66 insertions(+), 31 deletions(-) diff --git a/hassio/homeassistant.py b/hassio/homeassistant.py index 917f0298b..7a9b4b768 100644 --- a/hassio/homeassistant.py +++ b/hassio/homeassistant.py @@ -37,6 +37,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): self.coresys = coresys self.instance = DockerHomeAssistant(coresys) self.lock = asyncio.Lock(loop=coresys.loop) + self._error_state = False async def load(self): """Prepare HomeAssistant object.""" @@ -51,6 +52,11 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): """Return System Machines.""" return self.instance.machine + @property + def error_state(self): + """Return True if system is in error.""" + return self._error_state + @property def api_ip(self): """Return IP of HomeAssistant instance.""" @@ -207,6 +213,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): async def update(self, version=None): """Update HomeAssistant version.""" version = version or self.last_version + rollback = self.version running = await self.instance.is_running() exists = await self.instance.exists() @@ -214,11 +221,24 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): _LOGGER.warning("Version %s is already installed", version) return False - try: - return await self.instance.update(version) - finally: - if running: - await self._start() + # process a update + async def _update(to_version): + """Run Home Assistant update.""" + try: + return await self.instance.update(to_version) + finally: + if running: + await self._start() + + # Update Home Assistant + ret = await _update(version) + + # Update going wrong, revert it + if self.error_state and rollback: + _LOGGER.fatal("Home Assistant update fails -> rollback!") + ret = await _update(rollback) + + return ret async def _start(self): """Start HomeAssistant docker & wait.""" @@ -361,10 +381,20 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): pass while time.monotonic() - start_time < self.wait_boot: + # Check if API response if await self.sys_run_in_executor(check_port): _LOGGER.info("Detect a running Home-Assistant instance") + self._error_state = False return True + + # Check if Container is is_running + if not await self.instance.is_running(): + _LOGGER.error("Home Assistant is crashed!") + break + + # wait and don't hit the system await asyncio.sleep(10) _LOGGER.warning("Don't wait anymore of Home-Assistant startup!") + self._error_state = True return False diff --git a/hassio/tasks.py b/hassio/tasks.py index d5ddc298b..f83132f02 100644 --- a/hassio/tasks.py +++ b/hassio/tasks.py @@ -6,49 +6,51 @@ from .coresys import CoreSysAttributes _LOGGER = logging.getLogger(__name__) +HASS_WATCHDOG_API = 'HASS_WATCHDOG_API' + +RUN_UPDATE_SUPERVISOR = 29100 +RUN_UPDATE_ADDONS = 57600 + +RUN_RELOAD_ADDONS = 21600 +RUN_RELOAD_SNAPSHOTS = 72000 +RUN_RELOAD_HOST = 72000 +RUN_RELOAD_UPDATER = 21600 + +RUN_WATCHDOG_HOMEASSISTANT_DOCKER = 15 +RUN_WATCHDOG_HOMEASSISTANT_API = 300 + class Tasks(CoreSysAttributes): """Handle Tasks inside HassIO.""" - RUN_UPDATE_SUPERVISOR = 29100 - RUN_UPDATE_ADDONS = 57600 - - RUN_RELOAD_ADDONS = 21600 - RUN_RELOAD_SNAPSHOTS = 72000 - RUN_RELOAD_HOST = 72000 - RUN_RELOAD_UPDATER = 21600 - - RUN_WATCHDOG_HOMEASSISTANT_DOCKER = 15 - RUN_WATCHDOG_HOMEASSISTANT_API = 300 - def __init__(self, coresys): """Initialize Tasks.""" self.coresys = coresys self.jobs = set() - self._data = {} + self._cache = {} async def load(self): """Add Tasks to scheduler.""" self.jobs.add(self.sys_scheduler.register_task( - self._update_addons, self.RUN_UPDATE_ADDONS)) + self._update_addons, RUN_UPDATE_ADDONS)) self.jobs.add(self.sys_scheduler.register_task( - self._update_supervisor, self.RUN_UPDATE_SUPERVISOR)) + self._update_supervisor, RUN_UPDATE_SUPERVISOR)) self.jobs.add(self.sys_scheduler.register_task( - self.sys_addons.reload, self.RUN_RELOAD_ADDONS)) + self.sys_addons.reload, RUN_RELOAD_ADDONS)) self.jobs.add(self.sys_scheduler.register_task( - self.sys_updater.reload, self.RUN_RELOAD_UPDATER)) + self.sys_updater.reload, RUN_RELOAD_UPDATER)) self.jobs.add(self.sys_scheduler.register_task( - self.sys_snapshots.reload, self.RUN_RELOAD_SNAPSHOTS)) + self.sys_snapshots.reload, RUN_RELOAD_SNAPSHOTS)) self.jobs.add(self.sys_scheduler.register_task( - self.sys_host.reload, self.RUN_RELOAD_HOST)) + self.sys_host.reload, RUN_RELOAD_HOST)) self.jobs.add(self.sys_scheduler.register_task( self._watchdog_homeassistant_docker, - self.RUN_WATCHDOG_HOMEASSISTANT_DOCKER)) + RUN_WATCHDOG_HOMEASSISTANT_DOCKER)) self.jobs.add(self.sys_scheduler.register_task( self._watchdog_homeassistant_api, - self.RUN_WATCHDOG_HOMEASSISTANT_API)) + RUN_WATCHDOG_HOMEASSISTANT_API)) _LOGGER.info("All core tasks are scheduled") @@ -89,7 +91,8 @@ class Tasks(CoreSysAttributes): """Check running state of docker and start if they is close.""" # if Home-Assistant is active if not await self.sys_homeassistant.is_initialize() or \ - not self.sys_homeassistant.watchdog: + not self.sys_homeassistant.watchdog or \ + self.sys_homeassistant.error_state: return # if Home-Assistant is running @@ -106,13 +109,15 @@ class Tasks(CoreSysAttributes): Try 2 times to call API before we restart Home-Assistant. Maybe we had a delay in our system. """ - retry_scan = self._data.get('HASS_WATCHDOG_API', 0) - # If Home-Assistant is active if not await self.sys_homeassistant.is_initialize() or \ - not self.sys_homeassistant.watchdog: + not self.sys_homeassistant.watchdog or \ + self.sys_homeassistant.error_state: return + # Init cache data + retry_scan = self._cache.get(HASS_WATCHDOG_API, 0) + # If Home-Assistant API is up if self.sys_homeassistant.in_progress or \ await self.sys_homeassistant.check_api_state(): @@ -121,10 +126,10 @@ class Tasks(CoreSysAttributes): # Look like we run into a problem retry_scan += 1 if retry_scan == 1: - self._data['HASS_WATCHDOG_API'] = retry_scan + self._cache[HASS_WATCHDOG_API] = retry_scan _LOGGER.warning("Watchdog miss API response from Home-Assistant") return _LOGGER.error("Watchdog found a problem with Home-Assistant API!") await self.sys_homeassistant.restart() - self._data['HASS_WATCHDOG_API'] = 0 + self._cache[HASS_WATCHDOG_API] = 0