From e50515a17c7d2278fe18bf0626737b62fbd49d71 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 2 Jun 2020 11:26:10 +0200 Subject: [PATCH] Slowdown docker stop/start to avoid issue on slow IO (#1763) * Slowdown docker stop/start to avoid issue on slow IO * Fix lint * Address comments * Fix restore logic & Style * Fix type * Slow plugins --- supervisor/addons/__init__.py | 43 ++++++++++++++++++++++++---------- supervisor/addons/model.py | 5 ++-- supervisor/addons/validate.py | 12 ++++------ supervisor/const.py | 24 ++++++++----------- supervisor/core.py | 30 ++++++++++-------------- supervisor/misc/tasks.py | 20 +++++++++------- supervisor/plugins/__init__.py | 33 +++++++++++++++++--------- 7 files changed, 95 insertions(+), 72 deletions(-) diff --git a/supervisor/addons/__init__.py b/supervisor/addons/__init__.py index 34140dfc1..a3f84ede2 100644 --- a/supervisor/addons/__init__.py +++ b/supervisor/addons/__init__.py @@ -5,7 +5,7 @@ import logging import tarfile from typing import Dict, List, Optional, Union -from ..const import BOOT_AUTO, STATE_STARTED +from ..const import BOOT_AUTO, STATE_STARTED, AddonStartup from ..coresys import CoreSys, CoreSysAttributes from ..exceptions import ( AddonsError, @@ -78,30 +78,49 @@ class AddonManager(CoreSysAttributes): # Sync DNS await self.sync_dns() - async def boot(self, stage: str) -> None: + async def boot(self, stage: AddonStartup) -> None: """Boot add-ons with mode auto.""" - tasks = [] + tasks: List[Addon] = [] for addon in self.installed: if addon.boot != BOOT_AUTO or addon.startup != stage: continue - tasks.append(addon.start()) + tasks.append(addon) + # Evaluate add-ons which need to be started _LOGGER.info("Phase '%s' start %d add-ons", stage, len(tasks)) - if tasks: - await asyncio.wait(tasks) - await asyncio.sleep(self.sys_config.wait_boot) + if not tasks: + return - async def shutdown(self, stage: str) -> None: + # Start Add-ons sequential + # avoid issue on slow IO + for addon in tasks: + try: + await addon.start() + except Exception as err: # pylint: disable=broad-except + _LOGGER.warning("Can't start Add-on %s: %s", addon.slug, err) + + await asyncio.sleep(self.sys_config.wait_boot) + + async def shutdown(self, stage: AddonStartup) -> None: """Shutdown addons.""" - tasks = [] + tasks: List[Addon] = [] for addon in self.installed: if await addon.state() != STATE_STARTED or addon.startup != stage: continue - tasks.append(addon.stop()) + tasks.append(addon) + # Evaluate add-ons which need to be stopped _LOGGER.info("Phase '%s' stop %d add-ons", stage, len(tasks)) - if tasks: - await asyncio.wait(tasks) + if not tasks: + return + + # Stop Add-ons sequential + # avoid issue on slow IO + for addon in tasks: + try: + await addon.stop() + except Exception as err: # pylint: disable=broad-except + _LOGGER.warning("Can't stop Add-on %s: %s", addon.slug, err) async def install(self, slug: str) -> None: """Install an add-on.""" diff --git a/supervisor/addons/model.py b/supervisor/addons/model.py index 2fe2547f5..64b31ac87 100644 --- a/supervisor/addons/model.py +++ b/supervisor/addons/model.py @@ -64,6 +64,7 @@ from ..const import ( SECURITY_DISABLE, SECURITY_PROFILE, AddonStages, + AddonStartup, ) from ..coresys import CoreSys, CoreSysAttributes from .validate import RE_SERVICE, RE_VOLUME, schema_ui_options, validate_options @@ -193,9 +194,9 @@ class AddonModel(CoreSysAttributes): return True @property - def startup(self) -> Optional[str]: + def startup(self) -> AddonStartup: """Return startup type of add-on.""" - return self.data.get(ATTR_STARTUP) + return self.data[ATTR_STARTUP] @property def advanced(self) -> bool: diff --git a/supervisor/addons/validate.py b/supervisor/addons/validate.py index a44eac36e..246b18b46 100644 --- a/supervisor/addons/validate.py +++ b/supervisor/addons/validate.py @@ -85,12 +85,10 @@ from ..const import ( PRIVILEGED_ALL, ROLE_ALL, ROLE_DEFAULT, - STARTUP_ALL, - STARTUP_APPLICATION, - STARTUP_SERVICES, STATE_STARTED, STATE_STOPPED, AddonStages, + AddonStartup, ) from ..coresys import CoreSys from ..discovery.validate import valid_discovery_service @@ -169,12 +167,12 @@ MACHINE_ALL = [ ] -def _simple_startup(value): +def _simple_startup(value) -> str: """Define startup schema.""" if value == "before": - return STARTUP_SERVICES + return AddonStartup.SERVICES.value if value == "after": - return STARTUP_APPLICATION + return AddonStartup.APPLICATION.value return value @@ -188,7 +186,7 @@ SCHEMA_ADDON_CONFIG = vol.Schema( vol.Required(ATTR_ARCH): [vol.In(ARCH_ALL)], vol.Optional(ATTR_MACHINE): [vol.In(MACHINE_ALL)], vol.Optional(ATTR_URL): vol.Url(), - vol.Required(ATTR_STARTUP): vol.All(_simple_startup, vol.In(STARTUP_ALL)), + vol.Required(ATTR_STARTUP): vol.All(_simple_startup, vol.Coerce(AddonStartup)), vol.Required(ATTR_BOOT): vol.In([BOOT_AUTO, BOOT_MANUAL]), vol.Optional(ATTR_INIT, default=True): vol.Boolean(), vol.Optional(ATTR_ADVANCED, default=False): vol.Boolean(), diff --git a/supervisor/const.py b/supervisor/const.py index b05132d68..25377ad76 100644 --- a/supervisor/const.py +++ b/supervisor/const.py @@ -245,20 +245,6 @@ PROVIDE_SERVICE = "provide" NEED_SERVICE = "need" WANT_SERVICE = "want" -STARTUP_INITIALIZE = "initialize" -STARTUP_SYSTEM = "system" -STARTUP_SERVICES = "services" -STARTUP_APPLICATION = "application" -STARTUP_ONCE = "once" - -STARTUP_ALL = [ - STARTUP_ONCE, - STARTUP_INITIALIZE, - STARTUP_SYSTEM, - STARTUP_SERVICES, - STARTUP_APPLICATION, -] - BOOT_AUTO = "auto" BOOT_MANUAL = "manual" @@ -339,6 +325,16 @@ CHAN_ID = "chan_id" CHAN_TYPE = "chan_type" +class AddonStartup(str, Enum): + """Startup types of Add-on.""" + + INITIALIZE = "initialize" + SYSTEM = "system" + SERVICES = "services" + APPLICATION = "application" + ONCE = "once" + + class AddonStages(str, Enum): """Stage types of add-on.""" diff --git a/supervisor/core.py b/supervisor/core.py index aa6cda192..5e000a161 100644 --- a/supervisor/core.py +++ b/supervisor/core.py @@ -5,13 +5,7 @@ import logging import async_timeout -from .const import ( - STARTUP_APPLICATION, - STARTUP_INITIALIZE, - STARTUP_SERVICES, - STARTUP_SYSTEM, - CoreStates, -) +from .const import AddonStartup, CoreStates from .coresys import CoreSys, CoreSysAttributes from .exceptions import HassioError, HomeAssistantError, SupervisorUpdateError @@ -133,7 +127,7 @@ class Core(CoreSysAttributes): ) # Start addon mark as initialize - await self.sys_addons.boot(STARTUP_INITIALIZE) + await self.sys_addons.boot(AddonStartup.INITIALIZE) try: # HomeAssistant is already running / supervisor have only reboot @@ -145,10 +139,10 @@ class Core(CoreSysAttributes): self.sys_services.reset() # start addon mark as system - await self.sys_addons.boot(STARTUP_SYSTEM) + await self.sys_addons.boot(AddonStartup.SYSTEM) # start addon mark as services - await self.sys_addons.boot(STARTUP_SERVICES) + await self.sys_addons.boot(AddonStartup.SERVICES) # run HomeAssistant if ( @@ -161,7 +155,7 @@ class Core(CoreSysAttributes): _LOGGER.info("Skip start of Home Assistant") # start addon mark as application - await self.sys_addons.boot(STARTUP_APPLICATION) + await self.sys_addons.boot(AddonStartup.APPLICATION) # store new last boot self._update_last_boot() @@ -214,22 +208,24 @@ class Core(CoreSysAttributes): async def shutdown(self): """Shutdown all running containers in correct order.""" # don't process scheduler anymore - self.state = CoreStates.STOPPING + if self.state == CoreStates.RUNNING: + self.state = CoreStates.STOPPING # Shutdown Application Add-ons, using Home Assistant API - await self.sys_addons.shutdown(STARTUP_APPLICATION) + await self.sys_addons.shutdown(AddonStartup.APPLICATION) # Close Home Assistant with suppress(HassioError): await self.sys_homeassistant.stop() # Shutdown System Add-ons - await self.sys_addons.shutdown(STARTUP_SERVICES) - await self.sys_addons.shutdown(STARTUP_SYSTEM) - await self.sys_addons.shutdown(STARTUP_INITIALIZE) + await self.sys_addons.shutdown(AddonStartup.SERVICES) + await self.sys_addons.shutdown(AddonStartup.SYSTEM) + await self.sys_addons.shutdown(AddonStartup.INITIALIZE) # Shutdown all Plugins - await self.sys_plugins.shutdown() + if self.state == CoreStates.STOPPING: + await self.sys_plugins.shutdown() def _update_last_boot(self): """Update last boot time.""" diff --git a/supervisor/misc/tasks.py b/supervisor/misc/tasks.py index 412c52ab7..e3610517c 100644 --- a/supervisor/misc/tasks.py +++ b/supervisor/misc/tasks.py @@ -1,9 +1,9 @@ """A collection of tasks.""" -import asyncio import logging from ..coresys import CoreSysAttributes from ..exceptions import ( + AddonsError, AudioError, CliError, CoreDNSError, @@ -131,24 +131,26 @@ class Tasks(CoreSysAttributes): async def _update_addons(self): """Check if an update is available for an Add-on and update it.""" - tasks = [] for addon in self.sys_addons.all: if not addon.is_installed or not addon.auto_update: continue + # Evaluate available updates if addon.version == addon.latest_version: continue - - if addon.test_update_schema(): - tasks.append(addon.update()) - else: + if not addon.test_update_schema(): _LOGGER.warning( "Add-on %s will be ignored, schema tests fails", addon.slug ) + continue - if tasks: - _LOGGER.info("Add-on auto update process %d tasks", len(tasks)) - await asyncio.wait(tasks) + # Run Add-on update sequential + # avoid issue on slow IO + _LOGGER.info("Add-on auto update process %s", addon.slug) + try: + await addon.update() + except AddonsError: + _LOGGER.error("Can't auto update Add-on %s", addon.slug) async def _update_supervisor(self): """Check and run update of Supervisor Supervisor.""" diff --git a/supervisor/plugins/__init__.py b/supervisor/plugins/__init__.py index 674a3eef9..81a746c30 100644 --- a/supervisor/plugins/__init__.py +++ b/supervisor/plugins/__init__.py @@ -51,9 +51,17 @@ class PluginManager(CoreSysAttributes): async def load(self): """Load Supervisor plugins.""" - await asyncio.wait( - [self.dns.load(), self.audio.load(), self.cli.load(), self.multicast.load()] - ) + # Sequential to avoid issue on slow IO + for plugin in ( + self.dns, + self.audio, + self.cli, + self.multicast, + ): + try: + await plugin.load() + except Exception: # pylint: disable=broad-except + _LOGGER.warning("Can't load plugin %s", type(plugin).__name__) # Check requirements for plugin, required_version in ( @@ -103,11 +111,14 @@ class PluginManager(CoreSysAttributes): async def shutdown(self) -> None: """Shutdown Supervisor plugin.""" - await asyncio.wait( - [ - self.dns.stop(), - self.audio.stop(), - self.cli.stop(), - self.multicast.stop(), - ] - ) + # Sequential to avoid issue on slow IO + for plugin in ( + self.audio, + self.cli, + self.multicast, + self.dns, + ): + try: + await plugin.stop() + except Exception: # pylint: disable=broad-except + _LOGGER.warning("Can't stop plugin %s", type(plugin).__name__)