Slowdown docker stop/start to avoid issue on slow IO (#1763)

* Slowdown docker stop/start to avoid issue on slow IO

* Fix lint

* Address comments

* Fix restore logic & Style

* Fix type

* Slow plugins

supervisor/addons/__init__.py

@@ -5,7 +5,7 @@ import logging
 import tarfile
 from typing import Dict, List, Optional, Union
 
-from ..const import BOOT_AUTO, STATE_STARTED
+from ..const import BOOT_AUTO, STATE_STARTED, AddonStartup
 from ..coresys import CoreSys, CoreSysAttributes
 from ..exceptions import (
     AddonsError,
@@ -78,30 +78,49 @@ class AddonManager(CoreSysAttributes):
         # Sync DNS
         await self.sync_dns()
 
-    async def boot(self, stage: str) -> None:
+    async def boot(self, stage: AddonStartup) -> None:
         """Boot add-ons with mode auto."""
-        tasks = []
+        tasks: List[Addon] = []
         for addon in self.installed:
             if addon.boot != BOOT_AUTO or addon.startup != stage:
                 continue
-            tasks.append(addon.start())
+            tasks.append(addon)
 
+        # Evaluate add-ons which need to be started
         _LOGGER.info("Phase '%s' start %d add-ons", stage, len(tasks))
-        if tasks:
+        if not tasks:
-            await asyncio.wait(tasks)
+            return
-            await asyncio.sleep(self.sys_config.wait_boot)
 
-    async def shutdown(self, stage: str) -> None:
+        # Start Add-ons sequential
+        # avoid issue on slow IO
+        for addon in tasks:
+            try:
+                await addon.start()
+            except Exception as err:  # pylint: disable=broad-except
+                _LOGGER.warning("Can't start Add-on %s: %s", addon.slug, err)
+
+        await asyncio.sleep(self.sys_config.wait_boot)
+
+    async def shutdown(self, stage: AddonStartup) -> None:
         """Shutdown addons."""
-        tasks = []
+        tasks: List[Addon] = []
         for addon in self.installed:
             if await addon.state() != STATE_STARTED or addon.startup != stage:
                 continue
-            tasks.append(addon.stop())
+            tasks.append(addon)
 
+        # Evaluate add-ons which need to be stopped
         _LOGGER.info("Phase '%s' stop %d add-ons", stage, len(tasks))
-        if tasks:
+        if not tasks:
-            await asyncio.wait(tasks)
+            return
+
+        # Stop Add-ons sequential
+        # avoid issue on slow IO
+        for addon in tasks:
+            try:
+                await addon.stop()
+            except Exception as err:  # pylint: disable=broad-except
+                _LOGGER.warning("Can't stop Add-on %s: %s", addon.slug, err)
 
     async def install(self, slug: str) -> None:
         """Install an add-on."""

supervisor/addons/model.py

@@ -64,6 +64,7 @@ from ..const import (
     SECURITY_DISABLE,
     SECURITY_PROFILE,
     AddonStages,
+    AddonStartup,
 )
 from ..coresys import CoreSys, CoreSysAttributes
 from .validate import RE_SERVICE, RE_VOLUME, schema_ui_options, validate_options
@@ -193,9 +194,9 @@ class AddonModel(CoreSysAttributes):
         return True
 
     @property
-    def startup(self) -> Optional[str]:
+    def startup(self) -> AddonStartup:
         """Return startup type of add-on."""
-        return self.data.get(ATTR_STARTUP)
+        return self.data[ATTR_STARTUP]
 
     @property
     def advanced(self) -> bool:

supervisor/addons/validate.py

@@ -85,12 +85,10 @@ from ..const import (
     PRIVILEGED_ALL,
     ROLE_ALL,
     ROLE_DEFAULT,
-    STARTUP_ALL,
-    STARTUP_APPLICATION,
-    STARTUP_SERVICES,
     STATE_STARTED,
     STATE_STOPPED,
     AddonStages,
+    AddonStartup,
 )
 from ..coresys import CoreSys
 from ..discovery.validate import valid_discovery_service
@@ -169,12 +167,12 @@ MACHINE_ALL = [
 ]
 
 
-def _simple_startup(value):
+def _simple_startup(value) -> str:
     """Define startup schema."""
     if value == "before":
-        return STARTUP_SERVICES
+        return AddonStartup.SERVICES.value
     if value == "after":
-        return STARTUP_APPLICATION
+        return AddonStartup.APPLICATION.value
     return value
 
 
@@ -188,7 +186,7 @@ SCHEMA_ADDON_CONFIG = vol.Schema(
         vol.Required(ATTR_ARCH): [vol.In(ARCH_ALL)],
         vol.Optional(ATTR_MACHINE): [vol.In(MACHINE_ALL)],
         vol.Optional(ATTR_URL): vol.Url(),
-        vol.Required(ATTR_STARTUP): vol.All(_simple_startup, vol.In(STARTUP_ALL)),
+        vol.Required(ATTR_STARTUP): vol.All(_simple_startup, vol.Coerce(AddonStartup)),
         vol.Required(ATTR_BOOT): vol.In([BOOT_AUTO, BOOT_MANUAL]),
         vol.Optional(ATTR_INIT, default=True): vol.Boolean(),
         vol.Optional(ATTR_ADVANCED, default=False): vol.Boolean(),

supervisor/const.py

@@ -245,20 +245,6 @@ PROVIDE_SERVICE = "provide"
 NEED_SERVICE = "need"
 WANT_SERVICE = "want"
 
-STARTUP_INITIALIZE = "initialize"
-STARTUP_SYSTEM = "system"
-STARTUP_SERVICES = "services"
-STARTUP_APPLICATION = "application"
-STARTUP_ONCE = "once"
-
-STARTUP_ALL = [
-    STARTUP_ONCE,
-    STARTUP_INITIALIZE,
-    STARTUP_SYSTEM,
-    STARTUP_SERVICES,
-    STARTUP_APPLICATION,
-]
-
 BOOT_AUTO = "auto"
 BOOT_MANUAL = "manual"
 
@@ -339,6 +325,16 @@ CHAN_ID = "chan_id"
 CHAN_TYPE = "chan_type"
 
 
+class AddonStartup(str, Enum):
+    """Startup types of Add-on."""
+
+    INITIALIZE = "initialize"
+    SYSTEM = "system"
+    SERVICES = "services"
+    APPLICATION = "application"
+    ONCE = "once"
+
+
 class AddonStages(str, Enum):
     """Stage types of add-on."""
 

supervisor/core.py

@@ -5,13 +5,7 @@ import logging
 
 import async_timeout
 
-from .const import (
+from .const import AddonStartup, CoreStates
-    STARTUP_APPLICATION,
-    STARTUP_INITIALIZE,
-    STARTUP_SERVICES,
-    STARTUP_SYSTEM,
-    CoreStates,
-)
 from .coresys import CoreSys, CoreSysAttributes
 from .exceptions import HassioError, HomeAssistantError, SupervisorUpdateError
 
@@ -133,7 +127,7 @@ class Core(CoreSysAttributes):
                 )
 
         # Start addon mark as initialize
-        await self.sys_addons.boot(STARTUP_INITIALIZE)
+        await self.sys_addons.boot(AddonStartup.INITIALIZE)
 
         try:
             # HomeAssistant is already running / supervisor have only reboot
@@ -145,10 +139,10 @@ class Core(CoreSysAttributes):
             self.sys_services.reset()
 
             # start addon mark as system
-            await self.sys_addons.boot(STARTUP_SYSTEM)
+            await self.sys_addons.boot(AddonStartup.SYSTEM)
 
             # start addon mark as services
-            await self.sys_addons.boot(STARTUP_SERVICES)
+            await self.sys_addons.boot(AddonStartup.SERVICES)
 
             # run HomeAssistant
             if (
@@ -161,7 +155,7 @@ class Core(CoreSysAttributes):
                 _LOGGER.info("Skip start of Home Assistant")
 
             # start addon mark as application
-            await self.sys_addons.boot(STARTUP_APPLICATION)
+            await self.sys_addons.boot(AddonStartup.APPLICATION)
 
             # store new last boot
             self._update_last_boot()
@@ -214,22 +208,24 @@ class Core(CoreSysAttributes):
     async def shutdown(self):
         """Shutdown all running containers in correct order."""
         # don't process scheduler anymore
-        self.state = CoreStates.STOPPING
+        if self.state == CoreStates.RUNNING:
+            self.state = CoreStates.STOPPING
 
         # Shutdown Application Add-ons, using Home Assistant API
-        await self.sys_addons.shutdown(STARTUP_APPLICATION)
+        await self.sys_addons.shutdown(AddonStartup.APPLICATION)
 
         # Close Home Assistant
         with suppress(HassioError):
             await self.sys_homeassistant.stop()
 
         # Shutdown System Add-ons
-        await self.sys_addons.shutdown(STARTUP_SERVICES)
+        await self.sys_addons.shutdown(AddonStartup.SERVICES)
-        await self.sys_addons.shutdown(STARTUP_SYSTEM)
+        await self.sys_addons.shutdown(AddonStartup.SYSTEM)
-        await self.sys_addons.shutdown(STARTUP_INITIALIZE)
+        await self.sys_addons.shutdown(AddonStartup.INITIALIZE)
 
         # Shutdown all Plugins
-        await self.sys_plugins.shutdown()
+        if self.state == CoreStates.STOPPING:
+            await self.sys_plugins.shutdown()
 
     def _update_last_boot(self):
         """Update last boot time."""

supervisor/misc/tasks.py

@@ -1,9 +1,9 @@
 """A collection of tasks."""
-import asyncio
 import logging
 
 from ..coresys import CoreSysAttributes
 from ..exceptions import (
+    AddonsError,
     AudioError,
     CliError,
     CoreDNSError,
@@ -131,24 +131,26 @@ class Tasks(CoreSysAttributes):
 
     async def _update_addons(self):
         """Check if an update is available for an Add-on and update it."""
-        tasks = []
         for addon in self.sys_addons.all:
             if not addon.is_installed or not addon.auto_update:
                 continue
 
+            # Evaluate available updates
             if addon.version == addon.latest_version:
                 continue
-
+            if not addon.test_update_schema():
-            if addon.test_update_schema():
-                tasks.append(addon.update())
-            else:
                 _LOGGER.warning(
                     "Add-on %s will be ignored, schema tests fails", addon.slug
                 )
+                continue
 
-        if tasks:
+            # Run Add-on update sequential
-            _LOGGER.info("Add-on auto update process %d tasks", len(tasks))
+            # avoid issue on slow IO
-            await asyncio.wait(tasks)
+            _LOGGER.info("Add-on auto update process %s", addon.slug)
+            try:
+                await addon.update()
+            except AddonsError:
+                _LOGGER.error("Can't auto update Add-on %s", addon.slug)
 
     async def _update_supervisor(self):
         """Check and run update of Supervisor Supervisor."""

supervisor/plugins/__init__.py

@@ -51,9 +51,17 @@ class PluginManager(CoreSysAttributes):
 
     async def load(self):
         """Load Supervisor plugins."""
-        await asyncio.wait(
+        # Sequential to avoid issue on slow IO
-            [self.dns.load(), self.audio.load(), self.cli.load(), self.multicast.load()]
+        for plugin in (
-        )
+            self.dns,
+            self.audio,
+            self.cli,
+            self.multicast,
+        ):
+            try:
+                await plugin.load()
+            except Exception:  # pylint: disable=broad-except
+                _LOGGER.warning("Can't load plugin %s", type(plugin).__name__)
 
         # Check requirements
         for plugin, required_version in (
@@ -103,11 +111,14 @@ class PluginManager(CoreSysAttributes):
 
     async def shutdown(self) -> None:
         """Shutdown Supervisor plugin."""
-        await asyncio.wait(
+        # Sequential to avoid issue on slow IO
-            [
+        for plugin in (
-                self.dns.stop(),
+            self.audio,
-                self.audio.stop(),
+            self.cli,
-                self.cli.stop(),
+            self.multicast,
-                self.multicast.stop(),
+            self.dns,
-            ]
+        ):
-        )
+            try:
+                await plugin.stop()
+            except Exception:  # pylint: disable=broad-except
+                _LOGGER.warning("Can't stop plugin %s", type(plugin).__name__)