Fix watchdog & scheduler (#1757)

* Fix watchdog & scheduler

* Update supervisor/misc/scheduler.py

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

* Fix callback

* hmm

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

supervisor/bootstrap.py

@@ -27,15 +27,16 @@ from .discovery import Discovery
 from .hassos import HassOS
 from .homeassistant import HomeAssistant
 from .host import HostManager
-from .hwmon import HwMonitor
 from .ingress import Ingress
+from .misc.hwmon import HwMonitor
+from .misc.scheduler import Scheduler
+from .misc.secrets import SecretsManager
+from .misc.tasks import Tasks
 from .plugins import PluginManager
-from .secrets import SecretsManager
 from .services import ServiceManager
 from .snapshots import SnapshotManager
 from .store import StoreManager
 from .supervisor import Supervisor
-from .tasks import Tasks
 from .updater import Updater
 from .utils.dt import fetch_timezone
 
@@ -70,6 +71,7 @@ async def initialize_coresys() -> None:
     coresys.dbus = DBusManager(coresys)
     coresys.hassos = HassOS(coresys)
     coresys.secrets = SecretsManager(coresys)
+    coresys.scheduler = Scheduler(coresys)
 
     # bootstrap config
     initialize_system_data(coresys)

supervisor/const.py

@@ -362,6 +362,7 @@ class CoreStates(str, Enum):
     STARTUP = "startup"
     RUNNING = "running"
     FREEZE = "freeze"
+    STOPPING = "stopping"
 
 
 class LogLevel(str, Enum):

supervisor/core.py

@@ -187,7 +187,7 @@ class Core(CoreSysAttributes):
     async def stop(self):
         """Stop a running orchestration."""
         # don't process scheduler anymore
-        self.sys_scheduler.suspend = True
+        self.state = CoreStates.STOPPING
 
         # store new last boot / prevent time adjustments
         if self.state == CoreStates.RUNNING:
@@ -213,12 +213,17 @@ class Core(CoreSysAttributes):
 
     async def shutdown(self):
         """Shutdown all running containers in correct order."""
+        # don't process scheduler anymore
+        self.state = CoreStates.STOPPING
+
+        # Shutdown Application Add-ons, using Home Assistant API
         await self.sys_addons.shutdown(STARTUP_APPLICATION)
 
         # Close Home Assistant
         with suppress(HassioError):
             await self.sys_homeassistant.stop()
 
+        # Shutdown System Add-ons
         await self.sys_addons.shutdown(STARTUP_SERVICES)
         await self.sys_addons.shutdown(STARTUP_SYSTEM)
         await self.sys_addons.shutdown(STARTUP_INITIALIZE)

supervisor/coresys.py

@@ -10,7 +10,6 @@ from .config import CoreConfig
 from .const import UpdateChannels
 from .docker import DockerAPI
 from .misc.hardware import Hardware
-from .misc.scheduler import Scheduler
 
 if TYPE_CHECKING:
     from .addons import AddonManager
@@ -21,16 +20,17 @@ if TYPE_CHECKING:
     from .dbus import DBusManager
     from .discovery import Discovery
     from .hassos import HassOS
-    from .hwmon import HwMonitor
+    from .misc.scheduler import Scheduler
+    from .misc.hwmon import HwMonitor
+    from .misc.secrets import SecretsManager
+    from .misc.tasks import Tasks
     from .homeassistant import HomeAssistant
     from .host import HostManager
     from .ingress import Ingress
-    from .secrets import SecretsManager
     from .services import ServiceManager
     from .snapshots import SnapshotManager
     from .supervisor import Supervisor
     from .store import StoreManager
-    from .tasks import Tasks
     from .updater import Updater
     from .plugins import PluginManager
 
@@ -58,7 +58,6 @@ class CoreSys:
         self._config: CoreConfig = CoreConfig()
         self._hardware: Hardware = Hardware()
         self._docker: DockerAPI = DockerAPI()
-        self._scheduler: Scheduler = Scheduler()
 
         # Internal objects pointers
         self._core: Optional[Core] = None
@@ -77,6 +76,7 @@ class CoreSys:
         self._hassos: Optional[HassOS] = None
         self._services: Optional[ServiceManager] = None
         self._secrets: Optional[SecretsManager] = None
+        self._scheduler: Optional[Scheduler] = None
         self._store: Optional[StoreManager] = None
         self._discovery: Optional[Discovery] = None
         self._hwmonitor: Optional[HwMonitor] = None
@@ -127,8 +127,17 @@ class CoreSys:
     @property
     def scheduler(self) -> Scheduler:
         """Return Scheduler object."""
+        if self._scheduler is None:
+            raise RuntimeError("Scheduler not set!")
         return self._scheduler
 
+    @scheduler.setter
+    def scheduler(self, value: Scheduler) -> None:
+        """Set a Scheduler object."""
+        if self._scheduler:
+            raise RuntimeError("Scheduler already set!")
+        self._scheduler = value
+
     @property
     def core(self) -> Core:
         """Return core object."""

supervisor/misc/hwmon.py

@@ -6,8 +6,8 @@ from typing import Optional
 
 import pyudev
 
-from .coresys import CoreSys, CoreSysAttributes
+from ..coresys import CoreSys, CoreSysAttributes
-from .utils import AsyncCallFilter
+from ..utils import AsyncCallFilter
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 

supervisor/misc/scheduler.py

@@ -1,8 +1,10 @@
 """Schedule for Supervisor."""
-import asyncio
 from datetime import date, datetime, time, timedelta
 import logging
 
+from ..const import CoreStates
+from ..coresys import CoreSys, CoreSysAttributes
+
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 
 INTERVAL = "interval"
@@ -11,14 +13,13 @@ CALL = "callback"
 TASK = "task"
 
 
-class Scheduler:
+class Scheduler(CoreSysAttributes):
     """Schedule task inside Supervisor."""
 
-    def __init__(self):
+    def __init__(self, coresys: CoreSys):
         """Initialize task schedule."""
-        self.loop = asyncio.get_running_loop()
+        self.coresys: CoreSys = coresys
         self._data = {}
-        self.suspend = False
 
     def register_task(self, coro_callback, interval, repeat=True):
         """Schedule a coroutine.
@@ -40,8 +41,8 @@ class Scheduler:
         """Run a scheduled task."""
         data = self._data[task_id]
 
-        if not self.suspend:
+        if self.sys_core.state == CoreStates.RUNNING:
-            self.loop.create_task(data[CALL]())
+            self.sys_create_task(data[CALL]())
 
         if data[REPEAT]:
             self._schedule_task(data[INTERVAL], task_id)
@@ -51,7 +52,7 @@ class Scheduler:
     def _schedule_task(self, interval, task_id):
         """Schedule a task on loop."""
         if isinstance(interval, (int, float)):
-            job = self.loop.call_later(interval, self._run_task, task_id)
+            job = self.sys_loop.call_later(interval, self._run_task, task_id)
         elif isinstance(interval, time):
             today = datetime.combine(date.today(), interval)
             tomorrow = datetime.combine(date.today() + timedelta(days=1), interval)
@@ -62,7 +63,7 @@ class Scheduler:
             else:
                 calc = tomorrow
 
-            job = self.loop.call_at(calc.timestamp(), self._run_task, task_id)
+            job = self.sys_loop.call_at(calc.timestamp(), self._run_task, task_id)
         else:
             _LOGGER.critical(
                 "Unknown interval %s (type: %s) for scheduler %s",

supervisor/misc/secrets.py

@@ -6,8 +6,8 @@ from typing import Dict, Optional, Union
 
 from ruamel.yaml import YAML, YAMLError
 
-from .coresys import CoreSys, CoreSysAttributes
+from ..coresys import CoreSys, CoreSysAttributes
-from .utils import AsyncThrottle
+from ..utils import AsyncThrottle
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 

supervisor/misc/tasks.py

@@ -2,8 +2,8 @@
 import asyncio
 import logging
 
-from .coresys import CoreSysAttributes
+from ..coresys import CoreSysAttributes
-from .exceptions import (
+from ..exceptions import (
     AudioError,
     CliError,
     CoreDNSError,

supervisor/snapshots/__init__.py

@@ -3,7 +3,7 @@ import asyncio
 import logging
 from pathlib import Path
 
-from ..const import FOLDER_HOMEASSISTANT, SNAPSHOT_FULL, SNAPSHOT_PARTIAL
+from ..const import FOLDER_HOMEASSISTANT, SNAPSHOT_FULL, SNAPSHOT_PARTIAL, CoreStates
 from ..coresys import CoreSysAttributes
 from ..utils.dt import utcnow
 from .snapshot import Snapshot
@@ -125,7 +125,7 @@ class SnapshotManager(CoreSysAttributes):
         snapshot = self._create_snapshot(name, SNAPSHOT_FULL, password)
         _LOGGER.info("Full-Snapshot %s start", snapshot.slug)
         try:
-            self.sys_scheduler.suspend = True
+            self.sys_core.state = CoreStates.FREEZE
             await self.lock.acquire()
 
             async with snapshot:
@@ -147,7 +147,7 @@ class SnapshotManager(CoreSysAttributes):
             return snapshot
 
         finally:
-            self.sys_scheduler.suspend = False
+            self.sys_core.state = CoreStates.RUNNING
             self.lock.release()
 
     async def do_snapshot_partial(
@@ -164,7 +164,7 @@ class SnapshotManager(CoreSysAttributes):
 
         _LOGGER.info("Partial-Snapshot %s start", snapshot.slug)
         try:
-            self.sys_scheduler.suspend = True
+            self.sys_core.state = CoreStates.FREEZE
             await self.lock.acquire()
 
             async with snapshot:
@@ -196,7 +196,7 @@ class SnapshotManager(CoreSysAttributes):
             return snapshot
 
         finally:
-            self.sys_scheduler.suspend = False
+            self.sys_core.state = CoreStates.RUNNING
             self.lock.release()
 
     async def do_restore_full(self, snapshot, password=None):
@@ -215,7 +215,7 @@ class SnapshotManager(CoreSysAttributes):
 
         _LOGGER.info("Full-Restore %s start", snapshot.slug)
         try:
-            self.sys_scheduler.suspend = True
+            self.sys_core.state = CoreStates.FREEZE
             await self.lock.acquire()
 
             async with snapshot:
@@ -267,7 +267,7 @@ class SnapshotManager(CoreSysAttributes):
             return True
 
         finally:
-            self.sys_scheduler.suspend = False
+            self.sys_core.state = CoreStates.RUNNING
             self.lock.release()
 
     async def do_restore_partial(
@@ -287,7 +287,7 @@ class SnapshotManager(CoreSysAttributes):
 
         _LOGGER.info("Partial-Restore %s start", snapshot.slug)
         try:
-            self.sys_scheduler.suspend = True
+            self.sys_core.state = CoreStates.FREEZE
             await self.lock.acquire()
 
             async with snapshot:
@@ -339,5 +339,5 @@ class SnapshotManager(CoreSysAttributes):
             return True
 
         finally:
-            self.sys_scheduler.suspend = False
+            self.sys_core.state = CoreStates.RUNNING
             self.lock.release()