Improve error handling with job condition (#2322)

* Improve error handling with job condition

* fix

* first patch

* last patch

* Address comments

* Revert strange replace

supervisor/addons/__init__.py

@@ -10,6 +10,7 @@ from ..coresys import CoreSys, CoreSysAttributes
 from ..exceptions import (
     AddonConfigurationError,
     AddonsError,
+    AddonsJobError,
     AddonsNotSupportedError,
     CoreDNSError,
     DockerAPIError,
@@ -147,7 +148,8 @@ class AddonManager(CoreSysAttributes):
             JobCondition.FREE_SPACE,
             JobCondition.INTERNET_HOST,
             JobCondition.HEALTHY,
-        ]
+        ],
+        on_condition=AddonsJobError,
     )
     async def install(self, slug: str) -> None:
         """Install an add-on."""
@@ -248,7 +250,8 @@ class AddonManager(CoreSysAttributes):
             JobCondition.FREE_SPACE,
             JobCondition.INTERNET_HOST,
             JobCondition.HEALTHY,
-        ]
+        ],
+        on_condition=AddonsJobError,
     )
     async def update(self, slug: str) -> None:
         """Update add-on."""
@@ -297,7 +300,8 @@ class AddonManager(CoreSysAttributes):
             JobCondition.FREE_SPACE,
             JobCondition.INTERNET_HOST,
             JobCondition.HEALTHY,
-        ]
+        ],
+        on_condition=AddonsJobError,
     )
     async def rebuild(self, slug: str) -> None:
         """Perform a rebuild of local build add-on."""
@@ -339,7 +343,8 @@ class AddonManager(CoreSysAttributes):
             JobCondition.FREE_SPACE,
             JobCondition.INTERNET_HOST,
             JobCondition.HEALTHY,
-        ]
+        ],
+        on_condition=AddonsJobError,
     )
     async def restore(self, slug: str, tar_file: tarfile.TarFile) -> None:
         """Restore state of an add-on."""

supervisor/exceptions.py

@@ -9,6 +9,13 @@ class HassioNotSupportedError(HassioError):
     """Function is not supported."""
 
 
+# JobManager
+
+
+class JobException(HassioError):
+    """Base job exception."""
+
+
 # HomeAssistant
 
 
@@ -32,6 +39,10 @@ class HomeAssistantAuthError(HomeAssistantAPIError):
     """Home Assistant Auth API exception."""
 
 
+class HomeAssistantJobError(HomeAssistantError, JobException):
+    """Raise on Home Assistant job error."""
+
+
 # Supervisor
 
 
@@ -43,6 +54,10 @@ class SupervisorUpdateError(SupervisorError):
     """Supervisor update error."""
 
 
+class SupervisorJobError(SupervisorError, JobException):
+    """Raise on job errors."""
+
+
 # HassOS
 
 
@@ -128,6 +143,10 @@ class AddonsNotSupportedError(HassioNotSupportedError):
     """Addons don't support a function."""
 
 
+class AddonsJobError(AddonsError, JobException):
+    """Raise on job errors."""
+
+
 # Arch
 
 
@@ -138,10 +157,14 @@ class HassioArchNotFound(HassioNotSupportedError):
 # Updater
 
 
-class HassioUpdaterError(HassioError):
+class UpdaterError(HassioError):
     """Error on Updater."""
 
 
+class UpdaterJobError(UpdaterError, JobException):
+    """Raise on job error."""
+
+
 # Auth
 
 
@@ -312,10 +335,3 @@ class StoreGitError(StoreError):
 
 class StoreNotFound(StoreError):
     """Raise if slug is not known."""
-
-
-# JobManager
-
-
-class JobException(HassioError):
-    """Base job exception."""

supervisor/homeassistant/core.py

@@ -19,6 +19,7 @@ from ..exceptions import (
     DockerError,
     HomeAssistantCrashError,
     HomeAssistantError,
+    HomeAssistantJobError,
     HomeAssistantUpdateError,
 )
 from ..jobs.decorator import Job, JobCondition
@@ -158,7 +159,8 @@ class HomeAssistantCore(CoreSysAttributes):
             JobCondition.FREE_SPACE,
             JobCondition.HEALTHY,
             JobCondition.INTERNET_HOST,
-        ]
+        ],
+        on_condition=HomeAssistantJobError,
     )
     async def update(self, version: Optional[str] = None) -> None:
         """Update HomeAssistant version."""

supervisor/jobs/decorator.py

@@ -1,6 +1,6 @@
 """Job decorator."""
 import logging
-from typing import List, Optional
+from typing import Any, List, Optional
 
 import sentry_sdk
 
@@ -21,11 +21,13 @@ class Job:
         name: Optional[str] = None,
         conditions: Optional[List[JobCondition]] = None,
         cleanup: bool = True,
+        on_condition: Optional[JobException] = None,
     ):
         """Initialize the Job class."""
         self.name = name
         self.conditions = conditions
         self.cleanup = cleanup
+        self.on_condition = on_condition
         self._coresys: Optional[CoreSys] = None
         self._method = None
 
@@ -33,23 +35,28 @@ class Job:
         """Call the wrapper logic."""
         self._method = method
 
-        async def wrapper(*args, **kwargs):
+        async def wrapper(*args, **kwargs) -> Any:
             """Wrap the method."""
             if self.name is None:
                 self.name = str(self._method.__qualname__).lower().replace(".", "_")
+
+            # Evaluate coresys
             try:
                 self._coresys = args[0].coresys
             except AttributeError:
-                return False
+                pass
-
             if not self._coresys:
                 raise JobException(f"coresys is missing on {self.name}")
 
             job = self._coresys.jobs.get_job(self.name)
 
+            # Handle condition
             if self.conditions and not self._check_conditions():
-                return False
+                if self.on_condition is None:
+                    return
+                raise self.on_condition()
 
+            # Execute Job
             try:
                 return await self._method(*args, **kwargs)
             except HassioError as err:

supervisor/snapshots/__init__.py

@@ -122,7 +122,7 @@ class SnapshotManager(CoreSysAttributes):
         self.snapshots_obj[snapshot.slug] = snapshot
         return snapshot
 
-    @Job(conditions=[JobCondition.FREE_SPACE])
+    @Job(conditions=[JobCondition.FREE_SPACE, JobCondition.RUNNING])
     async def do_snapshot_full(self, name="", password=None):
         """Create a full snapshot."""
         if self.lock.locked():
@@ -144,9 +144,9 @@ class SnapshotManager(CoreSysAttributes):
                 _LOGGER.info("Snapshotting %s store folders", snapshot.slug)
                 await snapshot.store_folders()
 
-        except Exception as excep:  # pylint: disable=broad-except
+        except Exception as err:  # pylint: disable=broad-except
             _LOGGER.exception("Snapshot %s error", snapshot.slug)
-            print(excep)
+            self.sys_capture_exception(err)
             return None
 
         else:
@@ -158,7 +158,7 @@ class SnapshotManager(CoreSysAttributes):
             self.sys_core.state = CoreState.RUNNING
             self.lock.release()
 
-    @Job(conditions=[JobCondition.FREE_SPACE])
+    @Job(conditions=[JobCondition.FREE_SPACE, JobCondition.RUNNING])
     async def do_snapshot_partial(
         self, name="", addons=None, folders=None, password=None
     ):
@@ -195,8 +195,9 @@ class SnapshotManager(CoreSysAttributes):
                     _LOGGER.info("Snapshotting %s store folders", snapshot.slug)
                     await snapshot.store_folders(folders)
 
-        except Exception:  # pylint: disable=broad-except
+        except Exception as err:  # pylint: disable=broad-except
             _LOGGER.exception("Snapshot %s error", snapshot.slug)
+            self.sys_capture_exception(err)
             return None
 
         else:
@@ -216,6 +217,7 @@ class SnapshotManager(CoreSysAttributes):
             JobCondition.HEALTHY,
             JobCondition.INTERNET_HOST,
             JobCondition.INTERNET_SYSTEM,
+            JobCondition.RUNNING,
         ]
     )
     async def do_restore_full(self, snapshot, password=None):
@@ -282,8 +284,9 @@ class SnapshotManager(CoreSysAttributes):
                 await task_hass
                 await self.sys_homeassistant.core.start()
 
-        except Exception:  # pylint: disable=broad-except
+        except Exception as err:  # pylint: disable=broad-except
             _LOGGER.exception("Restore %s error", snapshot.slug)
+            self.sys_capture_exception(err)
             return False
 
         else:
@@ -300,6 +303,7 @@ class SnapshotManager(CoreSysAttributes):
             JobCondition.HEALTHY,
             JobCondition.INTERNET_HOST,
             JobCondition.INTERNET_SYSTEM,
+            JobCondition.RUNNING,
         ]
     )
     async def do_restore_partial(
@@ -368,8 +372,9 @@ class SnapshotManager(CoreSysAttributes):
                     _LOGGER.warning("Need restart HomeAssistant for API")
                     await self.sys_homeassistant.core.restart()
 
-        except Exception:  # pylint: disable=broad-except
+        except Exception as err:  # pylint: disable=broad-except
             _LOGGER.exception("Restore %s error", snapshot.slug)
+            self.sys_capture_exception(err)
             return False
 
         else:

supervisor/supervisor.py

@@ -21,6 +21,7 @@ from .exceptions import (
     DockerError,
     HostAppArmorError,
     SupervisorError,
+    SupervisorJobError,
     SupervisorUpdateError,
 )
 from .resolution.const import ContextType, IssueType
@@ -147,7 +148,7 @@ class Supervisor(CoreSysAttributes):
             await self.update_apparmor()
         self.sys_create_task(self.sys_core.stop())
 
-    @Job(conditions=[JobCondition.RUNNING])
+    @Job(conditions=[JobCondition.RUNNING], on_condition=SupervisorJobError)
     async def restart(self) -> None:
         """Restart Supervisor soft."""
         self.sys_core.exit_code = 100

supervisor/updater.py

@@ -25,7 +25,7 @@ from .const import (
     UpdateChannel,
 )
 from .coresys import CoreSysAttributes
-from .exceptions import HassioUpdaterError
+from .exceptions import UpdaterError, UpdaterJobError
 from .jobs.decorator import Job, JobCondition
 from .utils import AsyncThrottle
 from .utils.json import JsonConfig
@@ -44,12 +44,12 @@ class Updater(JsonConfig, CoreSysAttributes):
 
     async def load(self) -> None:
         """Update internal data."""
-        with suppress(HassioUpdaterError):
+        with suppress(UpdaterError):
             await self.fetch_data()
 
     async def reload(self) -> None:
         """Update internal data."""
-        with suppress(HassioUpdaterError):
+        with suppress(UpdaterJobError):
             await self.fetch_data()
 
     @property
@@ -165,7 +165,10 @@ class Updater(JsonConfig, CoreSysAttributes):
         self._data[ATTR_CHANNEL] = value
 
     @AsyncThrottle(timedelta(seconds=30))
-    @Job(conditions=[JobCondition.INTERNET_SYSTEM])
+    @Job(
+        conditions=[JobCondition.INTERNET_SYSTEM],
+        on_condition=UpdaterJobError,
+    )
     async def fetch_data(self):
         """Fetch current versions from Github.
 
@@ -181,16 +184,16 @@ class Updater(JsonConfig, CoreSysAttributes):
 
         except (aiohttp.ClientError, asyncio.TimeoutError) as err:
             _LOGGER.warning("Can't fetch versions from %s: %s", url, err)
-            raise HassioUpdaterError() from err
+            raise UpdaterError() from err
 
         except json.JSONDecodeError as err:
             _LOGGER.warning("Can't parse versions from %s: %s", url, err)
-            raise HassioUpdaterError() from err
+            raise UpdaterError() from err
 
         # data valid?
         if not data or data.get(ATTR_CHANNEL) != self.channel:
             _LOGGER.warning("Invalid data from %s", url)
-            raise HassioUpdaterError()
+            raise UpdaterError()
 
         try:
             # Update supervisor version
@@ -222,7 +225,7 @@ class Updater(JsonConfig, CoreSysAttributes):
 
         except KeyError as err:
             _LOGGER.warning("Can't process version data: %s", err)
-            raise HassioUpdaterError() from err
+            raise UpdaterError() from err
 
         else:
             self.save_data()

tests/jobs/test_job_decorator.py

@@ -232,3 +232,28 @@ async def test_ignore_conditions(coresys: CoreSys):
 
     coresys.jobs.ignore_conditions = [JobCondition.RUNNING]
     assert await test.execute()
+
+
+async def test_exception_conditions(coresys: CoreSys):
+    """Test the ignore conditions decorator."""
+
+    class TestClass:
+        """Test class."""
+
+        def __init__(self, coresys: CoreSys):
+            """Initialize the test class."""
+            self.coresys = coresys
+
+        @Job(conditions=[JobCondition.RUNNING], on_condition=HassioError)
+        async def execute(self):
+            """Execute the class method."""
+            return True
+
+    test = TestClass(coresys)
+
+    coresys.core.state = CoreState.RUNNING
+    assert await test.execute()
+
+    coresys.core.state = CoreState.FREEZE
+    with pytest.raises(HassioError):
+        await test.execute()