Add JobManager API ignore (#2290)

* Disable job condition for unhealth & unsupported systems

* Add JobManager API ignore

* Apply suggestions from code review

Co-authored-by: Paulus Schoutsen <balloob@gmail.com>

* Update tests/resolution/evaluation/test_evaluate_job_conditions.py

Co-authored-by: Paulus Schoutsen <balloob@gmail.com>

* fix names

* address comments

* Update decorator.py

* adjust security

* add reset

* Apply suggestions from code review

Co-authored-by: Joakim Sørensen <joasoe@gmail.com>

Co-authored-by: Paulus Schoutsen <balloob@gmail.com>
Co-authored-by: Joakim Sørensen <joasoe@gmail.com>

supervisor/api/__init__.py

@@ -18,6 +18,7 @@ from .homeassistant import APIHomeAssistant
 from .host import APIHost
 from .info import APIInfo
 from .ingress import APIIngress
+from .jobs import APIJobs
 from .multicast import APIMulticast
 from .network import APINetwork
 from .observer import APIObserver
@@ -72,6 +73,7 @@ class RestAPI(CoreSysAttributes):
         self._register_network()
         self._register_observer()
         self._register_os()
+        self._register_jobs()
         self._register_panel()
         self._register_proxy()
         self._register_resolution()
@@ -141,6 +143,19 @@ class RestAPI(CoreSysAttributes):
             ]
         )
 
+    def _register_jobs(self) -> None:
+        """Register Jobs functions."""
+        api_jobs = APIJobs()
+        api_jobs.coresys = self.coresys
+
+        self.webapp.add_routes(
+            [
+                web.get("/jobs/info", api_jobs.info),
+                web.post("/jobs/options", api_jobs.options),
+                web.post("/jobs/reset", api_jobs.reset),
+            ]
+        )
+
     def _register_cli(self) -> None:
         """Register HA cli functions."""
         api_cli = APICli()

supervisor/api/jobs.py

@@ -0,0 +1,42 @@
+"""Init file for Supervisor Jobs RESTful API."""
+import logging
+from typing import Any, Dict
+
+from aiohttp import web
+import voluptuous as vol
+
+from ..coresys import CoreSysAttributes
+from ..jobs.const import ATTR_IGNORE_CONDITIONS, JobCondition
+from .utils import api_process, api_validate
+
+_LOGGER: logging.Logger = logging.getLogger(__name__)
+
+SCHEMA_OPTIONS = vol.Schema(
+    {vol.Optional(ATTR_IGNORE_CONDITIONS): [vol.Coerce(JobCondition)]}
+)
+
+
+class APIJobs(CoreSysAttributes):
+    """Handle RESTful API for OS functions."""
+
+    @api_process
+    async def info(self, request: web.Request) -> Dict[str, Any]:
+        """Return JobManager information."""
+        return {
+            ATTR_IGNORE_CONDITIONS: self.sys_jobs.ignore_conditions,
+        }
+
+    @api_process
+    async def options(self, request: web.Request) -> None:
+        """Set options for JobManager."""
+        body = await api_validate(SCHEMA_OPTIONS, request)
+
+        if ATTR_IGNORE_CONDITIONS in body:
+            self.sys_jobs.ignore_conditions = body[ATTR_IGNORE_CONDITIONS]
+
+        self.sys_jobs.save_data()
+
+    @api_process
+    async def reset(self, request: web.Request) -> None:
+        """Reset options for JobManager."""
+        self.sys_jobs.reset_data()

supervisor/api/security.py

@@ -87,6 +87,7 @@ ADDONS_ROLE_ACCESS = {
         r"|/core/.+"
         r"|/dns/.+"
         r"|/docker/.+"
+        r"|/jobs/.+"
         r"|/hardware/.+"
         r"|/hassos/.+"
         r"|/homeassistant/.+"

supervisor/jobs/__init__.py

@@ -3,6 +3,9 @@ import logging
 from typing import Dict, List, Optional
 
 from ..coresys import CoreSys, CoreSysAttributes
+from ..utils.json import JsonConfig
+from .const import ATTR_IGNORE_CONDITIONS, FILE_CONFIG_JOBS, JobCondition
+from .validate import SCHEMA_JOBS_CONFIG
 
 _LOGGER: logging.Logger = logging.getLogger(__package__)
 
@@ -46,11 +49,12 @@ class SupervisorJob(CoreSysAttributes):
         )
 
 
-class JobManager(CoreSysAttributes):
+class JobManager(JsonConfig, CoreSysAttributes):
     """Job class."""
 
     def __init__(self, coresys: CoreSys):
         """Initialize the JobManager class."""
+        super().__init__(FILE_CONFIG_JOBS, SCHEMA_JOBS_CONFIG)
         self.coresys: CoreSys = coresys
         self._jobs: Dict[str, SupervisorJob] = {}
 
@@ -59,6 +63,16 @@ class JobManager(CoreSysAttributes):
         """Return a list of current jobs."""
         return self._jobs
 
+    @property
+    def ignore_conditions(self) -> List[JobCondition]:
+        """Return a list of ingore condition."""
+        return self._data[ATTR_IGNORE_CONDITIONS]
+
+    @ignore_conditions.setter
+    def ignore_conditions(self, value: List[JobCondition]) -> None:
+        """Set a list of ignored condition."""
+        self._data[ATTR_IGNORE_CONDITIONS] = value
+
     def get_job(self, name: str) -> SupervisorJob:
         """Return a job, create one if it does not exsist."""
         if name not in self._jobs:

supervisor/jobs/const.py

@@ -0,0 +1,19 @@
+"""Jobs constants."""
+from enum import Enum
+from pathlib import Path
+
+from ..const import SUPERVISOR_DATA
+
+FILE_CONFIG_JOBS = Path(SUPERVISOR_DATA, "jobs.json")
+
+ATTR_IGNORE_CONDITIONS = "ignore_conditions"
+
+
+class JobCondition(str, Enum):
+    """Job condition enum."""
+
+    FREE_SPACE = "free_space"
+    HEALTHY = "healthy"
+    INTERNET_SYSTEM = "internet_system"
+    INTERNET_HOST = "internet_host"
+    RUNNING = "running"

supervisor/jobs/decorator.py

@@ -1,5 +1,4 @@
 """Job decorator."""
-from enum import Enum
 import logging
 from typing import List, Optional
 
@@ -9,20 +8,11 @@ from ..const import CoreState
 from ..coresys import CoreSys
 from ..exceptions import HassioError, JobException
 from ..resolution.const import MINIMUM_FREE_SPACE_THRESHOLD, ContextType, IssueType
+from .const import JobCondition
 
 _LOGGER: logging.Logger = logging.getLogger(__package__)
 
 
-class JobCondition(str, Enum):
-    """Job condition enum."""
-
-    FREE_SPACE = "free_space"
-    HEALTHY = "healthy"
-    INTERNET_SYSTEM = "internet_system"
-    INTERNET_HOST = "internet_host"
-    RUNNING = "running"
-
-
 class Job:
     """Supervisor job decorator."""
 
@@ -76,61 +66,67 @@ class Job:
 
     def _check_conditions(self):
         """Check conditions."""
-        if JobCondition.HEALTHY in self.conditions:
-            if not self._coresys.core.healthy:
+        used_conditions = set(self.conditions) - set(
+            self._coresys.jobs.ignore_conditions
+        )
+        ignored_conditions = set(self.conditions) & set(
+            self._coresys.jobs.ignore_conditions
+        )
+
+        # Check if somethings is ignored
+        if ignored_conditions:
+            _LOGGER.critical(
+                "The following job conditions are ignored and will make the system unstable when they occur: %s",
+                ignored_conditions,
+            )
+
+        if JobCondition.HEALTHY in used_conditions and not self._coresys.core.healthy:
             _LOGGER.warning(
                 "'%s' blocked from execution, system is not healthy",
                 self._method.__qualname__,
             )
             return False
 
-        if JobCondition.RUNNING in self.conditions:
-            if self._coresys.core.state != CoreState.RUNNING:
+        if (
+            JobCondition.RUNNING in used_conditions
+            and self._coresys.core.state != CoreState.RUNNING
+        ):
             _LOGGER.warning(
                 "'%s' blocked from execution, system is not running",
                 self._method.__qualname__,
             )
             return False
 
-        if JobCondition.FREE_SPACE in self.conditions:
-            free_space = self._coresys.host.info.free_space
-            if free_space < MINIMUM_FREE_SPACE_THRESHOLD:
+        if (
+            JobCondition.FREE_SPACE in used_conditions
+            and self._coresys.host.info.free_space < MINIMUM_FREE_SPACE_THRESHOLD
+        ):
             _LOGGER.warning(
                 "'%s' blocked from execution, not enough free space (%sGB) left on the device",
                 self._method.__qualname__,
-                    free_space,
+                self._coresys.host.info.free_space,
             )
             self._coresys.resolution.create_issue(
                 IssueType.FREE_SPACE, ContextType.SYSTEM
             )
             return False
 
-        if any(
-            internet in self.conditions
-            for internet in (
-                JobCondition.INTERNET_SYSTEM,
-                JobCondition.INTERNET_HOST,
-            )
-        ):
-            if self._coresys.core.state not in (
-                CoreState.SETUP,
-                CoreState.RUNNING,
-            ):
-                return True
-
         if (
             JobCondition.INTERNET_SYSTEM in self.conditions
             and not self._coresys.supervisor.connectivity
+            and self._coresys.core.state in (CoreState.SETUP, CoreState.RUNNING)
         ):
             _LOGGER.warning(
                 "'%s' blocked from execution, no supervisor internet connection",
                 self._method.__qualname__,
             )
             return False
-            elif (
+
+        if (
             JobCondition.INTERNET_HOST in self.conditions
             and self._coresys.host.network.connectivity is not None
             and not self._coresys.host.network.connectivity
+            and self._coresys.core.state in (CoreState.SETUP, CoreState.RUNNING)
         ):
             _LOGGER.warning(
                 "'%s' blocked from execution, no host internet connection",

supervisor/jobs/validate.py

@@ -0,0 +1,12 @@
+"""Validate services schema."""
+
+import voluptuous as vol
+
+from .const import ATTR_IGNORE_CONDITIONS, JobCondition
+
+SCHEMA_JOBS_CONFIG = vol.Schema(
+    {
+        vol.Optional(ATTR_IGNORE_CONDITIONS, default=list): [vol.Coerce(JobCondition)],
+    },
+    extra=vol.REMOVE_EXTRA,
+)

supervisor/resolution/const.py

@@ -31,6 +31,7 @@ class UnsupportedReason(str, Enum):
     OS = "os"
     PRIVILEGED = "privileged"
     SYSTEMD = "systemd"
+    JOB_CONDITIONS = "job_conditions"
 
 
 class UnhealthyReason(str, Enum):

supervisor/resolution/evaluate.py

@@ -7,6 +7,7 @@ from .evaluations.container import EvaluateContainer
 from .evaluations.dbus import EvaluateDbus
 from .evaluations.docker_configuration import EvaluateDockerConfiguration
 from .evaluations.docker_version import EvaluateDockerVersion
+from .evaluations.job_conditions import EvaluateJobConditions
 from .evaluations.lxc import EvaluateLxc
 from .evaluations.network_manager import EvaluateNetworkManager
 from .evaluations.operating_system import EvaluateOperatingSystem
@@ -39,6 +40,7 @@ class ResolutionEvaluation(CoreSysAttributes):
         self._operating_system = EvaluateOperatingSystem(coresys)
         self._privileged = EvaluatePrivileged(coresys)
         self._systemd = EvaluateSystemd(coresys)
+        self._job_conditions = EvaluateJobConditions(coresys)
 
     async def evaluate_system(self) -> None:
         """Evaluate the system."""
@@ -52,6 +54,7 @@ class ResolutionEvaluation(CoreSysAttributes):
         await self._operating_system()
         await self._privileged()
         await self._systemd()
+        await self._job_conditions()
 
         if any(reason in self.sys_resolution.unsupported for reason in UNHEALTHY):
             self.sys_resolution.unhealthy = UnhealthyReason.DOCKER

supervisor/resolution/evaluations/job_conditions.py

@@ -0,0 +1,29 @@
+"""Evaluation class for Job Conditions."""
+from typing import List
+
+from ...const import CoreState
+from ..const import UnsupportedReason
+from .base import EvaluateBase
+
+
+class EvaluateJobConditions(EvaluateBase):
+    """Evaluate job conditions."""
+
+    @property
+    def reason(self) -> UnsupportedReason:
+        """Return a UnsupportedReason enum."""
+        return UnsupportedReason.JOB_CONDITIONS
+
+    @property
+    def on_failure(self) -> str:
+        """Return a string that is printed when self.evaluate is False."""
+        return "Found unsupported job conditions settings."
+
+    @property
+    def states(self) -> List[CoreState]:
+        """Return a list of valid states when this evaluation can run."""
+        return [CoreState.INITIALIZE, CoreState.SETUP, CoreState.RUNNING]
+
+    async def evaluate(self) -> None:
+        """Run evaluation."""
+        return len(self.sys_jobs.ignore_conditions) > 0

tests/api/test_jobs.py

@@ -0,0 +1,52 @@
+"""Test Docker API."""
+import pytest
+
+from supervisor.jobs.const import ATTR_IGNORE_CONDITIONS, JobCondition
+
+
+@pytest.mark.asyncio
+async def test_api_jobs_info(api_client):
+    """Test jobs info api."""
+    resp = await api_client.get("/jobs/info")
+    result = await resp.json()
+
+    assert result["data"][ATTR_IGNORE_CONDITIONS] == []
+
+
+@pytest.mark.asyncio
+async def test_api_jobs_options(api_client, coresys):
+    """Test jobs options api."""
+    resp = await api_client.post(
+        "/jobs/options", json={ATTR_IGNORE_CONDITIONS: [JobCondition.HEALTHY]}
+    )
+    result = await resp.json()
+    assert result["result"] == "ok"
+
+    resp = await api_client.get("/jobs/info")
+    result = await resp.json()
+    assert result["data"][ATTR_IGNORE_CONDITIONS] == [JobCondition.HEALTHY]
+
+    assert coresys.jobs.save_data.called
+
+
+@pytest.mark.asyncio
+async def test_api_jobs_reset(api_client, coresys):
+    """Test jobs reset api."""
+    resp = await api_client.post(
+        "/jobs/options", json={ATTR_IGNORE_CONDITIONS: [JobCondition.HEALTHY]}
+    )
+    result = await resp.json()
+    assert result["result"] == "ok"
+
+    resp = await api_client.get("/jobs/info")
+    result = await resp.json()
+    assert result["data"][ATTR_IGNORE_CONDITIONS] == [JobCondition.HEALTHY]
+
+    assert coresys.jobs.save_data.called
+    assert coresys.jobs.ignore_conditions == [JobCondition.HEALTHY]
+
+    resp = await api_client.post("/jobs/reset")
+    result = await resp.json()
+    assert result["result"] == "ok"
+
+    assert coresys.jobs.ignore_conditions == []

tests/conftest.py

@@ -131,6 +131,7 @@ async def coresys(loop, docker, network_manager, aiohttp_client) -> CoreSys:
     coresys_obj._auth.save_data = MagicMock()
     coresys_obj._updater.save_data = MagicMock()
     coresys_obj._config.save_data = MagicMock()
+    coresys_obj._jobs.save_data = MagicMock()
 
     # Mock test client
     coresys_obj.arch._default_arch = "amd64"

tests/jobs/test_job_decorator.py

@@ -205,3 +205,30 @@ async def test_running(coresys: CoreSys):
 
     coresys.core.state = CoreState.FREEZE
     assert not await test.execute()
+
+
+async def test_ignore_conditions(coresys: CoreSys):
+    """Test the ignore conditions decorator."""
+
+    class TestClass:
+        """Test class."""
+
+        def __init__(self, coresys: CoreSys):
+            """Initialize the test class."""
+            self.coresys = coresys
+
+        @Job(conditions=[JobCondition.RUNNING])
+        async def execute(self):
+            """Execute the class method."""
+            return True
+
+    test = TestClass(coresys)
+
+    coresys.core.state = CoreState.RUNNING
+    assert await test.execute()
+
+    coresys.core.state = CoreState.FREEZE
+    assert not await test.execute()
+
+    coresys.jobs.ignore_conditions = [JobCondition.RUNNING]
+    assert await test.execute()

tests/resolution/evaluation/test_evaluate_job_conditions.py

@@ -0,0 +1,46 @@
+"""Test evaluation base."""
+# pylint: disable=import-error,protected-access
+from unittest.mock import patch
+
+from supervisor.const import CoreState
+from supervisor.coresys import CoreSys
+from supervisor.jobs.const import JobCondition
+from supervisor.resolution.evaluations.job_conditions import EvaluateJobConditions
+
+
+async def test_evaluation(coresys: CoreSys):
+    """Test evaluation."""
+    job_conditions = EvaluateJobConditions(coresys)
+    coresys.core.state = CoreState.SETUP
+
+    await job_conditions()
+    assert job_conditions.reason not in coresys.resolution.unsupported
+
+    coresys.jobs.ignore_conditions = [JobCondition.HEALTHY]
+    await job_conditions()
+    assert job_conditions.reason in coresys.resolution.unsupported
+
+
+async def test_did_run(coresys: CoreSys):
+    """Test that the evaluation ran as expected."""
+    job_conditions = EvaluateJobConditions(coresys)
+    should_run = job_conditions.states
+    should_not_run = [state for state in CoreState if state not in should_run]
+    assert len(should_run) != 0
+    assert len(should_not_run) != 0
+
+    with patch(
+        "supervisor.resolution.evaluations.job_conditions.EvaluateJobConditions.evaluate",
+        return_value=None,
+    ) as evaluate:
+        for state in should_run:
+            coresys.core.state = state
+            await job_conditions()
+            evaluate.assert_called_once()
+            evaluate.reset_mock()
+
+        for state in should_not_run:
+            coresys.core.state = state
+            await job_conditions()
+            evaluate.assert_not_called()
+            evaluate.reset_mock()