Add restart policy evaluation (#3886)

* Add restart policy evaluation

* No container meta does not fail evaluation

supervisor/docker/const.py

@@ -28,6 +28,15 @@ class ContainerState(str, Enum):
     UNKNOWN = "unknown"
 
 
+class RestartPolicy(str, Enum):
+    """Restart policy of container."""
+
+    NO = "no"
+    ON_FAILURE = "on-failure"
+    UNLESS_STOPPED = "unless-stopped"
+    ALWAYS = "always"
+
+
 DBUS_PATH = "/run/dbus"
 DBUS_VOLUME = {"bind": DBUS_PATH, "mode": "ro"}
 

supervisor/docker/interface.py

@@ -35,7 +35,7 @@ from ..exceptions import (
 )
 from ..resolution.const import ContextType, IssueType, SuggestionType
 from ..utils import process_lock
-from .const import ContainerState
+from .const import ContainerState, RestartPolicy
 from .manager import CommandReturn
 from .monitor import DockerContainerStateEvent
 from .stats import DockerStats
@@ -134,6 +134,15 @@ class DockerInterface(CoreSysAttributes):
         """Return True if a task is in progress."""
         return self.lock.locked()
 
+    @property
+    def restart_policy(self) -> RestartPolicy | None:
+        """Return restart policy of container."""
+        if "RestartPolicy" not in self.meta_host:
+            return None
+
+        policy = self.meta_host["RestartPolicy"].get("Name")
+        return policy if policy else RestartPolicy.NO
+
     @property
     def security_opt(self) -> list[str]:
         """Control security options."""

supervisor/docker/observer.py

@@ -3,7 +3,7 @@ import logging
 
 from ..const import DOCKER_NETWORK_MASK
 from ..coresys import CoreSysAttributes
-from .const import ENV_TIME, ENV_TOKEN
+from .const import ENV_TIME, ENV_TOKEN, RestartPolicy
 from .interface import DockerInterface
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
@@ -46,7 +46,7 @@ class DockerObserver(DockerInterface, CoreSysAttributes):
             hostname=self.name.replace("_", "-"),
             detach=True,
             security_opt=self.security_opt,
-            restart_policy={"Name": "always"},
+            restart_policy={"Name": RestartPolicy.ALWAYS.value},
             extra_hosts={"supervisor": self.sys_docker.network.supervisor},
             environment={
                 ENV_TIME: self.sys_timezone,

supervisor/resolution/const.py

@@ -45,6 +45,7 @@ class UnsupportedReason(str, Enum):
     OS = "os"
     OS_AGENT = "os_agent"
     PRIVILEGED = "privileged"
+    RESTART_POLICY = "restart_policy"
     SOFTWARE = "software"
     SOURCE_MODS = "source_mods"
     SUPERVISOR_VERSION = "supervisor_version"

supervisor/resolution/evaluations/restart_policy.py

@@ -0,0 +1,72 @@
+"""Evaluation class for restart policy."""
+from supervisor.docker.const import RestartPolicy
+from supervisor.docker.interface import DockerInterface
+
+from ...const import CoreState
+from ...coresys import CoreSys
+from ..const import UnsupportedReason
+from .base import EvaluateBase
+
+
+def setup(coresys: CoreSys) -> EvaluateBase:
+    """Initialize evaluation-setup function."""
+    return EvaluateRestartPolicy(coresys)
+
+
+class EvaluateRestartPolicy(EvaluateBase):
+    """Evaluate restart policy of containers."""
+
+    def __init__(self, coresys: CoreSys) -> None:
+        """Initialize the evaluation class."""
+        super().__init__(coresys)
+        self.coresys = coresys
+        self._containers: list[str] = []
+
+    @property
+    def reason(self) -> UnsupportedReason:
+        """Return a UnsupportedReason enum."""
+        return UnsupportedReason.RESTART_POLICY
+
+    @property
+    def on_failure(self) -> str:
+        """Return a string that is printed when self.evaluate is True."""
+        return f"Found containers with unsupported restart policy: {self._containers}"
+
+    @property
+    def states(self) -> list[CoreState]:
+        """Return a list of valid states when this evaluation can run."""
+        return [CoreState.RUNNING]
+
+    @property
+    def no_restart_expected(self) -> set[DockerInterface]:
+        """Docker interfaces where no restart is expected policy."""
+        return {
+            self.sys_supervisor.instance,
+            self.sys_homeassistant.core.instance,
+            *{
+                plug.instance
+                for plug in self.sys_plugins.all_plugins
+                if plug != self.sys_plugins.observer
+            },
+            *{addon.instance for addon in self.sys_addons.installed},
+        }
+
+    @property
+    def always_restart_expected(self) -> set[DockerInterface]:
+        """Docker interfaces where always restart is expected policy."""
+        return {self.sys_plugins.observer.instance}
+
+    async def evaluate(self) -> bool:
+        """Run evaluation, return true if system fails."""
+        self._containers = {
+            instance.name
+            for instance in self.no_restart_expected
+            if instance.restart_policy and instance.restart_policy != RestartPolicy.NO
+        } | {
+            instance.name
+            for instance in self.always_restart_expected
+            if instance.restart_policy
+            and instance.restart_policy != RestartPolicy.ALWAYS
+        }
+
+        return len(self._containers) > 0

tests/fixtures/container_attrs.json

@@ -0,0 +1,253 @@
+{
+  "Id": "986e5efadb228654f1719735e802fecc099136e4640155887946246a87fc584a",
+  "Created": "2022-09-21T18:54:13.269240742Z",
+  "Path": "/init",
+  "Args": [],
+  "State": {
+    "Status": "running",
+    "Running": true,
+    "Paused": false,
+    "Restarting": false,
+    "OOMKilled": false,
+    "Dead": false,
+    "Pid": 2723,
+    "ExitCode": 0,
+    "Error": "",
+    "StartedAt": "2022-09-21T18:54:14.124021953Z",
+    "FinishedAt": "0001-01-01T00:00:00Z"
+  },
+  "Image": "sha256:bc34c81c040474bdc005e52857c04103702b081f58b70d5dbcdfc8261a03a4d9",
+  "ResolvConfPath": "/mnt/data/docker/containers/986e5efadb228654f1719735e802fecc099136e4640155887946246a87fc584a/resolv.conf",
+  "HostnamePath": "/mnt/data/docker/containers/986e5efadb228654f1719735e802fecc099136e4640155887946246a87fc584a/hostname",
+  "HostsPath": "/mnt/data/docker/containers/986e5efadb228654f1719735e802fecc099136e4640155887946246a87fc584a/hosts",
+  "LogPath": "",
+  "Name": "/addon_core_mosquitto",
+  "RestartCount": 0,
+  "Driver": "overlay2",
+  "Platform": "linux",
+  "MountLabel": "",
+  "ProcessLabel": "",
+  "AppArmorProfile": "docker-default",
+  "ExecIDs": null,
+  "HostConfig": {
+    "Binds": [
+      "/dev:/dev:ro",
+      "/mnt/data/supervisor/addons/data/core_mosquitto:/data:rw",
+      "/mnt/data/supervisor/ssl:/ssl:ro",
+      "/mnt/data/supervisor/share:/share:ro"
+    ],
+    "ContainerIDFile": "",
+    "LogConfig": { "Type": "journald", "Config": { "tag": "{{.Name}}" } },
+    "NetworkMode": "default",
+    "PortBindings": { "1883/tcp": [{ "HostIp": "", "HostPort": "1883" }] },
+    "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 },
+    "AutoRemove": false,
+    "VolumeDriver": "",
+    "VolumesFrom": null,
+    "CapAdd": null,
+    "CapDrop": null,
+    "CgroupnsMode": "private",
+    "Dns": ["172.30.32.3"],
+    "DnsOptions": null,
+    "DnsSearch": ["local.hass.io"],
+    "ExtraHosts": ["hassio:172.30.32.2", "supervisor:172.30.32.2"],
+    "GroupAdd": null,
+    "IpcMode": "private",
+    "Cgroup": "",
+    "Links": null,
+    "OomScoreAdj": 200,
+    "PidMode": "",
+    "Privileged": false,
+    "PublishAllPorts": false,
+    "ReadonlyRootfs": false,
+    "SecurityOpt": ["seccomp=unconfined"],
+    "Tmpfs": { "/dev/shm": "" },
+    "UTSMode": "",
+    "UsernsMode": "",
+    "ShmSize": 67108864,
+    "Runtime": "runc",
+    "ConsoleSize": [0, 0],
+    "Isolation": "",
+    "CpuShares": 0,
+    "Memory": 0,
+    "NanoCpus": 0,
+    "CgroupParent": "",
+    "BlkioWeight": 0,
+    "BlkioWeightDevice": null,
+    "BlkioDeviceReadBps": null,
+    "BlkioDeviceWriteBps": null,
+    "BlkioDeviceReadIOps": null,
+    "BlkioDeviceWriteIOps": null,
+    "CpuPeriod": 0,
+    "CpuQuota": 0,
+    "CpuRealtimePeriod": 0,
+    "CpuRealtimeRuntime": 0,
+    "CpusetCpus": "",
+    "CpusetMems": "",
+    "Devices": null,
+    "DeviceCgroupRules": null,
+    "DeviceRequests": null,
+    "KernelMemory": 0,
+    "KernelMemoryTCP": 0,
+    "MemoryReservation": 0,
+    "MemorySwap": 0,
+    "MemorySwappiness": null,
+    "OomKillDisable": null,
+    "PidsLimit": null,
+    "Ulimits": null,
+    "CpuCount": 0,
+    "CpuPercent": 0,
+    "IOMaximumIOps": 0,
+    "IOMaximumBandwidth": 0,
+    "MaskedPaths": [
+      "/proc/asound",
+      "/proc/acpi",
+      "/proc/kcore",
+      "/proc/keys",
+      "/proc/latency_stats",
+      "/proc/timer_list",
+      "/proc/timer_stats",
+      "/proc/sched_debug",
+      "/proc/scsi",
+      "/sys/firmware"
+    ],
+    "ReadonlyPaths": [
+      "/proc/bus",
+      "/proc/fs",
+      "/proc/irq",
+      "/proc/sys",
+      "/proc/sysrq-trigger"
+    ],
+    "Init": false
+  },
+  "GraphDriver": {
+    "Data": {
+      "LowerDir": "/mnt/data/docker/overlay2/1b79280bcdd879c0df64ee41755c70610cf3b650cea299c6f30081338bf745e5-init/diff:/mnt/data/docker/overlay2/2c22d6a27a68fb384f85d4bb833f7ac7956fcbb93be9f731e6deb76bb9420011/diff:/mnt/data/docker/overlay2/42ce3dc994391b83f90bf599f887a8e0fdc2ee24cec9b14f491412f1dae11714/diff:/mnt/data/docker/overlay2/9f7c5f5a2aa9a324c75f9c54fbee648bee86d963c4ad6fde2cb3ca0d3f886287/diff:/mnt/data/docker/overlay2/8a4cd2bb72bb673a42186a1b94aaf1bbba1f1e35953fea91e6ccc2c08b0c7dc8/diff:/mnt/data/docker/overlay2/eb50a9462c19acb312d44e3455579f84f3ef256e410c4992554f1527fbd2b9dc/diff:/mnt/data/docker/overlay2/d38688df678f243a89c055f768b5ec80f1ce0686ed28940fcc4430edc5b311a0/diff",
+      "MergedDir": "/mnt/data/docker/overlay2/1b79280bcdd879c0df64ee41755c70610cf3b650cea299c6f30081338bf745e5/merged",
+      "UpperDir": "/mnt/data/docker/overlay2/1b79280bcdd879c0df64ee41755c70610cf3b650cea299c6f30081338bf745e5/diff",
+      "WorkDir": "/mnt/data/docker/overlay2/1b79280bcdd879c0df64ee41755c70610cf3b650cea299c6f30081338bf745e5/work"
+    },
+    "Name": "overlay2"
+  },
+  "Mounts": [
+    {
+      "Type": "bind",
+      "Source": "/dev",
+      "Destination": "/dev",
+      "Mode": "ro",
+      "RW": false,
+      "Propagation": "rprivate"
+    },
+    {
+      "Type": "bind",
+      "Source": "/mnt/data/supervisor/addons/data/core_mosquitto",
+      "Destination": "/data",
+      "Mode": "rw",
+      "RW": true,
+      "Propagation": "rprivate"
+    },
+    {
+      "Type": "bind",
+      "Source": "/mnt/data/supervisor/ssl",
+      "Destination": "/ssl",
+      "Mode": "ro",
+      "RW": false,
+      "Propagation": "rprivate"
+    },
+    {
+      "Type": "bind",
+      "Source": "/mnt/data/supervisor/share",
+      "Destination": "/share",
+      "Mode": "ro",
+      "RW": false,
+      "Propagation": "rprivate"
+    }
+  ],
+  "Config": {
+    "Hostname": "core-mosquitto",
+    "Domainname": "local.hass.io",
+    "User": "",
+    "AttachStdin": false,
+    "AttachStdout": false,
+    "AttachStderr": false,
+    "ExposedPorts": { "1883/tcp": {} },
+    "Tty": false,
+    "OpenStdin": false,
+    "StdinOnce": false,
+    "Env": [
+      "TZ=America/New_York",
+      "SUPERVISOR_TOKEN=abc123",
+      "HASSIO_TOKEN=abc123",
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+      "LANG=C.UTF-8",
+      "DEBIAN_FRONTEND=noninteractive",
+      "CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt",
+      "S6_BEHAVIOUR_IF_STAGE2_FAILS=2",
+      "S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0",
+      "S6_CMD_WAIT_FOR_SERVICES=1"
+    ],
+    "Cmd": null,
+    "Image": "homeassistant/aarch64-addon-mosquitto:6.1.3",
+    "Volumes": { "/data": {}, "/dev": {}, "/share": {}, "/ssl": {} },
+    "WorkingDir": "/",
+    "Entrypoint": ["/init"],
+    "OnBuild": null,
+    "Labels": {
+      "io.hass.arch": "aarch64",
+      "io.hass.base.arch": "aarch64",
+      "io.hass.base.image": "arm64v8/debian:bullseye-slim",
+      "io.hass.base.name": "debian",
+      "io.hass.base.version": "2022.08.0",
+      "io.hass.description": "An Open Source MQTT broker",
+      "io.hass.name": "Mosquitto broker",
+      "io.hass.type": "addon",
+      "io.hass.url": "https://github.com/home-assistant/hassio-addons/tree/master/mosquitto",
+      "io.hass.version": "6.1.3",
+      "org.opencontainers.image.created": "2022-08-30 07:33:03+00:00",
+      "org.opencontainers.image.source": "https://github.com/home-assistant/docker-base",
+      "org.opencontainers.image.version": "6.1.3",
+      "supervisor_managed": ""
+    }
+  },
+  "NetworkSettings": {
+    "Bridge": "",
+    "SandboxID": "067cd11a63f96d227dcc0f01d3e4f5053c368021becd0b4b2da4f301cfda3d29",
+    "HairpinMode": false,
+    "LinkLocalIPv6Address": "",
+    "LinkLocalIPv6PrefixLen": 0,
+    "Ports": {
+      "1883/tcp": [
+        { "HostIp": "0.0.0.0", "HostPort": "1883" },
+        { "HostIp": "::", "HostPort": "1883" }
+      ]
+    },
+    "SandboxKey": "/var/run/docker/netns/067cd11a63f9",
+    "SecondaryIPAddresses": null,
+    "SecondaryIPv6Addresses": null,
+    "EndpointID": "",
+    "Gateway": "",
+    "GlobalIPv6Address": "",
+    "GlobalIPv6PrefixLen": 0,
+    "IPAddress": "",
+    "IPPrefixLen": 0,
+    "IPv6Gateway": "",
+    "MacAddress": "",
+    "Networks": {
+      "hassio": {
+        "IPAMConfig": null,
+        "Links": null,
+        "Aliases": ["core-mosquitto", "986e5efadb22"],
+        "NetworkID": "cd10b1eb5f4a1cd5179839f81ccdadd29545eaa0b921454d1a2e0452c12d6935",
+        "EndpointID": "9b2c58f4595618241bb45df028c95f2713bb0b1b6326d3bdab2366e2caadbe7b",
+        "Gateway": "172.30.32.1",
+        "IPAddress": "172.30.33.1",
+        "IPPrefixLen": 23,
+        "IPv6Gateway": "",
+        "GlobalIPv6Address": "",
+        "GlobalIPv6PrefixLen": 0,
+        "MacAddress": "02:42:ac:1e:21:01",
+        "DriverOpts": null
+      }
+    }
+  }
+}

tests/resolution/evaluation/test_restart_policy.py

@@ -0,0 +1,81 @@
+"""Test evaluate restart policy.."""
+
+from unittest.mock import MagicMock, patch
+
+from awesomeversion import AwesomeVersion
+
+from supervisor.addons.addon import Addon
+from supervisor.const import CoreState
+from supervisor.coresys import CoreSys
+from supervisor.resolution.evaluations.restart_policy import EvaluateRestartPolicy
+
+from tests.common import load_json_fixture
+
+TEST_VERSION = AwesomeVersion("1.0.0")
+
+
+async def test_evaluation(coresys: CoreSys, install_addon_ssh: Addon):
+    """Test evaluation."""
+    restart_policy = EvaluateRestartPolicy(coresys)
+    coresys.core.state = CoreState.RUNNING
+
+    await restart_policy()
+    assert restart_policy.reason not in coresys.resolution.unsupported
+
+    no_restart_attrs = load_json_fixture("container_attrs.json")
+    always_restart_attrs = load_json_fixture("container_attrs.json")
+    always_restart_attrs["HostConfig"]["RestartPolicy"]["Name"] = "always"
+    addon_attrs = no_restart_attrs
+    observer_attrs = always_restart_attrs
+
+    def get_container(name: str):
+        meta = MagicMock()
+        meta.attrs = observer_attrs if name == "hassio_observer" else addon_attrs
+        return meta
+
+    coresys.docker.containers.get = get_container
+    await coresys.plugins.observer.instance.attach(TEST_VERSION)
+    await install_addon_ssh.instance.attach(TEST_VERSION)
+
+    await restart_policy()
+    assert restart_policy.reason not in coresys.resolution.unsupported
+
+    addon_attrs = always_restart_attrs
+    await install_addon_ssh.instance.attach(TEST_VERSION)
+    await restart_policy()
+    assert restart_policy.reason in coresys.resolution.unsupported
+
+    addon_attrs = no_restart_attrs
+    await install_addon_ssh.instance.attach(TEST_VERSION)
+    await restart_policy()
+    assert restart_policy.reason not in coresys.resolution.unsupported
+
+    observer_attrs = no_restart_attrs
+    await coresys.plugins.observer.instance.attach(TEST_VERSION)
+    await restart_policy()
+    assert restart_policy.reason in coresys.resolution.unsupported
+
+
+async def test_did_run(coresys: CoreSys):
+    """Test that the evaluation ran as expected."""
+    restart_policy = EvaluateRestartPolicy(coresys)
+    should_run = restart_policy.states
+    should_not_run = [state for state in CoreState if state not in should_run]
+    assert len(should_run) != 0
+    assert len(should_not_run) != 0
+
+    with patch(
+        "supervisor.resolution.evaluations.restart_policy.EvaluateRestartPolicy.evaluate",
+        return_value=False,
+    ) as evaluate:
+        for state in should_run:
+            coresys.core.state = state
+            await restart_policy()
+            evaluate.assert_called_once()
+            evaluate.reset_mock()
+
+        for state in should_not_run:
+            coresys.core.state = state
+            await restart_policy()
+            evaluate.assert_not_called()
+            evaluate.reset_mock()