Add issues/suggestion to resolution center / start with diskspace (#2125)

Co-authored-by: Pascal Vizeli <pvizeli@syshack.ch>

supervisor/api/__init__.py

@@ -197,7 +197,16 @@ class RestAPI(CoreSysAttributes):
         api_resolution = APIResoulution()
         api_resolution.coresys = self.coresys
 
-        self.webapp.add_routes([web.get("/resolution", api_resolution.base)])
+        self.webapp.add_routes(
+            [
+                web.get("/resolution", api_resolution.base),
+                web.post("/resolution/{suggestion}", api_resolution.apply_suggestion),
+                web.post(
+                    "/resolution/{suggestion}/dismiss",
+                    api_resolution.dismiss_suggestion,
+                ),
+            ]
+        )
 
     def _register_auth(self) -> None:
         """Register auth functions."""

supervisor/api/resolution.py

@@ -3,8 +3,10 @@ from typing import Any, Dict
 
 from aiohttp import web
 
-from ..const import ATTR_UNSUPPORTED
+from ..const import ATTR_ISSUES, ATTR_SUGGESTIONS, ATTR_UNSUPPORTED
 from ..coresys import CoreSysAttributes
+from ..exceptions import APIError
+from ..resolution.const import Suggestion
 from .utils import api_process
 
 
@@ -14,4 +16,26 @@ class APIResoulution(CoreSysAttributes):
     @api_process
     async def base(self, request: web.Request) -> Dict[str, Any]:
         """Return network information."""
-        return {ATTR_UNSUPPORTED: self.sys_resolution.unsupported}
+        return {
+            ATTR_UNSUPPORTED: self.sys_resolution.unsupported,
+            ATTR_SUGGESTIONS: self.sys_resolution.suggestions,
+            ATTR_ISSUES: self.sys_resolution.issues,
+        }
+
+    @api_process
+    async def apply_suggestion(self, request: web.Request) -> None:
+        """Apply suggestion."""
+        try:
+            suggestion = Suggestion(request.match_info.get("suggestion"))
+            await self.sys_resolution.apply_suggestion(suggestion)
+        except ValueError:
+            raise APIError("Suggestion is not valid") from None
+
+    @api_process
+    async def dismiss_suggestion(self, request: web.Request) -> None:
+        """Dismiss suggestion."""
+        try:
+            suggestion = Suggestion(request.match_info.get("suggestion"))
+            await self.sys_resolution.dismiss_suggestion(suggestion)
+        except ValueError:
+            raise APIError("Suggestion is not valid") from None

supervisor/const.py

@@ -162,6 +162,7 @@ ATTR_HOST_NETWORK = "host_network"
 ATTR_HOST_PID = "host_pid"
 ATTR_HOSTNAME = "hostname"
 ATTR_ICON = "icon"
+ATTR_ISSUES = "issues"
 ATTR_ID = "id"
 ATTR_IMAGE = "image"
 ATTR_IMAGES = "images"
@@ -249,6 +250,7 @@ ATTR_STATE = "state"
 ATTR_STATIC = "static"
 ATTR_STDIN = "stdin"
 ATTR_STORAGE = "storage"
+ATTR_SUGGESTIONS = "suggestions"
 ATTR_SUPERVISOR = "supervisor"
 ATTR_SUPPORTED = "supported"
 ATTR_SUPPORTED_ARCH = "supported_arch"
@@ -429,17 +431,3 @@ class HostFeature(str, Enum):
     REBOOT = "reboot"
     SERVICES = "services"
     SHUTDOWN = "shutdown"
-
-
-class UnsupportedReason(str, Enum):
-    """Reasons for unsupported status."""
-
-    CONTAINER = "container"
-    DBUS = "dbus"
-    DOCKER_CONFIGURATION = "docker_configuration"
-    DOCKER_VERSION = "docker_version"
-    LXC = "lxc"
-    NETWORK_MANAGER = "network_manager"
-    OS = "os"
-    PRIVILEGED = "privileged"
-    SYSTEMD = "systemd"

supervisor/core.py

@@ -13,7 +13,6 @@ from .const import (
     AddonStartup,
     CoreState,
     HostFeature,
-    UnsupportedReason,
 )
 from .coresys import CoreSys, CoreSysAttributes
 from .exceptions import (
@@ -23,6 +22,7 @@ from .exceptions import (
     HomeAssistantError,
     SupervisorUpdateError,
 )
+from .resolution.const import UnsupportedReason
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 
@@ -159,6 +159,9 @@ class Core(CoreSysAttributes):
         # Load ingress
         await self.sys_ingress.load()
 
+        # Load Resoulution
+        await self.sys_resolution.load()
+
         # Check supported OS
         if not self.sys_hassos.available:
             if self.sys_host.info.operating_system not in SUPERVISED_SUPPORTED_OS:

supervisor/misc/tasks.py

@@ -12,6 +12,7 @@ from ..exceptions import (
     MulticastError,
     ObserverError,
 )
+from ..resolution.const import MINIMUM_FREE_SPACE_THRESHOLD, IssueType
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 
@@ -127,6 +128,11 @@ class Tasks(CoreSysAttributes):
                 )
                 continue
 
+            if self.sys_host.info.free_space > MINIMUM_FREE_SPACE_THRESHOLD:
+                _LOGGER.warning("Not enough free space, pausing add-on updates")
+                self.sys_resolution.issues = IssueType.FREE_SPACE
+                return
+
             # Run Add-on update sequential
             # avoid issue on slow IO
             _LOGGER.info("Add-on auto update process %s", addon.slug)
@@ -145,6 +151,11 @@ class Tasks(CoreSysAttributes):
             _LOGGER.warning("Ignore Supervisor update on dev channel!")
             return
 
+        if self.sys_host.info.free_space > MINIMUM_FREE_SPACE_THRESHOLD:
+            _LOGGER.warning("Not enough free space, pausing supervisor update")
+            self.sys_resolution.issues = IssueType.FREE_SPACE
+            return
+
         _LOGGER.info("Found new Supervisor version")
         await self.sys_supervisor.update()
 

supervisor/resolution/__init__.py

@@ -1,8 +1,14 @@
 """Supervisor resolution center."""
+import logging
 from typing import List
 
-from ..const import UnsupportedReason
 from ..coresys import CoreSys, CoreSysAttributes
+from ..resolution.const import UnsupportedReason
+from .const import SCHEDULED_HEALTHCHECK, IssueType, Suggestion
+from .free_space import ResolutionStorage
+from .notify import ResolutionNotify
+
+_LOGGER: logging.Logger = logging.getLogger(__name__)
 
 
 class ResolutionManager(CoreSysAttributes):
@@ -11,8 +17,45 @@ class ResolutionManager(CoreSysAttributes):
     def __init__(self, coresys: CoreSys):
         """Initialize Resolution manager."""
         self.coresys: CoreSys = coresys
+        self._notify = ResolutionNotify(coresys)
+        self._storage = ResolutionStorage(coresys)
+        self._dismissed_suggestions: List[Suggestion] = []
+        self._suggestions: List[Suggestion] = []
+        self._issues: List[IssueType] = []
         self._unsupported: List[UnsupportedReason] = []
 
+    @property
+    def storage(self) -> ResolutionStorage:
+        """Return the ResolutionStorage class."""
+        return self._storage
+
+    @property
+    def notify(self) -> ResolutionNotify:
+        """Return the ResolutionNotify class."""
+        return self._notify
+
+    @property
+    def issues(self) -> List[IssueType]:
+        """Return a list of issues."""
+        return self._issues
+
+    @issues.setter
+    def issues(self, issue: IssueType) -> None:
+        """Add issues."""
+        if issue not in self._issues:
+            self._issues.append(issue)
+
+    @property
+    def suggestions(self) -> List[Suggestion]:
+        """Return a list of suggestions that can handled."""
+        return [x for x in self._suggestions if x not in self._dismissed_suggestions]
+
+    @suggestions.setter
+    def suggestions(self, suggestion: Suggestion) -> None:
+        """Add suggestion."""
+        if suggestion not in self._suggestions:
+            self._suggestions.append(suggestion)
+
     @property
     def unsupported(self) -> List[UnsupportedReason]:
         """Return a list of unsupported reasons."""
@@ -21,4 +64,45 @@ class ResolutionManager(CoreSysAttributes):
     @unsupported.setter
     def unsupported(self, reason: UnsupportedReason) -> None:
         """Add a reason for unsupported."""
+        if reason not in self._unsupported:
             self._unsupported.append(reason)
+
+    async def load(self):
+        """Load the resoulution manager."""
+        # Initial healthcheck when the manager is loaded
+        await self.healthcheck()
+
+        # Schedule the healthcheck
+        self.sys_scheduler.register_task(self.healthcheck, SCHEDULED_HEALTHCHECK)
+
+    async def healthcheck(self):
+        """Scheduled task to check for known issues."""
+        # Check free space
+        self.sys_run_in_executor(self.storage.check_free_space)
+
+        # Create notification for any known issues
+        await self.notify.issue_notifications()
+
+    async def apply_suggestion(self, suggestion: Suggestion) -> None:
+        """Apply suggested action."""
+        if suggestion not in self.suggestions:
+            _LOGGER.warning("Suggestion %s is not valid", suggestion)
+            return
+
+        if suggestion == Suggestion.CLEAR_FULL_SNAPSHOT:
+            self.storage.clean_full_snapshots()
+
+        elif suggestion == Suggestion.CREATE_FULL_SNAPSHOT:
+            await self.sys_snapshots.do_snapshot_full()
+
+        self._suggestions.remove(suggestion)
+        await self.healthcheck()
+
+    async def dismiss_suggestion(self, suggestion: Suggestion) -> None:
+        """Dismiss suggested action."""
+        if suggestion not in self.suggestions:
+            _LOGGER.warning("Suggestion %s is not valid", suggestion)
+            return
+
+        if suggestion not in self._dismissed_suggestions:
+            self._dismissed_suggestions.append(suggestion)

supervisor/resolution/const.py

@@ -0,0 +1,34 @@
+"""Constants for the resoulution manager."""
+from enum import Enum
+
+SCHEDULED_HEALTHCHECK = 3600
+
+MINIMUM_FREE_SPACE_THRESHOLD = 1
+MINIMUM_FULL_SNAPSHOTS = 2
+
+
+class UnsupportedReason(str, Enum):
+    """Reasons for unsupported status."""
+
+    CONTAINER = "container"
+    DBUS = "dbus"
+    DOCKER_CONFIGURATION = "docker_configuration"
+    DOCKER_VERSION = "docker_version"
+    LXC = "lxc"
+    NETWORK_MANAGER = "network_manager"
+    OS = "os"
+    PRIVILEGED = "privileged"
+    SYSTEMD = "systemd"
+
+
+class IssueType(str, Enum):
+    """Issue type."""
+
+    FREE_SPACE = "free_space"
+
+
+class Suggestion(str, Enum):
+    """Sugestion."""
+
+    CLEAR_FULL_SNAPSHOT = "clear_full_snapshot"
+    CREATE_FULL_SNAPSHOT = "create_full_snapshot"

supervisor/resolution/free_space.py

@@ -0,0 +1,57 @@
+"""Helpers to check and fix issues with free space."""
+import logging
+
+from ..const import SNAPSHOT_FULL
+from ..coresys import CoreSys, CoreSysAttributes
+from .const import (
+    MINIMUM_FREE_SPACE_THRESHOLD,
+    MINIMUM_FULL_SNAPSHOTS,
+    IssueType,
+    Suggestion,
+)
+
+_LOGGER: logging.Logger = logging.getLogger(__name__)
+
+
+class ResolutionStorage(CoreSysAttributes):
+    """Storage class for resolution."""
+
+    def __init__(self, coresys: CoreSys):
+        """Initialize the storage class."""
+        self.coresys = coresys
+
+    def check_free_space(self) -> None:
+        """Check free space."""
+        if self.sys_host.info.free_space > MINIMUM_FREE_SPACE_THRESHOLD:
+            return
+
+        self.sys_resolution.issues = IssueType.FREE_SPACE
+
+        if (
+            len(
+                [
+                    x
+                    for x in self.sys_snapshots.list_snapshots
+                    if x.sys_type == SNAPSHOT_FULL
+                ]
+            )
+            >= MINIMUM_FULL_SNAPSHOTS
+        ):
+            self.sys_resolution.suggestions = Suggestion.CLEAR_FULL_SNAPSHOT
+
+        elif len(self.sys_snapshots.list_snapshots) == 0:
+            # No snapshots, let's suggest the user to create one!
+            self.sys_resolution.suggestions = Suggestion.CREATE_FULL_SNAPSHOT
+
+    def clean_full_snapshots(self):
+        """Clean out all old full snapshots, but keep the most recent."""
+        full_snapshots = [
+            x for x in self.sys_snapshots.list_snapshots if x.sys_type == SNAPSHOT_FULL
+        ]
+
+        if len(full_snapshots) < MINIMUM_FULL_SNAPSHOTS:
+            return
+
+        _LOGGER.info("Starting removal of old full snapshots")
+        for snapshot in sorted(full_snapshots, key=lambda x: x.date)[:-1]:
+            self.sys_snapshots.remove(snapshot)

supervisor/resolution/notify.py

@@ -0,0 +1,55 @@
+"""
+Helper to notify Core about issues.
+
+This helper creates persistant notification in the Core UI.
+In the future we want to remove this in favour of a "center" in the UI.
+"""
+import logging
+
+from ..coresys import CoreSys, CoreSysAttributes
+from ..exceptions import HomeAssistantAPIError
+from .const import IssueType
+
+_LOGGER: logging.Logger = logging.getLogger(__name__)
+
+
+class ResolutionNotify(CoreSysAttributes):
+    """Notify class for resolution."""
+
+    def __init__(self, coresys: CoreSys):
+        """Initialize the notify class."""
+        self.coresys = coresys
+
+    async def issue_notifications(self):
+        """Create persistant notifications about issues."""
+        if (
+            not self.sys_resolution.issues
+            or not self.sys_homeassistant.api.check_api_state()
+        ):
+            return
+
+        issues = []
+
+        for issue in self.sys_resolution.issues:
+            if issue == IssueType.FREE_SPACE:
+                issues.append(
+                    {
+                        "title": "Available space is less than 1GB!",
+                        "message": f"Available space is {self.sys_host.info.free_space}GB, see https://www.home-assistant.io/more-info/free-space for more information.",
+                        "notification_id": "supervisor_issue_free_space",
+                    }
+                )
+
+        for issue in issues:
+            try:
+                async with self.sys_homeassistant.api.make_request(
+                    "post",
+                    "api/services/persistent_notification/create",
+                    json=issue,
+                ) as resp:
+                    if resp.status in (200, 201):
+                        _LOGGER.debug("Sucessfully created persistent_notification")
+                    else:
+                        _LOGGER.error("Can't create persistant notification")
+            except HomeAssistantAPIError:
+                _LOGGER.error("Can't create persistant notification")

supervisor/snapshots/__init__.py

@@ -2,6 +2,7 @@
 import asyncio
 import logging
 from pathlib import Path
+from typing import Set
 
 from ..const import FOLDER_HOMEASSISTANT, SNAPSHOT_FULL, SNAPSHOT_PARTIAL, CoreState
 from ..coresys import CoreSysAttributes
@@ -23,7 +24,7 @@ class SnapshotManager(CoreSysAttributes):
         self.lock = asyncio.Lock()
 
     @property
-    def list_snapshots(self):
+    def list_snapshots(self) -> Set[Snapshot]:
         """Return a list of all snapshot object."""
         return set(self.snapshots_obj.values())
 
@@ -139,8 +140,9 @@ class SnapshotManager(CoreSysAttributes):
                 _LOGGER.info("Snapshot %s store folders", snapshot.slug)
                 await snapshot.store_folders()
 
-        except Exception:  # pylint: disable=broad-except
+        except Exception as excep:  # pylint: disable=broad-except
             _LOGGER.exception("Snapshot %s error", snapshot.slug)
+            print(excep)
             return None
 
         else:

tests/api/test_resolution.py

@@ -1,13 +1,47 @@
 """Test Resolution API."""
+from unittest.mock import MagicMock, patch
+
 import pytest
 
-from supervisor.const import ATTR_UNSUPPORTED, UnsupportedReason
+from supervisor.const import ATTR_ISSUES, ATTR_SUGGESTIONS, ATTR_UNSUPPORTED
+from supervisor.coresys import CoreSys
+from supervisor.resolution.const import IssueType, Suggestion, UnsupportedReason
 
 
 @pytest.mark.asyncio
-async def test_api_resolution_base(coresys, api_client):
+async def test_api_resolution_base(coresys: CoreSys, api_client):
     """Test resolution manager api."""
     coresys.resolution.unsupported = UnsupportedReason.OS
+    coresys.resolution.suggestions = Suggestion.CLEAR_FULL_SNAPSHOT
+    coresys.resolution.issues = IssueType.FREE_SPACE
     resp = await api_client.get("/resolution")
     result = await resp.json()
     assert UnsupportedReason.OS in result["data"][ATTR_UNSUPPORTED]
+    assert Suggestion.CLEAR_FULL_SNAPSHOT in result["data"][ATTR_SUGGESTIONS]
+    assert IssueType.FREE_SPACE in result["data"][ATTR_ISSUES]
+
+
+@pytest.mark.asyncio
+async def test_api_resolution_dismiss_suggestion(coresys: CoreSys, api_client):
+    """Test resolution manager suggestion apply api."""
+    coresys.resolution.suggestions = Suggestion.CLEAR_FULL_SNAPSHOT
+
+    assert Suggestion.CLEAR_FULL_SNAPSHOT in coresys.resolution.suggestions
+    await coresys.resolution.dismiss_suggestion(Suggestion.CLEAR_FULL_SNAPSHOT)
+    assert Suggestion.CLEAR_FULL_SNAPSHOT not in coresys.resolution.suggestions
+
+
+@pytest.mark.asyncio
+async def test_api_resolution_apply_suggestion(coresys: CoreSys, api_client):
+    """Test resolution manager suggestion apply api."""
+    coresys.resolution.suggestions = Suggestion.CLEAR_FULL_SNAPSHOT
+    coresys.resolution.suggestions = Suggestion.CREATE_FULL_SNAPSHOT
+
+    with patch("supervisor.snapshots.SnapshotManager", return_value=MagicMock()):
+        await coresys.resolution.apply_suggestion(Suggestion.CLEAR_FULL_SNAPSHOT)
+        await coresys.resolution.apply_suggestion(Suggestion.CREATE_FULL_SNAPSHOT)
+
+        assert Suggestion.CLEAR_FULL_SNAPSHOT not in coresys.resolution.suggestions
+        assert Suggestion.CREATE_FULL_SNAPSHOT not in coresys.resolution.suggestions
+
+    await coresys.resolution.apply_suggestion(Suggestion.CLEAR_FULL_SNAPSHOT)

tests/misc/test_filter_data.py

@@ -4,9 +4,10 @@ from unittest.mock import patch
 
 import pytest
 
-from supervisor.const import SUPERVISOR_VERSION, CoreState, UnsupportedReason
+from supervisor.const import SUPERVISOR_VERSION, CoreState
 from supervisor.exceptions import AddonConfigurationError
 from supervisor.misc.filter import filter_data
+from supervisor.resolution.const import UnsupportedReason
 
 SAMPLE_EVENT = {"sample": "event", "extra": {"Test": "123"}}
 

tests/resolution/test_resolution_manager.py

@@ -1,8 +1,18 @@
 """Tests for resolution manager."""
+from pathlib import Path
 
-
-from supervisor.const import UnsupportedReason
+from supervisor.const import (
+    ATTR_DATE,
+    ATTR_SLUG,
+    ATTR_TYPE,
+    SNAPSHOT_FULL,
+    SNAPSHOT_PARTIAL,
+)
 from supervisor.coresys import CoreSys
+from supervisor.resolution.const import UnsupportedReason
+from supervisor.snapshots.snapshot import Snapshot
+from supervisor.utils.dt import utcnow
+from supervisor.utils.tar import SecureTarFile
 
 
 def test_properies(coresys: CoreSys):
@@ -12,3 +22,39 @@ def test_properies(coresys: CoreSys):
 
     coresys.resolution.unsupported = UnsupportedReason.OS
     assert not coresys.core.supported
+
+
+async def test_clear_snapshots(coresys: CoreSys, tmp_path):
+    """Test snapshot cleanup."""
+    for slug in ["sn1", "sn2", "sn3", "sn4", "sn5"]:
+        temp_tar = Path(tmp_path, f"{slug}.tar")
+        with SecureTarFile(temp_tar, "w"):
+            pass
+        snapshot = Snapshot(coresys, temp_tar)
+        snapshot._data = {  # pylint: disable=protected-access
+            ATTR_SLUG: slug,
+            ATTR_DATE: utcnow().isoformat(),
+            ATTR_TYPE: SNAPSHOT_PARTIAL
+            if "1" in slug or "5" in slug
+            else SNAPSHOT_FULL,
+        }
+        coresys.snapshots.snapshots_obj[snapshot.slug] = snapshot
+
+    newest_full_snapshot = coresys.snapshots.snapshots_obj["sn4"]
+
+    assert newest_full_snapshot in coresys.snapshots.list_snapshots
+    assert (
+        len(
+            [x for x in coresys.snapshots.list_snapshots if x.sys_type == SNAPSHOT_FULL]
+        )
+        == 3
+    )
+
+    coresys.resolution.storage.clean_full_snapshots()
+    assert newest_full_snapshot in coresys.snapshots.list_snapshots
+    assert (
+        len(
+            [x for x in coresys.snapshots.list_snapshots if x.sys_type == SNAPSHOT_FULL]
+        )
+        == 1
+    )