Repair / fixup docker overlayfs issues (#1170)

* Add a repair modus

* Add repair to add-ons

* repair to cli

* Add API call

* fix sync call

* Clean all images

* Fix repair

* Fix supervisor

* Add new function to core

* fix tagging

* better style

* use retag

* new retag function

* Fix lint

* Fix import export

API.md

@@ -112,6 +112,10 @@ Output is the raw docker log.
 }
 ```
 
+- GET `/supervisor/repair`
+
+Repair overlayfs issue and restore lost images
+
 ### Snapshot
 
 - GET `/snapshots`

hassio/addons/__init__.py

@@ -256,3 +256,38 @@ class AddonManager(CoreSysAttributes):
 
         _LOGGER.info("Detect new Add-on after restore %s", slug)
         self.local[slug] = addon
+
+    async def repair(self) -> None:
+        """Repair local add-ons."""
+        needs_repair: List[Addon] = []
+
+        # Evaluate Add-ons to repair
+        for addon in self.installed:
+            if await addon.instance.exists():
+                continue
+            needs_repair.append(addon)
+
+        _LOGGER.info("Found %d add-ons to repair", len(needs_repair))
+        if not needs_repair:
+            return
+
+        for addon in needs_repair:
+            _LOGGER.info("Start repair for add-on: %s", addon.slug)
+
+            with suppress(DockerAPIError, KeyError):
+                # Need pull a image again
+                if not addon.need_build:
+                    await addon.instance.install(addon.version, addon.image)
+                    continue
+
+                # Need local lookup
+                elif addon.need_build and not addon.is_detached:
+                    store = self.store[addon.slug]
+                    # If this add-on is available for rebuild
+                    if addon.version == store.version:
+                        await addon.instance.install(addon.version, addon.image)
+                        continue
+
+            _LOGGER.error("Can't repair %s", addon.slug)
+            with suppress(AddonsError):
+                await self.uninstall(addon.slug)

hassio/addons/addon.py

@@ -618,7 +618,7 @@ class Addon(AddonModel):
                 image_file = Path(temp, "image.tar")
                 if image_file.is_file():
                     with suppress(DockerAPIError):
-                        await self.instance.import_image(image_file, version)
+                        await self.instance.import_image(image_file)
                 else:
                     with suppress(DockerAPIError):
                         await self.instance.install(version, restore_image)

hassio/api/__init__.py

@@ -130,6 +130,7 @@ class RestAPI(CoreSysAttributes):
                 web.post("/supervisor/update", api_supervisor.update),
                 web.post("/supervisor/reload", api_supervisor.reload),
                 web.post("/supervisor/options", api_supervisor.options),
+                web.post("/supervisor/repair", api_supervisor.repair),
             ]
         )
 

hassio/api/supervisor.py

@@ -161,6 +161,11 @@ class APISupervisor(CoreSysAttributes):
         """Reload add-ons, configuration, etc."""
         return asyncio.shield(self.sys_updater.reload())
 
+    @api_process
+    def repair(self, request: web.Request) -> Awaitable[None]:
+        """Try to repair the local setup / overlayfs."""
+        return asyncio.shield(self.sys_core.repair())
+
     @api_process_raw(CONTENT_TYPE_BINARY)
     def logs(self, request: web.Request) -> Awaitable[bytes]:
         """Return supervisor Docker logs."""

hassio/core.py

@@ -170,3 +170,18 @@ class HassIO(CoreSysAttributes):
         """Update last boot time."""
         self.sys_config.last_boot = self.sys_hardware.last_boot
         self.sys_config.save_data()
+
+    async def repair(self):
+        """Repair system integrity."""
+        await self.sys_run_in_executor(self.sys_docker.repair)
+
+        # Restore core functionality
+        await self.sys_addons.repair()
+        await self.sys_homeassistant.repair()
+
+        # Fix HassOS specific
+        if self.sys_hassos.available:
+            await self.sys_hassos.repair_cli()
+
+        # Tag version for latest
+        await self.sys_supervisor.repair()

hassio/docker/__init__.py

@@ -139,3 +139,34 @@ class DockerAPI:
                 container.remove(force=True)
 
         return CommandReturn(result.get("StatusCode"), output)
+
+    def repair(self) -> None:
+        """Repair local docker overlayfs2 issues."""
+
+        _LOGGER.info("Prune stale containers")
+        try:
+            output = self.docker.api.prune_containers()
+            _LOGGER.debug("Containers prune: %s", output)
+        except docker.errors.APIError as err:
+            _LOGGER.warning("Error for containers prune: %s", err)
+
+        _LOGGER.info("Prune stale images")
+        try:
+            output = self.docker.api.prune_images(filters={"dangling": False})
+            _LOGGER.debug("Images prune: %s", output)
+        except docker.errors.APIError as err:
+            _LOGGER.warning("Error for images prune: %s", err)
+
+        _LOGGER.info("Prune stale builds")
+        try:
+            output = self.docker.api.prune_builds()
+            _LOGGER.debug("Builds prune: %s", output)
+        except docker.errors.APIError as err:
+            _LOGGER.warning("Error for builds prune: %s", err)
+
+        _LOGGER.info("Prune stale volumes")
+        try:
+            output = self.docker.api.prune_builds()
+            _LOGGER.debug("Volumes prune: %s", output)
+        except docker.errors.APIError as err:
+            _LOGGER.warning("Error for volumes prune: %s", err)

hassio/docker/addon.py

@@ -397,7 +397,7 @@ class DockerAddon(DockerInterface):
         Need run inside executor.
         """
         try:
-            image = self.sys_docker.api.get_image(self.image)
+            image = self.sys_docker.api.get_image(f"{self.image}:{self.version}")
         except docker.errors.DockerException as err:
             _LOGGER.error("Can't fetch image %s: %s", self.image, err)
             raise DockerAPIError() from None
@@ -414,11 +414,11 @@ class DockerAddon(DockerInterface):
         _LOGGER.info("Export image %s done", self.image)
 
     @process_lock
-    def import_image(self, tar_file: Path, tag: str) -> Awaitable[None]:
+    def import_image(self, tar_file: Path) -> Awaitable[None]:
         """Import a tar file as image."""
-        return self.sys_run_in_executor(self._import_image, tar_file, tag)
+        return self.sys_run_in_executor(self._import_image, tar_file)
 
-    def _import_image(self, tar_file: Path, tag: str) -> None:
+    def _import_image(self, tar_file: Path) -> None:
         """Import a tar file as image.
 
         Need run inside executor.
@@ -427,14 +427,13 @@ class DockerAddon(DockerInterface):
             with tar_file.open("rb") as read_tar:
                 self.sys_docker.api.load_image(read_tar, quiet=True)
 
-            docker_image = self.sys_docker.images.get(self.image)
+            docker_image = self.sys_docker.images.get(f"{self.image}:{self.version}")
-            docker_image.tag(self.image, tag=tag)
         except (docker.errors.DockerException, OSError) as err:
             _LOGGER.error("Can't import image %s: %s", self.image, err)
             raise DockerAPIError() from None
 
-        _LOGGER.info("Import image %s and tag %s", tar_file, tag)
         self._meta = docker_image.attrs
+        _LOGGER.info("Import image %s and version %s", tar_file, self.version)
 
         with suppress(DockerAPIError):
             self._cleanup()

hassio/docker/interface.py

@@ -103,13 +103,10 @@ class DockerInterface(CoreSysAttributes):
 
         Need run inside executor.
         """
-        try:
+        with suppress(docker.errors.DockerException):
-            docker_image = self.sys_docker.images.get(self.image)
+            self.sys_docker.images.get(f"{self.image}:{self.version}")
-            assert f"{self.image}:{self.version}" in docker_image.tags
+            return True
-        except (docker.errors.DockerException, AssertionError):
+        return False
-            return False
-
-        return True
 
     def is_running(self) -> Awaitable[bool]:
         """Return True if Docker is running.

hassio/docker/supervisor.py

@@ -2,6 +2,7 @@
 from ipaddress import IPv4Address
 import logging
 import os
+from typing import Awaitable
 
 import docker
 
@@ -49,3 +50,21 @@ class DockerSupervisor(DockerInterface, CoreSysAttributes):
         self.sys_docker.network.attach_container(
             docker_container, alias=["hassio"], ipv4=self.sys_docker.network.supervisor
         )
+
+    def retag(self) -> Awaitable[None]:
+        """Retag latest image to version."""
+        return self.sys_run_in_executor(self._retag)
+
+    def _retag(self) -> None:
+        """Retag latest image to version.
+
+        Need run inside executor.
+        """
+        try:
+            docker_container = self.sys_docker.containers.get(self.name)
+
+            docker_container.image.tag(self.image, tag=self.version)
+            docker_container.image.tag(self.image, tag="latest")
+        except docker.errors.DockerException as err:
+            _LOGGER.error("Can't retag supervisor version: %s", err)
+            raise DockerAPIError() from None

hassio/hassos.py

@@ -195,3 +195,14 @@ class HassOS(CoreSysAttributes):
         except DockerAPIError:
             _LOGGER.error("HassOS CLI update fails")
             raise HassOSUpdateError() from None
+
+    async def repair_cli(self) -> None:
+        """Repair CLI container."""
+        if await self.instance.exists():
+            return
+
+        _LOGGER.info("Repair HassOS CLI %s", self.version_cli)
+        try:
+            await self.instance.install(self.version_cli, latest=True)
+        except DockerAPIError:
+            _LOGGER.error("Repairing of HassOS CLI fails")

hassio/homeassistant.py

@@ -597,3 +597,14 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
 
         self._error_state = True
         raise HomeAssistantError()
+
+    async def repair(self):
+        """Repair local Home Assistant data."""
+        if await self.instance.exists():
+            return
+
+        _LOGGER.info("Repair Home Assistant %s", self.version)
+        try:
+            await self.instance.install(self.version)
+        except DockerAPIError:
+            _LOGGER.error("Repairing of Home Assistant fails")

hassio/supervisor.py

@@ -9,7 +9,7 @@ from typing import Awaitable, Optional
 
 import aiohttp
 
-from .const import URL_HASSIO_APPARMOR
+from .const import URL_HASSIO_APPARMOR, HASSIO_VERSION
 from .coresys import CoreSys, CoreSysAttributes
 from .docker.stats import DockerStats
 from .docker.supervisor import DockerSupervisor
@@ -54,7 +54,7 @@ class Supervisor(CoreSysAttributes):
     @property
     def version(self) -> str:
         """Return version of running Home Assistant."""
-        return self.instance.version
+        return HASSIO_VERSION
 
     @property
     def latest_version(self) -> str:
@@ -136,3 +136,14 @@ class Supervisor(CoreSysAttributes):
             return await self.instance.stats()
         except DockerAPIError:
             raise SupervisorError() from None
+
+    async def repair(self):
+        """Repair local Supervisor data."""
+        if await self.instance.exists():
+            return
+
+        _LOGGER.info("Repair Supervisor %s", self.version)
+        try:
+            await self.instance.retag()
+        except DockerAPIError:
+            _LOGGER.error("Repairing of Supervisor fails")