From 2fc5e3b7d994d24405405c0db4f644c4abfd9ec7 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 7 Aug 2019 17:26:32 +0200 Subject: [PATCH] Repair / fixup docker overlayfs issues (#1170) * Add a repair modus * Add repair to add-ons * repair to cli * Add API call * fix sync call * Clean all images * Fix repair * Fix supervisor * Add new function to core * fix tagging * better style * use retag * new retag function * Fix lint * Fix import export --- API.md | 4 ++++ hassio/addons/__init__.py | 35 +++++++++++++++++++++++++++++++++++ hassio/addons/addon.py | 2 +- hassio/api/__init__.py | 1 + hassio/api/supervisor.py | 5 +++++ hassio/core.py | 15 +++++++++++++++ hassio/docker/__init__.py | 31 +++++++++++++++++++++++++++++++ hassio/docker/addon.py | 13 ++++++------- hassio/docker/interface.py | 11 ++++------- hassio/docker/supervisor.py | 19 +++++++++++++++++++ hassio/hassos.py | 11 +++++++++++ hassio/homeassistant.py | 11 +++++++++++ hassio/supervisor.py | 15 +++++++++++++-- 13 files changed, 156 insertions(+), 17 deletions(-) diff --git a/API.md b/API.md index 48c2cc5f5..6d5e76f38 100644 --- a/API.md +++ b/API.md @@ -112,6 +112,10 @@ Output is the raw docker log. } ``` +- GET `/supervisor/repair` + +Repair overlayfs issue and restore lost images + ### Snapshot - GET `/snapshots` diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index 84e64a28f..e522da543 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -256,3 +256,38 @@ class AddonManager(CoreSysAttributes): _LOGGER.info("Detect new Add-on after restore %s", slug) self.local[slug] = addon + + async def repair(self) -> None: + """Repair local add-ons.""" + needs_repair: List[Addon] = [] + + # Evaluate Add-ons to repair + for addon in self.installed: + if await addon.instance.exists(): + continue + needs_repair.append(addon) + + _LOGGER.info("Found %d add-ons to repair", len(needs_repair)) + if not needs_repair: + return + + for addon in needs_repair: + _LOGGER.info("Start repair for add-on: %s", addon.slug) + + with suppress(DockerAPIError, KeyError): + # Need pull a image again + if not addon.need_build: + await addon.instance.install(addon.version, addon.image) + continue + + # Need local lookup + elif addon.need_build and not addon.is_detached: + store = self.store[addon.slug] + # If this add-on is available for rebuild + if addon.version == store.version: + await addon.instance.install(addon.version, addon.image) + continue + + _LOGGER.error("Can't repair %s", addon.slug) + with suppress(AddonsError): + await self.uninstall(addon.slug) diff --git a/hassio/addons/addon.py b/hassio/addons/addon.py index ec0e8cb7f..d694bd916 100644 --- a/hassio/addons/addon.py +++ b/hassio/addons/addon.py @@ -618,7 +618,7 @@ class Addon(AddonModel): image_file = Path(temp, "image.tar") if image_file.is_file(): with suppress(DockerAPIError): - await self.instance.import_image(image_file, version) + await self.instance.import_image(image_file) else: with suppress(DockerAPIError): await self.instance.install(version, restore_image) diff --git a/hassio/api/__init__.py b/hassio/api/__init__.py index 0d36fd530..aebad8bba 100644 --- a/hassio/api/__init__.py +++ b/hassio/api/__init__.py @@ -130,6 +130,7 @@ class RestAPI(CoreSysAttributes): web.post("/supervisor/update", api_supervisor.update), web.post("/supervisor/reload", api_supervisor.reload), web.post("/supervisor/options", api_supervisor.options), + web.post("/supervisor/repair", api_supervisor.repair), ] ) diff --git a/hassio/api/supervisor.py b/hassio/api/supervisor.py index 46b4cb3fb..fafe40d63 100644 --- a/hassio/api/supervisor.py +++ b/hassio/api/supervisor.py @@ -161,6 +161,11 @@ class APISupervisor(CoreSysAttributes): """Reload add-ons, configuration, etc.""" return asyncio.shield(self.sys_updater.reload()) + @api_process + def repair(self, request: web.Request) -> Awaitable[None]: + """Try to repair the local setup / overlayfs.""" + return asyncio.shield(self.sys_core.repair()) + @api_process_raw(CONTENT_TYPE_BINARY) def logs(self, request: web.Request) -> Awaitable[bytes]: """Return supervisor Docker logs.""" diff --git a/hassio/core.py b/hassio/core.py index ce8e4c0e0..a49b97db8 100644 --- a/hassio/core.py +++ b/hassio/core.py @@ -170,3 +170,18 @@ class HassIO(CoreSysAttributes): """Update last boot time.""" self.sys_config.last_boot = self.sys_hardware.last_boot self.sys_config.save_data() + + async def repair(self): + """Repair system integrity.""" + await self.sys_run_in_executor(self.sys_docker.repair) + + # Restore core functionality + await self.sys_addons.repair() + await self.sys_homeassistant.repair() + + # Fix HassOS specific + if self.sys_hassos.available: + await self.sys_hassos.repair_cli() + + # Tag version for latest + await self.sys_supervisor.repair() diff --git a/hassio/docker/__init__.py b/hassio/docker/__init__.py index 13d893291..0b86ef359 100644 --- a/hassio/docker/__init__.py +++ b/hassio/docker/__init__.py @@ -139,3 +139,34 @@ class DockerAPI: container.remove(force=True) return CommandReturn(result.get("StatusCode"), output) + + def repair(self) -> None: + """Repair local docker overlayfs2 issues.""" + + _LOGGER.info("Prune stale containers") + try: + output = self.docker.api.prune_containers() + _LOGGER.debug("Containers prune: %s", output) + except docker.errors.APIError as err: + _LOGGER.warning("Error for containers prune: %s", err) + + _LOGGER.info("Prune stale images") + try: + output = self.docker.api.prune_images(filters={"dangling": False}) + _LOGGER.debug("Images prune: %s", output) + except docker.errors.APIError as err: + _LOGGER.warning("Error for images prune: %s", err) + + _LOGGER.info("Prune stale builds") + try: + output = self.docker.api.prune_builds() + _LOGGER.debug("Builds prune: %s", output) + except docker.errors.APIError as err: + _LOGGER.warning("Error for builds prune: %s", err) + + _LOGGER.info("Prune stale volumes") + try: + output = self.docker.api.prune_builds() + _LOGGER.debug("Volumes prune: %s", output) + except docker.errors.APIError as err: + _LOGGER.warning("Error for volumes prune: %s", err) diff --git a/hassio/docker/addon.py b/hassio/docker/addon.py index df7b4c799..51d1dda85 100644 --- a/hassio/docker/addon.py +++ b/hassio/docker/addon.py @@ -397,7 +397,7 @@ class DockerAddon(DockerInterface): Need run inside executor. """ try: - image = self.sys_docker.api.get_image(self.image) + image = self.sys_docker.api.get_image(f"{self.image}:{self.version}") except docker.errors.DockerException as err: _LOGGER.error("Can't fetch image %s: %s", self.image, err) raise DockerAPIError() from None @@ -414,11 +414,11 @@ class DockerAddon(DockerInterface): _LOGGER.info("Export image %s done", self.image) @process_lock - def import_image(self, tar_file: Path, tag: str) -> Awaitable[None]: + def import_image(self, tar_file: Path) -> Awaitable[None]: """Import a tar file as image.""" - return self.sys_run_in_executor(self._import_image, tar_file, tag) + return self.sys_run_in_executor(self._import_image, tar_file) - def _import_image(self, tar_file: Path, tag: str) -> None: + def _import_image(self, tar_file: Path) -> None: """Import a tar file as image. Need run inside executor. @@ -427,14 +427,13 @@ class DockerAddon(DockerInterface): with tar_file.open("rb") as read_tar: self.sys_docker.api.load_image(read_tar, quiet=True) - docker_image = self.sys_docker.images.get(self.image) - docker_image.tag(self.image, tag=tag) + docker_image = self.sys_docker.images.get(f"{self.image}:{self.version}") except (docker.errors.DockerException, OSError) as err: _LOGGER.error("Can't import image %s: %s", self.image, err) raise DockerAPIError() from None - _LOGGER.info("Import image %s and tag %s", tar_file, tag) self._meta = docker_image.attrs + _LOGGER.info("Import image %s and version %s", tar_file, self.version) with suppress(DockerAPIError): self._cleanup() diff --git a/hassio/docker/interface.py b/hassio/docker/interface.py index 1b0593778..a59cbfeab 100644 --- a/hassio/docker/interface.py +++ b/hassio/docker/interface.py @@ -103,13 +103,10 @@ class DockerInterface(CoreSysAttributes): Need run inside executor. """ - try: - docker_image = self.sys_docker.images.get(self.image) - assert f"{self.image}:{self.version}" in docker_image.tags - except (docker.errors.DockerException, AssertionError): - return False - - return True + with suppress(docker.errors.DockerException): + self.sys_docker.images.get(f"{self.image}:{self.version}") + return True + return False def is_running(self) -> Awaitable[bool]: """Return True if Docker is running. diff --git a/hassio/docker/supervisor.py b/hassio/docker/supervisor.py index 4fbe92d86..6104d3a76 100644 --- a/hassio/docker/supervisor.py +++ b/hassio/docker/supervisor.py @@ -2,6 +2,7 @@ from ipaddress import IPv4Address import logging import os +from typing import Awaitable import docker @@ -49,3 +50,21 @@ class DockerSupervisor(DockerInterface, CoreSysAttributes): self.sys_docker.network.attach_container( docker_container, alias=["hassio"], ipv4=self.sys_docker.network.supervisor ) + + def retag(self) -> Awaitable[None]: + """Retag latest image to version.""" + return self.sys_run_in_executor(self._retag) + + def _retag(self) -> None: + """Retag latest image to version. + + Need run inside executor. + """ + try: + docker_container = self.sys_docker.containers.get(self.name) + + docker_container.image.tag(self.image, tag=self.version) + docker_container.image.tag(self.image, tag="latest") + except docker.errors.DockerException as err: + _LOGGER.error("Can't retag supervisor version: %s", err) + raise DockerAPIError() from None diff --git a/hassio/hassos.py b/hassio/hassos.py index 2e0617e61..0237ce942 100644 --- a/hassio/hassos.py +++ b/hassio/hassos.py @@ -195,3 +195,14 @@ class HassOS(CoreSysAttributes): except DockerAPIError: _LOGGER.error("HassOS CLI update fails") raise HassOSUpdateError() from None + + async def repair_cli(self) -> None: + """Repair CLI container.""" + if await self.instance.exists(): + return + + _LOGGER.info("Repair HassOS CLI %s", self.version_cli) + try: + await self.instance.install(self.version_cli, latest=True) + except DockerAPIError: + _LOGGER.error("Repairing of HassOS CLI fails") diff --git a/hassio/homeassistant.py b/hassio/homeassistant.py index 253896582..c4702b3a1 100644 --- a/hassio/homeassistant.py +++ b/hassio/homeassistant.py @@ -597,3 +597,14 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): self._error_state = True raise HomeAssistantError() + + async def repair(self): + """Repair local Home Assistant data.""" + if await self.instance.exists(): + return + + _LOGGER.info("Repair Home Assistant %s", self.version) + try: + await self.instance.install(self.version) + except DockerAPIError: + _LOGGER.error("Repairing of Home Assistant fails") diff --git a/hassio/supervisor.py b/hassio/supervisor.py index fb63bcc5b..644dc30f8 100644 --- a/hassio/supervisor.py +++ b/hassio/supervisor.py @@ -9,7 +9,7 @@ from typing import Awaitable, Optional import aiohttp -from .const import URL_HASSIO_APPARMOR +from .const import URL_HASSIO_APPARMOR, HASSIO_VERSION from .coresys import CoreSys, CoreSysAttributes from .docker.stats import DockerStats from .docker.supervisor import DockerSupervisor @@ -54,7 +54,7 @@ class Supervisor(CoreSysAttributes): @property def version(self) -> str: """Return version of running Home Assistant.""" - return self.instance.version + return HASSIO_VERSION @property def latest_version(self) -> str: @@ -136,3 +136,14 @@ class Supervisor(CoreSysAttributes): return await self.instance.stats() except DockerAPIError: raise SupervisorError() from None + + async def repair(self): + """Repair local Supervisor data.""" + if await self.instance.exists(): + return + + _LOGGER.info("Repair Supervisor %s", self.version) + try: + await self.instance.retag() + except DockerAPIError: + _LOGGER.error("Repairing of Supervisor fails")