From aa5297026faa0a2c3e47056f2878ef872b7a4b8e Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sat, 28 Nov 2020 15:03:44 +0100 Subject: [PATCH 01/27] Handle Store suggestion (#2306) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Handle Store suggestion * Add fixup * Add more fixup & list * Enable fixups * Add tests * fix index * fix break * fix import * Load it anyway * Run suFix ccestion on load too * fix error message * fix error message * Fix remove * Finishing * Add tests * Fix error * fix cleanup stale stuff * Fix source * use source as url * add test for url * Apply suggestions from code review Co-authored-by: Joakim Sørensen Co-authored-by: Joakim Sørensen --- scripts/test_env.sh | 2 + supervisor/api/supervisor.py | 20 ++++- supervisor/exceptions.py | 4 + supervisor/resolution/const.py | 2 +- supervisor/resolution/fixup.py | 14 ++- supervisor/resolution/fixups/base.py | 24 +++-- .../resolution/fixups/clear_full_snapshot.py | 8 +- .../resolution/fixups/store_execute_reload.py | 50 +++++++++++ .../resolution/fixups/store_execute_remove.py | 52 +++++++++++ .../resolution/fixups/store_execute_reset.py | 52 +++++++++++ supervisor/store/__init__.py | 66 ++++++++------ supervisor/store/const.py | 10 +++ supervisor/store/git.py | 27 +++--- supervisor/store/repository.py | 88 ++++++++++++------- supervisor/utils/__init__.py | 24 +++++ tests/conftest.py | 1 - .../fixup/test_store_execute_reload.py | 32 +++++++ .../fixup/test_store_execute_remove.py | 32 +++++++ .../fixup/test_store_execute_reset.py | 38 ++++++++ tests/store/test_custom_repository.py | 37 +++++++- 20 files changed, 480 insertions(+), 103 deletions(-) create mode 100644 supervisor/resolution/fixups/store_execute_reload.py create mode 100644 supervisor/resolution/fixups/store_execute_remove.py create mode 100644 supervisor/resolution/fixups/store_execute_reset.py create mode 100644 supervisor/store/const.py create mode 100644 tests/resolution/fixup/test_store_execute_reload.py create mode 100644 tests/resolution/fixup/test_store_execute_remove.py create mode 100644 tests/resolution/fixup/test_store_execute_reset.py diff --git a/scripts/test_env.sh b/scripts/test_env.sh index fb2b93567..08c52a535 100755 --- a/scripts/test_env.sh +++ b/scripts/test_env.sh @@ -125,6 +125,8 @@ echo "Start Test-Env" start_docker trap "stop_docker" ERR +docker system prune -f + build_supervisor cleanup_lastboot cleanup_docker diff --git a/supervisor/api/supervisor.py b/supervisor/api/supervisor.py index be053b83c..d6dd6f31c 100644 --- a/supervisor/api/supervisor.py +++ b/supervisor/api/supervisor.py @@ -6,6 +6,8 @@ from typing import Any, Awaitable, Dict from aiohttp import web import voluptuous as vol +from supervisor.resolution.const import ContextType, SuggestionType + from ..const import ( ATTR_ADDONS, ATTR_ADDONS_REPOSITORIES, @@ -143,10 +145,20 @@ class APISupervisor(CoreSysAttributes): if ATTR_ADDONS_REPOSITORIES in body: new = set(body[ATTR_ADDONS_REPOSITORIES]) await asyncio.shield(self.sys_store.update_repositories(new)) - if sorted(body[ATTR_ADDONS_REPOSITORIES]) != sorted( - self.sys_config.addons_repositories - ): - raise APIError("Not a valid add-on repository") + + # Fix invalid repository + found_invalid = False + for suggestion in self.sys_resolution.suggestions: + if ( + suggestion.type != SuggestionType.EXECUTE_REMOVE + and suggestion.context != ContextType + ): + continue + found_invalid = True + await self.sys_resolution.apply_suggestion(suggestion) + + if found_invalid: + raise APIError("Invalid Add-on repository!") self.sys_updater.save_data() self.sys_config.save_data() diff --git a/supervisor/exceptions.py b/supervisor/exceptions.py index e18e385cc..f832fa9e4 100644 --- a/supervisor/exceptions.py +++ b/supervisor/exceptions.py @@ -310,6 +310,10 @@ class StoreGitError(StoreError): """Raise if something on git is happening.""" +class StoreNotFound(StoreError): + """Raise if slug is not known.""" + + # JobManager diff --git a/supervisor/resolution/const.py b/supervisor/resolution/const.py index c553aa0ef..ff3c83076 100644 --- a/supervisor/resolution/const.py +++ b/supervisor/resolution/const.py @@ -66,5 +66,5 @@ class SuggestionType(str, Enum): EXECUTE_REPAIR = "execute_repair" EXECUTE_RESET = "execute_reset" EXECUTE_RELOAD = "execute_reload" + EXECUTE_REMOVE = "execute_remove" REGISTRY_LOGIN = "registry_login" - NEW_INITIALIZE = "new_initialize" diff --git a/supervisor/resolution/fixup.py b/supervisor/resolution/fixup.py index 696e4adb7..d07dd2837 100644 --- a/supervisor/resolution/fixup.py +++ b/supervisor/resolution/fixup.py @@ -9,6 +9,9 @@ from ..coresys import CoreSys, CoreSysAttributes from .fixups.base import FixupBase from .fixups.clear_full_snapshot import FixupClearFullSnapshot from .fixups.create_full_snapshot import FixupCreateFullSnapshot +from .fixups.store_execute_reload import FixupStoreExecuteReload +from .fixups.store_execute_remove import FixupStoreExecuteRemove +from .fixups.store_execute_reset import FixupStoreExecuteReset _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -22,11 +25,20 @@ class ResolutionFixup(CoreSysAttributes): self._create_full_snapshot = FixupCreateFullSnapshot(coresys) self._clear_full_snapshot = FixupClearFullSnapshot(coresys) + self._store_execute_reset = FixupStoreExecuteReset(coresys) + self._store_execute_reload = FixupStoreExecuteReload(coresys) + self._store_execute_remove = FixupStoreExecuteRemove(coresys) @property def all_fixes(self) -> List[FixupBase]: """Return a list of all fixups.""" - return [self._create_full_snapshot, self._clear_full_snapshot] + return [ + self._create_full_snapshot, + self._clear_full_snapshot, + self._store_execute_reload, + self._store_execute_reset, + self._store_execute_remove, + ] async def run_autofix(self) -> None: """Run all startup fixes.""" diff --git a/supervisor/resolution/fixups/base.py b/supervisor/resolution/fixups/base.py index d1b188f5f..38ef12f86 100644 --- a/supervisor/resolution/fixups/base.py +++ b/supervisor/resolution/fixups/base.py @@ -1,11 +1,10 @@ """Baseclass for system fixup.""" from abc import ABC, abstractmethod, abstractproperty -from contextlib import suppress import logging -from typing import Optional +from typing import List, Optional from ...coresys import CoreSys, CoreSysAttributes -from ...exceptions import ResolutionError, ResolutionFixupError +from ...exceptions import ResolutionFixupError from ..const import ContextType, IssueType, SuggestionType from ..data import Issue, Suggestion @@ -42,13 +41,12 @@ class FixupBase(ABC, CoreSysAttributes): self.sys_resolution.dismiss_suggestion(fixing_suggestion) - if self.issue is None: - return - - with suppress(ResolutionError): - self.sys_resolution.dismiss_issue( - Issue(self.issue, self.context, fixing_suggestion.reference) - ) + # Cleanup issue + for issue_type in self.issues: + issue = Issue(issue_type, self.context, fixing_suggestion.reference) + if issue not in self.sys_resolution.issues: + continue + self.sys_resolution.dismiss_issue(issue) @abstractmethod async def process_fixup(self, reference: Optional[str] = None) -> None: @@ -65,9 +63,9 @@ class FixupBase(ABC, CoreSysAttributes): """Return a ContextType enum.""" @property - def issue(self) -> Optional[IssueType]: - """Return a IssueType enum.""" - return None + def issues(self) -> List[IssueType]: + """Return a IssueType enum list.""" + return [] @property def auto(self) -> bool: diff --git a/supervisor/resolution/fixups/clear_full_snapshot.py b/supervisor/resolution/fixups/clear_full_snapshot.py index add696000..72b5e4652 100644 --- a/supervisor/resolution/fixups/clear_full_snapshot.py +++ b/supervisor/resolution/fixups/clear_full_snapshot.py @@ -1,6 +1,6 @@ """Helpers to check and fix issues with free space.""" import logging -from typing import Optional +from typing import List, Optional from ...const import SNAPSHOT_FULL from ..const import MINIMUM_FULL_SNAPSHOTS, ContextType, IssueType, SuggestionType @@ -36,6 +36,6 @@ class FixupClearFullSnapshot(FixupBase): return ContextType.SYSTEM @property - def issue(self) -> IssueType: - """Return a IssueType enum.""" - return IssueType.FREE_SPACE + def issues(self) -> List[IssueType]: + """Return a IssueType enum list.""" + return [IssueType.FREE_SPACE] diff --git a/supervisor/resolution/fixups/store_execute_reload.py b/supervisor/resolution/fixups/store_execute_reload.py new file mode 100644 index 000000000..97c848e3a --- /dev/null +++ b/supervisor/resolution/fixups/store_execute_reload.py @@ -0,0 +1,50 @@ +"""Helpers to check and fix issues with free space.""" +import logging +from typing import List, Optional + +from supervisor.exceptions import ResolutionFixupError, StoreError, StoreNotFound + +from ..const import ContextType, IssueType, SuggestionType +from .base import FixupBase + +_LOGGER: logging.Logger = logging.getLogger(__name__) + + +class FixupStoreExecuteReload(FixupBase): + """Storage class for fixup.""" + + async def process_fixup(self, reference: Optional[str] = None) -> None: + """Initialize the fixup class.""" + _LOGGER.info("Reload Store: %s", reference) + try: + repository = self.sys_store.get(reference) + except StoreNotFound: + _LOGGER.warning("Can't find store %s for fixup", reference) + return + + # Load data again + try: + await repository.load() + await repository.update() + except StoreError: + raise ResolutionFixupError() from None + + @property + def suggestion(self) -> SuggestionType: + """Return a SuggestionType enum.""" + return SuggestionType.EXECUTE_RELOAD + + @property + def context(self) -> ContextType: + """Return a ContextType enum.""" + return ContextType.STORE + + @property + def issues(self) -> List[IssueType]: + """Return a IssueType enum list.""" + return [IssueType.FATAL_ERROR] + + @property + def auto(self) -> bool: + """Return if a fixup can be apply as auto fix.""" + return True diff --git a/supervisor/resolution/fixups/store_execute_remove.py b/supervisor/resolution/fixups/store_execute_remove.py new file mode 100644 index 000000000..4e7ef7f98 --- /dev/null +++ b/supervisor/resolution/fixups/store_execute_remove.py @@ -0,0 +1,52 @@ +"""Helpers to check and fix issues with free space.""" +import logging +from typing import List, Optional + +from supervisor.exceptions import ResolutionFixupError, StoreError, StoreNotFound + +from ..const import ContextType, IssueType, SuggestionType +from .base import FixupBase + +_LOGGER: logging.Logger = logging.getLogger(__name__) + + +class FixupStoreExecuteRemove(FixupBase): + """Storage class for fixup.""" + + async def process_fixup(self, reference: Optional[str] = None) -> None: + """Initialize the fixup class.""" + _LOGGER.info("Remove invalid Store: %s", reference) + try: + repository = self.sys_store.get(reference) + except StoreNotFound: + _LOGGER.warning("Can't find store %s for fixup", reference) + return + + # Remove repository + try: + await repository.remove() + except StoreError: + raise ResolutionFixupError() from None + + self.sys_config.drop_addon_repository(repository.source) + self.sys_config.save_data() + + @property + def suggestion(self) -> SuggestionType: + """Return a SuggestionType enum.""" + return SuggestionType.EXECUTE_REMOVE + + @property + def context(self) -> ContextType: + """Return a ContextType enum.""" + return ContextType.STORE + + @property + def issues(self) -> List[IssueType]: + """Return a IssueType enum list.""" + return [IssueType.CORRUPT_REPOSITORY] + + @property + def auto(self) -> bool: + """Return if a fixup can be apply as auto fix.""" + return True diff --git a/supervisor/resolution/fixups/store_execute_reset.py b/supervisor/resolution/fixups/store_execute_reset.py new file mode 100644 index 000000000..1a1829546 --- /dev/null +++ b/supervisor/resolution/fixups/store_execute_reset.py @@ -0,0 +1,52 @@ +"""Helpers to check and fix issues with free space.""" +import logging +from typing import List, Optional + +from supervisor.exceptions import ResolutionFixupError, StoreError, StoreNotFound + +from ...utils import remove_folder +from ..const import ContextType, IssueType, SuggestionType +from .base import FixupBase + +_LOGGER: logging.Logger = logging.getLogger(__name__) + + +class FixupStoreExecuteReset(FixupBase): + """Storage class for fixup.""" + + async def process_fixup(self, reference: Optional[str] = None) -> None: + """Initialize the fixup class.""" + _LOGGER.info("Reset corrupt Store: %s", reference) + try: + repository = self.sys_store.get(reference) + except StoreNotFound: + _LOGGER.warning("Can't find store %s for fixup", reference) + return + + await remove_folder(repository.git.path) + + # Load data again + try: + await repository.load() + except StoreError: + raise ResolutionFixupError() from None + + @property + def suggestion(self) -> SuggestionType: + """Return a SuggestionType enum.""" + return SuggestionType.EXECUTE_RESET + + @property + def context(self) -> ContextType: + """Return a ContextType enum.""" + return ContextType.STORE + + @property + def issues(self) -> List[IssueType]: + """Return a IssueType enum list.""" + return [IssueType.CORRUPT_REPOSITORY, IssueType.FATAL_ERROR] + + @property + def auto(self) -> bool: + """Return if a fixup can be apply as auto fix.""" + return True diff --git a/supervisor/store/__init__.py b/supervisor/store/__init__.py index 53835de9b..ed2dd527e 100644 --- a/supervisor/store/__init__.py +++ b/supervisor/store/__init__.py @@ -1,25 +1,20 @@ """Add-on Store handler.""" import asyncio import logging -from pathlib import Path from typing import Dict, List -import voluptuous as vol - -from supervisor.store.validate import SCHEMA_REPOSITORY_CONFIG -from supervisor.utils.json import read_json_file - -from ..const import REPOSITORY_CORE, REPOSITORY_LOCAL from ..coresys import CoreSys, CoreSysAttributes -from ..exceptions import JsonFileError, StoreGitError +from ..exceptions import StoreGitError, StoreNotFound from ..jobs.decorator import Job, JobCondition +from ..resolution.const import ContextType, IssueType, SuggestionType from .addon import AddonStore +from .const import StoreType from .data import StoreData from .repository import Repository _LOGGER: logging.Logger = logging.getLogger(__name__) -BUILTIN_REPOSITORIES = {REPOSITORY_CORE, REPOSITORY_LOCAL} +BUILTIN_REPOSITORIES = {StoreType.CORE.value, StoreType.LOCAL.value} class StoreManager(CoreSysAttributes): @@ -36,6 +31,20 @@ class StoreManager(CoreSysAttributes): """Return list of add-on repositories.""" return list(self.repositories.values()) + def get(self, slug: str) -> Repository: + """Return Repository with slug.""" + if slug not in self.repositories: + raise StoreNotFound() + return self.repositories[slug] + + def get_from_url(self, url: str) -> Repository: + """Return Repository with slug.""" + for repository in self.all: + if repository.source != url: + continue + return repository + raise StoreNotFound() + async def load(self) -> None: """Start up add-on management.""" self.data.update() @@ -48,7 +57,7 @@ class StoreManager(CoreSysAttributes): async def reload(self) -> None: """Update add-ons from repository and reload list.""" - tasks = [repository.update() for repository in self.repositories.values()] + tasks = [repository.update() for repository in self.all] if tasks: await asyncio.wait(tasks) @@ -61,35 +70,33 @@ class StoreManager(CoreSysAttributes): """Add a new custom repository.""" job = self.sys_jobs.get_job("storemanager_update_repositories") new_rep = set(list_repositories) - old_rep = set(self.repositories) + old_rep = {repository.source for repository in self.all} # add new repository async def _add_repository(url: str, step: int): """Add a repository.""" job.update(progress=job.progress + step, stage=f"Checking {url} started") repository = Repository(self.coresys, url) + + # Load the repository try: await repository.load() except StoreGitError: _LOGGER.error("Can't load data from repository %s", url) - return - - # don't add built-in repository to config - if url not in BUILTIN_REPOSITORIES: - # Verify that it is a add-on repository - repository_file = Path(repository.git.path, "repository.json") - try: - await self.sys_run_in_executor( - SCHEMA_REPOSITORY_CONFIG, read_json_file(repository_file) + else: + if not repository.validate(): + _LOGGER.error("%s is not a valid add-on repository", url) + self.sys_resolution.create_issue( + IssueType.CORRUPT_REPOSITORY, + ContextType.STORE, + reference=repository.slug, + suggestions=[SuggestionType.EXECUTE_REMOVE], ) - except (JsonFileError, vol.Invalid) as err: - _LOGGER.error("%s is not a valid add-on repository. %s", url, err) - await repository.remove() - return - self.sys_config.add_addon_repository(url) - - self.repositories[url] = repository + # Add Repository to list + if repository.type == StoreType.GIT: + self.sys_config.add_addon_repository(repository.source) + self.repositories[repository.slug] = repository job.update(progress=10, stage="Check repositories") repos = new_rep - old_rep @@ -97,9 +104,10 @@ class StoreManager(CoreSysAttributes): if tasks: await asyncio.wait(tasks) - # del new repository + # Delete stale repositories for url in old_rep - new_rep - BUILTIN_REPOSITORIES: - await self.repositories.pop(url).remove() + repository = self.get_from_url(url) + await self.repositories.pop(repository.slug).remove() self.sys_config.drop_addon_repository(url) # update data diff --git a/supervisor/store/const.py b/supervisor/store/const.py new file mode 100644 index 000000000..43cbc989d --- /dev/null +++ b/supervisor/store/const.py @@ -0,0 +1,10 @@ +"""Constants for the add-on store.""" +from enum import Enum + + +class StoreType(str, Enum): + """Store Types.""" + + CORE = "core" + LOCAL = "local" + GIT = "git" diff --git a/supervisor/store/git.py b/supervisor/store/git.py index 40e24ca26..11f76e984 100644 --- a/supervisor/store/git.py +++ b/supervisor/store/git.py @@ -3,7 +3,6 @@ import asyncio import functools as ft import logging from pathlib import Path -import shutil from typing import Dict, Optional import git @@ -13,6 +12,7 @@ from ..coresys import CoreSys, CoreSysAttributes from ..exceptions import StoreGitError from ..jobs.decorator import Job, JobCondition from ..resolution.const import ContextType, IssueType, SuggestionType +from ..utils import remove_folder from ..validate import RE_REPOSITORY from .utils import get_hash_from_repository @@ -30,7 +30,6 @@ class GitRepo(CoreSysAttributes): self.lock: asyncio.Lock = asyncio.Lock() self.data: Dict[str, str] = RE_REPOSITORY.match(url).groupdict() - self.slug: str = url @property def url(self) -> str: @@ -59,11 +58,12 @@ class GitRepo(CoreSysAttributes): git.NoSuchPathError, git.GitCommandError, ) as err: - _LOGGER.error("Can't load %s repo: %s.", self.path, err) + _LOGGER.error("Can't load %s", self.path) self.sys_resolution.create_issue( IssueType.FATAL_ERROR, ContextType.STORE, - reference=self.slug, + reference=self.path.stem, + suggestions=[SuggestionType.EXECUTE_RESET], ) raise StoreGitError() from err @@ -77,7 +77,7 @@ class GitRepo(CoreSysAttributes): self.sys_resolution.create_issue( IssueType.CORRUPT_REPOSITORY, ContextType.STORE, - reference=self.slug, + reference=self.path.stem, suggestions=[SuggestionType.EXECUTE_RESET], ) raise StoreGitError() from err @@ -114,8 +114,8 @@ class GitRepo(CoreSysAttributes): self.sys_resolution.create_issue( IssueType.FATAL_ERROR, ContextType.STORE, - reference=self.slug, - suggestions=[SuggestionType.NEW_INITIALIZE], + reference=self.path.stem, + suggestions=[SuggestionType.EXECUTE_RELOAD], ) raise StoreGitError() from err @@ -156,8 +156,8 @@ class GitRepo(CoreSysAttributes): self.sys_resolution.create_issue( IssueType.CORRUPT_REPOSITORY, ContextType.STORE, - reference=self.slug, - suggestions=[SuggestionType.EXECUTE_RELOAD], + reference=self.path.stem, + suggestions=[SuggestionType.EXECUTE_RESET], ) raise StoreGitError() from err @@ -169,14 +169,7 @@ class GitRepo(CoreSysAttributes): if not self.path.is_dir(): return - - def log_err(funct, path, _): - """Log error.""" - _LOGGER.warning("Can't remove %s", path) - - await self.sys_run_in_executor( - ft.partial(shutil.rmtree, self.path, onerror=log_err) - ) + await remove_folder(self.path) class GitRepoHassIO(GitRepo): diff --git a/supervisor/store/repository.py b/supervisor/store/repository.py index 7a6019e6e..59f6df341 100644 --- a/supervisor/store/repository.py +++ b/supervisor/store/repository.py @@ -1,75 +1,103 @@ """Represent a Supervisor repository.""" -from ..const import ( - ATTR_MAINTAINER, - ATTR_NAME, - ATTR_URL, - REPOSITORY_CORE, - REPOSITORY_LOCAL, -) -from ..coresys import CoreSysAttributes -from ..exceptions import APIError +import logging +from pathlib import Path +from typing import Dict, Optional + +import voluptuous as vol + +from ..const import ATTR_MAINTAINER, ATTR_NAME, ATTR_URL +from ..coresys import CoreSys, CoreSysAttributes +from ..exceptions import JsonFileError, StoreError +from ..utils.json import read_json_file +from .const import StoreType from .git import GitRepoCustom, GitRepoHassIO from .utils import get_hash_from_repository +from .validate import SCHEMA_REPOSITORY_CONFIG +_LOGGER: logging.Logger = logging.getLogger(__name__) UNKNOWN = "unknown" class Repository(CoreSysAttributes): """Repository in Supervisor.""" - slug: str = None - - def __init__(self, coresys, repository): + def __init__(self, coresys: CoreSys, repository: str): """Initialize repository object.""" - self.coresys = coresys - self.source = None - self.git = None + self.coresys: CoreSys = coresys + self.git: Optional[str] = None - if repository == REPOSITORY_LOCAL: - self.slug = repository - elif repository == REPOSITORY_CORE: - self.slug = repository + self.source: str = repository + if repository == StoreType.LOCAL: + self._slug = repository + self._type = StoreType.LOCAL + elif repository == StoreType.CORE: self.git = GitRepoHassIO(coresys) + self._slug = repository + self._type = StoreType.CORE else: - self.slug = get_hash_from_repository(repository) self.git = GitRepoCustom(coresys, repository) self.source = repository + self._slug = get_hash_from_repository(repository) + self._type = StoreType.GIT @property - def data(self): + def slug(self) -> str: + """Return repo slug.""" + return self._slug + + @property + def type(self) -> StoreType: + """Return type of the store.""" + return self._type + + @property + def data(self) -> Dict: """Return data struct repository.""" return self.sys_store.data.repositories.get(self.slug, {}) @property - def name(self): + def name(self) -> str: """Return name of repository.""" return self.data.get(ATTR_NAME, UNKNOWN) @property - def url(self): + def url(self) -> str: """Return URL of repository.""" return self.data.get(ATTR_URL, self.source) @property - def maintainer(self): + def maintainer(self) -> str: """Return url of repository.""" return self.data.get(ATTR_MAINTAINER, UNKNOWN) - async def load(self): + def validate(self) -> bool: + """Check if store is valid.""" + if self.type != StoreType.GIT: + return True + + repository_file = Path(self.git.path, "repository.json") + try: + SCHEMA_REPOSITORY_CONFIG(read_json_file(repository_file)) + except (JsonFileError, vol.Invalid): + return False + return True + + async def load(self) -> None: """Load addon repository.""" if not self.git: return await self.git.load() - async def update(self): + async def update(self) -> None: """Update add-on repository.""" - if not self.git: + if self.type == StoreType.LOCAL: return await self.git.pull() - async def remove(self): + async def remove(self) -> None: """Remove add-on repository.""" - if self.slug in (REPOSITORY_CORE, REPOSITORY_LOCAL): - raise APIError("Can't remove built-in repositories!") + if self.type != StoreType.GIT: + _LOGGER.error("Can't remove built-in repositories!") + raise StoreError() await self.git.remove() diff --git a/supervisor/utils/__init__.py b/supervisor/utils/__init__.py index ca7bc9d6c..c27e0feb9 100644 --- a/supervisor/utils/__init__.py +++ b/supervisor/utils/__init__.py @@ -3,6 +3,7 @@ import asyncio from datetime import datetime from ipaddress import IPv4Address import logging +from pathlib import Path import re import socket from typing import Any, Optional @@ -132,3 +133,26 @@ def get_message_from_exception_chain(err: Exception) -> str: return "" return get_message_from_exception_chain(err.__context__) + + +async def remove_folder(folder: Path, content_only: bool = False) -> None: + """Remove folder and reset privileged. + + Is needed to avoid issue with: + - CAP_DAC_OVERRIDE + - CAP_DAC_READ_SEARCH + """ + del_folder = f"{folder}" + "/{,.[!.],..?}*" if content_only else f"{folder}" + try: + proc = await asyncio.create_subprocess_exec( + "bash", "-c", f"rm -rf {del_folder}", stdout=asyncio.subprocess.DEVNULL + ) + + _, error_msg = await proc.communicate() + except OSError as err: + error_msg = str(err) + else: + if proc.returncode == 0: + return + + _LOGGER.error("Can't remove folder %s: %s", folder, error_msg) diff --git a/tests/conftest.py b/tests/conftest.py index 7015907ad..af35fdff9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -182,7 +182,6 @@ async def api_client(aiohttp_client, coresys: CoreSys): def store_manager(coresys: CoreSys): """Fixture for the store manager.""" sm_obj = coresys.store - sm_obj.repositories = set(coresys.config.addons_repositories) with patch("supervisor.store.data.StoreData.update", return_value=MagicMock()): yield sm_obj diff --git a/tests/resolution/fixup/test_store_execute_reload.py b/tests/resolution/fixup/test_store_execute_reload.py new file mode 100644 index 000000000..177bf5192 --- /dev/null +++ b/tests/resolution/fixup/test_store_execute_reload.py @@ -0,0 +1,32 @@ +"""Test evaluation base.""" +# pylint: disable=import-error,protected-access +from unittest.mock import AsyncMock + +from supervisor.coresys import CoreSys +from supervisor.resolution.const import ContextType, IssueType, SuggestionType +from supervisor.resolution.data import Issue, Suggestion +from supervisor.resolution.fixups.store_execute_reload import FixupStoreExecuteReload + + +async def test_fixup(coresys: CoreSys): + """Test fixup.""" + store_execute_reload = FixupStoreExecuteReload(coresys) + + assert store_execute_reload.auto + + coresys.resolution.suggestions = Suggestion( + SuggestionType.EXECUTE_RELOAD, ContextType.STORE, reference="test" + ) + coresys.resolution.issues = Issue( + IssueType.FATAL_ERROR, ContextType.STORE, reference="test" + ) + + mock_repositorie = AsyncMock() + coresys.store.repositories["test"] = mock_repositorie + + await store_execute_reload() + + assert mock_repositorie.load.called + assert mock_repositorie.update.called + assert len(coresys.resolution.suggestions) == 0 + assert len(coresys.resolution.issues) == 0 diff --git a/tests/resolution/fixup/test_store_execute_remove.py b/tests/resolution/fixup/test_store_execute_remove.py new file mode 100644 index 000000000..75de4c5dc --- /dev/null +++ b/tests/resolution/fixup/test_store_execute_remove.py @@ -0,0 +1,32 @@ +"""Test evaluation base.""" +# pylint: disable=import-error,protected-access +from unittest.mock import AsyncMock + +from supervisor.coresys import CoreSys +from supervisor.resolution.const import ContextType, IssueType, SuggestionType +from supervisor.resolution.data import Issue, Suggestion +from supervisor.resolution.fixups.store_execute_remove import FixupStoreExecuteRemove + + +async def test_fixup(coresys: CoreSys): + """Test fixup.""" + store_execute_remove = FixupStoreExecuteRemove(coresys) + + assert store_execute_remove.auto + + coresys.resolution.suggestions = Suggestion( + SuggestionType.EXECUTE_REMOVE, ContextType.STORE, reference="test" + ) + coresys.resolution.issues = Issue( + IssueType.CORRUPT_REPOSITORY, ContextType.STORE, reference="test" + ) + + mock_repositorie = AsyncMock() + coresys.store.repositories["test"] = mock_repositorie + + await store_execute_remove() + + assert mock_repositorie.remove.called + assert coresys.config.save_data.called + assert len(coresys.resolution.suggestions) == 0 + assert len(coresys.resolution.issues) == 0 diff --git a/tests/resolution/fixup/test_store_execute_reset.py b/tests/resolution/fixup/test_store_execute_reset.py new file mode 100644 index 000000000..7c2ff331b --- /dev/null +++ b/tests/resolution/fixup/test_store_execute_reset.py @@ -0,0 +1,38 @@ +"""Test evaluation base.""" +# pylint: disable=import-error,protected-access +from pathlib import Path +from unittest.mock import AsyncMock + +from supervisor.coresys import CoreSys +from supervisor.resolution.const import ContextType, IssueType, SuggestionType +from supervisor.resolution.data import Issue, Suggestion +from supervisor.resolution.fixups.store_execute_reset import FixupStoreExecuteReset + + +async def test_fixup(coresys: CoreSys, tmp_path): + """Test fixup.""" + store_execute_reset = FixupStoreExecuteReset(coresys) + test_repo = Path(tmp_path, "test_repo") + + assert store_execute_reset.auto + + coresys.resolution.suggestions = Suggestion( + SuggestionType.EXECUTE_RESET, ContextType.STORE, reference="test" + ) + coresys.resolution.issues = Issue( + IssueType.CORRUPT_REPOSITORY, ContextType.STORE, reference="test" + ) + + test_repo.mkdir() + assert test_repo.exists() + + mock_repositorie = AsyncMock() + mock_repositorie.git.path = test_repo + coresys.store.repositories["test"] = mock_repositorie + + await store_execute_reset() + + assert not test_repo.exists() + assert mock_repositorie.load.called + assert len(coresys.resolution.suggestions) == 0 + assert len(coresys.resolution.issues) == 0 diff --git a/tests/store/test_custom_repository.py b/tests/store/test_custom_repository.py index 3c808a235..2ca1ff1e2 100644 --- a/tests/store/test_custom_repository.py +++ b/tests/store/test_custom_repository.py @@ -4,16 +4,35 @@ from unittest.mock import patch import pytest +from supervisor.resolution.const import SuggestionType +from supervisor.store import BUILTIN_REPOSITORIES + @pytest.mark.asyncio async def test_add_valid_repository(coresys, store_manager): """Test add custom repository.""" current = coresys.config.addons_repositories - with patch("supervisor.store.repository.Repository.load", return_value=True), patch( + with patch("supervisor.store.repository.Repository.load", return_value=None), patch( "pathlib.Path.read_text", return_value=json.dumps({"name": "Awesome repository"}), ): await store_manager.update_repositories(current + ["http://example.com"]) + assert store_manager.get_from_url("http://example.com").validate() + assert "http://example.com" in coresys.config.addons_repositories + + +@pytest.mark.asyncio +async def test_add_valid_repository_url(coresys, store_manager): + """Test add custom repository.""" + current = coresys.config.addons_repositories + with patch("supervisor.store.repository.Repository.load", return_value=None), patch( + "pathlib.Path.read_text", + return_value=json.dumps( + {"name": "Awesome repository", "url": "http://example2.com/docs"} + ), + ): + await store_manager.update_repositories(current + ["http://example.com"]) + assert store_manager.get_from_url("http://example.com").validate() assert "http://example.com" in coresys.config.addons_repositories @@ -21,9 +40,21 @@ async def test_add_valid_repository(coresys, store_manager): async def test_add_invalid_repository(coresys, store_manager): """Test add custom repository.""" current = coresys.config.addons_repositories - with patch("supervisor.store.repository.Repository.load", return_value=True), patch( + with patch("supervisor.store.repository.Repository.load", return_value=None), patch( "pathlib.Path.read_text", return_value="", ): await store_manager.update_repositories(current + ["http://example.com"]) - assert "http://example.com" not in coresys.config.addons_repositories + assert not store_manager.get_from_url("http://example.com").validate() + + assert "http://example.com" in coresys.config.addons_repositories + assert coresys.resolution.suggestions[-1].type == SuggestionType.EXECUTE_REMOVE + + +@pytest.mark.asyncio +async def test_preinstall_valid_repository(coresys, store_manager): + """Test add core repository valid.""" + with patch("supervisor.store.repository.Repository.load", return_value=None): + await store_manager.update_repositories(BUILTIN_REPOSITORIES) + assert store_manager.get("core").validate() + assert store_manager.get("local").validate() From 5c25fcd84c5b9f8a17938b6521db9dd2ccde8421 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sat, 28 Nov 2020 15:57:00 +0100 Subject: [PATCH 02/27] Reset on invalid filesystem (#2307) --- supervisor/store/data.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/supervisor/store/data.py b/supervisor/store/data.py index b810493bb..dd3cd636b 100644 --- a/supervisor/store/data.py +++ b/supervisor/store/data.py @@ -16,7 +16,7 @@ from ..const import ( ) from ..coresys import CoreSys, CoreSysAttributes from ..exceptions import JsonFileError -from ..resolution.const import ContextType, IssueType +from ..resolution.const import ContextType, IssueType, SuggestionType from ..utils.json import read_json_file from .utils import extract_hash_from_path from .validate import SCHEMA_REPOSITORY_CONFIG @@ -33,7 +33,7 @@ class StoreData(CoreSysAttributes): self.repositories: Dict[str, Any] = {} self.addons: Dict[str, Any] = {} - def update(self): + def update(self) -> None: """Read data from add-on repository.""" self.repositories.clear() self.addons.clear() @@ -52,7 +52,7 @@ class StoreData(CoreSysAttributes): if repository_element.is_dir(): self._read_git_repository(repository_element) - def _read_git_repository(self, path): + def _read_git_repository(self, path: Path) -> None: """Process a custom repository folder.""" slug = extract_hash_from_path(path) @@ -73,7 +73,7 @@ class StoreData(CoreSysAttributes): self.repositories[slug] = repository_info self._read_addons_folder(path, slug) - def _read_addons_folder(self, path, repository): + def _read_addons_folder(self, path: Path, repository: Dict) -> None: """Read data from add-ons folder.""" try: # Generate a list without artefact, safe for corruptions @@ -84,7 +84,10 @@ class StoreData(CoreSysAttributes): ] except OSError as err: self.sys_resolution.create_issue( - IssueType.CORRUPT_REPOSITORY, ContextType.SYSTEM + IssueType.CORRUPT_REPOSITORY, + ContextType.STORE, + reference=path.stem, + suggestions=[SuggestionType.EXECUTE_RESET], ) _LOGGER.critical( "Can't process %s because of Filesystem issues: %s", repository, err From 06fae59fc83afc6513b4c2ac5002d8802c208dea Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sat, 28 Nov 2020 18:11:56 +0100 Subject: [PATCH 03/27] Remove corrupt FS on store sentry report (#2308) --- supervisor/store/data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/supervisor/store/data.py b/supervisor/store/data.py index dd3cd636b..dd1fe0dd5 100644 --- a/supervisor/store/data.py +++ b/supervisor/store/data.py @@ -92,7 +92,6 @@ class StoreData(CoreSysAttributes): _LOGGER.critical( "Can't process %s because of Filesystem issues: %s", repository, err ) - self.sys_capture_exception(err) return for addon in addon_list: From f6bf44de1ce3ff51670b23630db40935ca38cacc Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sun, 29 Nov 2020 12:34:33 +0100 Subject: [PATCH 04/27] Add tests for remove function (#2309) --- tests/utils/test_remove_folder.py | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/utils/test_remove_folder.py diff --git a/tests/utils/test_remove_folder.py b/tests/utils/test_remove_folder.py new file mode 100644 index 000000000..af70b25e7 --- /dev/null +++ b/tests/utils/test_remove_folder.py @@ -0,0 +1,45 @@ +"""test json.""" +from pathlib import Path +import shutil + +import pytest + +from supervisor.utils import remove_folder + + +@pytest.mark.asyncio +async def test_remove_all(tmp_path): + """Test remove folder.""" + # Prepair test folder + temp_orig = tmp_path.joinpath("orig") + fixture_data = Path(__file__).parents[1].joinpath("fixtures/tar_data") + shutil.copytree(fixture_data, temp_orig, symlinks=True) + + assert temp_orig.exists() + await remove_folder(temp_orig) + assert not temp_orig.exists() + + +@pytest.mark.asyncio +async def test_remove_content(tmp_path): + """Test remove content of folder.""" + # Prepair test folder + temp_orig = tmp_path.joinpath("orig") + fixture_data = Path(__file__).parents[1].joinpath("fixtures/tar_data") + shutil.copytree(fixture_data, temp_orig, symlinks=True) + + test_folder = Path(temp_orig, "test1") + test_file = Path(temp_orig, "README.md") + test_hidden = Path(temp_orig, ".hidden") + + test_hidden.touch() + + assert test_folder.exists() + assert test_file.exists() + assert test_hidden.exists() + + await remove_folder(temp_orig, content_only=True) + + assert not test_folder.exists() + assert not test_file.exists() + assert not test_hidden.exists() From 19620d6808113747f815ef89cd9542c603ac320e Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sun, 29 Nov 2020 14:00:29 +0100 Subject: [PATCH 05/27] Fix: cleanup repository from list (#2310) * Fix: cleanup repository from list * Add repr * Avoid not exists error message --- .../resolution/fixups/store_execute_remove.py | 2 ++ supervisor/store/repository.py | 10 ++++++++++ .../fixup/test_store_execute_remove.py | 4 ++++ tests/store/test_custom_repository.py | 19 +++++++++++++++++-- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/supervisor/resolution/fixups/store_execute_remove.py b/supervisor/resolution/fixups/store_execute_remove.py index 4e7ef7f98..2dcf3524b 100644 --- a/supervisor/resolution/fixups/store_execute_remove.py +++ b/supervisor/resolution/fixups/store_execute_remove.py @@ -27,6 +27,8 @@ class FixupStoreExecuteRemove(FixupBase): await repository.remove() except StoreError: raise ResolutionFixupError() from None + else: + self.sys_store.repositories.pop(repository.slug, None) self.sys_config.drop_addon_repository(repository.source) self.sys_config.save_data() diff --git a/supervisor/store/repository.py b/supervisor/store/repository.py index 59f6df341..5269c05f1 100644 --- a/supervisor/store/repository.py +++ b/supervisor/store/repository.py @@ -40,6 +40,10 @@ class Repository(CoreSysAttributes): self._slug = get_hash_from_repository(repository) self._type = StoreType.GIT + def __repr__(self) -> str: + """Return internal representation.""" + return f"" + @property def slug(self) -> str: """Return repo slug.""" @@ -75,11 +79,17 @@ class Repository(CoreSysAttributes): if self.type != StoreType.GIT: return True + # If exists? repository_file = Path(self.git.path, "repository.json") + if not repository_file.exists(): + return False + + # If valid? try: SCHEMA_REPOSITORY_CONFIG(read_json_file(repository_file)) except (JsonFileError, vol.Invalid): return False + return True async def load(self) -> None: diff --git a/tests/resolution/fixup/test_store_execute_remove.py b/tests/resolution/fixup/test_store_execute_remove.py index 75de4c5dc..4139f4c3b 100644 --- a/tests/resolution/fixup/test_store_execute_remove.py +++ b/tests/resolution/fixup/test_store_execute_remove.py @@ -22,6 +22,8 @@ async def test_fixup(coresys: CoreSys): ) mock_repositorie = AsyncMock() + mock_repositorie.slug = "test" + coresys.store.repositories["test"] = mock_repositorie await store_execute_remove() @@ -30,3 +32,5 @@ async def test_fixup(coresys: CoreSys): assert coresys.config.save_data.called assert len(coresys.resolution.suggestions) == 0 assert len(coresys.resolution.issues) == 0 + + assert "test" not in coresys.store.repositories diff --git a/tests/store/test_custom_repository.py b/tests/store/test_custom_repository.py index 2ca1ff1e2..32d39b1c5 100644 --- a/tests/store/test_custom_repository.py +++ b/tests/store/test_custom_repository.py @@ -15,7 +15,7 @@ async def test_add_valid_repository(coresys, store_manager): with patch("supervisor.store.repository.Repository.load", return_value=None), patch( "pathlib.Path.read_text", return_value=json.dumps({"name": "Awesome repository"}), - ): + ), patch("pathlib.Path.exists", return_value=True): await store_manager.update_repositories(current + ["http://example.com"]) assert store_manager.get_from_url("http://example.com").validate() assert "http://example.com" in coresys.config.addons_repositories @@ -30,7 +30,7 @@ async def test_add_valid_repository_url(coresys, store_manager): return_value=json.dumps( {"name": "Awesome repository", "url": "http://example2.com/docs"} ), - ): + ), patch("pathlib.Path.exists", return_value=True): await store_manager.update_repositories(current + ["http://example.com"]) assert store_manager.get_from_url("http://example.com").validate() assert "http://example.com" in coresys.config.addons_repositories @@ -51,6 +51,21 @@ async def test_add_invalid_repository(coresys, store_manager): assert coresys.resolution.suggestions[-1].type == SuggestionType.EXECUTE_REMOVE +@pytest.mark.asyncio +async def test_add_invalid_repository_file(coresys, store_manager): + """Test add custom repository.""" + current = coresys.config.addons_repositories + with patch("supervisor.store.repository.Repository.load", return_value=None), patch( + "pathlib.Path.read_text", + return_value=json.dumps({"name": "Awesome repository"}), + ), patch("pathlib.Path.exists", return_value=False): + await store_manager.update_repositories(current + ["http://example.com"]) + assert not store_manager.get_from_url("http://example.com").validate() + + assert "http://example.com" in coresys.config.addons_repositories + assert coresys.resolution.suggestions[-1].type == SuggestionType.EXECUTE_REMOVE + + @pytest.mark.asyncio async def test_preinstall_valid_repository(coresys, store_manager): """Test add core repository valid.""" From 49853e92a45935fd8c213f66529a42afc067e2ef Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 30 Nov 2020 11:01:10 +0100 Subject: [PATCH 06/27] Fix sentry spam (#2313) --- supervisor/bootstrap.py | 1 - supervisor/misc/filter.py | 4 ++++ tests/misc/test_filter_data.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/supervisor/bootstrap.py b/supervisor/bootstrap.py index 56676c7b2..9fbf492c2 100644 --- a/supervisor/bootstrap.py +++ b/supervisor/bootstrap.py @@ -303,7 +303,6 @@ def setup_diagnostics(coresys: CoreSys) -> None: sentry_sdk.init( dsn="https://9c6ea70f49234442b4746e447b24747e@o427061.ingest.sentry.io/5370612", before_send=lambda event, hint: filter_data(coresys, event, hint), - auto_enabling_integrations=False, integrations=[AioHttpIntegration(), sentry_logging], release=SUPERVISOR_VERSION, max_breadcrumbs=30, diff --git a/supervisor/misc/filter.py b/supervisor/misc/filter.py index 05127bcd1..10622a171 100644 --- a/supervisor/misc/filter.py +++ b/supervisor/misc/filter.py @@ -76,6 +76,10 @@ def filter_data(coresys: CoreSys, event: dict, hint: dict) -> dict: }, "resolution": { "issues": [attr.asdict(issue) for issue in coresys.resolution.issues], + "suggestions": [ + attr.asdict(suggestion) + for suggestion in coresys.resolution.suggestions + ], "unhealthy": coresys.resolution.unhealthy, }, } diff --git a/tests/misc/test_filter_data.py b/tests/misc/test_filter_data.py index 804533314..94eed42da 100644 --- a/tests/misc/test_filter_data.py +++ b/tests/misc/test_filter_data.py @@ -10,6 +10,7 @@ from supervisor.misc.filter import filter_data from supervisor.resolution.const import ( ContextType, IssueType, + SuggestionType, UnhealthyReason, UnsupportedReason, ) @@ -124,6 +125,34 @@ def test_issues_on_report(coresys): assert event["contexts"]["resolution"]["issues"][0]["context"] == ContextType.SYSTEM +def test_suggestions_on_report(coresys): + """Attach suggestion to report.""" + + coresys.resolution.create_issue( + IssueType.FATAL_ERROR, + ContextType.SYSTEM, + suggestions=[SuggestionType.EXECUTE_RELOAD], + ) + + coresys.config.diagnostics = True + coresys.core.state = CoreState.RUNNING + + with patch("shutil.disk_usage", return_value=(42, 42, 2 * (1024.0 ** 3))): + event = filter_data(coresys, SAMPLE_EVENT, {}) + + assert "issues" in event["contexts"]["resolution"] + assert event["contexts"]["resolution"]["issues"][0]["type"] == IssueType.FATAL_ERROR + assert event["contexts"]["resolution"]["issues"][0]["context"] == ContextType.SYSTEM + assert ( + event["contexts"]["resolution"]["suggestions"][0]["type"] + == SuggestionType.EXECUTE_RELOAD + ) + assert ( + event["contexts"]["resolution"]["suggestions"][0]["context"] + == ContextType.SYSTEM + ) + + def test_unhealthy_on_report(coresys): """Attach unhealthy to report.""" From d9c4dae739cfff6d1b9369eee24daf7031f69127 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 30 Nov 2020 10:28:05 +0000 Subject: [PATCH 07/27] Send error to sentry from update --- supervisor/supervisor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/supervisor/supervisor.py b/supervisor/supervisor.py index afb643c24..63360b625 100644 --- a/supervisor/supervisor.py +++ b/supervisor/supervisor.py @@ -137,6 +137,7 @@ class Supervisor(CoreSysAttributes): self.sys_resolution.create_issue( IssueType.UPDATE_FAILED, ContextType.SUPERVISOR ) + self.sys_capture_exception(err) raise SupervisorUpdateError() from err else: self.sys_config.version = version From e09a8391485889f10dbceaa16baf6975e19619f6 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 30 Nov 2020 15:22:02 +0100 Subject: [PATCH 08/27] Fix byte order for network DNS settings (#2315) --- supervisor/dbus/payloads/generate.py | 17 ++++++++++++++++- supervisor/dbus/payloads/interface_update.tmpl | 4 ++-- .../payloads/test_interface_update_payload.py | 6 +++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/supervisor/dbus/payloads/generate.py b/supervisor/dbus/payloads/generate.py index f2e432940..1c4051f89 100644 --- a/supervisor/dbus/payloads/generate.py +++ b/supervisor/dbus/payloads/generate.py @@ -1,7 +1,9 @@ """Payload generators for DBUS communication.""" from __future__ import annotations +from ipaddress import IPv4Address, IPv6Address from pathlib import Path +import socket from typing import TYPE_CHECKING, Optional from uuid import uuid4 @@ -22,7 +24,20 @@ def interface_update_payload( interface: Interface, name: Optional[str] = None, uuid: Optional[str] = None ) -> str: """Generate a payload for network interface update.""" - template = jinja2.Template(INTERFACE_UPDATE_TEMPLATE.read_text()) + env = jinja2.Environment() + + def ipv4_to_int(ip_address: IPv4Address) -> int: + """Convert an ipv4 to an int.""" + return socket.htonl(int(ip_address)) + + def ipv6_to_byte(ip_address: IPv6Address) -> str: + """Convert an ipv6 to an byte array.""" + return f'[byte {", ".join("0x{:02x}".format(val) for val in reversed(ip_address.packed))}]' + + # Init template + env.filters["ipv4_to_int"] = ipv4_to_int + env.filters["ipv6_to_byte"] = ipv6_to_byte + template: jinja2.Template = env.from_string(INTERFACE_UPDATE_TEMPLATE.read_text()) # Generate UUID if not uuid: diff --git a/supervisor/dbus/payloads/interface_update.tmpl b/supervisor/dbus/payloads/interface_update.tmpl index db57669bd..8b65e0faa 100644 --- a/supervisor/dbus/payloads/interface_update.tmpl +++ b/supervisor/dbus/payloads/interface_update.tmpl @@ -21,7 +21,7 @@ 'method': <'disabled'> {% else %} 'method': <'manual'>, - 'dns': <[uint32 {{ interface.ipv4.nameservers | map("int") | join(",") }}]>, + 'dns': <[uint32 {{ interface.ipv4.nameservers | map("ipv4_to_int") | join(",") }}]>, 'address-data': <[ {% for address in interface.ipv4.address %} { @@ -44,7 +44,7 @@ 'method': <'disabled'> {% else %} 'method': <'manual'>, - 'dns': <[uint32 {{ interface.ipv6.nameservers | map("int") | join(",") }}]>, + 'dns': <[{{ interface.ipv6.nameservers | map("ipv6_to_byte") | join(",") }}]>, 'address-data': <[ {% for address in interface.ipv6.address if not address.with_prefixlen.startswith("fe80::") %} { diff --git a/tests/dbus/payloads/test_interface_update_payload.py b/tests/dbus/payloads/test_interface_update_payload.py index 6458db79e..7b7ade14d 100644 --- a/tests/dbus/payloads/test_interface_update_payload.py +++ b/tests/dbus/payloads/test_interface_update_payload.py @@ -49,7 +49,7 @@ async def test_interface_update_payload_ethernet_ipv4(coresys): DBus.parse_gvariant(data)["ipv4"]["address-data"][0]["address"] == "192.168.1.1" ) assert DBus.parse_gvariant(data)["ipv4"]["address-data"][0]["prefix"] == 24 - assert DBus.parse_gvariant(data)["ipv4"]["dns"] == [16843009, 16777473] + assert DBus.parse_gvariant(data)["ipv4"]["dns"] == [16843009, 16842753] assert ( DBus.parse_gvariant(data)["connection"]["uuid"] == inet.settings.connection.uuid ) @@ -129,8 +129,8 @@ async def test_interface_update_payload_ethernet_ipv6(coresys): ) assert DBus.parse_gvariant(data)["ipv6"]["address-data"][0]["prefix"] == 64 assert DBus.parse_gvariant(data)["ipv6"]["dns"] == [ - 50543257694033307102031451402929176676, - 50543257694033307102031451402929202176, + [100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 71, 6, 38], + [0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 71, 6, 38], ] assert ( DBus.parse_gvariant(data)["connection"]["uuid"] == inet.settings.connection.uuid From 2d294f68418fbbcdbad776cdd9c614ef952074c4 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 30 Nov 2020 18:00:12 +0100 Subject: [PATCH 09/27] Make evaluation of container better (#2316) --- supervisor/bootstrap.py | 1 + supervisor/resolution/check.py | 3 +-- supervisor/resolution/evaluate.py | 4 +--- supervisor/resolution/evaluations/container.py | 16 ++++++++++------ supervisor/resolution/fixup.py | 6 ++---- .../evaluation/test_evaluate_container.py | 16 +++++++++++----- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/supervisor/bootstrap.py b/supervisor/bootstrap.py index 9fbf492c2..56676c7b2 100644 --- a/supervisor/bootstrap.py +++ b/supervisor/bootstrap.py @@ -303,6 +303,7 @@ def setup_diagnostics(coresys: CoreSys) -> None: sentry_sdk.init( dsn="https://9c6ea70f49234442b4746e447b24747e@o427061.ingest.sentry.io/5370612", before_send=lambda event, hint: filter_data(coresys, event, hint), + auto_enabling_integrations=False, integrations=[AioHttpIntegration(), sentry_logging], release=SUPERVISOR_VERSION, max_breadcrumbs=30, diff --git a/supervisor/resolution/check.py b/supervisor/resolution/check.py index 9196de8a6..eca0de579 100644 --- a/supervisor/resolution/check.py +++ b/supervisor/resolution/check.py @@ -3,7 +3,6 @@ import logging from typing import List from ..coresys import CoreSys, CoreSysAttributes -from ..exceptions import HassioError from .checks.base import CheckBase from .checks.free_space import CheckFreeSpace @@ -31,7 +30,7 @@ class ResolutionCheck(CoreSysAttributes): for test in self.all_tests: try: await test() - except HassioError as err: + except Exception as err: # pylint: disable=broad-except _LOGGER.warning("Error during processing %s: %s", test.issue, err) self.sys_capture_exception(err) diff --git a/supervisor/resolution/evaluate.py b/supervisor/resolution/evaluate.py index c7cbb60e4..7c14092c0 100644 --- a/supervisor/resolution/evaluate.py +++ b/supervisor/resolution/evaluate.py @@ -2,8 +2,6 @@ import logging from typing import List -from supervisor.exceptions import HassioError - from ..coresys import CoreSys, CoreSysAttributes from .const import UnhealthyReason, UnsupportedReason from .evaluations.base import EvaluateBase @@ -69,7 +67,7 @@ class ResolutionEvaluation(CoreSysAttributes): for evaluation in self.all_evalutions: try: await evaluation() - except HassioError as err: + except Exception as err: # pylint: disable=broad-except _LOGGER.warning( "Error during processing %s: %s", evaluation.reason, err ) diff --git a/supervisor/resolution/evaluations/container.py b/supervisor/resolution/evaluations/container.py index b0645d80a..2c02139a2 100644 --- a/supervisor/resolution/evaluations/container.py +++ b/supervisor/resolution/evaluations/container.py @@ -13,9 +13,8 @@ from .base import EvaluateBase _LOGGER: logging.Logger = logging.getLogger(__name__) DOCKER_IMAGE_DENYLIST = [ - "containrrr/watchtower", - "pyouroboros/ouroboros", - "v2tec/watchtower", + "watchtower", + "ouroboros", ] @@ -41,16 +40,21 @@ class EvaluateContainer(EvaluateBase): @property def states(self) -> List[CoreState]: """Return a list of valid states when this evaluation can run.""" - return [CoreState.SETUP, CoreState.RUNNING] + return [CoreState.SETUP, CoreState.RUNNING, CoreState.INITIALIZE] async def evaluate(self) -> None: """Run evaluation.""" self._images.clear() for image in await self.sys_run_in_executor(self._get_images): for tag in image.tags: - image_name = tag.split(":")[0] + image_name = tag.partition(":")[0].split("/")[-1] + + # Evalue system if ( - image_name in DOCKER_IMAGE_DENYLIST + any( + image_name.startswith(deny_name) + for deny_name in DOCKER_IMAGE_DENYLIST + ) and image_name not in self._images ): self._images.add(image_name) diff --git a/supervisor/resolution/fixup.py b/supervisor/resolution/fixup.py index d07dd2837..d82ece6fd 100644 --- a/supervisor/resolution/fixup.py +++ b/supervisor/resolution/fixup.py @@ -2,10 +2,8 @@ import logging from typing import List -from supervisor.exceptions import HassioError -from supervisor.resolution.data import Suggestion - from ..coresys import CoreSys, CoreSysAttributes +from .data import Suggestion from .fixups.base import FixupBase from .fixups.clear_full_snapshot import FixupClearFullSnapshot from .fixups.create_full_snapshot import FixupCreateFullSnapshot @@ -49,7 +47,7 @@ class ResolutionFixup(CoreSysAttributes): continue try: await fix() - except HassioError as err: + except Exception as err: # pylint: disable=broad-except _LOGGER.warning("Error during processing %s: %s", fix.suggestion, err) self.sys_capture_exception(err) diff --git a/tests/resolution/evaluation/test_evaluate_container.py b/tests/resolution/evaluation/test_evaluate_container.py index 15c59e1e7..75d52659d 100644 --- a/tests/resolution/evaluation/test_evaluate_container.py +++ b/tests/resolution/evaluation/test_evaluate_container.py @@ -6,10 +6,7 @@ from docker.errors import DockerException from supervisor.const import CoreState from supervisor.coresys import CoreSys -from supervisor.resolution.evaluations.container import ( - DOCKER_IMAGE_DENYLIST, - EvaluateContainer, -) +from supervisor.resolution.evaluations.container import EvaluateContainer def test_get_images(coresys: CoreSys): @@ -37,7 +34,16 @@ async def test_evaluation(coresys: CoreSys): with patch( "supervisor.resolution.evaluations.container.EvaluateContainer._get_images", - return_value=[MagicMock(tags=[f"{DOCKER_IMAGE_DENYLIST[0]}:latest"])], + return_value=[ + MagicMock( + tags=[ + "armhfbuild/watchtower:latest", + "concerco/watchtowerv6:10.0.2", + "containrrr/watchtower:1.1", + "pyouroboros/ouroboros:1.4.3", + ] + ) + ], ): await container() assert container.reason in coresys.resolution.unsupported From fb4386a7ada95339b397294576e5ff34cc8eef9c Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 1 Dec 2020 09:27:49 +0100 Subject: [PATCH 10/27] Use persistent MAC address instead clone / don't touch default (#2317) --- supervisor/dbus/network/configuration.py | 1 + supervisor/dbus/network/setting.py | 2 ++ supervisor/dbus/payloads/interface_update.tmpl | 4 ++-- supervisor/host/network.py | 11 ++++++++--- tests/api/test_network.py | 14 ++++++++++++-- .../dbus/payloads/test_interface_update_payload.py | 6 ++++-- 6 files changed, 29 insertions(+), 9 deletions(-) diff --git a/supervisor/dbus/network/configuration.py b/supervisor/dbus/network/configuration.py index a203a5378..887e3caf3 100644 --- a/supervisor/dbus/network/configuration.py +++ b/supervisor/dbus/network/configuration.py @@ -32,6 +32,7 @@ class ConnectionProperties: id: Optional[str] = attr.ib() uuid: Optional[str] = attr.ib() type: Optional[str] = attr.ib() + interface_name: Optional[str] = attr.ib() @attr.s(slots=True) diff --git a/supervisor/dbus/network/setting.py b/supervisor/dbus/network/setting.py index a1e190c4c..83d1ff3df 100644 --- a/supervisor/dbus/network/setting.py +++ b/supervisor/dbus/network/setting.py @@ -31,6 +31,7 @@ ATTR_ASSIGNED_MAC = "assigned-mac-address" ATTR_POWERSAVE = "powersave" ATTR_AUTH_ALGO = "auth-algo" ATTR_KEY_MGMT = "key-mgmt" +ATTR_INTERFACE_NAME = "interface-name" class NetworkSetting(DBusInterfaceProxy): @@ -109,6 +110,7 @@ class NetworkSetting(DBusInterfaceProxy): data[CONF_ATTR_CONNECTION].get(ATTR_ID), data[CONF_ATTR_CONNECTION].get(ATTR_UUID), data[CONF_ATTR_CONNECTION].get(ATTR_TYPE), + data[CONF_ATTR_CONNECTION].get(ATTR_INTERFACE_NAME), ) if CONF_ATTR_802_ETHERNET in data: diff --git a/supervisor/dbus/payloads/interface_update.tmpl b/supervisor/dbus/payloads/interface_update.tmpl index 8b65e0faa..a2af64ede 100644 --- a/supervisor/dbus/payloads/interface_update.tmpl +++ b/supervisor/dbus/payloads/interface_update.tmpl @@ -61,7 +61,7 @@ , '802-3-ethernet': { - 'assigned-mac-address': <'stable'> + 'assigned-mac-address': <'preserve'> } {% endif %} @@ -78,7 +78,7 @@ , '802-11-wireless': { - 'assigned-mac-address': <'stable'>, + 'assigned-mac-address': <'preserve'>, 'ssid': <[byte {{ interface.wifi.ssid }}]>, 'mode': <'{{ interface.wifi.mode.value }}'>, 'powersave': diff --git a/supervisor/host/network.py b/supervisor/host/network.py index 952e34e46..e55607384 100644 --- a/supervisor/host/network.py +++ b/supervisor/host/network.py @@ -111,7 +111,12 @@ class NetworkManager(CoreSysAttributes): inet = self.sys_dbus.network.interfaces.get(interface.name) # Update exist configuration - if inet and inet.settings and interface.enabled: + if ( + inet + and inet.settings + and inet.settings.connection.interface_name == interface.name + and interface.enabled + ): settings = interface_update_payload( interface, name=inet.settings.connection.id, @@ -279,7 +284,7 @@ class Interface: inet.connection.ipv4.nameservers, ) if inet.connection and inet.connection.ipv4 - else None, + else IpConfig(InterfaceMethod.DISABLED, [], None, []), IpConfig( Interface._map_nm_method(inet.settings.ipv6.method), inet.connection.ipv6.address, @@ -287,7 +292,7 @@ class Interface: inet.connection.ipv6.nameservers, ) if inet.connection and inet.connection.ipv6 - else None, + else IpConfig(InterfaceMethod.DISABLED, [], None, []), Interface._map_nm_wifi(inet), Interface._map_nm_vlan(inet), ) diff --git a/tests/api/test_network.py b/tests/api/test_network.py index f4f2739ae..fc5c1aaa2 100644 --- a/tests/api/test_network.py +++ b/tests/api/test_network.py @@ -26,8 +26,18 @@ async def test_api_network_info(api_client, coresys): assert interface["ipv4"]["gateway"] == "192.168.2.1" if interface["interface"] == TEST_INTERFACE_WLAN: assert not interface["primary"] - assert interface["ipv4"] is None - assert interface["ipv6"] is None + assert interface["ipv4"] == { + "address": [], + "gateway": None, + "method": "disabled", + "nameservers": [], + } + assert interface["ipv6"] == { + "address": [], + "gateway": None, + "method": "disabled", + "nameservers": [], + } assert result["data"]["docker"]["interface"] == DOCKER_NETWORK assert result["data"]["docker"]["address"] == str(DOCKER_NETWORK_MASK) diff --git a/tests/dbus/payloads/test_interface_update_payload.py b/tests/dbus/payloads/test_interface_update_payload.py index 7b7ade14d..9540d4237 100644 --- a/tests/dbus/payloads/test_interface_update_payload.py +++ b/tests/dbus/payloads/test_interface_update_payload.py @@ -21,7 +21,8 @@ async def test_interface_update_payload_ethernet(coresys): assert DBus.parse_gvariant(data)["ipv6"]["method"] == "auto" assert ( - DBus.parse_gvariant(data)["802-3-ethernet"]["assigned-mac-address"] == "stable" + DBus.parse_gvariant(data)["802-3-ethernet"]["assigned-mac-address"] + == "preserve" ) assert DBus.parse_gvariant(data)["connection"]["mdns"] == 2 @@ -243,7 +244,8 @@ async def test_interface_update_payload_wireless_open(coresys): assert DBus.parse_gvariant(data)["802-11-wireless"]["ssid"] == [84, 101, 115, 116] assert DBus.parse_gvariant(data)["802-11-wireless"]["mode"] == "infrastructure" assert ( - DBus.parse_gvariant(data)["802-11-wireless"]["assigned-mac-address"] == "stable" + DBus.parse_gvariant(data)["802-11-wireless"]["assigned-mac-address"] + == "preserve" ) assert "802-11-wireless-security" not in DBus.parse_gvariant(data) From 06e10fdd3cd62d63fbc6cd750a16f8aa30c21cee Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 1 Dec 2020 12:11:20 +0100 Subject: [PATCH 11/27] Fix NM naming schema (#2318) * Fix NM naming schema * Address comments * Add test --- supervisor/dbus/payloads/generate.py | 8 ++++---- .../dbus/payloads/test_interface_update_payload.py | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/supervisor/dbus/payloads/generate.py b/supervisor/dbus/payloads/generate.py index 1c4051f89..7ee049cb9 100644 --- a/supervisor/dbus/payloads/generate.py +++ b/supervisor/dbus/payloads/generate.py @@ -43,11 +43,11 @@ def interface_update_payload( if not uuid: uuid = str(uuid4()) - # Generate ID/name - if not name and interface.type != InterfaceType.VLAN: + # Generate/Update ID/name + if not name or not name.startswith("Supervisor"): name = f"Supervisor {interface.name}" - elif not name: - name = f"Supervisor {interface.name}.{interface.vlan.id}" + if interface.type == InterfaceType.VLAN: + name = f"{name}.{interface.vlan.id}" # Fix SSID if interface.wifi: diff --git a/tests/dbus/payloads/test_interface_update_payload.py b/tests/dbus/payloads/test_interface_update_payload.py index 9540d4237..439fa12c1 100644 --- a/tests/dbus/payloads/test_interface_update_payload.py +++ b/tests/dbus/payloads/test_interface_update_payload.py @@ -54,7 +54,7 @@ async def test_interface_update_payload_ethernet_ipv4(coresys): assert ( DBus.parse_gvariant(data)["connection"]["uuid"] == inet.settings.connection.uuid ) - assert DBus.parse_gvariant(data)["connection"]["id"] == inet.settings.connection.id + assert DBus.parse_gvariant(data)["connection"]["id"] == "Supervisor eth0" assert DBus.parse_gvariant(data)["connection"]["type"] == "802-3-ethernet" assert DBus.parse_gvariant(data)["connection"]["interface-name"] == interface.name assert DBus.parse_gvariant(data)["ipv4"]["gateway"] == "192.168.1.1" @@ -77,7 +77,7 @@ async def test_interface_update_payload_ethernet_ipv4_disabled(coresys): assert ( DBus.parse_gvariant(data)["connection"]["uuid"] == inet.settings.connection.uuid ) - assert DBus.parse_gvariant(data)["connection"]["id"] == inet.settings.connection.id + assert DBus.parse_gvariant(data)["connection"]["id"] == "Supervisor eth0" assert DBus.parse_gvariant(data)["connection"]["type"] == "802-3-ethernet" assert DBus.parse_gvariant(data)["connection"]["interface-name"] == interface.name @@ -99,7 +99,7 @@ async def test_interface_update_payload_ethernet_ipv4_auto(coresys): assert ( DBus.parse_gvariant(data)["connection"]["uuid"] == inet.settings.connection.uuid ) - assert DBus.parse_gvariant(data)["connection"]["id"] == inet.settings.connection.id + assert DBus.parse_gvariant(data)["connection"]["id"] == "Supervisor eth0" assert DBus.parse_gvariant(data)["connection"]["type"] == "802-3-ethernet" assert DBus.parse_gvariant(data)["connection"]["interface-name"] == interface.name @@ -136,7 +136,7 @@ async def test_interface_update_payload_ethernet_ipv6(coresys): assert ( DBus.parse_gvariant(data)["connection"]["uuid"] == inet.settings.connection.uuid ) - assert DBus.parse_gvariant(data)["connection"]["id"] == inet.settings.connection.id + assert DBus.parse_gvariant(data)["connection"]["id"] == "Supervisor eth0" assert DBus.parse_gvariant(data)["connection"]["type"] == "802-3-ethernet" assert DBus.parse_gvariant(data)["connection"]["interface-name"] == interface.name assert DBus.parse_gvariant(data)["ipv6"]["gateway"] == "fe80::da58:d7ff:fe00:9c69" @@ -158,7 +158,7 @@ async def test_interface_update_payload_ethernet_ipv6_disabled(coresys): assert ( DBus.parse_gvariant(data)["connection"]["uuid"] == inet.settings.connection.uuid ) - assert DBus.parse_gvariant(data)["connection"]["id"] == inet.settings.connection.id + assert DBus.parse_gvariant(data)["connection"]["id"] == "Supervisor eth0" assert DBus.parse_gvariant(data)["connection"]["type"] == "802-3-ethernet" assert DBus.parse_gvariant(data)["connection"]["interface-name"] == interface.name @@ -179,7 +179,7 @@ async def test_interface_update_payload_ethernet_ipv6_auto(coresys): assert ( DBus.parse_gvariant(data)["connection"]["uuid"] == inet.settings.connection.uuid ) - assert DBus.parse_gvariant(data)["connection"]["id"] == inet.settings.connection.id + assert DBus.parse_gvariant(data)["connection"]["id"] == "Supervisor eth0" assert DBus.parse_gvariant(data)["connection"]["type"] == "802-3-ethernet" assert DBus.parse_gvariant(data)["connection"]["interface-name"] == interface.name @@ -265,4 +265,5 @@ async def test_interface_update_payload_vlan(coresys): assert DBus.parse_gvariant(data)["vlan"]["id"] == 10 assert DBus.parse_gvariant(data)["vlan"]["parent"] == interface.name assert DBus.parse_gvariant(data)["connection"]["type"] == "vlan" + assert DBus.parse_gvariant(data)["connection"]["id"] == "Supervisor eth0.10" assert "interface-name" not in DBus.parse_gvariant(data)["connection"] From e2a473baa3441b4e91785cfaee1d814e3084b108 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 1 Dec 2020 14:03:59 +0100 Subject: [PATCH 12/27] Add network reload to api (#2319) --- supervisor/api/__init__.py | 1 + supervisor/api/network.py | 7 ++++++- tests/api/test_network.py | 9 +++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/supervisor/api/__init__.py b/supervisor/api/__init__.py index 004f2604c..3ea30f0e0 100644 --- a/supervisor/api/__init__.py +++ b/supervisor/api/__init__.py @@ -112,6 +112,7 @@ class RestAPI(CoreSysAttributes): self.webapp.add_routes( [ web.get("/network/info", api_network.info), + web.post("/network/reload", api_network.reload), web.get( "/network/interface/{interface}/info", api_network.interface_info ), diff --git a/supervisor/api/network.py b/supervisor/api/network.py index d0e78bd5c..e49f0e9b4 100644 --- a/supervisor/api/network.py +++ b/supervisor/api/network.py @@ -1,7 +1,7 @@ """REST API for network.""" import asyncio from ipaddress import ip_address, ip_interface -from typing import Any, Dict +from typing import Any, Awaitable, Dict from aiohttp import web import attr @@ -207,6 +207,11 @@ class APINetwork(CoreSysAttributes): await asyncio.shield(self.sys_host.network.apply_changes(interface)) + @api_process + def reload(self, request: web.Request) -> Awaitable[None]: + """Reload network data.""" + return asyncio.shield(self.sys_host.network.update()) + @api_process async def scan_accesspoints(self, request: web.Request) -> Dict[str, Any]: """Scan and return a list of available networks.""" diff --git a/tests/api/test_network.py b/tests/api/test_network.py index fc5c1aaa2..2a0fae161 100644 --- a/tests/api/test_network.py +++ b/tests/api/test_network.py @@ -185,3 +185,12 @@ async def test_api_network_wireless_scan(api_client): ap["ssid"] for ap in result["data"]["accesspoints"] ] assert [47, 63] == [ap["signal"] for ap in result["data"]["accesspoints"]] + + +@pytest.mark.asyncio +async def test_api_network_reload(api_client, coresys): + """Test network manager reload api.""" + resp = await api_client.post("/network/reload") + result = await resp.json() + + assert result["result"] == "ok" From 0b085354db9677fb65e450fe488c4f8dbfb99215 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 1 Dec 2020 14:29:37 +0100 Subject: [PATCH 13/27] Send list of images to sentry (#2321) --- supervisor/misc/filter.py | 1 + supervisor/resolution/evaluate.py | 4 +++- supervisor/resolution/evaluations/container.py | 12 ++++++++++-- tests/misc/test_filter_data.py | 14 ++++++++++++++ .../evaluation/test_evaluate_container.py | 9 +++++++++ 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/supervisor/misc/filter.py b/supervisor/misc/filter.py index 10622a171..209d85b7a 100644 --- a/supervisor/misc/filter.py +++ b/supervisor/misc/filter.py @@ -62,6 +62,7 @@ def filter_data(coresys: CoreSys, event: dict, hint: dict) -> dict: "host": coresys.host.info.operating_system, "kernel": coresys.host.info.kernel, "machine": coresys.machine, + "images": coresys.resolution.evaluate.cached_images, }, "versions": { "audio": coresys.plugins.audio.version, diff --git a/supervisor/resolution/evaluate.py b/supervisor/resolution/evaluate.py index 7c14092c0..1947cf819 100644 --- a/supervisor/resolution/evaluate.py +++ b/supervisor/resolution/evaluate.py @@ -1,6 +1,6 @@ """Helpers to evaluate the system.""" import logging -from typing import List +from typing import List, Set from ..coresys import CoreSys, CoreSysAttributes from .const import UnhealthyReason, UnsupportedReason @@ -33,6 +33,8 @@ class ResolutionEvaluation(CoreSysAttributes): """Initialize the evaluation class.""" self.coresys = coresys + self.cached_images: Set[str] = set() + self._container = EvaluateContainer(coresys) self._dbus = EvaluateDbus(coresys) self._docker_configuration = EvaluateDockerConfiguration(coresys) diff --git a/supervisor/resolution/evaluations/container.py b/supervisor/resolution/evaluations/container.py index 2c02139a2..af107ae51 100644 --- a/supervisor/resolution/evaluations/container.py +++ b/supervisor/resolution/evaluations/container.py @@ -7,7 +7,7 @@ from requests import RequestException from ...const import CoreState from ...coresys import CoreSys -from ..const import UnsupportedReason +from ..const import ContextType, IssueType, SuggestionType, UnsupportedReason from .base import EvaluateBase _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -44,12 +44,15 @@ class EvaluateContainer(EvaluateBase): async def evaluate(self) -> None: """Run evaluation.""" + self.sys_resolution.evaluate.cached_images.clear() self._images.clear() + for image in await self.sys_run_in_executor(self._get_images): for tag in image.tags: - image_name = tag.partition(":")[0].split("/")[-1] + self.sys_resolution.evaluate.cached_images.add(tag) # Evalue system + image_name = tag.partition(":")[0].split("/")[-1] if ( any( image_name.startswith(deny_name) @@ -68,5 +71,10 @@ class EvaluateContainer(EvaluateBase): images = self.sys_docker.images.list() except (DockerException, RequestException) as err: _LOGGER.error("Corrupt docker overlayfs detect: %s", err) + self.sys_resolution.create_issue( + IssueType.CORRUPT_DOCKER, + ContextType.SYSTEM, + suggestions=[SuggestionType.EXECUTE_REPAIR], + ) return images diff --git a/tests/misc/test_filter_data.py b/tests/misc/test_filter_data.py index 94eed42da..e8c2dcc78 100644 --- a/tests/misc/test_filter_data.py +++ b/tests/misc/test_filter_data.py @@ -165,3 +165,17 @@ def test_unhealthy_on_report(coresys): assert "issues" in event["contexts"]["resolution"] assert event["contexts"]["resolution"]["unhealthy"][-1] == UnhealthyReason.DOCKER + + +def test_images_report(coresys): + """Attach image to report.""" + + coresys.config.diagnostics = True + coresys.core.state = CoreState.RUNNING + coresys.resolution.evaluate.cached_images.add("my/test:image") + + with patch("shutil.disk_usage", return_value=(42, 42, 2 * (1024.0 ** 3))): + event = filter_data(coresys, SAMPLE_EVENT, {}) + + assert "issues" in event["contexts"]["resolution"] + assert event["contexts"]["host"]["images"] == {"my/test:image"} diff --git a/tests/resolution/evaluation/test_evaluate_container.py b/tests/resolution/evaluation/test_evaluate_container.py index 75d52659d..f75de3afd 100644 --- a/tests/resolution/evaluation/test_evaluate_container.py +++ b/tests/resolution/evaluation/test_evaluate_container.py @@ -48,6 +48,13 @@ async def test_evaluation(coresys: CoreSys): await container() assert container.reason in coresys.resolution.unsupported + assert coresys.resolution.evaluate.cached_images == { + "armhfbuild/watchtower:latest", + "concerco/watchtowerv6:10.0.2", + "containrrr/watchtower:1.1", + "pyouroboros/ouroboros:1.4.3", + } + with patch( "supervisor.resolution.evaluations.container.EvaluateContainer._get_images", return_value=[MagicMock(tags=[])], @@ -55,6 +62,8 @@ async def test_evaluation(coresys: CoreSys): await container() assert container.reason not in coresys.resolution.unsupported + assert coresys.resolution.evaluate.cached_images == set() + async def test_did_run(coresys: CoreSys): """Test that the evaluation ran as expected.""" From 59102afd452e4b42c793d25342cfc9f67c50e312 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 2 Dec 2020 12:43:38 +0100 Subject: [PATCH 14/27] Fix sentry with sets (#2323) --- supervisor/misc/filter.py | 2 +- tests/misc/test_filter_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/supervisor/misc/filter.py b/supervisor/misc/filter.py index 209d85b7a..2d3d1072f 100644 --- a/supervisor/misc/filter.py +++ b/supervisor/misc/filter.py @@ -62,7 +62,7 @@ def filter_data(coresys: CoreSys, event: dict, hint: dict) -> dict: "host": coresys.host.info.operating_system, "kernel": coresys.host.info.kernel, "machine": coresys.machine, - "images": coresys.resolution.evaluate.cached_images, + "images": list(coresys.resolution.evaluate.cached_images), }, "versions": { "audio": coresys.plugins.audio.version, diff --git a/tests/misc/test_filter_data.py b/tests/misc/test_filter_data.py index e8c2dcc78..9eaa2b0c1 100644 --- a/tests/misc/test_filter_data.py +++ b/tests/misc/test_filter_data.py @@ -178,4 +178,4 @@ def test_images_report(coresys): event = filter_data(coresys, SAMPLE_EVENT, {}) assert "issues" in event["contexts"]["resolution"] - assert event["contexts"]["host"]["images"] == {"my/test:image"} + assert event["contexts"]["host"]["images"] == ["my/test:image"] From 80763c4bbf74d8640e806c33928a8e8018a6d8d1 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 2 Dec 2020 14:59:42 +0100 Subject: [PATCH 15/27] Support OTA url over version files (#2324) * Support OTA url over version files * Fix new schema --- supervisor/const.py | 6 +----- supervisor/hassos.py | 9 ++++++--- supervisor/updater.py | 21 ++++++++++++++------- supervisor/validate.py | 3 +++ 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/supervisor/const.py b/supervisor/const.py index a09eb9166..4c153076b 100644 --- a/supervisor/const.py +++ b/supervisor/const.py @@ -9,11 +9,6 @@ URL_HASSIO_ADDONS = "https://github.com/home-assistant/addons" URL_HASSIO_APPARMOR = "https://version.home-assistant.io/apparmor.txt" URL_HASSIO_VERSION = "https://version.home-assistant.io/{channel}.json" -URL_HASSOS_OTA = ( - "https://github.com/home-assistant/operating-system/releases/download/" - "{version}/hassos_{board}-{version}.raucb" -) - SUPERVISOR_DATA = Path("/data") FILE_HASSIO_ADDONS = Path(SUPERVISOR_DATA, "addons.json") @@ -290,6 +285,7 @@ ATTR_MAC = "mac" ATTR_FREQUENCY = "frequency" ATTR_ACCESSPOINTS = "accesspoints" ATTR_UNHEALTHY = "unhealthy" +ATTR_OTA = "ota" PROVIDE_SERVICE = "provide" NEED_SERVICE = "need" diff --git a/supervisor/hassos.py b/supervisor/hassos.py index 2780616a7..3e9503d06 100644 --- a/supervisor/hassos.py +++ b/supervisor/hassos.py @@ -8,7 +8,6 @@ import aiohttp from cpe import CPE from packaging.version import parse as pkg_parse -from .const import URL_HASSOS_OTA from .coresys import CoreSys, CoreSysAttributes from .dbus.rauc import RaucState from .exceptions import DBusError, HassOSNotSupportedError, HassOSUpdateError @@ -64,10 +63,14 @@ class HassOS(CoreSysAttributes): async def _download_raucb(self, version: str) -> Path: """Download rauc bundle (OTA) from github.""" - url = URL_HASSOS_OTA.format(version=version, board=self.board) - raucb = Path(self.sys_config.path_tmp, f"hassos-{version}.raucb") + raw_url = self.sys_updater.ota_url + if raw_url is None: + _LOGGER.error("Don't have an URL for OTA updates!") + raise HassOSNotSupportedError() + url = raw_url.format(version=version, board=self.board) _LOGGER.info("Fetch OTA update from %s", url) + raucb = Path(self.sys_config.path_tmp, f"hassos-{version}.raucb") try: timeout = aiohttp.ClientTimeout(total=60 * 60, connect=180) async with self.sys_websession.get(url, timeout=timeout) as request: diff --git a/supervisor/updater.py b/supervisor/updater.py index 75dbac7e2..eb408b15d 100644 --- a/supervisor/updater.py +++ b/supervisor/updater.py @@ -18,6 +18,7 @@ from .const import ( ATTR_IMAGE, ATTR_MULTICAST, ATTR_OBSERVER, + ATTR_OTA, ATTR_SUPERVISOR, FILE_HASSIO_UPDATER, URL_HASSIO_VERSION, @@ -93,7 +94,7 @@ class Updater(JsonConfig, CoreSysAttributes): @property def image_homeassistant(self) -> Optional[str]: - """Return latest version of Home Assistant.""" + """Return image of Home Assistant docker.""" if ATTR_HOMEASSISTANT not in self._data[ATTR_IMAGE]: return None return self._data[ATTR_IMAGE][ATTR_HOMEASSISTANT].format( @@ -102,7 +103,7 @@ class Updater(JsonConfig, CoreSysAttributes): @property def image_supervisor(self) -> Optional[str]: - """Return latest version of Supervisor.""" + """Return image of Supervisor docker.""" if ATTR_SUPERVISOR not in self._data[ATTR_IMAGE]: return None return self._data[ATTR_IMAGE][ATTR_SUPERVISOR].format( @@ -111,28 +112,28 @@ class Updater(JsonConfig, CoreSysAttributes): @property def image_cli(self) -> Optional[str]: - """Return latest version of CLI.""" + """Return image of CLI docker.""" if ATTR_CLI not in self._data[ATTR_IMAGE]: return None return self._data[ATTR_IMAGE][ATTR_CLI].format(arch=self.sys_arch.supervisor) @property def image_dns(self) -> Optional[str]: - """Return latest version of DNS.""" + """Return image of DNS docker.""" if ATTR_DNS not in self._data[ATTR_IMAGE]: return None return self._data[ATTR_IMAGE][ATTR_DNS].format(arch=self.sys_arch.supervisor) @property def image_audio(self) -> Optional[str]: - """Return latest version of Audio.""" + """Return image of Audio docker.""" if ATTR_AUDIO not in self._data[ATTR_IMAGE]: return None return self._data[ATTR_IMAGE][ATTR_AUDIO].format(arch=self.sys_arch.supervisor) @property def image_observer(self) -> Optional[str]: - """Return latest version of Observer.""" + """Return image of Observer docker.""" if ATTR_OBSERVER not in self._data[ATTR_IMAGE]: return None return self._data[ATTR_IMAGE][ATTR_OBSERVER].format( @@ -141,13 +142,18 @@ class Updater(JsonConfig, CoreSysAttributes): @property def image_multicast(self) -> Optional[str]: - """Return latest version of Multicast.""" + """Return image of Multicast docker.""" if ATTR_MULTICAST not in self._data[ATTR_IMAGE]: return None return self._data[ATTR_IMAGE][ATTR_MULTICAST].format( arch=self.sys_arch.supervisor ) + @property + def ota_url(self) -> Optional[str]: + """Return OTA url for OS.""" + return self._data.get(ATTR_OTA) + @property def channel(self) -> UpdateChannel: """Return upstream channel of Supervisor instance.""" @@ -196,6 +202,7 @@ class Updater(JsonConfig, CoreSysAttributes): # Update HassOS version if self.sys_hassos.board: self._data[ATTR_HASSOS] = data["hassos"][self.sys_hassos.board] + self._data[ATTR_OTA] = data["ota"] # Update Home Assistant plugins self._data[ATTR_CLI] = data["cli"] diff --git a/supervisor/validate.py b/supervisor/validate.py index 48925c6ba..294a6f8ed 100644 --- a/supervisor/validate.py +++ b/supervisor/validate.py @@ -27,6 +27,7 @@ from .const import ( ATTR_LOGGING, ATTR_MULTICAST, ATTR_OBSERVER, + ATTR_OTA, ATTR_PASSWORD, ATTR_PORT, ATTR_PORTS, @@ -135,6 +136,7 @@ SCHEMA_HASS_CONFIG = vol.Schema( ) +# pylint: disable=no-value-for-parameter SCHEMA_UPDATER_CONFIG = vol.Schema( { vol.Optional(ATTR_CHANNEL, default=UpdateChannel.STABLE): vol.Coerce( @@ -160,6 +162,7 @@ SCHEMA_UPDATER_CONFIG = vol.Schema( }, extra=vol.REMOVE_EXTRA, ), + vol.Optional(ATTR_OTA): vol.Url(), }, extra=vol.REMOVE_EXTRA, ) From 3d79891249e5d061017f682e8d0b50c8c0934f44 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 2 Dec 2020 17:22:28 +0100 Subject: [PATCH 16/27] Make dbus connection more robust (#2325) * Make dbus connection more robust * move rauc down --- supervisor/dbus/__init__.py | 25 +++++++++++++++++-------- supervisor/dbus/hostname.py | 2 ++ supervisor/dbus/interface.py | 1 + supervisor/dbus/network/__init__.py | 2 ++ supervisor/dbus/rauc.py | 2 ++ supervisor/dbus/systemd.py | 2 ++ 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/supervisor/dbus/__init__.py b/supervisor/dbus/__init__.py index da89bffc8..4ad5bb44d 100644 --- a/supervisor/dbus/__init__.py +++ b/supervisor/dbus/__init__.py @@ -1,9 +1,11 @@ """D-Bus interface objects.""" import logging +from typing import List +from ..const import SOCKET_DBUS from ..coresys import CoreSys, CoreSysAttributes -from ..exceptions import DBusNotConnectedError from .hostname import Hostname +from .interface import DBusInterface from .network import NetworkManager from .rauc import Rauc from .systemd import Systemd @@ -45,15 +47,22 @@ class DBusManager(CoreSysAttributes): async def load(self) -> None: """Connect interfaces to D-Bus.""" - - try: - await self.systemd.connect() - await self.hostname.connect() - await self.rauc.connect() - await self.network.connect() - except DBusNotConnectedError: + if not SOCKET_DBUS.exists(): _LOGGER.error( "No D-Bus support on Host. Disabled any kind of host control!" ) + return + + dbus_loads: List[DBusInterface] = [ + self.systemd, + self.hostname, + self.network, + self.rauc, + ] + for dbus in dbus_loads: + try: + await dbus.connect() + except Exception as err: # pylint: disable=broad-except + _LOGGER.warning("Can't load dbus interface %s: %s", dbus.name, err) self.sys_host.supported_features.cache_clear() diff --git a/supervisor/dbus/hostname.py b/supervisor/dbus/hostname.py index 65d64cf5a..93ddbf656 100644 --- a/supervisor/dbus/hostname.py +++ b/supervisor/dbus/hostname.py @@ -23,6 +23,8 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) class Hostname(DBusInterface): """Handle D-Bus interface for hostname/system.""" + name = DBUS_NAME_HOSTNAME + def __init__(self): """Initialize Properties.""" self._hostname: Optional[str] = None diff --git a/supervisor/dbus/interface.py b/supervisor/dbus/interface.py index 8a3562e90..c0c085f33 100644 --- a/supervisor/dbus/interface.py +++ b/supervisor/dbus/interface.py @@ -9,6 +9,7 @@ class DBusInterface(ABC): """Handle D-Bus interface for hostname/system.""" dbus: Optional[DBus] = None + name: Optional[str] = None @property def is_connected(self): diff --git a/supervisor/dbus/network/__init__.py b/supervisor/dbus/network/__init__.py index 4b998e0d4..1e137eea1 100644 --- a/supervisor/dbus/network/__init__.py +++ b/supervisor/dbus/network/__init__.py @@ -27,6 +27,8 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) class NetworkManager(DBusInterface): """Handle D-Bus interface for Network Manager.""" + name = DBUS_NAME_NM + def __init__(self) -> None: """Initialize Properties.""" self._dns: NetworkManagerDNS = NetworkManagerDNS() diff --git a/supervisor/dbus/rauc.py b/supervisor/dbus/rauc.py index 0a29112ed..7eb1d18c6 100644 --- a/supervisor/dbus/rauc.py +++ b/supervisor/dbus/rauc.py @@ -25,6 +25,8 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) class Rauc(DBusInterface): """Handle D-Bus interface for rauc.""" + name = DBUS_NAME_RAUC + def __init__(self): """Initialize Properties.""" self._operation: Optional[str] = None diff --git a/supervisor/dbus/systemd.py b/supervisor/dbus/systemd.py index 659304c5c..6e5c4d2de 100644 --- a/supervisor/dbus/systemd.py +++ b/supervisor/dbus/systemd.py @@ -13,6 +13,8 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) class Systemd(DBusInterface): """Systemd function handler.""" + name = DBUS_NAME_SYSTEMD + async def connect(self): """Connect to D-Bus.""" try: From 6462eea2efb1ab0a5a1b00afa24cc28051ef6850 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 2 Dec 2020 17:24:40 +0100 Subject: [PATCH 17/27] Fix sentry stdlib spam (#2326) --- supervisor/bootstrap.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/supervisor/bootstrap.py b/supervisor/bootstrap.py index 56676c7b2..f99207490 100644 --- a/supervisor/bootstrap.py +++ b/supervisor/bootstrap.py @@ -8,7 +8,11 @@ import signal from colorlog import ColoredFormatter import sentry_sdk from sentry_sdk.integrations.aiohttp import AioHttpIntegration +from sentry_sdk.integrations.atexit import AtexitIntegration +from sentry_sdk.integrations.dedupe import DedupeIntegration +from sentry_sdk.integrations.excepthook import ExcepthookIntegration from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.threading import ThreadingIntegration from supervisor.jobs import JobManager @@ -295,16 +299,20 @@ def supervisor_debugger(coresys: CoreSys) -> None: def setup_diagnostics(coresys: CoreSys) -> None: """Sentry diagnostic backend.""" - sentry_logging = LoggingIntegration( - level=logging.WARNING, event_level=logging.CRITICAL - ) - _LOGGER.info("Initializing Supervisor Sentry") sentry_sdk.init( dsn="https://9c6ea70f49234442b4746e447b24747e@o427061.ingest.sentry.io/5370612", before_send=lambda event, hint: filter_data(coresys, event, hint), auto_enabling_integrations=False, - integrations=[AioHttpIntegration(), sentry_logging], + default_integrations=False, + integrations=[ + AioHttpIntegration(), + ExcepthookIntegration(), + DedupeIntegration(), + AtexitIntegration(), + ThreadingIntegration(), + LoggingIntegration(level=logging.WARNING, event_level=logging.CRITICAL), + ], release=SUPERVISOR_VERSION, max_breadcrumbs=30, ) From f8fd7b5933d38e6ec2b35d819bdc83de666e94a9 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 3 Dec 2020 12:24:32 +0100 Subject: [PATCH 18/27] Improve error handling with job condition (#2322) * Improve error handling with job condition * fix * first patch * last patch * Address comments * Revert strange replace --- supervisor/addons/__init__.py | 13 +++++++++---- supervisor/exceptions.py | 32 ++++++++++++++++++++++++-------- supervisor/homeassistant/core.py | 4 +++- supervisor/jobs/decorator.py | 17 ++++++++++++----- supervisor/snapshots/__init__.py | 19 ++++++++++++------- supervisor/supervisor.py | 3 ++- supervisor/updater.py | 19 +++++++++++-------- tests/jobs/test_job_decorator.py | 25 +++++++++++++++++++++++++ 8 files changed, 98 insertions(+), 34 deletions(-) diff --git a/supervisor/addons/__init__.py b/supervisor/addons/__init__.py index f3902e8b2..fcbc241b5 100644 --- a/supervisor/addons/__init__.py +++ b/supervisor/addons/__init__.py @@ -10,6 +10,7 @@ from ..coresys import CoreSys, CoreSysAttributes from ..exceptions import ( AddonConfigurationError, AddonsError, + AddonsJobError, AddonsNotSupportedError, CoreDNSError, DockerAPIError, @@ -147,7 +148,8 @@ class AddonManager(CoreSysAttributes): JobCondition.FREE_SPACE, JobCondition.INTERNET_HOST, JobCondition.HEALTHY, - ] + ], + on_condition=AddonsJobError, ) async def install(self, slug: str) -> None: """Install an add-on.""" @@ -248,7 +250,8 @@ class AddonManager(CoreSysAttributes): JobCondition.FREE_SPACE, JobCondition.INTERNET_HOST, JobCondition.HEALTHY, - ] + ], + on_condition=AddonsJobError, ) async def update(self, slug: str) -> None: """Update add-on.""" @@ -297,7 +300,8 @@ class AddonManager(CoreSysAttributes): JobCondition.FREE_SPACE, JobCondition.INTERNET_HOST, JobCondition.HEALTHY, - ] + ], + on_condition=AddonsJobError, ) async def rebuild(self, slug: str) -> None: """Perform a rebuild of local build add-on.""" @@ -339,7 +343,8 @@ class AddonManager(CoreSysAttributes): JobCondition.FREE_SPACE, JobCondition.INTERNET_HOST, JobCondition.HEALTHY, - ] + ], + on_condition=AddonsJobError, ) async def restore(self, slug: str, tar_file: tarfile.TarFile) -> None: """Restore state of an add-on.""" diff --git a/supervisor/exceptions.py b/supervisor/exceptions.py index f832fa9e4..0d8507046 100644 --- a/supervisor/exceptions.py +++ b/supervisor/exceptions.py @@ -9,6 +9,13 @@ class HassioNotSupportedError(HassioError): """Function is not supported.""" +# JobManager + + +class JobException(HassioError): + """Base job exception.""" + + # HomeAssistant @@ -32,6 +39,10 @@ class HomeAssistantAuthError(HomeAssistantAPIError): """Home Assistant Auth API exception.""" +class HomeAssistantJobError(HomeAssistantError, JobException): + """Raise on Home Assistant job error.""" + + # Supervisor @@ -43,6 +54,10 @@ class SupervisorUpdateError(SupervisorError): """Supervisor update error.""" +class SupervisorJobError(SupervisorError, JobException): + """Raise on job errors.""" + + # HassOS @@ -128,6 +143,10 @@ class AddonsNotSupportedError(HassioNotSupportedError): """Addons don't support a function.""" +class AddonsJobError(AddonsError, JobException): + """Raise on job errors.""" + + # Arch @@ -138,10 +157,14 @@ class HassioArchNotFound(HassioNotSupportedError): # Updater -class HassioUpdaterError(HassioError): +class UpdaterError(HassioError): """Error on Updater.""" +class UpdaterJobError(UpdaterError, JobException): + """Raise on job error.""" + + # Auth @@ -312,10 +335,3 @@ class StoreGitError(StoreError): class StoreNotFound(StoreError): """Raise if slug is not known.""" - - -# JobManager - - -class JobException(HassioError): - """Base job exception.""" diff --git a/supervisor/homeassistant/core.py b/supervisor/homeassistant/core.py index 97b71a7fe..61c80ee88 100644 --- a/supervisor/homeassistant/core.py +++ b/supervisor/homeassistant/core.py @@ -19,6 +19,7 @@ from ..exceptions import ( DockerError, HomeAssistantCrashError, HomeAssistantError, + HomeAssistantJobError, HomeAssistantUpdateError, ) from ..jobs.decorator import Job, JobCondition @@ -158,7 +159,8 @@ class HomeAssistantCore(CoreSysAttributes): JobCondition.FREE_SPACE, JobCondition.HEALTHY, JobCondition.INTERNET_HOST, - ] + ], + on_condition=HomeAssistantJobError, ) async def update(self, version: Optional[str] = None) -> None: """Update HomeAssistant version.""" diff --git a/supervisor/jobs/decorator.py b/supervisor/jobs/decorator.py index 05b39dd78..5c36a8236 100644 --- a/supervisor/jobs/decorator.py +++ b/supervisor/jobs/decorator.py @@ -1,6 +1,6 @@ """Job decorator.""" import logging -from typing import List, Optional +from typing import Any, List, Optional import sentry_sdk @@ -21,11 +21,13 @@ class Job: name: Optional[str] = None, conditions: Optional[List[JobCondition]] = None, cleanup: bool = True, + on_condition: Optional[JobException] = None, ): """Initialize the Job class.""" self.name = name self.conditions = conditions self.cleanup = cleanup + self.on_condition = on_condition self._coresys: Optional[CoreSys] = None self._method = None @@ -33,23 +35,28 @@ class Job: """Call the wrapper logic.""" self._method = method - async def wrapper(*args, **kwargs): + async def wrapper(*args, **kwargs) -> Any: """Wrap the method.""" if self.name is None: self.name = str(self._method.__qualname__).lower().replace(".", "_") + + # Evaluate coresys try: self._coresys = args[0].coresys except AttributeError: - return False - + pass if not self._coresys: raise JobException(f"coresys is missing on {self.name}") job = self._coresys.jobs.get_job(self.name) + # Handle condition if self.conditions and not self._check_conditions(): - return False + if self.on_condition is None: + return + raise self.on_condition() + # Execute Job try: return await self._method(*args, **kwargs) except HassioError as err: diff --git a/supervisor/snapshots/__init__.py b/supervisor/snapshots/__init__.py index af22e50f5..4cd2cfdce 100644 --- a/supervisor/snapshots/__init__.py +++ b/supervisor/snapshots/__init__.py @@ -122,7 +122,7 @@ class SnapshotManager(CoreSysAttributes): self.snapshots_obj[snapshot.slug] = snapshot return snapshot - @Job(conditions=[JobCondition.FREE_SPACE]) + @Job(conditions=[JobCondition.FREE_SPACE, JobCondition.RUNNING]) async def do_snapshot_full(self, name="", password=None): """Create a full snapshot.""" if self.lock.locked(): @@ -144,9 +144,9 @@ class SnapshotManager(CoreSysAttributes): _LOGGER.info("Snapshotting %s store folders", snapshot.slug) await snapshot.store_folders() - except Exception as excep: # pylint: disable=broad-except + except Exception as err: # pylint: disable=broad-except _LOGGER.exception("Snapshot %s error", snapshot.slug) - print(excep) + self.sys_capture_exception(err) return None else: @@ -158,7 +158,7 @@ class SnapshotManager(CoreSysAttributes): self.sys_core.state = CoreState.RUNNING self.lock.release() - @Job(conditions=[JobCondition.FREE_SPACE]) + @Job(conditions=[JobCondition.FREE_SPACE, JobCondition.RUNNING]) async def do_snapshot_partial( self, name="", addons=None, folders=None, password=None ): @@ -195,8 +195,9 @@ class SnapshotManager(CoreSysAttributes): _LOGGER.info("Snapshotting %s store folders", snapshot.slug) await snapshot.store_folders(folders) - except Exception: # pylint: disable=broad-except + except Exception as err: # pylint: disable=broad-except _LOGGER.exception("Snapshot %s error", snapshot.slug) + self.sys_capture_exception(err) return None else: @@ -216,6 +217,7 @@ class SnapshotManager(CoreSysAttributes): JobCondition.HEALTHY, JobCondition.INTERNET_HOST, JobCondition.INTERNET_SYSTEM, + JobCondition.RUNNING, ] ) async def do_restore_full(self, snapshot, password=None): @@ -282,8 +284,9 @@ class SnapshotManager(CoreSysAttributes): await task_hass await self.sys_homeassistant.core.start() - except Exception: # pylint: disable=broad-except + except Exception as err: # pylint: disable=broad-except _LOGGER.exception("Restore %s error", snapshot.slug) + self.sys_capture_exception(err) return False else: @@ -300,6 +303,7 @@ class SnapshotManager(CoreSysAttributes): JobCondition.HEALTHY, JobCondition.INTERNET_HOST, JobCondition.INTERNET_SYSTEM, + JobCondition.RUNNING, ] ) async def do_restore_partial( @@ -368,8 +372,9 @@ class SnapshotManager(CoreSysAttributes): _LOGGER.warning("Need restart HomeAssistant for API") await self.sys_homeassistant.core.restart() - except Exception: # pylint: disable=broad-except + except Exception as err: # pylint: disable=broad-except _LOGGER.exception("Restore %s error", snapshot.slug) + self.sys_capture_exception(err) return False else: diff --git a/supervisor/supervisor.py b/supervisor/supervisor.py index 63360b625..c2f97e3a7 100644 --- a/supervisor/supervisor.py +++ b/supervisor/supervisor.py @@ -21,6 +21,7 @@ from .exceptions import ( DockerError, HostAppArmorError, SupervisorError, + SupervisorJobError, SupervisorUpdateError, ) from .resolution.const import ContextType, IssueType @@ -147,7 +148,7 @@ class Supervisor(CoreSysAttributes): await self.update_apparmor() self.sys_create_task(self.sys_core.stop()) - @Job(conditions=[JobCondition.RUNNING]) + @Job(conditions=[JobCondition.RUNNING], on_condition=SupervisorJobError) async def restart(self) -> None: """Restart Supervisor soft.""" self.sys_core.exit_code = 100 diff --git a/supervisor/updater.py b/supervisor/updater.py index eb408b15d..985af95fc 100644 --- a/supervisor/updater.py +++ b/supervisor/updater.py @@ -25,7 +25,7 @@ from .const import ( UpdateChannel, ) from .coresys import CoreSysAttributes -from .exceptions import HassioUpdaterError +from .exceptions import UpdaterError, UpdaterJobError from .jobs.decorator import Job, JobCondition from .utils import AsyncThrottle from .utils.json import JsonConfig @@ -44,12 +44,12 @@ class Updater(JsonConfig, CoreSysAttributes): async def load(self) -> None: """Update internal data.""" - with suppress(HassioUpdaterError): + with suppress(UpdaterError): await self.fetch_data() async def reload(self) -> None: """Update internal data.""" - with suppress(HassioUpdaterError): + with suppress(UpdaterJobError): await self.fetch_data() @property @@ -165,7 +165,10 @@ class Updater(JsonConfig, CoreSysAttributes): self._data[ATTR_CHANNEL] = value @AsyncThrottle(timedelta(seconds=30)) - @Job(conditions=[JobCondition.INTERNET_SYSTEM]) + @Job( + conditions=[JobCondition.INTERNET_SYSTEM], + on_condition=UpdaterJobError, + ) async def fetch_data(self): """Fetch current versions from Github. @@ -181,16 +184,16 @@ class Updater(JsonConfig, CoreSysAttributes): except (aiohttp.ClientError, asyncio.TimeoutError) as err: _LOGGER.warning("Can't fetch versions from %s: %s", url, err) - raise HassioUpdaterError() from err + raise UpdaterError() from err except json.JSONDecodeError as err: _LOGGER.warning("Can't parse versions from %s: %s", url, err) - raise HassioUpdaterError() from err + raise UpdaterError() from err # data valid? if not data or data.get(ATTR_CHANNEL) != self.channel: _LOGGER.warning("Invalid data from %s", url) - raise HassioUpdaterError() + raise UpdaterError() try: # Update supervisor version @@ -222,7 +225,7 @@ class Updater(JsonConfig, CoreSysAttributes): except KeyError as err: _LOGGER.warning("Can't process version data: %s", err) - raise HassioUpdaterError() from err + raise UpdaterError() from err else: self.save_data() diff --git a/tests/jobs/test_job_decorator.py b/tests/jobs/test_job_decorator.py index 48422944d..d13a6f7a0 100644 --- a/tests/jobs/test_job_decorator.py +++ b/tests/jobs/test_job_decorator.py @@ -232,3 +232,28 @@ async def test_ignore_conditions(coresys: CoreSys): coresys.jobs.ignore_conditions = [JobCondition.RUNNING] assert await test.execute() + + +async def test_exception_conditions(coresys: CoreSys): + """Test the ignore conditions decorator.""" + + class TestClass: + """Test class.""" + + def __init__(self, coresys: CoreSys): + """Initialize the test class.""" + self.coresys = coresys + + @Job(conditions=[JobCondition.RUNNING], on_condition=HassioError) + async def execute(self): + """Execute the class method.""" + return True + + test = TestClass(coresys) + + coresys.core.state = CoreState.RUNNING + assert await test.execute() + + coresys.core.state = CoreState.FREEZE + with pytest.raises(HassioError): + await test.execute() From 0c55bf20fceb96f3974fc453dc1be58edebcaf0c Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 3 Dec 2020 21:06:48 +0100 Subject: [PATCH 19/27] Fix issue on store git clone (#2331) --- supervisor/bootstrap.py | 2 +- supervisor/core.py | 1 + supervisor/coresys.py | 2 +- supervisor/exceptions.py | 8 + supervisor/resolution/__init__.py | 182 +----------------- supervisor/resolution/fixup.py | 5 +- .../resolution/fixups/store_execute_reset.py | 13 +- supervisor/resolution/module.py | 181 +++++++++++++++++ supervisor/store/__init__.py | 10 +- supervisor/store/git.py | 24 ++- supervisor/store/repository.py | 2 +- 11 files changed, 238 insertions(+), 192 deletions(-) create mode 100644 supervisor/resolution/module.py diff --git a/supervisor/bootstrap.py b/supervisor/bootstrap.py index f99207490..5b1ade822 100644 --- a/supervisor/bootstrap.py +++ b/supervisor/bootstrap.py @@ -44,7 +44,7 @@ from .misc.hwmon import HwMonitor from .misc.scheduler import Scheduler from .misc.tasks import Tasks from .plugins import PluginManager -from .resolution import ResolutionManager +from .resolution.module import ResolutionManager from .services import ServiceManager from .snapshots import SnapshotManager from .store import StoreManager diff --git a/supervisor/core.py b/supervisor/core.py index 10dd24b6a..68b975c27 100644 --- a/supervisor/core.py +++ b/supervisor/core.py @@ -83,6 +83,7 @@ class Core(CoreSysAttributes): """Start setting up supervisor orchestration.""" self.state = CoreState.SETUP + # Order can be important! setup_loads: List[Awaitable[None]] = [ # rest api views self.sys_api.load(), diff --git a/supervisor/coresys.py b/supervisor/coresys.py index 60db6a3bf..954fb46c7 100644 --- a/supervisor/coresys.py +++ b/supervisor/coresys.py @@ -30,7 +30,7 @@ if TYPE_CHECKING: from .misc.scheduler import Scheduler from .misc.tasks import Tasks from .plugins import PluginManager - from .resolution import ResolutionManager + from .resolution.module import ResolutionManager from .services import ServiceManager from .snapshots import SnapshotManager from .store import StoreManager diff --git a/supervisor/exceptions.py b/supervisor/exceptions.py index 0d8507046..848e6a397 100644 --- a/supervisor/exceptions.py +++ b/supervisor/exceptions.py @@ -322,6 +322,10 @@ class ResolutionFixupError(HassioError): """Rasie if a fixup fails.""" +class ResolutionFixupJobError(ResolutionFixupError, JobException): + """Raise on job error.""" + + # Store @@ -335,3 +339,7 @@ class StoreGitError(StoreError): class StoreNotFound(StoreError): """Raise if slug is not known.""" + + +class StoreJobError(StoreError, JobException): + """Raise on job error with git.""" diff --git a/supervisor/resolution/__init__.py b/supervisor/resolution/__init__.py index cb1f85874..700244dda 100644 --- a/supervisor/resolution/__init__.py +++ b/supervisor/resolution/__init__.py @@ -1,181 +1 @@ -"""Supervisor resolution center.""" -from datetime import time -import logging -from typing import List, Optional - -from ..coresys import CoreSys, CoreSysAttributes -from ..exceptions import ResolutionError, ResolutionNotFound -from .check import ResolutionCheck -from .const import ( - SCHEDULED_HEALTHCHECK, - ContextType, - IssueType, - SuggestionType, - UnhealthyReason, - UnsupportedReason, -) -from .data import Issue, Suggestion -from .evaluate import ResolutionEvaluation -from .fixup import ResolutionFixup -from .notify import ResolutionNotify - -_LOGGER: logging.Logger = logging.getLogger(__name__) - - -class ResolutionManager(CoreSysAttributes): - """Resolution manager for supervisor.""" - - def __init__(self, coresys: CoreSys): - """Initialize Resolution manager.""" - self.coresys: CoreSys = coresys - self._evaluate = ResolutionEvaluation(coresys) - self._check = ResolutionCheck(coresys) - self._fixup = ResolutionFixup(coresys) - self._notify = ResolutionNotify(coresys) - - self._suggestions: List[Suggestion] = [] - self._issues: List[Issue] = [] - self._unsupported: List[UnsupportedReason] = [] - self._unhealthy: List[UnhealthyReason] = [] - - @property - def evaluate(self) -> ResolutionEvaluation: - """Return the ResolutionEvaluation class.""" - return self._evaluate - - @property - def check(self) -> ResolutionCheck: - """Return the ResolutionCheck class.""" - return self._check - - @property - def fixup(self) -> ResolutionFixup: - """Return the ResolutionFixup class.""" - return self._fixup - - @property - def notify(self) -> ResolutionNotify: - """Return the ResolutionNotify class.""" - return self._notify - - @property - def issues(self) -> List[Issue]: - """Return a list of issues.""" - return self._issues - - @issues.setter - def issues(self, issue: Issue) -> None: - """Add issues.""" - if issue not in self._issues: - self._issues.append(issue) - - @property - def suggestions(self) -> List[Suggestion]: - """Return a list of suggestions that can handled.""" - return self._suggestions - - @suggestions.setter - def suggestions(self, suggestion: Suggestion) -> None: - """Add suggestion.""" - if suggestion not in self._suggestions: - self._suggestions.append(suggestion) - - @property - def unsupported(self) -> List[UnsupportedReason]: - """Return a list of unsupported reasons.""" - return self._unsupported - - @unsupported.setter - def unsupported(self, reason: UnsupportedReason) -> None: - """Add a reason for unsupported.""" - if reason not in self._unsupported: - self._unsupported.append(reason) - - @property - def unhealthy(self) -> List[UnhealthyReason]: - """Return a list of unsupported reasons.""" - return self._unhealthy - - @unhealthy.setter - def unhealthy(self, reason: UnhealthyReason) -> None: - """Add a reason for unsupported.""" - if reason not in self._unhealthy: - self._unhealthy.append(reason) - - def get_suggestion(self, uuid: str) -> Suggestion: - """Return suggestion with uuid.""" - for suggestion in self._suggestions: - if suggestion.uuid != uuid: - continue - return suggestion - raise ResolutionNotFound() - - def get_issue(self, uuid: str) -> Issue: - """Return issue with uuid.""" - for issue in self._issues: - if issue.uuid != uuid: - continue - return issue - raise ResolutionNotFound() - - def create_issue( - self, - issue: IssueType, - context: ContextType, - reference: Optional[str] = None, - suggestions: Optional[List[SuggestionType]] = None, - ) -> None: - """Create issues and suggestion.""" - self.issues = Issue(issue, context, reference) - if not suggestions: - return - - # Add suggestions - for suggestion in suggestions: - self.suggestions = Suggestion(suggestion, context, reference) - - async def load(self): - """Load the resoulution manager.""" - # Initial healthcheck when the manager is loaded - await self.healthcheck() - - # Schedule the healthcheck - self.sys_scheduler.register_task(self.healthcheck, SCHEDULED_HEALTHCHECK) - self.sys_scheduler.register_task(self.fixup.run_autofix, time(hour=2)) - - async def healthcheck(self): - """Scheduled task to check for known issues.""" - await self.check.check_system() - - # Create notification for any known issues - await self.notify.issue_notifications() - - async def apply_suggestion(self, suggestion: Suggestion) -> None: - """Apply suggested action.""" - if suggestion not in self._suggestions: - _LOGGER.warning("Suggestion %s is not valid", suggestion.uuid) - raise ResolutionError() - - await self.fixup.apply_fixup(suggestion) - await self.healthcheck() - - def dismiss_suggestion(self, suggestion: Suggestion) -> None: - """Dismiss suggested action.""" - if suggestion not in self._suggestions: - _LOGGER.warning("The UUID %s is not valid suggestion", suggestion.uuid) - raise ResolutionError() - self._suggestions.remove(suggestion) - - def dismiss_issue(self, issue: Issue) -> None: - """Dismiss suggested action.""" - if issue not in self._issues: - _LOGGER.warning("The UUID %s is not a valid issue", issue.uuid) - raise ResolutionError() - self._issues.remove(issue) - - def dismiss_unsupported(self, reason: Issue) -> None: - """Dismiss a reason for unsupported.""" - if reason not in self._unsupported: - _LOGGER.warning("The reason %s is not active", reason) - raise ResolutionError() - self._unsupported.remove(reason) +"""Resolution Supervisor module.""" diff --git a/supervisor/resolution/fixup.py b/supervisor/resolution/fixup.py index d82ece6fd..69d0713e7 100644 --- a/supervisor/resolution/fixup.py +++ b/supervisor/resolution/fixup.py @@ -29,7 +29,10 @@ class ResolutionFixup(CoreSysAttributes): @property def all_fixes(self) -> List[FixupBase]: - """Return a list of all fixups.""" + """Return a list of all fixups. + + Order can be important! + """ return [ self._create_full_snapshot, self._clear_full_snapshot, diff --git a/supervisor/resolution/fixups/store_execute_reset.py b/supervisor/resolution/fixups/store_execute_reset.py index 1a1829546..b9cdac00c 100644 --- a/supervisor/resolution/fixups/store_execute_reset.py +++ b/supervisor/resolution/fixups/store_execute_reset.py @@ -2,8 +2,14 @@ import logging from typing import List, Optional -from supervisor.exceptions import ResolutionFixupError, StoreError, StoreNotFound - +from ...exceptions import ( + ResolutionFixupError, + ResolutionFixupJobError, + StoreError, + StoreNotFound, +) +from ...jobs.const import JobCondition +from ...jobs.decorator import Job from ...utils import remove_folder from ..const import ContextType, IssueType, SuggestionType from .base import FixupBase @@ -14,6 +20,9 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) class FixupStoreExecuteReset(FixupBase): """Storage class for fixup.""" + @Job( + conditions=[JobCondition.INTERNET_SYSTEM], on_condition=ResolutionFixupJobError + ) async def process_fixup(self, reference: Optional[str] = None) -> None: """Initialize the fixup class.""" _LOGGER.info("Reset corrupt Store: %s", reference) diff --git a/supervisor/resolution/module.py b/supervisor/resolution/module.py new file mode 100644 index 000000000..cb1f85874 --- /dev/null +++ b/supervisor/resolution/module.py @@ -0,0 +1,181 @@ +"""Supervisor resolution center.""" +from datetime import time +import logging +from typing import List, Optional + +from ..coresys import CoreSys, CoreSysAttributes +from ..exceptions import ResolutionError, ResolutionNotFound +from .check import ResolutionCheck +from .const import ( + SCHEDULED_HEALTHCHECK, + ContextType, + IssueType, + SuggestionType, + UnhealthyReason, + UnsupportedReason, +) +from .data import Issue, Suggestion +from .evaluate import ResolutionEvaluation +from .fixup import ResolutionFixup +from .notify import ResolutionNotify + +_LOGGER: logging.Logger = logging.getLogger(__name__) + + +class ResolutionManager(CoreSysAttributes): + """Resolution manager for supervisor.""" + + def __init__(self, coresys: CoreSys): + """Initialize Resolution manager.""" + self.coresys: CoreSys = coresys + self._evaluate = ResolutionEvaluation(coresys) + self._check = ResolutionCheck(coresys) + self._fixup = ResolutionFixup(coresys) + self._notify = ResolutionNotify(coresys) + + self._suggestions: List[Suggestion] = [] + self._issues: List[Issue] = [] + self._unsupported: List[UnsupportedReason] = [] + self._unhealthy: List[UnhealthyReason] = [] + + @property + def evaluate(self) -> ResolutionEvaluation: + """Return the ResolutionEvaluation class.""" + return self._evaluate + + @property + def check(self) -> ResolutionCheck: + """Return the ResolutionCheck class.""" + return self._check + + @property + def fixup(self) -> ResolutionFixup: + """Return the ResolutionFixup class.""" + return self._fixup + + @property + def notify(self) -> ResolutionNotify: + """Return the ResolutionNotify class.""" + return self._notify + + @property + def issues(self) -> List[Issue]: + """Return a list of issues.""" + return self._issues + + @issues.setter + def issues(self, issue: Issue) -> None: + """Add issues.""" + if issue not in self._issues: + self._issues.append(issue) + + @property + def suggestions(self) -> List[Suggestion]: + """Return a list of suggestions that can handled.""" + return self._suggestions + + @suggestions.setter + def suggestions(self, suggestion: Suggestion) -> None: + """Add suggestion.""" + if suggestion not in self._suggestions: + self._suggestions.append(suggestion) + + @property + def unsupported(self) -> List[UnsupportedReason]: + """Return a list of unsupported reasons.""" + return self._unsupported + + @unsupported.setter + def unsupported(self, reason: UnsupportedReason) -> None: + """Add a reason for unsupported.""" + if reason not in self._unsupported: + self._unsupported.append(reason) + + @property + def unhealthy(self) -> List[UnhealthyReason]: + """Return a list of unsupported reasons.""" + return self._unhealthy + + @unhealthy.setter + def unhealthy(self, reason: UnhealthyReason) -> None: + """Add a reason for unsupported.""" + if reason not in self._unhealthy: + self._unhealthy.append(reason) + + def get_suggestion(self, uuid: str) -> Suggestion: + """Return suggestion with uuid.""" + for suggestion in self._suggestions: + if suggestion.uuid != uuid: + continue + return suggestion + raise ResolutionNotFound() + + def get_issue(self, uuid: str) -> Issue: + """Return issue with uuid.""" + for issue in self._issues: + if issue.uuid != uuid: + continue + return issue + raise ResolutionNotFound() + + def create_issue( + self, + issue: IssueType, + context: ContextType, + reference: Optional[str] = None, + suggestions: Optional[List[SuggestionType]] = None, + ) -> None: + """Create issues and suggestion.""" + self.issues = Issue(issue, context, reference) + if not suggestions: + return + + # Add suggestions + for suggestion in suggestions: + self.suggestions = Suggestion(suggestion, context, reference) + + async def load(self): + """Load the resoulution manager.""" + # Initial healthcheck when the manager is loaded + await self.healthcheck() + + # Schedule the healthcheck + self.sys_scheduler.register_task(self.healthcheck, SCHEDULED_HEALTHCHECK) + self.sys_scheduler.register_task(self.fixup.run_autofix, time(hour=2)) + + async def healthcheck(self): + """Scheduled task to check for known issues.""" + await self.check.check_system() + + # Create notification for any known issues + await self.notify.issue_notifications() + + async def apply_suggestion(self, suggestion: Suggestion) -> None: + """Apply suggested action.""" + if suggestion not in self._suggestions: + _LOGGER.warning("Suggestion %s is not valid", suggestion.uuid) + raise ResolutionError() + + await self.fixup.apply_fixup(suggestion) + await self.healthcheck() + + def dismiss_suggestion(self, suggestion: Suggestion) -> None: + """Dismiss suggested action.""" + if suggestion not in self._suggestions: + _LOGGER.warning("The UUID %s is not valid suggestion", suggestion.uuid) + raise ResolutionError() + self._suggestions.remove(suggestion) + + def dismiss_issue(self, issue: Issue) -> None: + """Dismiss suggested action.""" + if issue not in self._issues: + _LOGGER.warning("The UUID %s is not a valid issue", issue.uuid) + raise ResolutionError() + self._issues.remove(issue) + + def dismiss_unsupported(self, reason: Issue) -> None: + """Dismiss a reason for unsupported.""" + if reason not in self._unsupported: + _LOGGER.warning("The reason %s is not active", reason) + raise ResolutionError() + self._unsupported.remove(reason) diff --git a/supervisor/store/__init__.py b/supervisor/store/__init__.py index ed2dd527e..c0e58402d 100644 --- a/supervisor/store/__init__.py +++ b/supervisor/store/__init__.py @@ -4,7 +4,7 @@ import logging from typing import Dict, List from ..coresys import CoreSys, CoreSysAttributes -from ..exceptions import StoreGitError, StoreNotFound +from ..exceptions import StoreGitError, StoreJobError, StoreNotFound from ..jobs.decorator import Job, JobCondition from ..resolution.const import ContextType, IssueType, SuggestionType from .addon import AddonStore @@ -83,6 +83,14 @@ class StoreManager(CoreSysAttributes): await repository.load() except StoreGitError: _LOGGER.error("Can't load data from repository %s", url) + except StoreJobError: + _LOGGER.warning("Skip update to later for %s", repository.slug) + self.sys_resolution.create_issue( + IssueType.FATAL_ERROR, + ContextType.STORE, + refrence=repository.slug, + suggestions=[SuggestionType.EXECUTE_RELOAD], + ) else: if not repository.validate(): _LOGGER.error("%s is not a valid add-on repository", url) diff --git a/supervisor/store/git.py b/supervisor/store/git.py index 11f76e984..34f234d6b 100644 --- a/supervisor/store/git.py +++ b/supervisor/store/git.py @@ -9,7 +9,7 @@ import git from ..const import ATTR_BRANCH, ATTR_URL, URL_HASSIO_ADDONS from ..coresys import CoreSys, CoreSysAttributes -from ..exceptions import StoreGitError +from ..exceptions import StoreGitError, StoreJobError from ..jobs.decorator import Job, JobCondition from ..resolution.const import ContextType, IssueType, SuggestionType from ..utils import remove_folder @@ -22,6 +22,8 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) class GitRepo(CoreSysAttributes): """Manage Add-on Git repository.""" + builtin: bool + def __init__(self, coresys: CoreSys, path: Path, url: str): """Initialize Git base wrapper.""" self.coresys: CoreSys = coresys @@ -82,7 +84,10 @@ class GitRepo(CoreSysAttributes): ) raise StoreGitError() from err - @Job(conditions=[JobCondition.FREE_SPACE, JobCondition.INTERNET_SYSTEM]) + @Job( + conditions=[JobCondition.FREE_SPACE, JobCondition.INTERNET_SYSTEM], + on_condition=StoreJobError, + ) async def clone(self) -> None: """Clone git add-on repository.""" async with self.lock: @@ -115,11 +120,18 @@ class GitRepo(CoreSysAttributes): IssueType.FATAL_ERROR, ContextType.STORE, reference=self.path.stem, - suggestions=[SuggestionType.EXECUTE_RELOAD], + suggestions=[ + SuggestionType.EXECUTE_RELOAD + if self.builtin + else SuggestionType.EXECUTE_REMOVE + ], ) raise StoreGitError() from err - @Job(conditions=[JobCondition.FREE_SPACE, JobCondition.INTERNET_SYSTEM]) + @Job( + conditions=[JobCondition.FREE_SPACE, JobCondition.INTERNET_SYSTEM], + on_condition=StoreJobError, + ) async def pull(self): """Pull Git add-on repo.""" if self.lock.locked(): @@ -175,6 +187,8 @@ class GitRepo(CoreSysAttributes): class GitRepoHassIO(GitRepo): """Supervisor add-ons repository.""" + builtin: bool = False + def __init__(self, coresys): """Initialize Git Supervisor add-on repository.""" super().__init__(coresys, coresys.config.path_addons_core, URL_HASSIO_ADDONS) @@ -183,6 +197,8 @@ class GitRepoHassIO(GitRepo): class GitRepoCustom(GitRepo): """Custom add-ons repository.""" + builtin: bool = False + def __init__(self, coresys, url): """Initialize custom Git Supervisor addo-n repository.""" path = Path(coresys.config.path_addons_git, get_hash_from_repository(url)) diff --git a/supervisor/store/repository.py b/supervisor/store/repository.py index 5269c05f1..18767b573 100644 --- a/supervisor/store/repository.py +++ b/supervisor/store/repository.py @@ -100,7 +100,7 @@ class Repository(CoreSysAttributes): async def update(self) -> None: """Update add-on repository.""" - if self.type == StoreType.LOCAL: + if self.type == StoreType.LOCAL or not self.validate(): return await self.git.pull() From 6245b6d82315a4961705dc97e935e01d95cca727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20S=C3=B8rensen?= Date: Thu, 3 Dec 2020 21:40:34 +0100 Subject: [PATCH 20/27] Fix KeyError in connectivity_enabled (#2336) --- supervisor/dbus/network/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervisor/dbus/network/__init__.py b/supervisor/dbus/network/__init__.py index 1e137eea1..f43e00070 100644 --- a/supervisor/dbus/network/__init__.py +++ b/supervisor/dbus/network/__init__.py @@ -55,7 +55,7 @@ class NetworkManager(DBusInterface): @property def connectivity_enabled(self) -> bool: """Return if connectivity check is enabled.""" - return self.properties[DBUS_ATTR_CONNECTION_ENABLED] + return self.properties.get(DBUS_ATTR_CONNECTION_ENABLED, False) @dbus_connected def activate_connection( From c8e00ba160b6867e062daeffeaf4b2ca0467c788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20S=C3=B8rensen?= Date: Fri, 4 Dec 2020 11:37:55 +0100 Subject: [PATCH 21/27] Adds vlan_struct (#2337) * Return VLAN ID instead of wifistruct for VLAN * Use vlan_struct * fix typing Co-authored-by: Pascal Vizeli --- supervisor/api/network.py | 18 ++++++++++++++---- supervisor/const.py | 2 ++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/supervisor/api/network.py b/supervisor/api/network.py index e49f0e9b4..5595d2a66 100644 --- a/supervisor/api/network.py +++ b/supervisor/api/network.py @@ -18,6 +18,7 @@ from ..const import ( ATTR_FREQUENCY, ATTR_GATEWAY, ATTR_HOST_INTERNET, + ATTR_ID, ATTR_INTERFACE, ATTR_INTERFACES, ATTR_IPV4, @@ -26,6 +27,7 @@ from ..const import ( ATTR_METHOD, ATTR_MODE, ATTR_NAMESERVERS, + ATTR_PARENT, ATTR_PRIMARY, ATTR_PSK, ATTR_SIGNAL, @@ -80,7 +82,7 @@ SCHEMA_UPDATE = vol.Schema( ) -def ipconfig_struct(config: IpConfig) -> dict: +def ipconfig_struct(config: IpConfig) -> Dict[str, Any]: """Return a dict with information about ip configuration.""" return { ATTR_METHOD: config.method, @@ -90,7 +92,7 @@ def ipconfig_struct(config: IpConfig) -> dict: } -def wifi_struct(config: WifiConfig) -> dict: +def wifi_struct(config: WifiConfig) -> Dict[str, Any]: """Return a dict with information about wifi configuration.""" return { ATTR_MODE: config.mode, @@ -100,7 +102,15 @@ def wifi_struct(config: WifiConfig) -> dict: } -def interface_struct(interface: Interface) -> dict: +def vlan_struct(config: VlanConfig) -> Dict[str, Any]: + """Return a dict with information about VLAN configuration.""" + return { + ATTR_ID: config.id, + ATTR_PARENT: config.interface, + } + + +def interface_struct(interface: Interface) -> Dict[str, Any]: """Return a dict with information of a interface to be used in th API.""" return { ATTR_INTERFACE: interface.name, @@ -115,7 +125,7 @@ def interface_struct(interface: Interface) -> dict: } -def accesspoint_struct(accesspoint: AccessPoint) -> dict: +def accesspoint_struct(accesspoint: AccessPoint) -> Dict[str, Any]: """Return a dict for AccessPoint.""" return { ATTR_MODE: accesspoint.mode, diff --git a/supervisor/const.py b/supervisor/const.py index 4c153076b..e81d22159 100644 --- a/supervisor/const.py +++ b/supervisor/const.py @@ -153,6 +153,7 @@ ATTR_HOST_NETWORK = "host_network" ATTR_HOST_PID = "host_pid" ATTR_HOSTNAME = "hostname" ATTR_ICON = "icon" +ATTR_ID = "id" ATTR_ISSUES = "issues" ATTR_IMAGE = "image" ATTR_IMAGES = "images" @@ -205,6 +206,7 @@ ATTR_PANEL_ICON = "panel_icon" ATTR_PANEL_TITLE = "panel_title" ATTR_PANELS = "panels" ATTR_PASSWORD = "password" +ATTR_PARENT = "parent" ATTR_PORT = "port" ATTR_PORTS = "ports" ATTR_PORTS_DESCRIPTION = "ports_description" From fab6fcd5acd5459f615feeed108461481248c857 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Fri, 4 Dec 2020 12:38:12 +0100 Subject: [PATCH 22/27] Check NetworkManager Version if it is supported (#2340) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Joakim Sørensen --- supervisor/dbus/const.py | 1 + supervisor/dbus/interface.py | 4 +++ supervisor/dbus/network/__init__.py | 40 ++++++++++++++++++++-- tests/dbus/network/test_network_manager.py | 17 +++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/supervisor/dbus/const.py b/supervisor/dbus/const.py index ca490e7e5..3cd04d154 100644 --- a/supervisor/dbus/const.py +++ b/supervisor/dbus/const.py @@ -65,6 +65,7 @@ DBUS_ATTR_STATIC_OPERATING_SYSTEM_CPE_NAME = "OperatingSystemCPEName" DBUS_ATTR_TYPE = "Type" DBUS_ATTR_UUID = "Uuid" DBUS_ATTR_VARIANT = "Variant" +DBUS_ATTR_VERSION = "Version" DBUS_ATTR_MANAGED = "Managed" DBUS_ATTR_CONNECTION_ENABLED = "ConnectivityCheckEnabled" diff --git a/supervisor/dbus/interface.py b/supervisor/dbus/interface.py index c0c085f33..d682aa104 100644 --- a/supervisor/dbus/interface.py +++ b/supervisor/dbus/interface.py @@ -20,6 +20,10 @@ class DBusInterface(ABC): async def connect(self): """Connect to D-Bus.""" + def disconnect(self): + """Disconnect from D-Bus.""" + self.dbus = None + class DBusInterfaceProxy(ABC): """Handle D-Bus interface proxy.""" diff --git a/supervisor/dbus/network/__init__.py b/supervisor/dbus/network/__init__.py index f43e00070..32ecab8b8 100644 --- a/supervisor/dbus/network/__init__.py +++ b/supervisor/dbus/network/__init__.py @@ -2,14 +2,21 @@ import logging from typing import Any, Awaitable, Dict +from packaging.version import parse as pkg_parse import sentry_sdk -from ...exceptions import DBusError, DBusInterfaceError, DBusProgramError +from ...exceptions import ( + DBusError, + DBusInterfaceError, + DBusProgramError, + HostNotSupportedError, +) from ...utils.gdbus import DBus from ..const import ( DBUS_ATTR_CONNECTION_ENABLED, DBUS_ATTR_DEVICES, DBUS_ATTR_PRIMARY_CONNECTION, + DBUS_ATTR_VERSION, DBUS_NAME_NM, DBUS_OBJECT_BASE, DBUS_OBJECT_NM, @@ -23,6 +30,8 @@ from .settings import NetworkManagerSettings _LOGGER: logging.Logger = logging.getLogger(__name__) +MINIMAL_VERSION = "1.14.6" + class NetworkManager(DBusInterface): """Handle D-Bus interface for Network Manager.""" @@ -55,7 +64,12 @@ class NetworkManager(DBusInterface): @property def connectivity_enabled(self) -> bool: """Return if connectivity check is enabled.""" - return self.properties.get(DBUS_ATTR_CONNECTION_ENABLED, False) + return self.properties[DBUS_ATTR_CONNECTION_ENABLED] + + @property + def version(self) -> bool: + """Return if connectivity check is enabled.""" + return self.properties[DBUS_ATTR_VERSION] @dbus_connected def activate_connection( @@ -93,6 +107,28 @@ class NetworkManager(DBusInterface): "No Network Manager support on the host. Local network functions have been disabled." ) + # Make Sure we only connect to supported version + if self.is_connected: + try: + await self._validate_version() + except (HostNotSupportedError, DBusError): + self.disconnect() + self.dns.disconnect() + self.settings.disconnect() + + async def _validate_version(self) -> None: + """Validate Version of NetworkManager.""" + self.properties = await self.dbus.get_properties(DBUS_NAME_NM) + + try: + if pkg_parse(self.version) >= pkg_parse(MINIMAL_VERSION): + return + except (TypeError, ValueError, KeyError): + pass + + _LOGGER.error("Version '%s' of NetworkManager is not supported!", self.version) + raise HostNotSupportedError() + @dbus_connected async def update(self): """Update Properties.""" diff --git a/tests/dbus/network/test_network_manager.py b/tests/dbus/network/test_network_manager.py index 7dc0ad3ea..fdaba20de 100644 --- a/tests/dbus/network/test_network_manager.py +++ b/tests/dbus/network/test_network_manager.py @@ -1,12 +1,29 @@ """Test NetwrokInterface.""" +from unittest.mock import AsyncMock + import pytest from supervisor.dbus.network import NetworkManager +from supervisor.exceptions import HostNotSupportedError from tests.const import TEST_INTERFACE +# pylint: disable=protected-access + @pytest.mark.asyncio async def test_network_manager(network_manager: NetworkManager): """Test network manager update.""" assert TEST_INTERFACE in network_manager.interfaces + + +@pytest.mark.asyncio +async def test_network_manager_version(network_manager: NetworkManager): + """Test if version validate work.""" + await network_manager._validate_version() + assert network_manager.version == "1.22.10" + + network_manager.dbus.get_properties = AsyncMock(return_value={"Version": "1.13.9"}) + with pytest.raises(HostNotSupportedError): + await network_manager._validate_version() + assert network_manager.version == "1.13.9" From 7a51c828c27ec48d85dfbd3ad11c45a76eaa42b1 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Fri, 4 Dec 2020 15:26:13 +0000 Subject: [PATCH 23/27] Fix wrong exception handler --- supervisor/updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervisor/updater.py b/supervisor/updater.py index 985af95fc..ae65c8ea1 100644 --- a/supervisor/updater.py +++ b/supervisor/updater.py @@ -49,7 +49,7 @@ class Updater(JsonConfig, CoreSysAttributes): async def reload(self) -> None: """Update internal data.""" - with suppress(UpdaterJobError): + with suppress(UpdaterError): await self.fetch_data() @property From 792bc610a3e5f082a9fc6920a5ee26d6ca809b32 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Fri, 4 Dec 2020 17:55:26 +0100 Subject: [PATCH 24/27] Fix NM byte switch for ipv6 nameserver (#2342) * Fix NM byte switch for ipv6 nameserver * remove not needed code --- supervisor/dbus/payloads/generate.py | 4 +++- tests/dbus/payloads/test_interface_update_payload.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/supervisor/dbus/payloads/generate.py b/supervisor/dbus/payloads/generate.py index 7ee049cb9..caf4ced07 100644 --- a/supervisor/dbus/payloads/generate.py +++ b/supervisor/dbus/payloads/generate.py @@ -32,7 +32,9 @@ def interface_update_payload( def ipv6_to_byte(ip_address: IPv6Address) -> str: """Convert an ipv6 to an byte array.""" - return f'[byte {", ".join("0x{:02x}".format(val) for val in reversed(ip_address.packed))}]' + return ( + f'[byte {", ".join("0x{:02x}".format(val) for val in ip_address.packed)}]' + ) # Init template env.filters["ipv4_to_int"] = ipv4_to_int diff --git a/tests/dbus/payloads/test_interface_update_payload.py b/tests/dbus/payloads/test_interface_update_payload.py index 439fa12c1..69945fc37 100644 --- a/tests/dbus/payloads/test_interface_update_payload.py +++ b/tests/dbus/payloads/test_interface_update_payload.py @@ -115,6 +115,7 @@ async def test_interface_update_payload_ethernet_ipv6(coresys): interface.ipv6.nameservers = [ ip_address("2606:4700:4700::64"), ip_address("2606:4700:4700::6400"), + ip_address("2606:4700:4700::1111"), ] interface.ipv6.gateway = ip_address("fe80::da58:d7ff:fe00:9c69") @@ -130,8 +131,9 @@ async def test_interface_update_payload_ethernet_ipv6(coresys): ) assert DBus.parse_gvariant(data)["ipv6"]["address-data"][0]["prefix"] == 64 assert DBus.parse_gvariant(data)["ipv6"]["dns"] == [ - [100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 71, 6, 38], - [0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 71, 6, 38], + [38, 6, 71, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100], + [38, 6, 71, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0], + [38, 6, 71, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17], ] assert ( DBus.parse_gvariant(data)["connection"]["uuid"] == inet.settings.connection.uuid From 1d4323621191ae8c36cfbef3c78a857d58fa680d Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sat, 5 Dec 2020 10:57:20 +0100 Subject: [PATCH 25/27] Upgrade json config handler logging to critical (#2344) --- supervisor/utils/json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/supervisor/utils/json.py b/supervisor/utils/json.py index 60da3ceb8..c3465e6aa 100644 --- a/supervisor/utils/json.py +++ b/supervisor/utils/json.py @@ -67,7 +67,7 @@ class JsonConfig: try: self._data = self._schema(self._data) except vol.Invalid as ex: - _LOGGER.error( + _LOGGER.critical( "Can't parse %s: %s", self._file, humanize_error(self._data, ex) ) @@ -81,7 +81,7 @@ class JsonConfig: try: self._data = self._schema(self._data) except vol.Invalid as ex: - _LOGGER.error("Can't parse data: %s", humanize_error(self._data, ex)) + _LOGGER.critical("Can't parse data: %s", humanize_error(self._data, ex)) # Load last valid data _LOGGER.warning("Resetting %s to last version", self._file) From dee2998ee33d9a484cefe35ad0824ee484ea6c4c Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sat, 5 Dec 2020 11:19:27 +0100 Subject: [PATCH 26/27] Fix issue with mark system wrong as rollback (#2345) --- supervisor/core.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/supervisor/core.py b/supervisor/core.py index 68b975c27..af9467770 100644 --- a/supervisor/core.py +++ b/supervisor/core.py @@ -66,18 +66,25 @@ class Core(CoreSysAttributes): await self.sys_resolution.evaluate.evaluate_system() # Check supervisor version/update - if self.sys_dev: - self.sys_config.version = self.sys_supervisor.version - elif self.sys_config.version != self.sys_supervisor.version: + if self.sys_config.version == self.sys_supervisor.version: + return + + # Somethings going wrong + _LOGGER.error( + "Update '%s' of Supervisor '%s' failed!", + self.sys_config.version, + self.sys_supervisor.version, + ) + + if self.sys_supervisor.need_update: self.sys_resolution.create_issue( IssueType.UPDATE_ROLLBACK, ContextType.SUPERVISOR ) self.sys_resolution.unhealthy = UnhealthyReason.SUPERVISOR - _LOGGER.error( - "Update '%s' of Supervisor '%s' failed!", - self.sys_config.version, - self.sys_supervisor.version, - ) + + # Fix wrong version in config / avoid boot loop on OS + self.sys_config.version = self.sys_supervisor.version + self.sys_config.save_data() async def setup(self): """Start setting up supervisor orchestration.""" From 4ff9da68ef5b9ba902b7a51ea1c9527df00bc19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20S=C3=B8rensen?= Date: Sat, 5 Dec 2020 13:04:37 +0100 Subject: [PATCH 27/27] Actually use the vlan_struct (#2347) --- supervisor/api/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervisor/api/network.py b/supervisor/api/network.py index 5595d2a66..de3db6b9e 100644 --- a/supervisor/api/network.py +++ b/supervisor/api/network.py @@ -121,7 +121,7 @@ def interface_struct(interface: Interface) -> Dict[str, Any]: ATTR_IPV4: ipconfig_struct(interface.ipv4) if interface.ipv4 else None, ATTR_IPV6: ipconfig_struct(interface.ipv6) if interface.ipv6 else None, ATTR_WIFI: wifi_struct(interface.wifi) if interface.wifi else None, - ATTR_VLAN: wifi_struct(interface.vlan) if interface.vlan else None, + ATTR_VLAN: vlan_struct(interface.vlan) if interface.vlan else None, }