diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 61551f188..02d1af0a8 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,11 +6,13 @@ "appPort": "9123:8123", "runArgs": [ "-e", - "GIT_EDITOR=\"code --wait\"", + "GIT_EDITOR=code --wait", "--privileged" ], "extensions": [ - "ms-python.python" + "ms-python.python", + "visualstudioexptteam.vscodeintellicode", + "esbenp.prettier-vscode" ], "settings": { "python.pythonPath": "/usr/local/bin/python", @@ -26,4 +28,4 @@ "editor.formatOnType": true, "files.trimTrailingWhitespace": true } -} +} \ No newline at end of file diff --git a/hassio/__main__.py b/hassio/__main__.py index 426845fa4..dd8a15aa0 100644 --- a/hassio/__main__.py +++ b/hassio/__main__.py @@ -12,6 +12,7 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) def initialize_event_loop(): """Attempt to use uvloop.""" try: + # pylint: disable=import-outside-toplevel import uvloop uvloop.install() diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index dba02f21f..fe95b1d29 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -285,6 +285,9 @@ class AddonManager(CoreSysAttributes): for addon in needs_repair: _LOGGER.info("Start repair for add-on: %s", addon.slug) + await self.sys_run_in_executor( + self.sys_docker.network.stale_cleanup, addon.instance.name + ) with suppress(DockerAPIError, KeyError): # Need pull a image again @@ -293,7 +296,7 @@ class AddonManager(CoreSysAttributes): continue # Need local lookup - elif addon.need_build and not addon.is_detached: + if addon.need_build and not addon.is_detached: store = self.store[addon.slug] # If this add-on is available for rebuild if addon.version == store.version: diff --git a/hassio/addons/addon.py b/hassio/addons/addon.py index 5b633946e..50e6f9eac 100644 --- a/hassio/addons/addon.py +++ b/hassio/addons/addon.py @@ -51,6 +51,7 @@ from ..exceptions import ( ) from ..utils.apparmor import adjust_profile from ..utils.json import read_json_file, write_json_file +from ..utils.tar import exclude_filter, secure_path from .model import AddonModel, Data from .utils import remove_data from .validate import SCHEMA_ADDON_SNAPSHOT, validate_options @@ -525,7 +526,7 @@ class Addon(AddonModel): async def snapshot(self, tar_file: tarfile.TarFile) -> None: """Snapshot state of an add-on.""" - with TemporaryDirectory(dir=str(self.sys_config.path_tmp)) as temp: + with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp: # store local image if self.need_build: try: @@ -560,8 +561,15 @@ class Addon(AddonModel): def _write_tarfile(): """Write tar inside loop.""" with tar_file as snapshot: + # Snapshot system snapshot.add(temp, arcname=".") - snapshot.add(self.path_data, arcname="data") + + # Snapshot data + snapshot.add( + self.path_data, + arcname="data", + filter=exclude_filter(self.snapshot_exclude), + ) try: _LOGGER.info("Build snapshot for add-on %s", self.slug) @@ -574,12 +582,12 @@ class Addon(AddonModel): async def restore(self, tar_file: tarfile.TarFile) -> None: """Restore state of an add-on.""" - with TemporaryDirectory(dir=str(self.sys_config.path_tmp)) as temp: + with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp: # extract snapshot def _extract_tarfile(): """Extract tar snapshot.""" with tar_file as snapshot: - snapshot.extractall(path=Path(temp)) + snapshot.extractall(path=Path(temp), members=secure_path(snapshot)) try: await self.sys_run_in_executor(_extract_tarfile) @@ -640,7 +648,7 @@ class Addon(AddonModel): # Restore data def _restore_data(): """Restore data.""" - shutil.copytree(str(Path(temp, "data")), str(self.path_data)) + shutil.copytree(Path(temp, "data"), self.path_data) _LOGGER.info("Restore data for addon %s", self.slug) if self.path_data.is_dir(): diff --git a/hassio/addons/model.py b/hassio/addons/model.py index ffdada1c9..6cc799b44 100644 --- a/hassio/addons/model.py +++ b/hassio/addons/model.py @@ -47,6 +47,7 @@ from ..const import ( ATTR_SCHEMA, ATTR_SERVICES, ATTR_SLUG, + ATTR_SNAPSHOT_EXCLUDE, ATTR_STARTUP, ATTR_STDIN, ATTR_TIMEOUT, @@ -324,6 +325,11 @@ class AddonModel(CoreSysAttributes): """Return Hass.io role for API.""" return self.data[ATTR_HASSIO_ROLE] + @property + def snapshot_exclude(self) -> List[str]: + """Return Exclude list for snapshot.""" + return self.data.get(ATTR_SNAPSHOT_EXCLUDE, []) + @property def with_stdin(self) -> bool: """Return True if the add-on access use stdin input.""" diff --git a/hassio/addons/validate.py b/hassio/addons/validate.py index 87b695887..d47c9e2d9 100644 --- a/hassio/addons/validate.py +++ b/hassio/addons/validate.py @@ -62,6 +62,7 @@ from ..const import ( ATTR_SCHEMA, ATTR_SERVICES, ATTR_SLUG, + ATTR_SNAPSHOT_EXCLUDE, ATTR_SQUASH, ATTR_STARTUP, ATTR_STATE, @@ -214,6 +215,7 @@ SCHEMA_ADDON_CONFIG = vol.Schema( vol.Optional(ATTR_AUTH_API, default=False): vol.Boolean(), vol.Optional(ATTR_SERVICES): [vol.Match(RE_SERVICE)], vol.Optional(ATTR_DISCOVERY): [valid_discovery_service], + vol.Optional(ATTR_SNAPSHOT_EXCLUDE): [vol.Coerce(str)], vol.Required(ATTR_OPTIONS): dict, vol.Required(ATTR_SCHEMA): vol.Any( vol.Schema( diff --git a/hassio/bootstrap.py b/hassio/bootstrap.py index a5587f5d0..35b35eee4 100644 --- a/hassio/bootstrap.py +++ b/hassio/bootstrap.py @@ -236,6 +236,7 @@ def supervisor_debugger(coresys: CoreSys) -> None: """Setup debugger if needed.""" if not coresys.config.debug: return + # pylint: disable=import-outside-toplevel import ptvsd _LOGGER.info("Initialize Hass.io debugger") diff --git a/hassio/const.py b/hassio/const.py index 681eff5dd..6194f771b 100644 --- a/hassio/const.py +++ b/hassio/const.py @@ -2,7 +2,7 @@ from pathlib import Path from ipaddress import ip_network -HASSIO_VERSION = "189" +HASSIO_VERSION = "190" URL_HASSIO_ADDONS = "https://github.com/home-assistant/hassio-addons" @@ -221,6 +221,7 @@ ATTR_SERVERS = "servers" ATTR_LOCALS = "locals" ATTR_UDEV = "udev" ATTR_VALUE = "value" +ATTR_SNAPSHOT_EXCLUDE = "snapshot_exclude" PROVIDE_SERVICE = "provide" NEED_SERVICE = "need" diff --git a/hassio/discovery/__init__.py b/hassio/discovery/__init__.py index 7fc5bfb49..3af8961ec 100644 --- a/hassio/discovery/__init__.py +++ b/hassio/discovery/__init__.py @@ -31,8 +31,8 @@ class Message: addon: str = attr.ib() service: str = attr.ib() - config: Dict[str, Any] = attr.ib(cmp=False) - uuid: UUID = attr.ib(factory=lambda: uuid4().hex, cmp=False) + config: Dict[str, Any] = attr.ib(eq=False) + uuid: UUID = attr.ib(factory=lambda: uuid4().hex, eq=False) class Discovery(CoreSysAttributes, JsonConfig): diff --git a/hassio/discovery/services/almond.py b/hassio/discovery/services/almond.py new file mode 100644 index 000000000..c8afa158e --- /dev/null +++ b/hassio/discovery/services/almond.py @@ -0,0 +1,11 @@ +"""Discovery service for Almond.""" +import voluptuous as vol + +from hassio.validate import NETWORK_PORT + +from ..const import ATTR_HOST, ATTR_PORT + + +SCHEMA = vol.Schema( + {vol.Required(ATTR_HOST): vol.Coerce(str), vol.Required(ATTR_PORT): NETWORK_PORT} +) diff --git a/hassio/discovery/services/home_panel.py b/hassio/discovery/services/home_panel.py new file mode 100644 index 000000000..c924239cf --- /dev/null +++ b/hassio/discovery/services/home_panel.py @@ -0,0 +1,11 @@ +"""Discovery service for Home Panel.""" +import voluptuous as vol + +from hassio.validate import NETWORK_PORT + +from ..const import ATTR_HOST, ATTR_PORT + + +SCHEMA = vol.Schema( + {vol.Required(ATTR_HOST): vol.Coerce(str), vol.Required(ATTR_PORT): NETWORK_PORT} +) diff --git a/hassio/docker/__init__.py b/hassio/docker/__init__.py index 66c0846ab..8274d506a 100644 --- a/hassio/docker/__init__.py +++ b/hassio/docker/__init__.py @@ -178,3 +178,10 @@ class DockerAPI: _LOGGER.debug("Volumes prune: %s", output) except docker.errors.APIError as err: _LOGGER.warning("Error for volumes prune: %s", err) + + _LOGGER.info("Prune stale networks") + try: + output = self.docker.api.prune_networks() + _LOGGER.debug("Networks prune: %s", output) + except docker.errors.APIError as err: + _LOGGER.warning("Error for networks prune: %s", err) diff --git a/hassio/docker/network.py b/hassio/docker/network.py index 3f4d0943b..610a4db61 100644 --- a/hassio/docker/network.py +++ b/hassio/docker/network.py @@ -1,4 +1,5 @@ """Internal network manager for Hass.io.""" +from contextlib import suppress from ipaddress import IPv4Address import logging from typing import List, Optional @@ -107,3 +108,11 @@ class DockerNetwork: except docker.errors.APIError as err: _LOGGER.warning("Can't disconnect container from default: %s", err) raise DockerAPIError() from None + + def stale_cleanup(self, container_name: str): + """Remove force a container from Network. + + Fix: https://github.com/moby/moby/issues/23302 + """ + with suppress(docker.errors.APIError): + self.network.disconnect(container_name, force=True) diff --git a/hassio/homeassistant.py b/hassio/homeassistant.py index d9fd3c7ff..672c71856 100644 --- a/hassio/homeassistant.py +++ b/hassio/homeassistant.py @@ -575,7 +575,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): migration_progress = True _LOGGER.info("Home Assistant record migration in progress") continue - elif migration_progress: + if migration_progress: migration_progress = False # Reset start time start_time = time.monotonic() _LOGGER.info("Home Assistant record migration done") @@ -586,7 +586,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): pip_progress = True _LOGGER.info("Home Assistant pip installation in progress") continue - elif pip_progress: + if pip_progress: pip_progress = False # Reset start time start_time = time.monotonic() _LOGGER.info("Home Assistant pip installation done") @@ -605,6 +605,11 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): return _LOGGER.info("Repair Home Assistant %s", self.version) + await self.sys_run_in_executor( + self.sys_docker.network.stale_cleanup, self.instance.name + ) + + # Pull image try: await self.instance.install(self.version) except DockerAPIError: diff --git a/hassio/host/alsa.py b/hassio/host/alsa.py index 180fde33c..e17082364 100644 --- a/hassio/host/alsa.py +++ b/hassio/host/alsa.py @@ -11,8 +11,13 @@ from ..coresys import CoreSysAttributes _LOGGER: logging.Logger = logging.getLogger(__name__) -# pylint: disable=invalid-name -DefaultConfig = attr.make_class("DefaultConfig", ["input", "output"]) + +@attr.s() +class DefaultConfig: + """Default config input/output ALSA channel.""" + + input: str = attr.ib() + output: str = attr.ib() AUDIODB_JSON: Path = Path(__file__).parents[1].joinpath("data/audiodb.json") diff --git a/hassio/host/services.py b/hassio/host/services.py index 5eb7c70ba..6dd6367f4 100644 --- a/hassio/host/services.py +++ b/hassio/host/services.py @@ -91,9 +91,9 @@ class ServiceManager(CoreSysAttributes): class ServiceInfo: """Represent a single Service.""" - name = attr.ib(type=str) - description = attr.ib(type=str) - state = attr.ib(type=str) + name: str = attr.ib() + description: str = attr.ib() + state: str = attr.ib() @staticmethod def read_from(unit): diff --git a/hassio/snapshots/snapshot.py b/hassio/snapshots/snapshot.py index 9437d748b..5ea4d3d5b 100644 --- a/hassio/snapshots/snapshot.py +++ b/hassio/snapshots/snapshot.py @@ -41,7 +41,7 @@ from ..const import ( from ..coresys import CoreSys, CoreSysAttributes from ..exceptions import AddonsError from ..utils.json import write_json_file -from ..utils.tar import SecureTarFile +from ..utils.tar import SecureTarFile, secure_path from .utils import key_to_iv, password_for_validating, password_to_key, remove_folder from .validate import ALL_FOLDERS, SCHEMA_SNAPSHOT @@ -248,7 +248,7 @@ class Snapshot(CoreSysAttributes): def _extract_snapshot(): """Extract a snapshot.""" with tarfile.open(self.tarfile, "r:") as tar: - tar.extractall(path=self._tmp.name) + tar.extractall(path=self._tmp.name, members=secure_path(tar)) await self.sys_run_in_executor(_extract_snapshot) @@ -396,7 +396,7 @@ class Snapshot(CoreSysAttributes): try: _LOGGER.info("Restore folder %s", name) with SecureTarFile(tar_name, "r", key=self._key) as tar_file: - tar_file.extractall(path=origin_dir) + tar_file.extractall(path=origin_dir, members=tar_file) _LOGGER.info("Restore folder %s done", name) except (tarfile.TarError, OSError) as err: _LOGGER.warning("Can't restore folder %s: %s", name, err) diff --git a/hassio/snapshots/utils.py b/hassio/snapshots/utils.py index 2f9d73bce..a2403a482 100644 --- a/hassio/snapshots/utils.py +++ b/hassio/snapshots/utils.py @@ -42,7 +42,7 @@ def remove_folder(folder): for obj in folder.iterdir(): try: if obj.is_dir(): - shutil.rmtree(str(obj), ignore_errors=True) + shutil.rmtree(obj, ignore_errors=True) else: obj.unlink() except (OSError, shutil.Error): diff --git a/hassio/store/git.py b/hassio/store/git.py index 7bff4f6ab..a60799230 100644 --- a/hassio/store/git.py +++ b/hassio/store/git.py @@ -137,7 +137,7 @@ class GitRepo(CoreSysAttributes): """Log error.""" _LOGGER.warning("Can't remove %s", path) - shutil.rmtree(str(self.path), onerror=log_err) + shutil.rmtree(self.path, onerror=log_err) class GitRepoHassIO(GitRepo): diff --git a/hassio/utils/tar.py b/hassio/utils/tar.py index af3b0a181..27f12fa94 100644 --- a/hassio/utils/tar.py +++ b/hassio/utils/tar.py @@ -1,19 +1,22 @@ """Tarfile fileobject handler for encrypted files.""" import hashlib +import logging import os from pathlib import Path import tarfile -from typing import IO, Optional +from typing import IO, Callable, Generator, List, Optional from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import padding from cryptography.hazmat.primitives.ciphers import ( - CipherContext, Cipher, + CipherContext, algorithms, modes, ) +_LOGGER: logging.Logger = logging.getLogger(__name__) + BLOCK_SIZE = 16 BLOCK_SIZE_BITS = 128 @@ -111,3 +114,39 @@ def _generate_iv(key: bytes, salt: bytes) -> bytes: for _ in range(100): temp_iv = hashlib.sha256(temp_iv).digest() return temp_iv[:16] + + +def secure_path(tar: tarfile.TarFile) -> Generator[tarfile.TarInfo, None, None]: + """Security safe check of path. + + Prevent ../ or absolut paths + """ + for member in tar: + file_path = Path(member.name) + try: + assert not file_path.is_absolute() + Path("/fake", file_path).resolve().relative_to("/fake") + except (ValueError, RuntimeError, AssertionError): + _LOGGER.warning("Issue with file %s", file_path) + continue + else: + yield member + + +def exclude_filter( + exclude_list: List[str] +) -> Callable[[tarfile.TarInfo], Optional[tarfile.TarInfo]]: + """Create callable filter function to check TarInfo for add.""" + + def my_filter(tar: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: + """Custom exclude filter.""" + file_path = Path(tar.name) + for exclude in exclude_list: + if not file_path.match(exclude): + continue + _LOGGER.debug("Ignore %s because of %s", file_path, exclude) + return None + + return tar + + return my_filter diff --git a/requirements.txt b/requirements.txt index 3279921c3..418f45365 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ -aiohttp==3.6.1 +aiohttp==3.6.2 async_timeout==3.0.1 -attrs==19.1.0 +attrs==19.3.0 cchardet==2.1.4 colorlog==4.0.2 cpe==1.2.1 -cryptography==2.7 -docker==4.0.2 -gitpython==3.0.2 +cryptography==2.8 +docker==4.1.0 +gitpython==3.0.4 packaging==19.2 -pytz==2019.2 +pytz==2019.3 pyudev==0.21.0 ruamel.yaml==0.15.100 uvloop==0.13.0 diff --git a/requirements_tests.txt b/requirements_tests.txt index 3e0d445ac..beffc951d 100644 --- a/requirements_tests.txt +++ b/requirements_tests.txt @@ -1,5 +1,5 @@ flake8==3.7.8 -pylint==2.3.1 -pytest==5.1.3 +pylint==2.4.3 +pytest==5.2.1 pytest-timeout==1.3.3 pytest-aiohttp==0.3.0 diff --git a/tests/discovery/test_almond.py b/tests/discovery/test_almond.py new file mode 100644 index 000000000..78a8e186c --- /dev/null +++ b/tests/discovery/test_almond.py @@ -0,0 +1,19 @@ +"""Test adguard discovery.""" + +import voluptuous as vol +import pytest + +from hassio.discovery.validate import valid_discovery_config + + +def test_good_config(): + """Test good deconz config.""" + + valid_discovery_config("almond", {"host": "test", "port": 3812}) + + +def test_bad_config(): + """Test good adguard config.""" + + with pytest.raises(vol.Invalid): + valid_discovery_config("almond", {"host": "test"}) diff --git a/tests/discovery/test_home_panel.py b/tests/discovery/test_home_panel.py new file mode 100644 index 000000000..883aeb678 --- /dev/null +++ b/tests/discovery/test_home_panel.py @@ -0,0 +1,19 @@ +"""Test adguard discovery.""" + +import voluptuous as vol +import pytest + +from hassio.discovery.validate import valid_discovery_config + + +def test_good_config(): + """Test good deconz config.""" + + valid_discovery_config("home_panel", {"host": "test", "port": 3812}) + + +def test_bad_config(): + """Test good adguard config.""" + + with pytest.raises(vol.Invalid): + valid_discovery_config("home_panel", {"host": "test"}) diff --git a/tests/utils/test_tarfile.py b/tests/utils/test_tarfile.py new file mode 100644 index 000000000..e0a5e31e7 --- /dev/null +++ b/tests/utils/test_tarfile.py @@ -0,0 +1,61 @@ +"""Test Tarfile functions.""" + +import attr +import pytest + +from hassio.utils.tar import secure_path, exclude_filter + + +@attr.s +class TarInfo: + """Fake TarInfo""" + + name: str = attr.ib() + + +def test_secure_path(): + """Test Secure Path.""" + test_list = [ + TarInfo("test.txt"), + TarInfo("data/xy.blob"), + TarInfo("bla/blu/ble"), + TarInfo("data/../xy.blob"), + ] + assert test_list == list(secure_path(test_list)) + + +def test_not_secure_path(): + """Test Not secure path.""" + test_list = [ + TarInfo("/test.txt"), + TarInfo("data/../../xy.blob"), + TarInfo("/bla/blu/ble"), + ] + assert [] == list(secure_path(test_list)) + + +def test_exclude_filter_good(): + """Test exclude filter.""" + filter_funct = exclude_filter(["not/match", "/dev/xy"]) + test_list = [ + TarInfo("test.txt"), + TarInfo("data/xy.blob"), + TarInfo("bla/blu/ble"), + TarInfo("data/../xy.blob"), + ] + + assert test_list == [filter_funct(result) for result in test_list] + + +def test_exclude_filter_bad(): + """Test exclude filter.""" + filter_funct = exclude_filter(["*.txt", "data/*", "bla/blu/ble"]) + test_list = [ + TarInfo("test.txt"), + TarInfo("data/xy.blob"), + TarInfo("bla/blu/ble"), + TarInfo("data/test_files/kk.txt"), + ] + + for info in [filter_funct(result) for result in test_list]: + assert info is None