Merge pull request #1343 from home-assistant/dev

Release 191

.devcontainer/devcontainer.json

@@ -6,11 +6,13 @@
 	"appPort": "9123:8123",
 	"runArgs": [
 		"-e",
-		"GIT_EDITOR=\"code --wait\"",
+		"GIT_EDITOR=code --wait",
 		"--privileged"
 	],
 	"extensions": [
-		"ms-python.python"
+		"ms-python.python",
+		"visualstudioexptteam.vscodeintellicode",
+		"esbenp.prettier-vscode"
 	],
 	"settings": {
 		"python.pythonPath": "/usr/local/bin/python",
@@ -26,4 +28,4 @@
 		"editor.formatOnType": true,
 		"files.trimTrailingWhitespace": true
 	}
-}
+}

hassio/__main__.py

@@ -12,6 +12,7 @@ _LOGGER: logging.Logger = logging.getLogger(__name__)
 def initialize_event_loop():
     """Attempt to use uvloop."""
     try:
+        # pylint: disable=import-outside-toplevel
         import uvloop
 
         uvloop.install()

hassio/addons/__init__.py

@@ -285,6 +285,9 @@ class AddonManager(CoreSysAttributes):
 
         for addon in needs_repair:
             _LOGGER.info("Start repair for add-on: %s", addon.slug)
+            await self.sys_run_in_executor(
+                self.sys_docker.network.stale_cleanup, addon.instance.name
+            )
 
             with suppress(DockerAPIError, KeyError):
                 # Need pull a image again
@@ -293,7 +296,7 @@ class AddonManager(CoreSysAttributes):
                     continue
 
                 # Need local lookup
-                elif addon.need_build and not addon.is_detached:
+                if addon.need_build and not addon.is_detached:
                     store = self.store[addon.slug]
                     # If this add-on is available for rebuild
                     if addon.version == store.version:

hassio/addons/addon.py

@@ -51,6 +51,7 @@ from ..exceptions import (
 )
 from ..utils.apparmor import adjust_profile
 from ..utils.json import read_json_file, write_json_file
+from ..utils.tar import exclude_filter, secure_path
 from .model import AddonModel, Data
 from .utils import remove_data
 from .validate import SCHEMA_ADDON_SNAPSHOT, validate_options
@@ -525,7 +526,7 @@ class Addon(AddonModel):
 
     async def snapshot(self, tar_file: tarfile.TarFile) -> None:
         """Snapshot state of an add-on."""
-        with TemporaryDirectory(dir=str(self.sys_config.path_tmp)) as temp:
+        with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
             # store local image
             if self.need_build:
                 try:
@@ -560,8 +561,15 @@ class Addon(AddonModel):
             def _write_tarfile():
                 """Write tar inside loop."""
                 with tar_file as snapshot:
+                    # Snapshot system
                     snapshot.add(temp, arcname=".")
-                    snapshot.add(self.path_data, arcname="data")
+
+                    # Snapshot data
+                    snapshot.add(
+                        self.path_data,
+                        arcname="data",
+                        filter=exclude_filter(self.snapshot_exclude),
+                    )
 
             try:
                 _LOGGER.info("Build snapshot for add-on %s", self.slug)
@@ -574,12 +582,12 @@ class Addon(AddonModel):
 
     async def restore(self, tar_file: tarfile.TarFile) -> None:
         """Restore state of an add-on."""
-        with TemporaryDirectory(dir=str(self.sys_config.path_tmp)) as temp:
+        with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
             # extract snapshot
             def _extract_tarfile():
                 """Extract tar snapshot."""
                 with tar_file as snapshot:
-                    snapshot.extractall(path=Path(temp))
+                    snapshot.extractall(path=Path(temp), members=secure_path(snapshot))
 
             try:
                 await self.sys_run_in_executor(_extract_tarfile)
@@ -640,7 +648,7 @@ class Addon(AddonModel):
             # Restore data
             def _restore_data():
                 """Restore data."""
-                shutil.copytree(str(Path(temp, "data")), str(self.path_data))
+                shutil.copytree(Path(temp, "data"), self.path_data)
 
             _LOGGER.info("Restore data for addon %s", self.slug)
             if self.path_data.is_dir():

hassio/addons/model.py

@@ -47,6 +47,7 @@ from ..const import (
     ATTR_SCHEMA,
     ATTR_SERVICES,
     ATTR_SLUG,
+    ATTR_SNAPSHOT_EXCLUDE,
     ATTR_STARTUP,
     ATTR_STDIN,
     ATTR_TIMEOUT,
@@ -324,6 +325,11 @@ class AddonModel(CoreSysAttributes):
         """Return Hass.io role for API."""
         return self.data[ATTR_HASSIO_ROLE]
 
+    @property
+    def snapshot_exclude(self) -> List[str]:
+        """Return Exclude list for snapshot."""
+        return self.data.get(ATTR_SNAPSHOT_EXCLUDE, [])
+
     @property
     def with_stdin(self) -> bool:
         """Return True if the add-on access use stdin input."""

hassio/addons/validate.py

@@ -2,6 +2,7 @@
 import logging
 import re
 import secrets
+from typing import Any, Dict
 import uuid
 
 import voluptuous as vol
@@ -61,6 +62,7 @@ from ..const import (
     ATTR_SCHEMA,
     ATTR_SERVICES,
     ATTR_SLUG,
+    ATTR_SNAPSHOT_EXCLUDE,
     ATTR_SQUASH,
     ATTR_STARTUP,
     ATTR_STATE,
@@ -85,6 +87,7 @@ from ..const import (
     STATE_STARTED,
     STATE_STOPPED,
 )
+from ..coresys import CoreSys
 from ..discovery.validate import valid_discovery_service
 from ..validate import (
     ALSA_DEVICE,
@@ -212,6 +215,7 @@ SCHEMA_ADDON_CONFIG = vol.Schema(
         vol.Optional(ATTR_AUTH_API, default=False): vol.Boolean(),
         vol.Optional(ATTR_SERVICES): [vol.Match(RE_SERVICE)],
         vol.Optional(ATTR_DISCOVERY): [valid_discovery_service],
+        vol.Optional(ATTR_SNAPSHOT_EXCLUDE): [vol.Coerce(str)],
         vol.Required(ATTR_OPTIONS): dict,
         vol.Required(ATTR_SCHEMA): vol.Any(
             vol.Schema(
@@ -310,7 +314,7 @@ SCHEMA_ADDON_SNAPSHOT = vol.Schema(
 )
 
 
-def validate_options(coresys, raw_schema):
+def validate_options(coresys: CoreSys, raw_schema: Dict[str, Any]):
     """Validate schema."""
 
     def validate(struct):
@@ -346,7 +350,7 @@ def validate_options(coresys, raw_schema):
 
 # pylint: disable=no-value-for-parameter
 # pylint: disable=inconsistent-return-statements
-def _single_validate(coresys, typ, value, key):
+def _single_validate(coresys: CoreSys, typ: str, value: Any, key: str):
     """Validate a single element."""
     # if required argument
     if value is None:
@@ -385,7 +389,7 @@ def _single_validate(coresys, typ, value, key):
         return NETWORK_PORT(value)
     elif typ.startswith(V_MATCH):
         return vol.Match(match.group("match"))(str(value))
-    elif typ.strartswith(V_LIST):
+    elif typ.startswith(V_LIST):
         return vol.In(match.group("list").split("|"))(str(value))
 
     raise vol.Invalid(f"Fatal error for {key} type {typ}")

hassio/bootstrap.py

@@ -236,6 +236,7 @@ def supervisor_debugger(coresys: CoreSys) -> None:
     """Setup debugger if needed."""
     if not coresys.config.debug:
         return
+    # pylint: disable=import-outside-toplevel
     import ptvsd
 
     _LOGGER.info("Initialize Hass.io debugger")

hassio/const.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from ipaddress import ip_network
 
-HASSIO_VERSION = "188"
+HASSIO_VERSION = "191"
 
 
 URL_HASSIO_ADDONS = "https://github.com/home-assistant/hassio-addons"
@@ -221,6 +221,7 @@ ATTR_SERVERS = "servers"
 ATTR_LOCALS = "locals"
 ATTR_UDEV = "udev"
 ATTR_VALUE = "value"
+ATTR_SNAPSHOT_EXCLUDE = "snapshot_exclude"
 
 PROVIDE_SERVICE = "provide"
 NEED_SERVICE = "need"

hassio/discovery/__init__.py

@@ -31,8 +31,8 @@ class Message:
 
     addon: str = attr.ib()
     service: str = attr.ib()
-    config: Dict[str, Any] = attr.ib(cmp=False)
-    uuid: UUID = attr.ib(factory=lambda: uuid4().hex, cmp=False)
+    config: Dict[str, Any] = attr.ib(eq=False)
+    uuid: UUID = attr.ib(factory=lambda: uuid4().hex, eq=False)
 
 
 class Discovery(CoreSysAttributes, JsonConfig):

hassio/discovery/services/almond.py

@@ -0,0 +1,11 @@
+"""Discovery service for Almond."""
+import voluptuous as vol
+
+from hassio.validate import NETWORK_PORT
+
+from ..const import ATTR_HOST, ATTR_PORT
+
+
+SCHEMA = vol.Schema(
+    {vol.Required(ATTR_HOST): vol.Coerce(str), vol.Required(ATTR_PORT): NETWORK_PORT}
+)

hassio/discovery/services/home_panel.py

@@ -0,0 +1,11 @@
+"""Discovery service for Home Panel."""
+import voluptuous as vol
+
+from hassio.validate import NETWORK_PORT
+
+from ..const import ATTR_HOST, ATTR_PORT
+
+
+SCHEMA = vol.Schema(
+    {vol.Required(ATTR_HOST): vol.Coerce(str), vol.Required(ATTR_PORT): NETWORK_PORT}
+)

hassio/docker/__init__.py

@@ -178,3 +178,10 @@ class DockerAPI:
             _LOGGER.debug("Volumes prune: %s", output)
         except docker.errors.APIError as err:
             _LOGGER.warning("Error for volumes prune: %s", err)
+
+        _LOGGER.info("Prune stale networks")
+        try:
+            output = self.docker.api.prune_networks()
+            _LOGGER.debug("Networks prune: %s", output)
+        except docker.errors.APIError as err:
+            _LOGGER.warning("Error for networks prune: %s", err)

hassio/docker/network.py

@@ -1,4 +1,5 @@
 """Internal network manager for Hass.io."""
+from contextlib import suppress
 from ipaddress import IPv4Address
 import logging
 from typing import List, Optional
@@ -107,3 +108,11 @@ class DockerNetwork:
         except docker.errors.APIError as err:
             _LOGGER.warning("Can't disconnect container from default: %s", err)
             raise DockerAPIError() from None
+
+    def stale_cleanup(self, container_name: str):
+        """Remove force a container from Network.
+
+        Fix: https://github.com/moby/moby/issues/23302
+        """
+        with suppress(docker.errors.APIError):
+            self.network.disconnect(container_name, force=True)

hassio/homeassistant.py

@@ -575,7 +575,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
                     migration_progress = True
                     _LOGGER.info("Home Assistant record migration in progress")
                 continue
-            elif migration_progress:
+            if migration_progress:
                 migration_progress = False  # Reset start time
                 start_time = time.monotonic()
                 _LOGGER.info("Home Assistant record migration done")
@@ -586,7 +586,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
                     pip_progress = True
                     _LOGGER.info("Home Assistant pip installation in progress")
                 continue
-            elif pip_progress:
+            if pip_progress:
                 pip_progress = False  # Reset start time
                 start_time = time.monotonic()
                 _LOGGER.info("Home Assistant pip installation done")
@@ -605,6 +605,11 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
             return
 
         _LOGGER.info("Repair Home Assistant %s", self.version)
+        await self.sys_run_in_executor(
+            self.sys_docker.network.stale_cleanup, self.instance.name
+        )
+
+        # Pull image
         try:
             await self.instance.install(self.version)
         except DockerAPIError:

hassio/host/alsa.py

@@ -11,8 +11,13 @@ from ..coresys import CoreSysAttributes
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 
-# pylint: disable=invalid-name
-DefaultConfig = attr.make_class("DefaultConfig", ["input", "output"])
+
+@attr.s()
+class DefaultConfig:
+    """Default config input/output ALSA channel."""
+
+    input: str = attr.ib()
+    output: str = attr.ib()
 
 
 AUDIODB_JSON: Path = Path(__file__).parents[1].joinpath("data/audiodb.json")

hassio/host/services.py

@@ -91,9 +91,9 @@ class ServiceManager(CoreSysAttributes):
 class ServiceInfo:
     """Represent a single Service."""
 
-    name = attr.ib(type=str)
-    description = attr.ib(type=str)
-    state = attr.ib(type=str)
+    name: str = attr.ib()
+    description: str = attr.ib()
+    state: str = attr.ib()
 
     @staticmethod
     def read_from(unit):

hassio/snapshots/snapshot.py

@@ -41,7 +41,7 @@ from ..const import (
 from ..coresys import CoreSys, CoreSysAttributes
 from ..exceptions import AddonsError
 from ..utils.json import write_json_file
-from ..utils.tar import SecureTarFile
+from ..utils.tar import SecureTarFile, secure_path
 from .utils import key_to_iv, password_for_validating, password_to_key, remove_folder
 from .validate import ALL_FOLDERS, SCHEMA_SNAPSHOT
 
@@ -248,7 +248,7 @@ class Snapshot(CoreSysAttributes):
         def _extract_snapshot():
             """Extract a snapshot."""
             with tarfile.open(self.tarfile, "r:") as tar:
-                tar.extractall(path=self._tmp.name)
+                tar.extractall(path=self._tmp.name, members=secure_path(tar))
 
         await self.sys_run_in_executor(_extract_snapshot)
 
@@ -396,7 +396,7 @@ class Snapshot(CoreSysAttributes):
             try:
                 _LOGGER.info("Restore folder %s", name)
                 with SecureTarFile(tar_name, "r", key=self._key) as tar_file:
-                    tar_file.extractall(path=origin_dir)
+                    tar_file.extractall(path=origin_dir, members=tar_file)
                 _LOGGER.info("Restore folder %s done", name)
             except (tarfile.TarError, OSError) as err:
                 _LOGGER.warning("Can't restore folder %s: %s", name, err)

hassio/snapshots/utils.py

@@ -42,7 +42,7 @@ def remove_folder(folder):
     for obj in folder.iterdir():
         try:
             if obj.is_dir():
-                shutil.rmtree(str(obj), ignore_errors=True)
+                shutil.rmtree(obj, ignore_errors=True)
             else:
                 obj.unlink()
         except (OSError, shutil.Error):

hassio/store/git.py

@@ -137,7 +137,7 @@ class GitRepo(CoreSysAttributes):
             """Log error."""
             _LOGGER.warning("Can't remove %s", path)
 
-        shutil.rmtree(str(self.path), onerror=log_err)
+        shutil.rmtree(self.path, onerror=log_err)
 
 
 class GitRepoHassIO(GitRepo):

hassio/utils/tar.py

@@ -1,19 +1,22 @@
 """Tarfile fileobject handler for encrypted files."""
 import hashlib
+import logging
 import os
 from pathlib import Path
 import tarfile
-from typing import IO, Optional
+from typing import IO, Callable, Generator, List, Optional
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import padding
 from cryptography.hazmat.primitives.ciphers import (
-    CipherContext,
     Cipher,
+    CipherContext,
     algorithms,
     modes,
 )
 
+_LOGGER: logging.Logger = logging.getLogger(__name__)
+
 BLOCK_SIZE = 16
 BLOCK_SIZE_BITS = 128
 
@@ -111,3 +114,39 @@ def _generate_iv(key: bytes, salt: bytes) -> bytes:
     for _ in range(100):
         temp_iv = hashlib.sha256(temp_iv).digest()
     return temp_iv[:16]
+
+
+def secure_path(tar: tarfile.TarFile) -> Generator[tarfile.TarInfo, None, None]:
+    """Security safe check of path.
+
+    Prevent ../ or absolut paths
+    """
+    for member in tar:
+        file_path = Path(member.name)
+        try:
+            assert not file_path.is_absolute()
+            Path("/fake", file_path).resolve().relative_to("/fake")
+        except (ValueError, RuntimeError, AssertionError):
+            _LOGGER.warning("Issue with file %s", file_path)
+            continue
+        else:
+            yield member
+
+
+def exclude_filter(
+    exclude_list: List[str]
+) -> Callable[[tarfile.TarInfo], Optional[tarfile.TarInfo]]:
+    """Create callable filter function to check TarInfo for add."""
+
+    def my_filter(tar: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
+        """Custom exclude filter."""
+        file_path = Path(tar.name)
+        for exclude in exclude_list:
+            if not file_path.match(exclude):
+                continue
+            _LOGGER.debug("Ignore %s because of %s", file_path, exclude)
+            return None
+
+        return tar
+
+    return my_filter

requirements.txt

@@ -1,16 +1,16 @@
-aiohttp==3.5.4
+aiohttp==3.6.1
 async_timeout==3.0.1
-attrs==19.1.0
+attrs==19.3.0
 cchardet==2.1.4
 colorlog==4.0.2
 cpe==1.2.1
-cryptography==2.7
-docker==4.0.2
-gitpython==3.0.2
-packaging==19.1
-pytz==2019.2
+cryptography==2.8
+docker==4.1.0
+gitpython==3.0.4
+packaging==19.2
+pytz==2019.3
 pyudev==0.21.0
 ruamel.yaml==0.15.100
-uvloop==0.12.2
+uvloop==0.13.0
 voluptuous==0.11.7
 ptvsd==4.3.2

requirements_tests.txt

@@ -1,5 +1,5 @@
 flake8==3.7.8
-pylint==2.3.1
-pytest==5.1.2
+pylint==2.4.3
+pytest==5.2.1
 pytest-timeout==1.3.3
 pytest-aiohttp==0.3.0

setup.py

@@ -19,7 +19,7 @@ setup(
         "Intended Audience :: Developers",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
-        "Topic :: Home Automation"
+        "Topic :: Home Automation",
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Topic :: Scientific/Engineering :: Atmospheric Science",
         "Development Status :: 5 - Production/Stable",

tests/discovery/test_almond.py

@@ -0,0 +1,19 @@
+"""Test adguard discovery."""
+
+import voluptuous as vol
+import pytest
+
+from hassio.discovery.validate import valid_discovery_config
+
+
+def test_good_config():
+    """Test good deconz config."""
+
+    valid_discovery_config("almond", {"host": "test", "port": 3812})
+
+
+def test_bad_config():
+    """Test good adguard config."""
+
+    with pytest.raises(vol.Invalid):
+        valid_discovery_config("almond", {"host": "test"})

tests/discovery/test_home_panel.py

@@ -0,0 +1,19 @@
+"""Test adguard discovery."""
+
+import voluptuous as vol
+import pytest
+
+from hassio.discovery.validate import valid_discovery_config
+
+
+def test_good_config():
+    """Test good deconz config."""
+
+    valid_discovery_config("home_panel", {"host": "test", "port": 3812})
+
+
+def test_bad_config():
+    """Test good adguard config."""
+
+    with pytest.raises(vol.Invalid):
+        valid_discovery_config("home_panel", {"host": "test"})

tests/utils/test_tarfile.py

@@ -0,0 +1,61 @@
+"""Test Tarfile functions."""
+
+import attr
+import pytest
+
+from hassio.utils.tar import secure_path, exclude_filter
+
+
+@attr.s
+class TarInfo:
+    """Fake TarInfo"""
+
+    name: str = attr.ib()
+
+
+def test_secure_path():
+    """Test Secure Path."""
+    test_list = [
+        TarInfo("test.txt"),
+        TarInfo("data/xy.blob"),
+        TarInfo("bla/blu/ble"),
+        TarInfo("data/../xy.blob"),
+    ]
+    assert test_list == list(secure_path(test_list))
+
+
+def test_not_secure_path():
+    """Test Not secure path."""
+    test_list = [
+        TarInfo("/test.txt"),
+        TarInfo("data/../../xy.blob"),
+        TarInfo("/bla/blu/ble"),
+    ]
+    assert [] == list(secure_path(test_list))
+
+
+def test_exclude_filter_good():
+    """Test exclude filter."""
+    filter_funct = exclude_filter(["not/match", "/dev/xy"])
+    test_list = [
+        TarInfo("test.txt"),
+        TarInfo("data/xy.blob"),
+        TarInfo("bla/blu/ble"),
+        TarInfo("data/../xy.blob"),
+    ]
+
+    assert test_list == [filter_funct(result) for result in test_list]
+
+
+def test_exclude_filter_bad():
+    """Test exclude filter."""
+    filter_funct = exclude_filter(["*.txt", "data/*", "bla/blu/ble"])
+    test_list = [
+        TarInfo("test.txt"),
+        TarInfo("data/xy.blob"),
+        TarInfo("bla/blu/ble"),
+        TarInfo("data/test_files/kk.txt"),
+    ]
+
+    for info in [filter_funct(result) for result in test_list]:
+        assert info is None