Use deepmerge for options (#3162)

* Use deepmerge for options

* fix issue

* Update supervisor/addons/addon.py

Co-authored-by: Joakim Sørensen <joasoe@gmail.com>

* Add tests

* Fix merge schema

* Make save for overwrites

* drop community

* more cleanup

* Fix tests

* Fix lists

* revert strategy

* protect overwritten lists

* Update tests/api/test_store.py

Co-authored-by: Joakim Sørensen <joasoe@gmail.com>

Co-authored-by: Joakim Sørensen <joasoe@gmail.com>

requirements.txt

@@ -10,6 +10,7 @@ colorlog==6.4.1
 cpe==1.2.1
 cryptography==3.4.6
 debugpy==1.4.3
+deepmerge==0.3.0
 docker==5.0.2
 gitpython==3.1.24
 jinja2==3.0.1

supervisor/addons/addon.py

@@ -10,9 +10,10 @@ import secrets
 import shutil
 import tarfile
 from tempfile import TemporaryDirectory
-from typing import Any, Awaitable, Optional
+from typing import Any, Awaitable, Final, Optional
 
 import aiohttp
+from deepmerge import Merger
 import voluptuous as vol
 from voluptuous.humanize import humanize_error
 
@@ -87,6 +88,12 @@ RE_OLD_AUDIO = re.compile(r"\d+,\d+")
 
 WATCHDOG_TIMEOUT = aiohttp.ClientTimeout(total=10)
 
+_OPTIONS_MERGER: Final = Merger(
+    type_strategies=[(dict, ["merge"])],
+    fallback_strategies=["override"],
+    type_conflict_strategies=["override"],
+)
+
 
 class Addon(AddonModel):
     """Hold data for add-on inside Supervisor."""
@@ -194,7 +201,9 @@ class Addon(AddonModel):
     @property
     def options(self) -> dict[str, Any]:
         """Return options with local changes."""
-        return {**self.data[ATTR_OPTIONS], **self.persist[ATTR_OPTIONS]}
+        return _OPTIONS_MERGER.merge(
+            deepcopy(self.data[ATTR_OPTIONS]), deepcopy(self.persist[ATTR_OPTIONS])
+        )
 
     @options.setter
     def options(self, value: Optional[dict[str, Any]]) -> None:

supervisor/store/__init__.py

@@ -68,14 +68,12 @@ class StoreManager(CoreSysAttributes):
     @Job(conditions=[JobCondition.INTERNET_SYSTEM])
     async def update_repositories(self, list_repositories):
         """Add a new custom repository."""
-        job = self.sys_jobs.get_job("storemanager_update_repositories")
         new_rep = set(list_repositories)
         old_rep = {repository.source for repository in self.all}
 
         # add new repository
-        async def _add_repository(url: str, step: int):
+        async def _add_repository(url: str):
             """Add a repository."""
-            job.update(progress=job.progress + step, stage=f"Checking {url} started")
             if url == URL_HASSIO_ADDONS:
                 url = StoreType.CORE
 
@@ -109,9 +107,8 @@ class StoreManager(CoreSysAttributes):
                 self.sys_config.add_addon_repository(repository.source)
             self.repositories[repository.slug] = repository
 
-        job.update(progress=10, stage="Check repositories")
         repos = new_rep - old_rep
-        tasks = [_add_repository(url, 80 / len(repos)) for url in repos]
+        tasks = [_add_repository(url) for url in repos]
         if tasks:
             await asyncio.wait(tasks)
 
@@ -122,14 +119,9 @@ class StoreManager(CoreSysAttributes):
             self.sys_config.drop_addon_repository(url)
 
         # update data
-        job.update(progress=90, stage="Update addons")
         self.data.update()
-
-        job.update(progress=95, stage="Read addons")
         self._read_addons()
 
-        job.update(progress=100)
-
     def _read_addons(self) -> None:
         """Reload add-ons inside store."""
         all_addons = set(self.data.addons)

tests/addons/test_addon.py

@@ -0,0 +1,58 @@
+"""Test Home Assistant Add-ons."""
+
+from supervisor.coresys import CoreSys
+
+from ..const import TEST_ADDON_SLUG
+
+
+def test_options_merge(coresys: CoreSys, install_addon_ssh) -> None:
+    """Test options merge."""
+    addon = coresys.addons.get(TEST_ADDON_SLUG)
+
+    assert addon.options == {
+        "apks": [],
+        "authorized_keys": [],
+        "password": "",
+        "server": {"tcp_forwarding": False},
+    }
+
+    addon.options = {"password": "test"}
+    assert addon.persist["options"] == {"password": "test"}
+    assert addon.options == {
+        "apks": [],
+        "authorized_keys": [],
+        "password": "test",
+        "server": {"tcp_forwarding": False},
+    }
+
+    addon.options = {"password": "test", "apks": ["gcc"]}
+    assert addon.persist["options"] == {"password": "test", "apks": ["gcc"]}
+    assert addon.options == {
+        "apks": ["gcc"],
+        "authorized_keys": [],
+        "password": "test",
+        "server": {"tcp_forwarding": False},
+    }
+
+    addon.options = {"password": "test", "server": {"tcp_forwarding": True}}
+    assert addon.persist["options"] == {
+        "password": "test",
+        "server": {"tcp_forwarding": True},
+    }
+    assert addon.options == {
+        "apks": [],
+        "authorized_keys": [],
+        "password": "test",
+        "server": {"tcp_forwarding": True},
+    }
+
+    # Test overwrite
+    test = addon.options
+    test["server"]["test"] = 1
+    assert addon.options == {
+        "apks": [],
+        "authorized_keys": [],
+        "password": "test",
+        "server": {"tcp_forwarding": True},
+    }
+    addon.options = {"password": "test", "server": {"tcp_forwarding": True}}

tests/api/test_store.py

@@ -14,8 +14,8 @@ async def test_api_store(
     resp = await api_client.get("/store")
     result = await resp.json()
 
-    assert result["data"]["addons"][0]["slug"] == store_addon.slug
-    assert result["data"]["repositories"][0]["slug"] == repository.slug
+    assert result["data"]["addons"][-1]["slug"] == store_addon.slug
+    assert result["data"]["repositories"][-1]["slug"] == repository.slug
 
 
 @pytest.mark.asyncio
@@ -26,7 +26,7 @@ async def test_api_store_addons(api_client: TestClient, store_addon: AddonStore)
     result = await resp.json()
     print(result)
 
-    assert result["data"][0]["slug"] == store_addon.slug
+    assert result["data"][-1]["slug"] == store_addon.slug
 
 
 @pytest.mark.asyncio
@@ -53,7 +53,7 @@ async def test_api_store_repositories(api_client: TestClient, repository: Reposi
     resp = await api_client.get("/store/repositories")
     result = await resp.json()
 
-    assert result["data"][0]["slug"] == repository.slug
+    assert result["data"][-1]["slug"] == repository.slug
 
 
 @pytest.mark.asyncio

tests/conftest.py

@@ -10,6 +10,8 @@ from aiohttp import web
 from awesomeversion import AwesomeVersion
 import pytest
 
+from supervisor import config as su_config
+from supervisor.addons.addon import Addon
 from supervisor.api import RestAPI
 from supervisor.bootstrap import initialize_coresys
 from supervisor.const import REQUEST_FROM
@@ -20,7 +22,8 @@ from supervisor.store.addon import AddonStore
 from supervisor.store.repository import Repository
 from supervisor.utils.gdbus import DBus
 
-from tests.common import exists_fixture, load_fixture, load_json_fixture
+from .common import exists_fixture, load_fixture, load_json_fixture
+from .const import TEST_ADDON_SLUG
 
 # pylint: disable=redefined-outer-name, protected-access
 
@@ -138,6 +141,7 @@ async def coresys(loop, docker, network_manager, aiohttp_client, run_dir) -> Cor
     coresys_obj._config.save_data = MagicMock()
     coresys_obj._jobs.save_data = MagicMock()
     coresys_obj._resolution.save_data = MagicMock()
+    coresys_obj._addons.data.save_data = MagicMock()
 
     # Mock test client
     coresys_obj.arch._default_arch = "amd64"
@@ -154,6 +158,17 @@ async def coresys(loop, docker, network_manager, aiohttp_client, run_dir) -> Cor
     coresys_obj.supervisor._connectivity = True
     coresys_obj.host.network._connectivity = True
 
+    # Fix Paths
+    su_config.ADDONS_CORE = Path(
+        Path(__file__).parent.joinpath("fixtures"), "addons/core"
+    )
+    su_config.ADDONS_LOCAL = Path(
+        Path(__file__).parent.joinpath("fixtures"), "addons/local"
+    )
+    su_config.ADDONS_GIT = Path(
+        Path(__file__).parent.joinpath("fixtures"), "addons/git"
+    )
+
     # WebSocket
     coresys_obj.homeassistant.api.check_api_state = mock_async_return_true
     coresys_obj.homeassistant._websocket._client = AsyncMock(
@@ -222,7 +237,7 @@ def run_dir(tmp_path):
 
 
 @pytest.fixture
-def store_addon(coresys: CoreSys, tmp_path):
+def store_addon(coresys: CoreSys, tmp_path, repository):
     """Store add-on fixture."""
     addon_obj = AddonStore(coresys, "test_store_addon")
 
@@ -232,8 +247,10 @@ def store_addon(coresys: CoreSys, tmp_path):
 
 
 @pytest.fixture
-def repository(coresys: CoreSys):
+async def repository(coresys: CoreSys):
     """Repository fixture."""
+    coresys.config.drop_addon_repository("https://github.com/hassio-addons/repository")
+    await coresys.store.load()
     repository_obj = Repository(
         coresys, "https://github.com/awesome-developer/awesome-repo"
     )
@@ -241,3 +258,12 @@ def repository(coresys: CoreSys):
     coresys.store.repositories[repository_obj.slug] = repository_obj
 
     yield repository_obj
+
+
+@pytest.fixture
+def install_addon_ssh(coresys: CoreSys, repository):
+    """Install local_ssh add-on."""
+    store = coresys.addons.store[TEST_ADDON_SLUG]
+    coresys.addons.data.install(store)
+    addon = Addon(coresys, store.slug)
+    coresys.addons.local[addon.slug] = addon

tests/const.py

@@ -3,3 +3,5 @@
 TEST_INTERFACE = "eth0"
 TEST_INTERFACE_WLAN = "wlan0"
 TEST_WS_URL = "ws://test.org:3000"
+
+TEST_ADDON_SLUG = "local_ssh"

tests/fixtures/addons/core/samba/build.yaml

@@ -0,0 +1,6 @@
+build_from:
+  aarch64: "ghcr.io/home-assistant/aarch64-base:3.13"
+  amd64: "ghcr.io/home-assistant/amd64-base:3.13"
+  armhf: "ghcr.io/home-assistant/armhf-base:3.13"
+  armv7: "ghcr.io/home-assistant/armv7-base:3.13"
+  i386: "ghcr.io/home-assistant/i386-base:3.13"

tests/fixtures/addons/core/samba/config.yaml

@@ -0,0 +1,47 @@
+name: Samba share
+version: 9.5.1
+slug: samba
+description: Expose Home Assistant folders with SMB/CIFS
+url: "https://github.com/home-assistant/hassio-addons/tree/master/samba"
+arch:
+  - armhf
+  - armv7
+  - aarch64
+  - amd64
+  - i386
+startup: services
+init: false
+hassio_api: true
+host_network: true
+map:
+  - "config:rw"
+  - "ssl:rw"
+  - "addons:rw"
+  - "share:rw"
+  - "backup:rw"
+  - "media:rw"
+options:
+  workgroup: WORKGROUP
+  username: homeassistant
+  password: null
+  allow_hosts:
+    - 10.0.0.0/8
+    - 172.16.0.0/12
+    - 192.168.0.0/16
+    - "fe80::/10"
+  veto_files:
+    - ._*
+    - .DS_Store
+    - Thumbs.db
+    - icon?
+    - .Trashes
+  compatibility_mode: false
+schema:
+  workgroup: str
+  username: str
+  password: password
+  allow_hosts:
+    - str
+  veto_files:
+    - str
+  compatibility_mode: bool

tests/fixtures/addons/local/ssh/build.yaml

@@ -0,0 +1,13 @@
+build_from:
+  aarch64: "ghcr.io/home-assistant/aarch64-base:3.14"
+  amd64: "ghcr.io/home-assistant/amd64-base:3.14"
+  armhf: "ghcr.io/home-assistant/armhf-base:3.14"
+  armv7: "ghcr.io/home-assistant/armv7-base:3.14"
+  i386: "ghcr.io/home-assistant/i386-base:3.14"
+labels:
+  org.opencontainers.image.title: "Home Assistant Add-on: Test add-on"
+  org.opencontainers.image.description: "Test add-on PyTest."
+  org.opencontainers.image.source: "https://github.com/home-assistant/supervisor"
+  org.opencontainers.image.licenses: "Apache License 2.0"
+args:
+  TEST_VERSION: "2021.05.0"

tests/fixtures/addons/local/ssh/config.yaml

@@ -0,0 +1,44 @@
+name: Terminal & SSH
+version: 9.2.1
+slug: ssh
+description: Allow logging in remotely to Home Assistant using SSH
+url: "https://github.com/home-assistant/hassio-addons/tree/master/ssh"
+arch:
+  - armhf
+  - armv7
+  - aarch64
+  - amd64
+  - i386
+init: false
+advanced: true
+host_dbus: true
+ingress: true
+panel_icon: "mdi:console"
+panel_title: Terminal
+hassio_api: true
+hassio_role: manager
+audio: true
+uart: true
+ports:
+  22/tcp: null
+map:
+  - "config:rw"
+  - "ssl:rw"
+  - "addons:rw"
+  - "share:rw"
+  - "backup:rw"
+  - "media:rw"
+options:
+  authorized_keys: []
+  apks: []
+  password: ""
+  server:
+    tcp_forwarding: false
+schema:
+  authorized_keys:
+    - str
+  password: password
+  server:
+    tcp_forwarding: bool
+  apks:
+    - str

tests/store/test_builtin_stores.py

@@ -0,0 +1,17 @@
+"""Test local and core store."""
+
+from supervisor.coresys import CoreSys
+
+
+def test_local_store(coresys: CoreSys, repository) -> None:
+    """Test loading from local store."""
+    assert coresys.store.get("local")
+
+    assert "local_ssh" in coresys.addons.store
+
+
+def test_core_store(coresys: CoreSys, repository) -> None:
+    """Test loading from core store."""
+    assert coresys.store.get("core")
+
+    assert "core_samba" in coresys.addons.store