Replace non-unicode characters for add-on static files (#5712)

* Replace non-unicode characters for add-on static files

Add-on documentation and changelog get read and returned as text file.
However, in case the original author used non-unicode characters, or
the file corrupted, loading currently fails with an UnicodeDecodeError.

Let's just use the built-in replace error handling of Python, so they
appear for the user as  non-unicode characters by replacing them with
the official unicode replacement character "�".

* Remove superflous parameter for binary files

* ruff format

* Add pytests

supervisor/api/store.py

@@ -69,12 +69,21 @@ SCHEMA_ADD_REPOSITORY = vol.Schema(
 )
 
 
-def _read_static_file(path: Path, binary: bool = False) -> Any:
+def _read_static_text_file(path: Path) -> Any:
-    """Read in a static file asset for API output.
+    """Read in a static text file asset for API output.
 
     Must be run in executor.
     """
-    with path.open("rb" if binary else "r") as asset:
+    with path.open("r", errors="replace") as asset:
+        return asset.read()
+
+
+def _read_static_binary_file(path: Path) -> Any:
+    """Read in a static binary file asset for API output.
+
+    Must be run in executor.
+    """
+    with path.open("rb") as asset:
         return asset.read()
 
 
@@ -247,7 +256,7 @@ class APIStore(CoreSysAttributes):
         if not addon.with_icon:
             raise APIError(f"No icon found for add-on {addon.slug}!")
 
-        return await self.sys_run_in_executor(_read_static_file, addon.path_icon, True)
+        return await self.sys_run_in_executor(_read_static_binary_file, addon.path_icon)
 
     @api_process_raw(CONTENT_TYPE_PNG)
     async def addons_addon_logo(self, request: web.Request) -> bytes:
@@ -256,7 +265,7 @@ class APIStore(CoreSysAttributes):
         if not addon.with_logo:
             raise APIError(f"No logo found for add-on {addon.slug}!")
 
-        return await self.sys_run_in_executor(_read_static_file, addon.path_logo, True)
+        return await self.sys_run_in_executor(_read_static_binary_file, addon.path_logo)
 
     @api_process_raw(CONTENT_TYPE_TEXT)
     async def addons_addon_changelog(self, request: web.Request) -> str:
@@ -270,7 +279,9 @@ class APIStore(CoreSysAttributes):
         if not addon.with_changelog:
             return f"No changelog found for add-on {addon.slug}!"
 
-        return await self.sys_run_in_executor(_read_static_file, addon.path_changelog)
+        return await self.sys_run_in_executor(
+            _read_static_text_file, addon.path_changelog
+        )
 
     @api_process_raw(CONTENT_TYPE_TEXT)
     async def addons_addon_documentation(self, request: web.Request) -> str:
@@ -285,7 +296,7 @@ class APIStore(CoreSysAttributes):
             return f"No documentation found for add-on {addon.slug}!"
 
         return await self.sys_run_in_executor(
-            _read_static_file, addon.path_documentation
+            _read_static_text_file, addon.path_documentation
         )
 
     @api_process

tests/api/test_store.py

@@ -349,3 +349,41 @@ async def test_repository_not_found(api_client: TestClient, method: str, url: st
     assert resp.status == 404
     body = await resp.json()
     assert body["message"] == "Repository bad does not exist in the store"
+
+
+@pytest.mark.parametrize("resource", ["store/addons", "addons"])
+async def test_api_store_addons_documentation_corrupted(
+    api_client: TestClient, coresys: CoreSys, store_addon: AddonStore, resource: str
+):
+    """Test /store/addons/{addon}/documentation REST API.
+
+    Test add-on with documentation file with byte sequences which cannot be decoded
+    using UTF-8.
+    """
+    store_addon.path_documentation.write_bytes(b"Text with an invalid UTF-8 char: \xff")
+    await store_addon.refresh_path_cache()
+    assert store_addon.with_documentation is True
+
+    resp = await api_client.get(f"/{resource}/{store_addon.slug}/documentation")
+    assert resp.status == 200
+    result = await resp.text()
+    assert result == "Text with an invalid UTF-8 char: <20>"
+
+
+@pytest.mark.parametrize("resource", ["store/addons", "addons"])
+async def test_api_store_addons_changelog_corrupted(
+    api_client: TestClient, coresys: CoreSys, store_addon: AddonStore, resource: str
+):
+    """Test /store/addons/{addon}/changelog REST API.
+
+    Test add-on with changelog file with byte sequences which cannot be decoded
+    using UTF-8.
+    """
+    store_addon.path_changelog.write_bytes(b"Text with an invalid UTF-8 char: \xff")
+    await store_addon.refresh_path_cache()
+    assert store_addon.with_changelog is True
+
+    resp = await api_client.get(f"/{resource}/{store_addon.slug}/changelog")
+    assert resp.status == 200
+    result = await resp.text()
+    assert result == "Text with an invalid UTF-8 char: <20>"

tests/conftest.py

@@ -516,6 +516,7 @@ def store_addon(coresys: CoreSys, tmp_path, repository):
     coresys.store.data.addons[addon_obj.slug] = SCHEMA_ADDON_SYSTEM(
         load_json_fixture("add-on.json")
     )
+    coresys.store.data.addons[addon_obj.slug]["location"] = tmp_path
     yield addon_obj