Allow use YAML for addon and repository config (#2645)

* Allow use YAML for addon and repository config

* pylint

supervisor/const.py

@@ -21,6 +21,8 @@ FILE_HASSIO_INGRESS = Path(SUPERVISOR_DATA, "ingress.json")
 FILE_HASSIO_SERVICES = Path(SUPERVISOR_DATA, "services.json")
 FILE_HASSIO_UPDATER = Path(SUPERVISOR_DATA, "updater.json")
 
+FILE_SUFFIX_CONFIGURATION = [".yaml", ".yml", ".json"]
+
 MACHINE_ID = Path("/etc/machine-id")
 SOCKET_DBUS = Path("/run/dbus/system_bus_socket")
 SOCKET_DOCKER = Path("/run/docker.sock")

supervisor/exceptions.py

@@ -279,7 +279,14 @@ class AppArmorInvalidError(AppArmorError):
 
 
 class JsonFileError(HassioError):
-    """Invalid json file."""
+    """Invalid JSON file."""
+
+
+# util/yaml
+
+
+class YamlFileError(HassioError):
+    """Invalid YAML file."""
 
 
 # util/pwned

supervisor/homeassistant/secrets.py

@@ -4,11 +4,10 @@ import logging
 from pathlib import Path
 from typing import Dict, Optional, Union
 
-from ruamel.yaml import YAML, YAMLError
-
 from ..coresys import CoreSys, CoreSysAttributes
 from ..jobs.const import JobExecutionLimit
 from ..jobs.decorator import Job
+from ..utils.yaml import read_yaml_file
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 
@@ -49,17 +48,8 @@ class HomeAssistantSecrets(CoreSysAttributes):
             return
 
         # Read secrets
-        try:
+        secrets = await self.sys_run_in_executor(read_yaml_file, self.path_secrets)
-            yaml = YAML()
+        self.secrets = {
-            yaml.allow_duplicate_keys = True
+            k: v for k, v in secrets.items() if isinstance(v, (bool, float, int, str))
-            data = await self.sys_run_in_executor(yaml.load, self.path_secrets) or {}
+        }
-
+        _LOGGER.debug("Reloading Home Assistant secrets: %s", len(self.secrets))
-            # Filter to only get supported values
-            self.secrets = {
-                k: v for k, v in data.items() if isinstance(v, (bool, float, int, str))
-            }
-
-        except (YAMLError, AttributeError) as err:
-            _LOGGER.error("Can't process Home Assistant secrets: %s", err)
-        else:
-            _LOGGER.debug("Reloading Home Assistant secrets: %s", len(self.secrets))

supervisor/store/data.py

@@ -11,12 +11,14 @@ from ..const import (
     ATTR_LOCATON,
     ATTR_REPOSITORY,
     ATTR_SLUG,
+    FILE_SUFFIX_CONFIGURATION,
     REPOSITORY_CORE,
     REPOSITORY_LOCAL,
 )
 from ..coresys import CoreSys, CoreSysAttributes
-from ..exceptions import JsonFileError
+from ..exceptions import JsonFileError, YamlFileError
 from ..resolution.const import ContextType, IssueType, SuggestionType
+from ..utils import find_one_filetype, read_json_or_yaml_file
 from ..utils.json import read_json_file
 from .const import StoreType
 from .utils import extract_hash_from_path
@@ -58,10 +60,15 @@ class StoreData(CoreSysAttributes):
         slug = extract_hash_from_path(path)
 
         # exists repository json
-        repository_file = Path(path, "repository.json")
+        repository_file = find_one_filetype(
+            path, "repository", FILE_SUFFIX_CONFIGURATION
+        )
+
         try:
-            repository_info = SCHEMA_REPOSITORY_CONFIG(read_json_file(repository_file))
+            repository_info = SCHEMA_REPOSITORY_CONFIG(
-        except JsonFileError:
+                read_json_or_yaml_file(repository_file)
+            )
+        except (JsonFileError, YamlFileError):
             _LOGGER.warning(
                 "Can't read repository information from %s", repository_file
             )
@@ -80,8 +87,10 @@ class StoreData(CoreSysAttributes):
             # Generate a list without artefact, safe for corruptions
             addon_list = [
                 addon
-                for addon in path.glob("**/config.json")
+                for addon in path.glob("**/config.*")
                 if ".git" not in addon.parts
+                and ".github" not in addon.parts
+                and addon.suffix in FILE_SUFFIX_CONFIGURATION
             ]
         except OSError as err:
             suggestion = None
@@ -100,7 +109,7 @@ class StoreData(CoreSysAttributes):
 
         for addon in addon_list:
             try:
-                addon_config = read_json_file(addon)
+                addon_config = read_json_or_yaml_file(addon)
             except JsonFileError:
                 _LOGGER.warning("Can't read %s from repository %s", addon, repository)
                 continue

supervisor/store/repository.py

@@ -5,10 +5,10 @@ from typing import Dict, Optional
 
 import voluptuous as vol
 
-from ..const import ATTR_MAINTAINER, ATTR_NAME, ATTR_URL
+from ..const import ATTR_MAINTAINER, ATTR_NAME, ATTR_URL, FILE_SUFFIX_CONFIGURATION
 from ..coresys import CoreSys, CoreSysAttributes
-from ..exceptions import JsonFileError, StoreError
+from ..exceptions import JsonFileError, StoreError, YamlFileError
-from ..utils.json import read_json_file
+from ..utils import read_json_or_yaml_file
 from .const import StoreType
 from .git import GitRepoCustom, GitRepoHassIO
 from .utils import get_hash_from_repository
@@ -80,14 +80,18 @@ class Repository(CoreSysAttributes):
             return True
 
         # If exists?
-        repository_file = Path(self.git.path, "repository.json")
+        for filetype in FILE_SUFFIX_CONFIGURATION:
+            repository_file = Path(self.git.path / f"repository{filetype}")
+            if not repository_file.exists():
+                continue
+
         if not repository_file.exists():
             return False
 
         # If valid?
         try:
-            SCHEMA_REPOSITORY_CONFIG(read_json_file(repository_file))
+            SCHEMA_REPOSITORY_CONFIG(read_json_or_yaml_file(repository_file))
-        except (JsonFileError, vol.Invalid):
+        except (JsonFileError, YamlFileError, vol.Invalid):
             return False
 
         return True

supervisor/utils/__init__.py

@@ -5,13 +5,38 @@ import logging
 from pathlib import Path
 import re
 import socket
-from typing import Any
+from typing import Any, List, Optional
+
+from ..exceptions import HassioError
+from .json import read_json_file
+from .yaml import read_yaml_file
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 
 RE_STRING: re.Pattern = re.compile(r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))")
 
 
+def find_one_filetype(
+    path: Path, filename: str, filetypes: List[str]
+) -> Optional[Path]:
+    """Find first file matching filetypes."""
+    for file in path.glob(f"**/{filename}.*"):
+        if file.suffix in filetypes:
+            return file
+    return None
+
+
+def read_json_or_yaml_file(path: Path) -> dict:
+    """Read JSON or YAML file."""
+    if path.suffix == ".json":
+        return read_json_file(path)
+
+    if path.suffix in [".yaml", ".yml"]:
+        return read_yaml_file(path)
+
+    raise HassioError(f"{path} is not JSON or YAML")
+
+
 def convert_to_ascii(raw: bytes) -> str:
     """Convert binary to ascii and remove colors."""
     return RE_STRING.sub("", raw.decode())

supervisor/utils/yaml.py

@@ -0,0 +1,22 @@
+"""Tools handle YAML files for Supervisor."""
+import logging
+from pathlib import Path
+
+from ruamel.yaml import YAML, YAMLError
+
+from ..exceptions import YamlFileError
+
+_YAML = YAML()
+_YAML.allow_duplicate_keys = True
+
+_LOGGER: logging.Logger = logging.getLogger(__name__)
+
+
+def read_yaml_file(path: Path) -> dict:
+    """Read YAML file from path."""
+    try:
+        return _YAML.load(path) or {}
+
+    except (YAMLError, AttributeError) as err:
+        _LOGGER.error("Can't read YAML file %s - %s", path, err)
+        raise YamlFileError() from err

tests/utils/test_yaml.py

@@ -0,0 +1,53 @@
+"""test yaml."""
+import json
+
+from ruamel.yaml import YAML as _YAML
+
+from supervisor.const import FILE_SUFFIX_CONFIGURATION
+from supervisor.utils import find_one_filetype, read_json_or_yaml_file, yaml
+
+YAML = _YAML()
+
+
+def test_reading_yaml(tmp_path):
+    """Test reading YAML file."""
+    tempfile = tmp_path / "test.yaml"
+    YAML.dump({"test": "test"}, tempfile)
+    yaml.read_yaml_file(tempfile)
+
+
+def test_get_file_from_type(tmp_path):
+    """Test get file from type."""
+    tempfile = tmp_path / "test1.yaml"
+    YAML.dump({"test": "test"}, tempfile)
+    found = find_one_filetype(tmp_path, "test1", FILE_SUFFIX_CONFIGURATION)
+    assert found.parts[-1] == "test1.yaml"
+
+    tempfile = tmp_path / "test2.yml"
+    YAML.dump({"test": "test"}, tempfile)
+    found = find_one_filetype(tmp_path, "test2", FILE_SUFFIX_CONFIGURATION)
+    assert found.parts[-1] == "test2.yml"
+
+    tempfile = tmp_path / "test3.json"
+    YAML.dump({"test": "test"}, tempfile)
+    found = find_one_filetype(tmp_path, "test3", FILE_SUFFIX_CONFIGURATION)
+    assert found.parts[-1] == "test3.json"
+
+    tempfile = tmp_path / "test.config"
+    YAML.dump({"test": "test"}, tempfile)
+    found = find_one_filetype(tmp_path, "test4", FILE_SUFFIX_CONFIGURATION)
+    assert not found
+
+
+def test_read_json_or_yaml_file(tmp_path):
+    """Read JSON or YAML file."""
+    tempfile = tmp_path / "test.json"
+    with open(tempfile, "w") as outfile:
+        json.dump({"test": "test"}, outfile)
+    read = read_json_or_yaml_file(tempfile)
+    assert read["test"] == "test"
+
+    tempfile = tmp_path / "test.yaml"
+    YAML.dump({"test": "test"}, tempfile)
+    read = read_json_or_yaml_file(tempfile)
+    assert read["test"] == "test"