From 8c34b72b62f69d850a54b0ec511f8b7638352776 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Tue, 1 Jul 2025 04:57:00 +0200 Subject: [PATCH] Jinja expressions in configs (Take #3) (#8955) --- esphome/components/packages/__init__.py | 3 +- esphome/components/substitutions/__init__.py | 95 ++++++++++--- esphome/components/substitutions/jinja.py | 99 ++++++++++++++ esphome/config.py | 1 - esphome/yaml_util.py | 2 - requirements.txt | 1 + .../fixtures/substitutions/.gitignore | 1 + .../substitutions/00-simple_var.approved.yaml | 19 +++ .../substitutions/00-simple_var.input.yaml | 21 +++ .../substitutions/01-include.approved.yaml | 15 +++ .../substitutions/01-include.input.yaml | 15 +++ .../02-expressions.approved.yaml | 24 ++++ .../substitutions/02-expressions.input.yaml | 22 +++ .../substitutions/03-closures.approved.yaml | 17 +++ .../substitutions/03-closures.input.yaml | 16 +++ .../04-display_example.approved.yaml | 5 + .../04-display_example.input.yaml | 7 + .../substitutions/closures_package.yaml | 3 + .../fixtures/substitutions/display.yaml | 11 ++ .../fixtures/substitutions/inc1.yaml | 8 ++ tests/unit_tests/test_substitutions.py | 125 ++++++++++++++++++ 21 files changed, 486 insertions(+), 24 deletions(-) create mode 100644 esphome/components/substitutions/jinja.py create mode 100644 tests/unit_tests/fixtures/substitutions/.gitignore create mode 100644 tests/unit_tests/fixtures/substitutions/00-simple_var.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/00-simple_var.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/01-include.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/01-include.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/02-expressions.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/02-expressions.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/03-closures.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/03-closures.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/04-display_example.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/04-display_example.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/closures_package.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/display.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/inc1.yaml create mode 100644 tests/unit_tests/test_substitutions.py diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 08ae798282..6eb746ec63 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -74,7 +74,7 @@ BASE_SCHEMA = cv.All( { cv.Required(CONF_PATH): validate_yaml_filename, cv.Optional(CONF_VARS, default={}): cv.Schema( - {cv.string: cv.string} + {cv.string: object} ), } ), @@ -148,7 +148,6 @@ def _process_base_package(config: dict) -> dict: raise cv.Invalid( f"Current ESPHome Version is too old to use this package: {ESPHOME_VERSION} < {min_version}" ) - vars = {k: str(v) for k, v in vars.items()} new_yaml = yaml_util.substitute_vars(new_yaml, vars) packages[f"{filename}{idx}"] = new_yaml except EsphomeError as e: diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index 41e49f70db..5878af43b2 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -5,6 +5,13 @@ from esphome.config_helpers import Extend, Remove, merge_config import esphome.config_validation as cv from esphome.const import CONF_SUBSTITUTIONS, VALID_SUBSTITUTIONS_CHARACTERS from esphome.yaml_util import ESPHomeDataBase, make_data_base +from .jinja import ( + Jinja, + JinjaStr, + has_jinja, + TemplateError, + TemplateRuntimeError, +) CODEOWNERS = ["@esphome/core"] _LOGGER = logging.getLogger(__name__) @@ -28,7 +35,7 @@ def validate_substitution_key(value): CONFIG_SCHEMA = cv.Schema( { - validate_substitution_key: cv.string_strict, + validate_substitution_key: object, } ) @@ -37,7 +44,42 @@ async def to_code(config): pass -def _expand_substitutions(substitutions, value, path, ignore_missing): +def _expand_jinja(value, orig_value, path, jinja, ignore_missing): + if has_jinja(value): + # If the original value passed in to this function is a JinjaStr, it means it contains an unresolved + # Jinja expression from a previous pass. + if isinstance(orig_value, JinjaStr): + # Rebuild the JinjaStr in case it was lost while replacing substitutions. + value = JinjaStr(value, orig_value.upvalues) + try: + # Invoke the jinja engine to evaluate the expression. + value, err = jinja.expand(value) + if err is not None: + if not ignore_missing and "password" not in path: + _LOGGER.warning( + "Found '%s' (see %s) which looks like an expression," + " but could not resolve all the variables: %s", + value, + "->".join(str(x) for x in path), + err.message, + ) + except ( + TemplateError, + TemplateRuntimeError, + RuntimeError, + ArithmeticError, + AttributeError, + TypeError, + ) as err: + raise cv.Invalid( + f"{type(err).__name__} Error evaluating jinja expression '{value}': {str(err)}." + f" See {'->'.join(str(x) for x in path)}", + path, + ) + return value + + +def _expand_substitutions(substitutions, value, path, jinja, ignore_missing): if "$" not in value: return value @@ -47,7 +89,8 @@ def _expand_substitutions(substitutions, value, path, ignore_missing): while True: m = cv.VARIABLE_PROG.search(value, i) if not m: - # Nothing more to match. Done + # No more variable substitutions found. See if the remainder looks like a jinja template + value = _expand_jinja(value, orig_value, path, jinja, ignore_missing) break i, j = m.span(0) @@ -67,8 +110,15 @@ def _expand_substitutions(substitutions, value, path, ignore_missing): continue sub = substitutions[name] + + if i == 0 and j == len(value): + # The variable spans the whole expression, e.g., "${varName}". Return its resolved value directly + # to conserve its type. + value = sub + break + tail = value[j:] - value = value[:i] + sub + value = value[:i] + str(sub) i = len(value) value += tail @@ -77,36 +127,40 @@ def _expand_substitutions(substitutions, value, path, ignore_missing): if isinstance(orig_value, ESPHomeDataBase): # even though string can get larger or smaller, the range should point # to original document marks - return make_data_base(value, orig_value) + value = make_data_base(value, orig_value) return value -def _substitute_item(substitutions, item, path, ignore_missing): +def _substitute_item(substitutions, item, path, jinja, ignore_missing): if isinstance(item, list): for i, it in enumerate(item): - sub = _substitute_item(substitutions, it, path + [i], ignore_missing) + sub = _substitute_item(substitutions, it, path + [i], jinja, ignore_missing) if sub is not None: item[i] = sub elif isinstance(item, dict): replace_keys = [] for k, v in item.items(): if path or k != CONF_SUBSTITUTIONS: - sub = _substitute_item(substitutions, k, path + [k], ignore_missing) + sub = _substitute_item( + substitutions, k, path + [k], jinja, ignore_missing + ) if sub is not None: replace_keys.append((k, sub)) - sub = _substitute_item(substitutions, v, path + [k], ignore_missing) + sub = _substitute_item(substitutions, v, path + [k], jinja, ignore_missing) if sub is not None: item[k] = sub for old, new in replace_keys: item[new] = merge_config(item.get(old), item.get(new)) del item[old] elif isinstance(item, str): - sub = _expand_substitutions(substitutions, item, path, ignore_missing) - if sub != item: + sub = _expand_substitutions(substitutions, item, path, jinja, ignore_missing) + if isinstance(sub, JinjaStr) or sub != item: return sub elif isinstance(item, (core.Lambda, Extend, Remove)): - sub = _expand_substitutions(substitutions, item.value, path, ignore_missing) + sub = _expand_substitutions( + substitutions, item.value, path, jinja, ignore_missing + ) if sub != item: item.value = sub return None @@ -116,11 +170,11 @@ def do_substitution_pass(config, command_line_substitutions, ignore_missing=Fals if CONF_SUBSTITUTIONS not in config and not command_line_substitutions: return - substitutions = config.get(CONF_SUBSTITUTIONS) - if substitutions is None: - substitutions = command_line_substitutions - elif command_line_substitutions: - substitutions = {**substitutions, **command_line_substitutions} + # Merge substitutions in config, overriding with substitutions coming from command line: + substitutions = { + **config.get(CONF_SUBSTITUTIONS, {}), + **(command_line_substitutions or {}), + } with cv.prepend_path("substitutions"): if not isinstance(substitutions, dict): raise cv.Invalid( @@ -133,7 +187,7 @@ def do_substitution_pass(config, command_line_substitutions, ignore_missing=Fals sub = validate_substitution_key(key) if sub != key: replace_keys.append((key, sub)) - substitutions[key] = cv.string_strict(value) + substitutions[key] = value for old, new in replace_keys: substitutions[new] = substitutions[old] del substitutions[old] @@ -141,4 +195,7 @@ def do_substitution_pass(config, command_line_substitutions, ignore_missing=Fals config[CONF_SUBSTITUTIONS] = substitutions # Move substitutions to the first place to replace substitutions in them correctly config.move_to_end(CONF_SUBSTITUTIONS, False) - _substitute_item(substitutions, config, [], ignore_missing) + + # Create a Jinja environment that will consider substitutions in scope: + jinja = Jinja(substitutions) + _substitute_item(substitutions, config, [], jinja, ignore_missing) diff --git a/esphome/components/substitutions/jinja.py b/esphome/components/substitutions/jinja.py new file mode 100644 index 0000000000..9ecdbab844 --- /dev/null +++ b/esphome/components/substitutions/jinja.py @@ -0,0 +1,99 @@ +import logging +import math +import re +import jinja2 as jinja +from jinja2.nativetypes import NativeEnvironment + +TemplateError = jinja.TemplateError +TemplateSyntaxError = jinja.TemplateSyntaxError +TemplateRuntimeError = jinja.TemplateRuntimeError +UndefinedError = jinja.UndefinedError +Undefined = jinja.Undefined + +_LOGGER = logging.getLogger(__name__) + +DETECT_JINJA = r"(\$\{)" +detect_jinja_re = re.compile( + r"<%.+?%>" # Block form expression: <% ... %> + r"|\$\{[^}]+\}", # Braced form expression: ${ ... } + flags=re.MULTILINE, +) + + +def has_jinja(st): + return detect_jinja_re.search(st) is not None + + +class JinjaStr(str): + """ + Wraps a string containing an unresolved Jinja expression, + storing the variables visible to it when it failed to resolve. + For example, an expression inside a package, `${ A * B }` may fail + to resolve at package parsing time if `A` is a local package var + but `B` is a substitution defined in the root yaml. + Therefore, we store the value of `A` as an upvalue bound + to the original string so we may be able to resolve `${ A * B }` + later in the main substitutions pass. + """ + + def __new__(cls, value: str, upvalues=None): + obj = super().__new__(cls, value) + obj.upvalues = upvalues or {} + return obj + + def __init__(self, value: str, upvalues=None): + self.upvalues = upvalues or {} + + +class Jinja: + """ + Wraps a Jinja environment + """ + + def __init__(self, context_vars): + self.env = NativeEnvironment( + trim_blocks=True, + lstrip_blocks=True, + block_start_string="<%", + block_end_string="%>", + line_statement_prefix="#", + line_comment_prefix="##", + variable_start_string="${", + variable_end_string="}", + undefined=jinja.StrictUndefined, + ) + self.env.add_extension("jinja2.ext.do") + self.env.globals["math"] = math # Inject entire math module + self.context_vars = {**context_vars} + self.env.globals = {**self.env.globals, **self.context_vars} + + def expand(self, content_str): + """ + Renders a string that may contain Jinja expressions or statements + Returns the resulting processed string if all values could be resolved. + Otherwise, it returns a tagged (JinjaStr) string that captures variables + in scope (upvalues), like a closure for later evaluation. + """ + result = None + override_vars = {} + if isinstance(content_str, JinjaStr): + # If `value` is already a JinjaStr, it means we are trying to evaluate it again + # in a parent pass. + # Hopefully, all required variables are visible now. + override_vars = content_str.upvalues + try: + template = self.env.from_string(content_str) + result = template.render(override_vars) + if isinstance(result, Undefined): + # This happens when the expression is simply an undefined variable. Jinja does not + # raise an exception, instead we get "Undefined". + # Trigger an UndefinedError exception so we skip to below. + print("" + result) + except (TemplateSyntaxError, UndefinedError) as err: + # `content_str` contains a Jinja expression that refers to a variable that is undefined + # in this scope. Perhaps it refers to a root substitution that is not visible yet. + # Therefore, return the original `content_str` as a JinjaStr, which contains the variables + # that are actually visible to it at this point to postpone evaluation. + return JinjaStr(content_str, {**self.context_vars, **override_vars}), err + + return result, None diff --git a/esphome/config.py b/esphome/config.py index ca3686a0e6..73cc7657cc 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -789,7 +789,6 @@ def validate_config( result.add_output_path([CONF_SUBSTITUTIONS], CONF_SUBSTITUTIONS) try: substitutions.do_substitution_pass(config, command_line_substitutions) - substitutions.do_substitution_pass(config, command_line_substitutions) except vol.Invalid as err: result.add_error(err) return result diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index bd1806affc..e52fc9e788 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -292,8 +292,6 @@ class ESPHomeLoaderMixin: if file is None: raise yaml.MarkedYAMLError("Must include 'file'", node.start_mark) vars = fields.get(CONF_VARS) - if vars: - vars = {k: str(v) for k, v in vars.items()} return file, vars if isinstance(node, yaml.nodes.MappingNode): diff --git a/requirements.txt b/requirements.txt index 12f3b84359..1010a311d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ esphome-glyphsets==0.2.0 pillow==10.4.0 cairosvg==2.8.2 freetype-py==2.5.1 +jinja2==3.1.6 # esp-idf requires this, but doesn't bundle it by default # https://github.com/espressif/esp-idf/blob/220590d599e134d7a5e7f1e683cc4550349ffbf8/requirements.txt#L24 diff --git a/tests/unit_tests/fixtures/substitutions/.gitignore b/tests/unit_tests/fixtures/substitutions/.gitignore new file mode 100644 index 0000000000..0b15cdb2b7 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/.gitignore @@ -0,0 +1 @@ +*.received.yaml \ No newline at end of file diff --git a/tests/unit_tests/fixtures/substitutions/00-simple_var.approved.yaml b/tests/unit_tests/fixtures/substitutions/00-simple_var.approved.yaml new file mode 100644 index 0000000000..c031399c37 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/00-simple_var.approved.yaml @@ -0,0 +1,19 @@ +substitutions: + var1: '1' + var2: '2' + var21: '79' +esphome: + name: test +test_list: + - '1' + - '1' + - '1' + - '1' + - 'Values: 1 2' + - 'Value: 79' + - 1 + 2 + - 1 * 2 + - 'Undefined var: ${undefined_var}' + - ${undefined_var} + - $undefined_var + - ${ undefined_var } diff --git a/tests/unit_tests/fixtures/substitutions/00-simple_var.input.yaml b/tests/unit_tests/fixtures/substitutions/00-simple_var.input.yaml new file mode 100644 index 0000000000..88a4ffb991 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/00-simple_var.input.yaml @@ -0,0 +1,21 @@ +esphome: + name: test + +substitutions: + var1: "1" + var2: "2" + var21: "79" + +test_list: + - "$var1" + - "${var1}" + - $var1 + - ${var1} + - "Values: $var1 ${var2}" + - "Value: ${var2${var1}}" + - "$var1 + $var2" + - "${ var1 } * ${ var2 }" + - "Undefined var: ${undefined_var}" + - ${undefined_var} + - $undefined_var + - ${ undefined_var } diff --git a/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml b/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml new file mode 100644 index 0000000000..a812fedcfd --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml @@ -0,0 +1,15 @@ +substitutions: + var1: '1' + var2: '2' + a: alpha +test_list: + - values: + - var1: '1' + - a: A + - b: B-default + - c: The value of C is C + - values: + - var1: '1' + - a: alpha + - b: beta + - c: The value of C is $c diff --git a/tests/unit_tests/fixtures/substitutions/01-include.input.yaml b/tests/unit_tests/fixtures/substitutions/01-include.input.yaml new file mode 100644 index 0000000000..d3daa681a4 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/01-include.input.yaml @@ -0,0 +1,15 @@ +substitutions: + var1: "1" + var2: "2" + a: "alpha" + +test_list: + - !include + file: inc1.yaml + vars: + a: "A" + c: "C" + - !include + file: inc1.yaml + vars: + b: "beta" diff --git a/tests/unit_tests/fixtures/substitutions/02-expressions.approved.yaml b/tests/unit_tests/fixtures/substitutions/02-expressions.approved.yaml new file mode 100644 index 0000000000..9e401ec5d6 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/02-expressions.approved.yaml @@ -0,0 +1,24 @@ +substitutions: + width: 7 + height: 8 + enabled: true + pin: &id001 + number: 18 + inverted: true + area: 25 + numberOne: 1 + var1: 79 +test_list: + - The area is 56 + - 56 + - 56 + 1 + - ENABLED + - list: + - 7 + - 8 + - width: 7 + height: 8 + - *id001 + - The pin number is 18 + - The square root is: 5.0 + - The number is 80 diff --git a/tests/unit_tests/fixtures/substitutions/02-expressions.input.yaml b/tests/unit_tests/fixtures/substitutions/02-expressions.input.yaml new file mode 100644 index 0000000000..1777b46f67 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/02-expressions.input.yaml @@ -0,0 +1,22 @@ +substitutions: + width: 7 + height: 8 + enabled: true + pin: + number: 18 + inverted: true + area: 25 + numberOne: 1 + var1: 79 + +test_list: + - "The area is ${width * height}" + - ${width * height} + - ${width * height} + 1 + - ${enabled and "ENABLED" or "DISABLED"} + - list: ${ [width, height] } + - "${ {'width': width, 'height': height} }" + - ${pin} + - The pin number is ${pin.number} + - The square root is: ${math.sqrt(area)} + - The number is ${var${numberOne} + 1} diff --git a/tests/unit_tests/fixtures/substitutions/03-closures.approved.yaml b/tests/unit_tests/fixtures/substitutions/03-closures.approved.yaml new file mode 100644 index 0000000000..c8f7d9976c --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/03-closures.approved.yaml @@ -0,0 +1,17 @@ +substitutions: + B: 5 + var7: 79 +package_result: + - The value of A*B is 35, where A is a package var and B is a substitution in the + root file + - Double substitution also works; the value of var7 is 79, where A is a package + var +local_results: + - The value of B is 5 + - 'You will see, however, that + + ${A} is not substituted here, since + + it is out of scope. + + ' diff --git a/tests/unit_tests/fixtures/substitutions/03-closures.input.yaml b/tests/unit_tests/fixtures/substitutions/03-closures.input.yaml new file mode 100644 index 0000000000..e0b2c39e52 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/03-closures.input.yaml @@ -0,0 +1,16 @@ +substitutions: + B: 5 + var7: 79 + +packages: + closures_package: !include + file: closures_package.yaml + vars: + A: 7 + +local_results: + - The value of B is ${B} + - | + You will see, however, that + ${A} is not substituted here, since + it is out of scope. diff --git a/tests/unit_tests/fixtures/substitutions/04-display_example.approved.yaml b/tests/unit_tests/fixtures/substitutions/04-display_example.approved.yaml new file mode 100644 index 0000000000..f559181b45 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/04-display_example.approved.yaml @@ -0,0 +1,5 @@ +display: + - platform: ili9xxx + dimensions: + width: 960 + height: 544 diff --git a/tests/unit_tests/fixtures/substitutions/04-display_example.input.yaml b/tests/unit_tests/fixtures/substitutions/04-display_example.input.yaml new file mode 100644 index 0000000000..9d8f64a253 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/04-display_example.input.yaml @@ -0,0 +1,7 @@ +# main.yaml +packages: + my_display: !include + file: display.yaml + vars: + high_dpi: true + native_height: 272 diff --git a/tests/unit_tests/fixtures/substitutions/closures_package.yaml b/tests/unit_tests/fixtures/substitutions/closures_package.yaml new file mode 100644 index 0000000000..e87908814d --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/closures_package.yaml @@ -0,0 +1,3 @@ +package_result: + - The value of A*B is ${A * B}, where A is a package var and B is a substitution in the root file + - Double substitution also works; the value of var7 is ${var$A}, where A is a package var diff --git a/tests/unit_tests/fixtures/substitutions/display.yaml b/tests/unit_tests/fixtures/substitutions/display.yaml new file mode 100644 index 0000000000..1e2249dddb --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/display.yaml @@ -0,0 +1,11 @@ +# display.yaml + +defaults: + native_width: 480 + native_height: 480 + +display: + - platform: ili9xxx + dimensions: + width: ${high_dpi and native_width * 2 or native_width} + height: ${high_dpi and native_height * 2 or native_height} diff --git a/tests/unit_tests/fixtures/substitutions/inc1.yaml b/tests/unit_tests/fixtures/substitutions/inc1.yaml new file mode 100644 index 0000000000..65b91a5e16 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/inc1.yaml @@ -0,0 +1,8 @@ +defaults: + b: "B-default" + +values: + - var1: $var1 + - a: $a + - b: ${b} + - c: The value of C is $c diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py new file mode 100644 index 0000000000..b377499d29 --- /dev/null +++ b/tests/unit_tests/test_substitutions.py @@ -0,0 +1,125 @@ +import glob +import logging +import os + +from esphome import yaml_util +from esphome.components import substitutions +from esphome.const import CONF_PACKAGES + +_LOGGER = logging.getLogger(__name__) + +# Set to True for dev mode behavior +# This will generate the expected version of the test files. + +DEV_MODE = False + + +def sort_dicts(obj): + """Recursively sort dictionaries for order-insensitive comparison.""" + if isinstance(obj, dict): + return {k: sort_dicts(obj[k]) for k in sorted(obj)} + elif isinstance(obj, list): + # Lists are not sorted; we preserve order + return [sort_dicts(i) for i in obj] + else: + return obj + + +def dict_diff(a, b, path=""): + """Recursively find differences between two dict/list structures.""" + diffs = [] + if isinstance(a, dict) and isinstance(b, dict): + a_keys = set(a) + b_keys = set(b) + for key in a_keys - b_keys: + diffs.append(f"{path}/{key} only in actual") + for key in b_keys - a_keys: + diffs.append(f"{path}/{key} only in expected") + for key in a_keys & b_keys: + diffs.extend(dict_diff(a[key], b[key], f"{path}/{key}")) + elif isinstance(a, list) and isinstance(b, list): + min_len = min(len(a), len(b)) + for i in range(min_len): + diffs.extend(dict_diff(a[i], b[i], f"{path}[{i}]")) + if len(a) > len(b): + for i in range(min_len, len(a)): + diffs.append(f"{path}[{i}] only in actual: {a[i]!r}") + elif len(b) > len(a): + for i in range(min_len, len(b)): + diffs.append(f"{path}[{i}] only in expected: {b[i]!r}") + else: + if a != b: + diffs.append(f"\t{path}: actual={a!r} expected={b!r}") + return diffs + + +def write_yaml(path, data): + with open(path, "w", encoding="utf-8") as f: + f.write(yaml_util.dump(data)) + + +def test_substitutions_fixtures(fixture_path): + base_dir = fixture_path / "substitutions" + sources = sorted(glob.glob(str(base_dir / "*.input.yaml"))) + assert sources, f"No input YAML files found in {base_dir}" + + failures = [] + for source_path in sources: + try: + expected_path = source_path.replace(".input.yaml", ".approved.yaml") + test_case = os.path.splitext(os.path.basename(source_path))[0].replace( + ".input", "" + ) + + # Load using ESPHome's YAML loader + config = yaml_util.load_yaml(source_path) + + if CONF_PACKAGES in config: + from esphome.components.packages import do_packages_pass + + config = do_packages_pass(config) + + substitutions.do_substitution_pass(config, None) + + # Also load expected using ESPHome's loader, or use {} if missing and DEV_MODE + if os.path.isfile(expected_path): + expected = yaml_util.load_yaml(expected_path) + elif DEV_MODE: + expected = {} + else: + assert os.path.isfile(expected_path), ( + f"Expected file missing: {expected_path}" + ) + + # Sort dicts only (not lists) for comparison + got_sorted = sort_dicts(config) + expected_sorted = sort_dicts(expected) + + if got_sorted != expected_sorted: + diff = "\n".join(dict_diff(got_sorted, expected_sorted)) + msg = ( + f"Substitution result mismatch for {os.path.basename(source_path)}\n" + f"Diff:\n{diff}\n\n" + f"Got: {got_sorted}\n" + f"Expected: {expected_sorted}" + ) + # Write out the received file when test fails + if DEV_MODE: + received_path = os.path.join( + os.path.dirname(source_path), f"{test_case}.received.yaml" + ) + write_yaml(received_path, config) + print(msg) + failures.append(msg) + else: + raise AssertionError(msg) + except Exception as err: + _LOGGER.error("Error in test file %s", source_path) + raise err + + if DEV_MODE and failures: + print(f"\n{len(failures)} substitution test case(s) failed.") + + if DEV_MODE: + _LOGGER.error("Tests passed, but Dev mode is enabled.") + assert not DEV_MODE # make sure DEV_MODE is disabled after you are finished.