Warn if template functions fail and no default is specified (#56453)

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>
Co-authored-by: Paulus Schoutsen <balloob@gmail.com>

homeassistant/helpers/template.py

@@ -6,7 +6,7 @@ import asyncio
 import base64
 import collections.abc
 from collections.abc import Callable, Generator, Iterable
-from contextlib import suppress
+from contextlib import contextmanager, suppress
 from contextvars import ContextVar
 from datetime import datetime, timedelta
 from functools import partial, wraps
@@ -88,7 +88,9 @@ _COLLECTABLE_STATE_ATTRIBUTES = {
 ALL_STATES_RATE_LIMIT = timedelta(minutes=1)
 DOMAIN_STATES_RATE_LIMIT = timedelta(seconds=1)
 
-template_cv: ContextVar[str | None] = ContextVar("template_cv", default=None)
+template_cv: ContextVar[tuple[str, str] | None] = ContextVar(
+    "template_cv", default=None
+)
 
 
 @bind_hass
@@ -336,13 +338,14 @@ class Template:
 
     def ensure_valid(self) -> None:
         """Return if template is valid."""
-        if self.is_static or self._compiled_code is not None:
+        with set_template(self.template, "compiling"):
-            return
+            if self.is_static or self._compiled_code is not None:
+                return
 
-        try:
+            try:
-            self._compiled_code = self._env.compile(self.template)  # type: ignore[no-untyped-call]
+                self._compiled_code = self._env.compile(self.template)  # type: ignore[no-untyped-call]
-        except jinja2.TemplateError as err:
+            except jinja2.TemplateError as err:
-            raise TemplateError(err) from err
+                raise TemplateError(err) from err
 
     def render(
         self,
@@ -1201,8 +1204,26 @@ def utcnow(hass: HomeAssistant) -> datetime:
     return dt_util.utcnow()
 
 
-def forgiving_round(value, precision=0, method="common"):
+def warn_no_default(function, value, default):
-    """Round accepted strings."""
+    """Log warning if no default is specified."""
+    template, action = template_cv.get() or ("", "rendering or compiling")
+    _LOGGER.warning(
+        (
+            "Template warning: '%s' got invalid input '%s' when %s template '%s' "
+            "but no default was specified. Currently '%s' will return '%s', however this template will fail "
+            "to render in Home Assistant core 2021.12"
+        ),
+        function,
+        value,
+        action,
+        template,
+        function,
+        default,
+    )
+
+
+def forgiving_round(value, precision=0, method="common", default=_SENTINEL):
+    """Filter to round a value."""
     try:
         # support rounding methods like jinja
         multiplier = float(10 ** precision)
@@ -1218,94 +1239,137 @@ def forgiving_round(value, precision=0, method="common"):
         return int(value) if precision == 0 else value
     except (ValueError, TypeError):
         # If value can't be converted to float
-        return value
+        if default is _SENTINEL:
+            warn_no_default("round", value, value)
+            return value
+        return default
 
 
-def multiply(value, amount):
+def multiply(value, amount, default=_SENTINEL):
     """Filter to convert value to float and multiply it."""
     try:
         return float(value) * amount
     except (ValueError, TypeError):
         # If value can't be converted to float
-        return value
+        if default is _SENTINEL:
+            warn_no_default("multiply", value, value)
+            return value
+        return default
 
 
-def logarithm(value, base=math.e):
+def logarithm(value, base=math.e, default=_SENTINEL):
-    """Filter to get logarithm of the value with a specific base."""
+    """Filter and function to get logarithm of the value with a specific base."""
     try:
         return math.log(float(value), float(base))
     except (ValueError, TypeError):
-        return value
+        if default is _SENTINEL:
+            warn_no_default("log", value, value)
+            return value
+        return default
 
 
-def sine(value):
+def sine(value, default=_SENTINEL):
-    """Filter to get sine of the value."""
+    """Filter and function to get sine of the value."""
     try:
         return math.sin(float(value))
     except (ValueError, TypeError):
-        return value
+        if default is _SENTINEL:
+            warn_no_default("sin", value, value)
+            return value
+        return default
 
 
-def cosine(value):
+def cosine(value, default=_SENTINEL):
-    """Filter to get cosine of the value."""
+    """Filter and function to get cosine of the value."""
     try:
         return math.cos(float(value))
     except (ValueError, TypeError):
-        return value
+        if default is _SENTINEL:
+            warn_no_default("cos", value, value)
+            return value
+        return default
 
 
-def tangent(value):
+def tangent(value, default=_SENTINEL):
-    """Filter to get tangent of the value."""
+    """Filter and function to get tangent of the value."""
     try:
         return math.tan(float(value))
     except (ValueError, TypeError):
-        return value
+        if default is _SENTINEL:
+            warn_no_default("tan", value, value)
+            return value
+        return default
 
 
-def arc_sine(value):
+def arc_sine(value, default=_SENTINEL):
-    """Filter to get arc sine of the value."""
+    """Filter and function to get arc sine of the value."""
     try:
         return math.asin(float(value))
     except (ValueError, TypeError):
-        return value
+        if default is _SENTINEL:
+            warn_no_default("asin", value, value)
+            return value
+        return default
 
 
-def arc_cosine(value):
+def arc_cosine(value, default=_SENTINEL):
-    """Filter to get arc cosine of the value."""
+    """Filter and function to get arc cosine of the value."""
     try:
         return math.acos(float(value))
     except (ValueError, TypeError):
-        return value
+        if default is _SENTINEL:
+            warn_no_default("acos", value, value)
+            return value
+        return default
 
 
-def arc_tangent(value):
+def arc_tangent(value, default=_SENTINEL):
-    """Filter to get arc tangent of the value."""
+    """Filter and function to get arc tangent of the value."""
     try:
         return math.atan(float(value))
     except (ValueError, TypeError):
-        return value
+        if default is _SENTINEL:
+            warn_no_default("atan", value, value)
+            return value
+        return default
 
 
-def arc_tangent2(*args):
+def arc_tangent2(*args, default=_SENTINEL):
-    """Filter to calculate four quadrant arc tangent of y / x."""
+    """Filter and function to calculate four quadrant arc tangent of y / x.
+
+    The parameters to atan2 may be passed either in an iterable or as separate arguments
+    The default value may be passed either as a positional or in a keyword argument
+    """
     try:
-        if len(args) == 1 and isinstance(args[0], (list, tuple)):
+        if 1 <= len(args) <= 2 and isinstance(args[0], (list, tuple)):
+            if len(args) == 2 and default is _SENTINEL:
+                # Default value passed as a positional argument
+                default = args[1]
             args = args[0]
+        elif len(args) == 3 and default is _SENTINEL:
+            # Default value passed as a positional argument
+            default = args[2]
 
         return math.atan2(float(args[0]), float(args[1]))
     except (ValueError, TypeError):
-        return args
+        if default is _SENTINEL:
+            warn_no_default("atan2", args, args)
+            return args
+        return default
 
 
-def square_root(value):
+def square_root(value, default=_SENTINEL):
-    """Filter to get square root of the value."""
+    """Filter and function to get square root of the value."""
     try:
         return math.sqrt(float(value))
     except (ValueError, TypeError):
-        return value
+        if default is _SENTINEL:
+            warn_no_default("sqrt", value, value)
+            return value
+        return default
 
 
-def timestamp_custom(value, date_format=DATE_STR_FORMAT, local=True):
+def timestamp_custom(value, date_format=DATE_STR_FORMAT, local=True, default=_SENTINEL):
     """Filter to convert given timestamp to format."""
     try:
         date = dt_util.utc_from_timestamp(value)
@@ -1316,10 +1380,13 @@ def timestamp_custom(value, date_format=DATE_STR_FORMAT, local=True):
         return date.strftime(date_format)
     except (ValueError, TypeError):
         # If timestamp can't be converted
-        return value
+        if default is _SENTINEL:
+            warn_no_default("timestamp_custom", value, value)
+            return value
+        return default
 
 
-def timestamp_local(value):
+def timestamp_local(value, default=_SENTINEL):
     """Filter to convert given timestamp to local date/time."""
     try:
         return dt_util.as_local(dt_util.utc_from_timestamp(value)).strftime(
@@ -1327,32 +1394,44 @@ def timestamp_local(value):
         )
     except (ValueError, TypeError):
         # If timestamp can't be converted
-        return value
+        if default is _SENTINEL:
+            warn_no_default("timestamp_local", value, value)
+            return value
+        return default
 
 
-def timestamp_utc(value):
+def timestamp_utc(value, default=_SENTINEL):
     """Filter to convert given timestamp to UTC date/time."""
     try:
         return dt_util.utc_from_timestamp(value).strftime(DATE_STR_FORMAT)
     except (ValueError, TypeError):
         # If timestamp can't be converted
-        return value
+        if default is _SENTINEL:
+            warn_no_default("timestamp_utc", value, value)
+            return value
+        return default
 
 
-def forgiving_as_timestamp(value):
+def forgiving_as_timestamp(value, default=_SENTINEL):
-    """Try to convert value to timestamp."""
+    """Filter and function which tries to convert value to timestamp."""
     try:
         return dt_util.as_timestamp(value)
     except (ValueError, TypeError):
-        return None
+        if default is _SENTINEL:
+            warn_no_default("as_timestamp", value, None)
+            return None
+        return default
 
 
-def strptime(string, fmt):
+def strptime(string, fmt, default=_SENTINEL):
     """Parse a time string to datetime."""
     try:
         return datetime.strptime(string, fmt)
     except (ValueError, AttributeError, TypeError):
-        return string
+        if default is _SENTINEL:
+            warn_no_default("strptime", string, string)
+            return string
+        return default
 
 
 def fail_when_undefined(value):
@@ -1362,12 +1441,26 @@ def fail_when_undefined(value):
     return value
 
 
-def forgiving_float(value):
+def forgiving_float(value, default=_SENTINEL):
     """Try to convert value to a float."""
     try:
         return float(value)
     except (ValueError, TypeError):
-        return value
+        if default is _SENTINEL:
+            warn_no_default("float", value, value)
+            return value
+        return default
+
+
+def forgiving_float_filter(value, default=_SENTINEL):
+    """Try to convert value to a float."""
+    try:
+        return float(value)
+    except (ValueError, TypeError):
+        if default is _SENTINEL:
+            warn_no_default("float", value, 0)
+            return 0
+        return default
 
 
 def is_number(value):
@@ -1493,22 +1586,33 @@ def urlencode(value):
     return urllib_urlencode(value).encode("utf-8")
 
 
+@contextmanager
+def set_template(template_str: str, action: str) -> Generator:
+    """Store template being parsed or rendered in a Contextvar to aid error handling."""
+    template_cv.set((template_str, action))
+    try:
+        yield
+    finally:
+        template_cv.set(None)
+
+
 def _render_with_context(
     template_str: str, template: jinja2.Template, **kwargs: Any
 ) -> str:
     """Store template being rendered in a ContextVar to aid error handling."""
-    template_cv.set(template_str)
+    with set_template(template_str, "rendering"):
-    return template.render(**kwargs)
+        return template.render(**kwargs)
 
 
 class LoggingUndefined(jinja2.Undefined):
     """Log on undefined variables."""
 
     def _log_message(self):
-        template = template_cv.get() or ""
+        template, action = template_cv.get() or ("", "rendering or compiling")
         _LOGGER.warning(
-            "Template variable warning: %s when rendering '%s'",
+            "Template variable warning: %s when %s '%s'",
             self._undefined_message,
+            action,
             template,
         )
 
@@ -1516,10 +1620,11 @@ class LoggingUndefined(jinja2.Undefined):
         try:
             return super()._fail_with_undefined_error(*args, **kwargs)
         except self._undefined_exception as ex:
-            template = template_cv.get() or ""
+            template, action = template_cv.get() or ("", "rendering or compiling")
             _LOGGER.error(
-                "Template variable error: %s when rendering '%s'",
+                "Template variable error: %s when %s '%s'",
                 self._undefined_message,
+                action,
                 template,
             )
             raise ex
@@ -1587,6 +1692,7 @@ class TemplateEnvironment(ImmutableSandboxedEnvironment):
         self.filters["bitwise_or"] = bitwise_or
         self.filters["ord"] = ord
         self.filters["is_number"] = is_number
+        self.filters["float"] = forgiving_float_filter
         self.globals["log"] = logarithm
         self.globals["sin"] = sine
         self.globals["cos"] = cosine

tests/helpers/test_template.py

@@ -38,6 +38,12 @@ def _set_up_units(hass):
     )
 
 
+def render(hass, template_str, variables=None):
+    """Create render info from template."""
+    tmp = template.Template(template_str, hass)
+    return tmp.async_render(variables)
+
+
 def render_to_info(hass, template_str, variables=None):
     """Create render info from template."""
     tmp = template.Template(template_str, hass)
@@ -196,8 +202,8 @@ def test_iterating_domain_states(hass):
     )
 
 
-def test_float(hass):
+def test_float_function(hass):
-    """Test float."""
+    """Test float function."""
     hass.states.async_set("sensor.temperature", "12")
 
     assert (
@@ -219,6 +225,20 @@ def test_float(hass):
         == "forgiving"
     )
 
+    assert render(hass, "{{ float('bad', 1) }}") == 1
+    assert render(hass, "{{ float('bad', default=1) }}") == 1
+
+
+def test_float_filter(hass):
+    """Test float filter."""
+    hass.states.async_set("sensor.temperature", "12")
+
+    assert render(hass, "{{ states.sensor.temperature.state | float }}") == 12.0
+    assert render(hass, "{{ states.sensor.temperature.state | float > 11 }}") is True
+    assert render(hass, "{{ 'bad' | float }}") == 0
+    assert render(hass, "{{ 'bad' | float(1) }}") == 1
+    assert render(hass, "{{ 'bad' | float(default=1) }}") == 1
+
 
 @pytest.mark.parametrize(
     "value, expected",
@@ -295,8 +315,8 @@ def test_rounding_value(hass):
     )
 
 
-def test_rounding_value_get_original_value_on_error(hass):
+def test_rounding_value_on_error(hass):
-    """Test rounding value get original value on error."""
+    """Test rounding value handling of error."""
     assert template.Template("{{ None | round }}", hass).async_render() is None
 
     assert (
@@ -304,6 +324,9 @@ def test_rounding_value_get_original_value_on_error(hass):
         == "no_number"
     )
 
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | round(default=1) }}") == 1
+
 
 def test_multiply(hass):
     """Test multiply."""
@@ -317,6 +340,10 @@ def test_multiply(hass):
             == out
         )
 
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | multiply(10, 1) }}") == 1
+    assert render(hass, "{{ 'no_number' | multiply(10, default=1) }}") == 1
+
 
 def test_logarithm(hass):
     """Test logarithm."""
@@ -343,6 +370,12 @@ def test_logarithm(hass):
             == expected
         )
 
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | log(10, 1) }}") == 1
+    assert render(hass, "{{ 'no_number' | log(10, default=1) }}") == 1
+    assert render(hass, "{{ log('no_number', 10, 1) }}") == 1
+    assert render(hass, "{{ log('no_number', 10, default=1) }}") == 1
+
 
 def test_sine(hass):
     """Test sine."""
@@ -360,6 +393,13 @@ def test_sine(hass):
             template.Template("{{ %s | sin | round(3) }}" % value, hass).async_render()
             == expected
         )
+        assert render(hass, f"{{{{ sin({value}) | round(3) }}}}") == expected
+
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | sin(1) }}") == 1
+    assert render(hass, "{{ 'no_number' | sin(default=1) }}") == 1
+    assert render(hass, "{{ sin('no_number', 1) }}") == 1
+    assert render(hass, "{{ sin('no_number', default=1) }}") == 1
 
 
 def test_cos(hass):
@@ -378,6 +418,13 @@ def test_cos(hass):
             template.Template("{{ %s | cos | round(3) }}" % value, hass).async_render()
             == expected
         )
+        assert render(hass, f"{{{{ cos({value}) | round(3) }}}}") == expected
+
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | sin(1) }}") == 1
+    assert render(hass, "{{ 'no_number' | sin(default=1) }}") == 1
+    assert render(hass, "{{ sin('no_number', 1) }}") == 1
+    assert render(hass, "{{ sin('no_number', default=1) }}") == 1
 
 
 def test_tan(hass):
@@ -396,6 +443,13 @@ def test_tan(hass):
             template.Template("{{ %s | tan | round(3) }}" % value, hass).async_render()
             == expected
         )
+        assert render(hass, f"{{{{ tan({value}) | round(3) }}}}") == expected
+
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | tan(1) }}") == 1
+    assert render(hass, "{{ 'no_number' | tan(default=1) }}") == 1
+    assert render(hass, "{{ tan('no_number', 1) }}") == 1
+    assert render(hass, "{{ tan('no_number', default=1) }}") == 1
 
 
 def test_sqrt(hass):
@@ -414,6 +468,13 @@ def test_sqrt(hass):
             template.Template("{{ %s | sqrt | round(3) }}" % value, hass).async_render()
             == expected
         )
+        assert render(hass, f"{{{{ sqrt({value}) | round(3) }}}}") == expected
+
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | sqrt(1) }}") == 1
+    assert render(hass, "{{ 'no_number' | sqrt(default=1) }}") == 1
+    assert render(hass, "{{ sqrt('no_number', 1) }}") == 1
+    assert render(hass, "{{ sqrt('no_number', default=1) }}") == 1
 
 
 def test_arc_sine(hass):
@@ -434,6 +495,13 @@ def test_arc_sine(hass):
             template.Template("{{ %s | asin | round(3) }}" % value, hass).async_render()
             == expected
         )
+        assert render(hass, f"{{{{ asin({value}) | round(3) }}}}") == expected
+
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | asin(1) }}") == 1
+    assert render(hass, "{{ 'no_number' | asin(default=1) }}") == 1
+    assert render(hass, "{{ asin('no_number', 1) }}") == 1
+    assert render(hass, "{{ asin('no_number', default=1) }}") == 1
 
 
 def test_arc_cos(hass):
@@ -454,6 +522,13 @@ def test_arc_cos(hass):
             template.Template("{{ %s | acos | round(3) }}" % value, hass).async_render()
             == expected
         )
+        assert render(hass, f"{{{{ acos({value}) | round(3) }}}}") == expected
+
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | acos(1) }}") == 1
+    assert render(hass, "{{ 'no_number' | acos(default=1) }}") == 1
+    assert render(hass, "{{ acos('no_number', 1) }}") == 1
+    assert render(hass, "{{ acos('no_number', default=1) }}") == 1
 
 
 def test_arc_tan(hass):
@@ -476,6 +551,13 @@ def test_arc_tan(hass):
             template.Template("{{ %s | atan | round(3) }}" % value, hass).async_render()
             == expected
         )
+        assert render(hass, f"{{{{ atan({value}) | round(3) }}}}") == expected
+
+    # Test handling of default return value
+    assert render(hass, "{{ 'no_number' | atan(1) }}") == 1
+    assert render(hass, "{{ 'no_number' | atan(default=1) }}") == 1
+    assert render(hass, "{{ atan('no_number', 1) }}") == 1
+    assert render(hass, "{{ atan('no_number', default=1) }}") == 1
 
 
 def test_arc_tan2(hass):
@@ -510,6 +592,12 @@ def test_arc_tan2(hass):
             == expected
         )
 
+    # Test handling of default return value
+    assert render(hass, "{{ ('duck', 'goose') | atan2(1) }}") == 1
+    assert render(hass, "{{ ('duck', 'goose') | atan2(default=1) }}") == 1
+    assert render(hass, "{{ atan2('duck', 'goose', 1) }}") == 1
+    assert render(hass, "{{ atan2('duck', 'goose', default=1) }}") == 1
+
 
 def test_strptime(hass):
     """Test the parse timestamp method."""
@@ -532,6 +620,10 @@ def test_strptime(hass):
 
         assert template.Template(temp, hass).async_render() == expected
 
+    # Test handling of default return value
+    assert render(hass, "{{ strptime('invalid', '%Y', 1) }}") == 1
+    assert render(hass, "{{ strptime('invalid', '%Y', default=1) }}") == 1
+
 
 def test_timestamp_custom(hass):
     """Test the timestamps to custom filter."""
@@ -554,6 +646,10 @@ def test_timestamp_custom(hass):
 
         assert template.Template(f"{{{{ {inp} | {fil} }}}}", hass).async_render() == out
 
+    # Test handling of default return value
+    assert render(hass, "{{ None | timestamp_custom('invalid', True, 1) }}") == 1
+    assert render(hass, "{{ None | timestamp_custom(default=1) }}") == 1
+
 
 def test_timestamp_local(hass):
     """Test the timestamps to local filter."""
@@ -565,6 +661,10 @@ def test_timestamp_local(hass):
             == out
         )
 
+    # Test handling of default return value
+    assert render(hass, "{{ None | timestamp_local(1) }}") == 1
+    assert render(hass, "{{ None | timestamp_local(default=1) }}") == 1
+
 
 @pytest.mark.parametrize(
     "input",
@@ -702,6 +802,10 @@ def test_timestamp_utc(hass):
             == out
         )
 
+    # Test handling of default return value
+    assert render(hass, "{{ None | timestamp_utc(1) }}") == 1
+    assert render(hass, "{{ None | timestamp_utc(default=1) }}") == 1
+
 
 def test_as_timestamp(hass):
     """Test the as_timestamp function."""
@@ -720,6 +824,12 @@ def test_as_timestamp(hass):
     )
     assert template.Template(tpl, hass).async_render() == 1706951424.0
 
+    # Test handling of default return value
+    assert render(hass, "{{ 'invalid' | as_timestamp(1) }}") == 1
+    assert render(hass, "{{ 'invalid' | as_timestamp(default=1) }}") == 1
+    assert render(hass, "{{ as_timestamp('invalid', 1) }}") == 1
+    assert render(hass, "{{ as_timestamp('invalid', default=1) }}") == 1
+
 
 @patch.object(random, "choice")
 def test_random_every_time(test_choice, hass):