Update template sensor to use async_track_template_result (#38940)

* Add template entity

* Update template tracking to work for template sensors

* add test for whitespace

* Update homeassistant/helpers/config_validation.py

* revert

* fix

* reduce

* fix _refresh missing decorator

* defer until start

* do not throw errors during startup

* defer tracking until start event

Co-authored-by: Swamp-Ig <github@ninjateaparty.com>

homeassistant/components/template/sensor.py

@@ -20,17 +20,15 @@ from homeassistant.const import (
     CONF_SENSORS,
     CONF_UNIQUE_ID,
     CONF_VALUE_TEMPLATE,
-    EVENT_HOMEASSISTANT_START,
-    MATCH_ALL,
 )
 from homeassistant.core import callback
 from homeassistant.exceptions import TemplateError
 import homeassistant.helpers.config_validation as cv
 from homeassistant.helpers.entity import Entity, async_generate_entity_id
-from homeassistant.helpers.event import async_track_state_change_event
+from homeassistant.helpers.template import result_as_boolean
 
-from . import extract_entities, initialise_templates
 from .const import CONF_AVAILABILITY_TEMPLATE
+from .template_entity import TemplateEntity
 
 CONF_ATTRIBUTE_TEMPLATES = "attribute_templates"
 
@@ -75,23 +73,6 @@ async def async_setup_platform(hass, config, async_add_entities, discovery_info=
         attribute_templates = device_config[CONF_ATTRIBUTE_TEMPLATES]
         unique_id = device_config.get(CONF_UNIQUE_ID)
 
-        templates = {
-            CONF_VALUE_TEMPLATE: state_template,
-            CONF_ICON_TEMPLATE: icon_template,
-            CONF_ENTITY_PICTURE_TEMPLATE: entity_picture_template,
-            CONF_FRIENDLY_NAME_TEMPLATE: friendly_name_template,
-            CONF_AVAILABILITY_TEMPLATE: availability_template,
-        }
-
-        initialise_templates(hass, templates, attribute_templates)
-        entity_ids = extract_entities(
-            device,
-            "sensor",
-            device_config.get(ATTR_ENTITY_ID),
-            templates,
-            attribute_templates,
-        )
-
         sensors.append(
             SensorTemplate(
                 hass,
@@ -103,7 +84,6 @@ async def async_setup_platform(hass, config, async_add_entities, discovery_info=
                 icon_template,
                 entity_picture_template,
                 availability_template,
-                entity_ids,
                 device_class,
                 attribute_templates,
                 unique_id,
@@ -115,7 +95,7 @@ async def async_setup_platform(hass, config, async_add_entities, discovery_info=
     return True
 
 
-class SensorTemplate(Entity):
+class SensorTemplate(TemplateEntity, Entity):
     """Representation of a Template Sensor."""
 
     def __init__(
@@ -129,7 +109,6 @@ class SensorTemplate(Entity):
         icon_template,
         entity_picture_template,
         availability_template,
-        entity_ids,
         device_class,
         attribute_templates,
         unique_id,
@@ -149,35 +128,66 @@ class SensorTemplate(Entity):
         self._availability_template = availability_template
         self._icon = None
         self._entity_picture = None
-        self._entities = entity_ids
         self._device_class = device_class
         self._available = True
         self._attribute_templates = attribute_templates
         self._attributes = {}
         self._unique_id = unique_id
+        super().__init__()
 
     async def async_added_to_hass(self):
         """Register callbacks."""
 
-        @callback
+        self.add_template_attribute("_state", self._template, None, self._update_state)
-        def template_sensor_state_listener(event):
+        if self._icon_template is not None:
-            """Handle device state changes."""
+            self.add_template_attribute(
-            self.async_schedule_update_ha_state(True)
+                "_icon", self._icon_template, vol.Or(cv.whitespace, cv.icon)
-
+            )
-        @callback
+        if self._entity_picture_template is not None:
-        def template_sensor_startup(event):
+            self.add_template_attribute(
-            """Update template on startup."""
+                "_entity_picture", self._entity_picture_template
-            if self._entities != MATCH_ALL:
+            )
-                # Track state change only for valid templates
+        if self._friendly_name_template is not None:
-                async_track_state_change_event(
+            self.add_template_attribute("_name", self._friendly_name_template)
-                    self.hass, self._entities, template_sensor_state_listener
+        if self._availability_template is not None:
+            self.add_template_attribute(
+                "_available", self._availability_template, None, self._update_available
             )
 
-            self.async_schedule_update_ha_state(True)
+        for key, value in self._attribute_templates.items():
+            self._add_attribute_template(key, value)
 
-        self.hass.bus.async_listen_once(
+        await super().async_added_to_hass()
-            EVENT_HOMEASSISTANT_START, template_sensor_startup
+
-        )
+    @callback
+    def _add_attribute_template(self, attribute_key, attribute_template):
+        """Create a template tracker for the attribute."""
+
+        def _update_attribute(result):
+            attr_result = None if isinstance(result, TemplateError) else result
+            self._attributes[attribute_key] = attr_result
+
+        self.add_template_attribute(None, attribute_template, None, _update_attribute)
+
+    @callback
+    def _update_state(self, result):
+        if isinstance(result, TemplateError):
+            if not self._availability_template:
+                self._available = False
+            self._state = None
+            return
+
+        if not self._availability_template:
+            self._available = True
+        self._state = result
+
+    @callback
+    def _update_available(self, result):
+        if isinstance(result, TemplateError):
+            self._available = True
+            return
+
+        self._available = result_as_boolean(result)
 
     @property
     def name(self):
@@ -228,69 +238,3 @@ class SensorTemplate(Entity):
     def should_poll(self):
         """No polling needed."""
         return False
-
-    async def async_update(self):
-        """Update the state from the template."""
-        try:
-            self._state = self._template.async_render()
-            self._available = True
-        except TemplateError as ex:
-            self._available = False
-            if ex.args and ex.args[0].startswith(
-                "UndefinedError: 'None' has no attribute"
-            ):
-                # Common during HA startup - so just a warning
-                _LOGGER.warning(
-                    "Could not render template %s, the state is unknown", self._name
-                )
-            else:
-                self._state = None
-                _LOGGER.error("Could not render template %s: %s", self._name, ex)
-
-        attrs = {}
-        for key, value in self._attribute_templates.items():
-            try:
-                attrs[key] = value.async_render()
-            except TemplateError as err:
-                _LOGGER.error("Error rendering attribute %s: %s", key, err)
-
-        self._attributes = attrs
-
-        templates = {
-            "_icon": self._icon_template,
-            "_entity_picture": self._entity_picture_template,
-            "_name": self._friendly_name_template,
-            "_available": self._availability_template,
-        }
-
-        for property_name, template in templates.items():
-            if template is None:
-                continue
-
-            try:
-                value = template.async_render()
-                if property_name == "_available":
-                    value = value.lower() == "true"
-                setattr(self, property_name, value)
-            except TemplateError as ex:
-                friendly_property_name = property_name[1:].replace("_", " ")
-                if ex.args and ex.args[0].startswith(
-                    "UndefinedError: 'None' has no attribute"
-                ):
-                    # Common during HA startup - so just a warning
-                    _LOGGER.warning(
-                        "Could not render %s template %s, the state is unknown",
-                        friendly_property_name,
-                        self._name,
-                    )
-                    continue
-
-                try:
-                    setattr(self, property_name, getattr(super(), property_name))
-                except AttributeError:
-                    _LOGGER.error(
-                        "Could not render %s template %s: %s",
-                        friendly_property_name,
-                        self._name,
-                        ex,
-                    )

homeassistant/components/template/template_entity.py

@@ -0,0 +1,179 @@
+"""TemplateEntity utility class."""
+
+import logging
+from typing import Any, Callable, Optional, Union
+
+import voluptuous as vol
+
+from homeassistant.core import EVENT_HOMEASSISTANT_START, callback
+from homeassistant.exceptions import TemplateError
+from homeassistant.helpers.config_validation import match_all
+from homeassistant.helpers.entity import Entity
+from homeassistant.helpers.event import Event, async_track_template_result
+from homeassistant.helpers.template import Template
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class _TemplateAttribute:
+    """Attribute value linked to template result."""
+
+    def __init__(
+        self,
+        entity: Entity,
+        attribute: str,
+        template: Template,
+        validator: Callable[[Any], Any] = match_all,
+        on_update: Optional[Callable[[Any], None]] = None,
+    ):
+        """Template attribute."""
+        self._entity = entity
+        self._attribute = attribute
+        self.template = template
+        self.validator = validator
+        self.on_update = on_update
+        self.async_update = None
+        self.add_complete = False
+
+    @callback
+    def async_setup(self):
+        """Config update path for the attribute."""
+        if self.on_update:
+            return
+
+        if not hasattr(self._entity, self._attribute):
+            raise AttributeError(f"Attribute '{self._attribute}' does not exist.")
+
+        self.on_update = self._default_update
+
+    @callback
+    def _default_update(self, result):
+        attr_result = None if isinstance(result, TemplateError) else result
+        setattr(self._entity, self._attribute, attr_result)
+
+    @callback
+    def _write_update_if_added(self):
+        if self.add_complete:
+            self._entity.async_write_ha_state()
+
+    @callback
+    def _handle_result(
+        self,
+        event: Optional[Event],
+        template: Template,
+        last_result: Optional[str],
+        result: Union[str, TemplateError],
+    ) -> None:
+        if isinstance(result, TemplateError):
+            _LOGGER.error(
+                "TemplateError('%s') "
+                "while processing template '%s' "
+                "for attribute '%s' in entity '%s'",
+                result,
+                self.template,
+                self._attribute,
+                self._entity.entity_id,
+            )
+            self.on_update(result)
+            self._write_update_if_added()
+
+            return
+
+        if not self.validator:
+            self.on_update(result)
+            self._write_update_if_added()
+            return
+
+        try:
+            validated = self.validator(result)
+        except vol.Invalid as ex:
+            _LOGGER.error(
+                "Error validating template result '%s' "
+                "from template '%s' "
+                "for attribute '%s' in entity %s "
+                "validation message '%s'",
+                result,
+                self.template,
+                self._attribute,
+                self._entity.entity_id,
+                ex.msg,
+            )
+            self.on_update(None)
+            self._write_update_if_added()
+            return
+
+        self.on_update(validated)
+        self._write_update_if_added()
+
+    @callback
+    def async_template_startup(self) -> None:
+        """Call from containing entity when added to hass."""
+        result_info = async_track_template_result(
+            self._entity.hass, self.template, self._handle_result
+        )
+        self.async_update = result_info.async_refresh
+
+        @callback
+        def _remove_from_hass():
+            result_info.async_remove()
+
+        return _remove_from_hass
+
+
+class TemplateEntity(Entity):
+    """Entity that uses templates to calculate attributes."""
+
+    def __init__(self):
+        """Template Entity."""
+        self._template_attrs = []
+
+    def add_template_attribute(
+        self,
+        attribute: str,
+        template: Template,
+        validator: Callable[[Any], Any] = match_all,
+        on_update: Optional[Callable[[Any], None]] = None,
+    ) -> None:
+        """
+        Call in the constructor to add a template linked to a attribute.
+
+        Parameters
+        ----------
+        attribute
+            The name of the attribute to link to. This attribute must exist
+            unless a custom on_update method is supplied.
+        template
+            The template to calculate.
+        validator
+            Validator function to parse the result and ensure it's valid.
+        on_update
+            Called to store the template result rather than storing it
+            the supplied attribute. Passed the result of the validator, or None
+            if the template or validator resulted in an error.
+
+        """
+        attribute = _TemplateAttribute(self, attribute, template, validator, on_update)
+        attribute.async_setup()
+        self._template_attrs.append(attribute)
+
+    async def _async_template_startup(self, _) -> None:
+        # async_update will not write state
+        # until "add_complete" is set on the attribute
+        for attribute in self._template_attrs:
+            self.async_on_remove(attribute.async_template_startup())
+        await self.async_update()
+        for attribute in self._template_attrs:
+            attribute.add_complete = True
+        self.async_write_ha_state()
+
+    async def async_added_to_hass(self) -> None:
+        """Run when entity about to be added to hass."""
+        self.hass.bus.async_listen_once(
+            EVENT_HOMEASSISTANT_START, self._async_template_startup
+        )
+
+    async def async_update(self) -> None:
+        """Call for forced update."""
+        for attribute in self._template_attrs:
+            if attribute.async_update:
+                attribute.async_update()

homeassistant/helpers/config_validation.py

@@ -156,6 +156,17 @@ def boolean(value: Any) -> bool:
     raise vol.Invalid(f"invalid boolean value {value}")
 
 
+_WS = re.compile("\\s*")
+
+
+def whitespace(value: Any) -> str:
+    """Validate result contains only whitespace."""
+    if isinstance(value, str) and _WS.fullmatch(value):
+        return value
+
+    raise vol.Invalid(f"contains non-whitespace: {value}")
+
+
 def isdevice(value: Any) -> str:
     """Validate that value is a real device."""
     try:

homeassistant/helpers/event.py

@@ -405,7 +405,11 @@ def async_track_template(
     ) -> None:
         """Check if condition is correct and run action."""
         if isinstance(result, TemplateError):
-            _LOGGER.exception(result)
+            _LOGGER.error(
+                "Error while processing template: %s",
+                template.template,
+                exc_info=result,
+            )
             return
 
         if result_as_boolean(last_result) or not result_as_boolean(result):
@@ -444,10 +448,10 @@ class _TrackTemplateResultInfo:
         """Handle removal / refresh of tracker init."""
         self.hass = hass
         self._template = template
+        self._template.hass = hass
         self._action = action
         self._variables = variables
-        self._last_result: Optional[str] = None
+        self._last_result: Optional[Union[str, TemplateError]] = None
-        self._last_exception = False
         self._all_listener: Optional[Callable] = None
         self._domains_listener: Optional[Callable] = None
         self._entities_listener: Optional[Callable] = None
@@ -458,8 +462,11 @@ class _TrackTemplateResultInfo:
         """Activation of template tracking."""
         self._info = self._template.async_render_to_info(self._variables)
         if self._info.exception:
-            self._last_exception = True
+            _LOGGER.error(
-            _LOGGER.exception(self._info.exception)
+                "Error while processing template: %s",
+                self._template.template,
+                exc_info=self._info.exception,
+            )
         self._create_listeners()
         self._last_info = self._info
 
@@ -593,26 +600,26 @@ class _TrackTemplateResultInfo:
             self._variables = variables
         self._refresh(None)
 
+    @callback
     def _refresh(self, event: Optional[Event]) -> None:
         self._info = self._template.async_render_to_info(self._variables)
         self._update_listeners()
         self._last_info = self._info
 
         try:
-            result = self._info.result
+            result: Union[str, TemplateError] = self._info.result
         except TemplateError as ex:
-            if not self._last_exception:
+            result = ex
-                self.hass.async_run_job(
-                    self._action, event, self._template, self._last_result, ex
-                )
-            self._last_exception = True
-            return
-        self._last_exception = False
 
         # Check to see if the result has changed
         if result == self._last_result:
             return
 
+        if isinstance(result, TemplateError) and isinstance(
+            self._last_result, TemplateError
+        ):
+            return
+
         self.hass.async_run_job(
             self._action, event, self._template, self._last_result, result
         )

tests/components/template/test_binary_sensor.py

@@ -3,6 +3,8 @@ from datetime import timedelta
 import unittest
 from unittest import mock
 
+import jinja2
+
 from homeassistant import setup
 from homeassistant.components.template import binary_sensor as template
 from homeassistant.const import (
@@ -318,10 +320,10 @@ class TestBinarySensorTemplate(unittest.TestCase):
             None,
             None,
         ).result()
-        mock_render.side_effect = TemplateError("foo")
+        mock_render.side_effect = TemplateError(jinja2.TemplateError("foo"))
         run_callback_threadsafe(self.hass.loop, vs.async_check_state).result()
         mock_render.side_effect = TemplateError(
-            "UndefinedError: 'None' has no attribute"
+            jinja2.TemplateError("UndefinedError: 'None' has no attribute")
         )
         run_callback_threadsafe(self.hass.loop, vs.async_check_state).result()
 

tests/components/template/test_sensor.py

@@ -544,9 +544,13 @@ async def test_invalid_attribute_template(hass, caplog):
     )
     await hass.async_block_till_done()
     assert len(hass.states.async_all()) == 2
+
+    hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
+    await hass.async_block_till_done()
     await hass.helpers.entity_component.async_update_entity("sensor.invalid_template")
 
-    assert ("Error rendering attribute test_attribute") in caplog.text
+    assert "TemplateError" in caplog.text
+    assert "test_attribute" in caplog.text
 
 
 async def test_invalid_availability_template_keeps_component_available(hass, caplog):
@@ -577,7 +581,7 @@ async def test_invalid_availability_template_keeps_component_available(hass, cap
 
 
 async def test_no_template_match_all(hass, caplog):
-    """Test that we do not allow sensors that match on all."""
+    """Test that we allow static templates."""
     hass.states.async_set("sensor.test_sensor", "startup")
 
     await async_setup_component(
@@ -617,31 +621,6 @@ async def test_no_template_match_all(hass, caplog):
 
     await hass.async_block_till_done()
     assert len(hass.states.async_all()) == 6
-    assert (
-        "Template sensor 'invalid_state' has no entity ids "
-        "configured to track nor were we able to extract the entities to "
-        "track from the value template"
-    ) in caplog.text
-    assert (
-        "Template sensor 'invalid_icon' has no entity ids "
-        "configured to track nor were we able to extract the entities to "
-        "track from the icon template"
-    ) in caplog.text
-    assert (
-        "Template sensor 'invalid_entity_picture' has no entity ids "
-        "configured to track nor were we able to extract the entities to "
-        "track from the entity_picture template"
-    ) in caplog.text
-    assert (
-        "Template sensor 'invalid_friendly_name' has no entity ids "
-        "configured to track nor were we able to extract the entities to "
-        "track from the friendly_name template"
-    ) in caplog.text
-    assert (
-        "Template sensor 'invalid_attribute' has no entity ids "
-        "configured to track nor were we able to extract the entities to "
-        "track from the test_attribute template"
-    ) in caplog.text
 
     assert hass.states.get("sensor.invalid_state").state == "unknown"
     assert hass.states.get("sensor.invalid_icon").state == "unknown"

tests/helpers/test_config_validation.py

@@ -1092,3 +1092,24 @@ def test_script(caplog):
             cv.script_action(data)
 
         assert msg in str(excinfo.value)
+
+
+def test_whitespace():
+    """Test whitespace validation."""
+    schema = vol.Schema(cv.whitespace)
+
+    for value in (
+        None,
+        "" "T",
+        "negative",
+        "lock",
+        "tr  ue",
+        [],
+        [1, 2],
+        {"one": "two"},
+    ):
+        with pytest.raises(vol.MultipleInvalid):
+            schema(value)
+
+    for value in ("  ", "   "):
+        assert schema(value)

tests/helpers/test_event.py

@@ -4,6 +4,7 @@ import asyncio
 from datetime import datetime, timedelta
 
 from astral import Astral
+import jinja2
 import pytest
 
 from homeassistant.components import sun
@@ -942,7 +943,7 @@ async def test_track_template_result_errors(hass, caplog):
     hass.states.async_set("switch.not_exist", "on")
     await hass.async_block_till_done()
 
-    assert len(syntax_error_runs) == 0
+    assert len(syntax_error_runs) == 1
     assert len(not_exist_runs) == 2
     assert not_exist_runs[1][0].data.get("entity_id") == "switch.not_exist"
     assert not_exist_runs[1][1] == template_not_exist
@@ -950,7 +951,7 @@ async def test_track_template_result_errors(hass, caplog):
     assert not_exist_runs[1][3] == "on"
 
     with patch.object(Template, "async_render") as render:
-        render.side_effect = TemplateError("Test")
+        render.side_effect = TemplateError(jinja2.TemplateError())
 
         hass.states.async_set("switch.not_exist", "off")
         await hass.async_block_till_done()