diff --git a/homeassistant/helpers/event.py b/homeassistant/helpers/event.py
index 464d8df6e0b..044f18c4255 100644
--- a/homeassistant/helpers/event.py
+++ b/homeassistant/helpers/event.py
@@ -758,15 +758,15 @@ class _TrackTemplateResultInfo:
         for track_template_ in self._track_templates:
             template = track_template_.template
             variables = track_template_.variables
+            self._info[template] = info = template.async_render_to_info(variables)
 
-            self._info[template] = template.async_render_to_info(variables)
-            if self._info[template].exception:
+            if info.exception:
                 if raise_on_template_error:
-                    raise self._info[template].exception
+                    raise info.exception
                 _LOGGER.error(
                     "Error while processing template: %s",
                     track_template_.template,
-                    exc_info=self._info[template].exception,
+                    exc_info=info.exception,
                 )
 
         self._track_state_changes = async_track_state_change_filtered(
@@ -817,9 +817,7 @@ class _TrackTemplateResultInfo:
         if event:
             info = self._info[template]
 
-            if not self._rate_limit.async_has_timer(
-                template
-            ) and not _event_triggers_rerender(event, info):
+            if not _event_triggers_rerender(event, info):
                 return False
 
             if self._rate_limit.async_schedule_action(
@@ -828,6 +826,8 @@ class _TrackTemplateResultInfo:
                 now,
                 self._refresh,
                 event,
+                (track_template_,),
+                True,
             ):
                 return False
 
@@ -838,10 +838,12 @@ class _TrackTemplateResultInfo:
             )
 
         self._rate_limit.async_triggered(template, now)
-        self._info[template] = template.async_render_to_info(track_template_.variables)
+        self._info[template] = info = template.async_render_to_info(
+            track_template_.variables
+        )
 
         try:
-            result: Union[str, TemplateError] = self._info[template].result()
+            result: Union[str, TemplateError] = info.result()
         except TemplateError as ex:
             result = ex
 
@@ -857,12 +859,29 @@ class _TrackTemplateResultInfo:
         return TrackTemplateResult(template, last_result, result)
 
     @callback
-    def _refresh(self, event: Optional[Event]) -> None:
+    def _refresh(
+        self,
+        event: Optional[Event],
+        track_templates: Optional[Iterable[TrackTemplate]] = None,
+        replayed: Optional[bool] = False,
+    ) -> None:
+        """Refresh the template.
+
+        The event is the state_changed event that caused the refresh
+        to be considered.
+
+        track_templates is an optional list of TrackTemplate objects
+        to refresh.  If not provided, all tracked templates will be
+        considered.
+
+        replayed is True if the event is being replayed because the
+        rate limit was hit.
+        """
         updates = []
         info_changed = False
-        now = dt_util.utcnow()
+        now = event.time_fired if not replayed and event else dt_util.utcnow()
 
-        for track_template_ in self._track_templates:
+        for track_template_ in track_templates or self._track_templates:
             update = self._render_template_if_ready(track_template_, now, event)
             if not update:
                 continue
diff --git a/homeassistant/helpers/ratelimit.py b/homeassistant/helpers/ratelimit.py
index 422ebdf2eee..d7099e9fe3d 100644
--- a/homeassistant/helpers/ratelimit.py
+++ b/homeassistant/helpers/ratelimit.py
@@ -26,6 +26,8 @@ class KeyedRateLimit:
     @callback
     def async_has_timer(self, key: Hashable) -> bool:
         """Check if a rate limit timer is running."""
+        if not self._rate_limit_timers:
+            return False
         return key in self._rate_limit_timers
 
     @callback
@@ -37,7 +39,7 @@ class KeyedRateLimit:
     @callback
     def async_cancel_timer(self, key: Hashable) -> None:
         """Cancel a rate limit time that will call the action."""
-        if not self.async_has_timer(key):
+        if not self._rate_limit_timers or not self.async_has_timer(key):
             return
 
         self._rate_limit_timers.pop(key).cancel()
@@ -71,10 +73,14 @@ class KeyedRateLimit:
 
             Return None
         """
-        if rate_limit is None or key not in self._last_triggered:
+        if rate_limit is None:
             return None
 
-        next_call_time = self._last_triggered[key] + rate_limit
+        last_triggered = self._last_triggered.get(key)
+        if not last_triggered:
+            return None
+
+        next_call_time = last_triggered + rate_limit
 
         if next_call_time <= now:
             self.async_cancel_timer(key)
diff --git a/homeassistant/helpers/template.py b/homeassistant/helpers/template.py
index b92f926f809..8a971f6805e 100644
--- a/homeassistant/helpers/template.py
+++ b/homeassistant/helpers/template.py
@@ -213,6 +213,10 @@ class RenderInfo:
             split_entity_id(entity_id)[0] in self.domains or entity_id in self.entities
         )
 
+    def _filter_entities(self, entity_id: str) -> bool:
+        """Template should re-render if the entity state changes when we match specific entities."""
+        return entity_id in self.entities
+
     def _filter_lifecycle_domains(self, entity_id: str) -> bool:
         """Template should re-render if the entity is added or removed with domains watched."""
         return split_entity_id(entity_id)[0] in self.domains_lifecycle
@@ -255,8 +259,10 @@ class RenderInfo:
         if self.all_states:
             return
 
-        if self.entities or self.domains:
+        if self.domains:
             self.filter = self._filter_domains_and_entities
+        elif self.entities:
+            self.filter = self._filter_entities
         else:
             self.filter = _false
 
@@ -264,6 +270,15 @@ class RenderInfo:
 class Template:
     """Class to hold a template and manage caching and rendering."""
 
+    __slots__ = (
+        "__weakref__",
+        "template",
+        "hass",
+        "is_static",
+        "_compiled_code",
+        "_compiled",
+    )
+
     def __init__(self, template, hass=None):
         """Instantiate a template."""
         if not isinstance(template, str):