Optimize sliding window history_stats to not re-query the database every interval (#143279)

Co-authored-by: J. Nick Koston <nick@koston.org>

homeassistant/components/history_stats/data.py

@@ -54,7 +54,7 @@ class HistoryStats:
         self._period = (MIN_TIME_UTC, MIN_TIME_UTC)
         self._state: HistoryStatsState = HistoryStatsState(None, None, self._period)
         self._history_current_period: list[HistoryState] = []
-        self._previous_run_before_start = False
+        self._has_recorder_data = False
         self._entity_states = set(entity_states)
         self._duration = duration
         self._start = start
@@ -88,20 +88,20 @@ class HistoryStats:
         if current_period_start_timestamp > now_timestamp:
             # History cannot tell the future
             self._history_current_period = []
-            self._previous_run_before_start = True
+            self._has_recorder_data = False
             self._state = HistoryStatsState(None, None, self._period)
             return self._state
         #
         # We avoid querying the database if the below did NOT happen:
         #
-        # - The previous run happened before the start time
+        # - No previous run occurred (uninitialized)
-        # - The start time changed
+        # - The start time moved back in time
-        # - The period shrank in size
+        # - The end time moved back in time
         # - The previous period ended before now
         #
         if (
-            not self._previous_run_before_start
+            self._has_recorder_data
-            and current_period_start_timestamp == previous_period_start_timestamp
+            and current_period_start_timestamp >= previous_period_start_timestamp
             and (
                 current_period_end_timestamp == previous_period_end_timestamp
                 or (
@@ -110,6 +110,12 @@ class HistoryStats:
                 )
             )
         ):
+            start_changed = (
+                current_period_start_timestamp != previous_period_start_timestamp
+            )
+            if start_changed:
+                self._prune_history_cache(current_period_start_timestamp)
+
             new_data = False
             if event and (new_state := event.data["new_state"]) is not None:
                 if (
@@ -121,7 +127,11 @@ class HistoryStats:
                         HistoryState(new_state.state, new_state.last_changed_timestamp)
                     )
                     new_data = True
-            if not new_data and current_period_end_timestamp < now_timestamp:
+            if (
+                not new_data
+                and current_period_end_timestamp < now_timestamp
+                and not start_changed
+            ):
                 # If period has not changed and current time after the period end...
                 # Don't compute anything as the value cannot have changed
                 return self._state
@@ -139,7 +149,7 @@ class HistoryStats:
                         HistoryState(new_state.state, new_state.last_changed_timestamp)
                     )
 
-            self._previous_run_before_start = False
+            self._has_recorder_data = True
 
         seconds_matched, match_count = self._async_compute_seconds_and_changes(
             now_timestamp,
@@ -223,3 +233,18 @@ class HistoryStats:
         # Save value in seconds
         seconds_matched = elapsed
         return seconds_matched, match_count
+
+    def _prune_history_cache(self, start_timestamp: float) -> None:
+        """Remove unnecessary old data from the history state cache from previous runs.
+
+        Update the timestamp of the last record from before the start to the current start time.
+        """
+        trim_count = 0
+        for i, history_state in enumerate(self._history_current_period):
+            if history_state.last_changed >= start_timestamp:
+                break
+            history_state.last_changed = start_timestamp
+            if i > 0:
+                trim_count += 1
+        if trim_count:  # Don't slice if no data was removed
+            self._history_current_period = self._history_current_period[trim_count:]

tests/components/history_stats/test_sensor.py

@@ -969,6 +969,135 @@ async def test_async_start_from_history_and_switch_to_watching_state_changes_mul
     assert hass.states.get("sensor.sensor4").state == "87.5"
 
 
+async def test_start_from_history_then_watch_state_changes_sliding(
+    recorder_mock: Recorder,
+    hass: HomeAssistant,
+) -> None:
+    """Test we startup from history and switch to watching state changes.
+
+    With a sliding window, history_stats does not requery the recorder.
+    """
+    await hass.config.async_set_time_zone("UTC")
+    utcnow = dt_util.utcnow()
+    start_time = utcnow.replace(hour=0, minute=0, second=0, microsecond=0)
+    time = start_time
+
+    def _fake_states(*args, **kwargs):
+        return {
+            "binary_sensor.state": [
+                ha.State(
+                    "binary_sensor.state",
+                    "off",
+                    last_changed=start_time - timedelta(hours=1),
+                    last_updated=start_time - timedelta(hours=1),
+                ),
+            ]
+        }
+
+    with (
+        patch(
+            "homeassistant.components.recorder.history.state_changes_during_period",
+            _fake_states,
+        ),
+        freeze_time(start_time),
+    ):
+        await async_setup_component(
+            hass,
+            "sensor",
+            {
+                "sensor": [
+                    {
+                        "platform": "history_stats",
+                        "entity_id": "binary_sensor.state",
+                        "name": f"sensor{i}",
+                        "state": "on",
+                        "end": "{{ utcnow() }}",
+                        "duration": {"hours": 1},
+                        "type": sensor_type,
+                    }
+                    for i, sensor_type in enumerate(["time", "ratio", "count"])
+                ]
+            },
+        )
+        await hass.async_block_till_done()
+
+        for i in range(3):
+            await async_update_entity(hass, f"sensor.sensor{i}")
+        await hass.async_block_till_done()
+
+    assert hass.states.get("sensor.sensor0").state == "0.0"
+    assert hass.states.get("sensor.sensor1").state == "0.0"
+    assert hass.states.get("sensor.sensor2").state == "0"
+
+    with freeze_time(time):
+        hass.states.async_set("binary_sensor.state", "on")
+        await hass.async_block_till_done()
+        async_fire_time_changed(hass, time)
+        await hass.async_block_till_done()
+
+    assert hass.states.get("sensor.sensor0").state == "0.0"
+    assert hass.states.get("sensor.sensor1").state == "0.0"
+    assert hass.states.get("sensor.sensor2").state == "1"
+
+    # After sensor has been on for 15 minutes, check state
+    time += timedelta(minutes=15)  # 00:15
+    with freeze_time(time):
+        async_fire_time_changed(hass, time)
+        await hass.async_block_till_done()
+
+    assert hass.states.get("sensor.sensor0").state == "0.25"
+    assert hass.states.get("sensor.sensor1").state == "25.0"
+    assert hass.states.get("sensor.sensor2").state == "1"
+
+    with freeze_time(time):
+        hass.states.async_set("binary_sensor.state", "off")
+        await hass.async_block_till_done()
+        async_fire_time_changed(hass, time)
+        await hass.async_block_till_done()
+
+    time += timedelta(minutes=30)  # 00:45
+
+    with freeze_time(time):
+        async_fire_time_changed(hass, time)
+        await hass.async_block_till_done()
+
+    assert hass.states.get("sensor.sensor0").state == "0.25"
+    assert hass.states.get("sensor.sensor1").state == "25.0"
+    assert hass.states.get("sensor.sensor2").state == "1"
+
+    time += timedelta(minutes=20)  # 01:05
+
+    with freeze_time(time):
+        async_fire_time_changed(hass, time)
+        await hass.async_block_till_done()
+
+    # Sliding window will have started to erase the initial on period, so now it will only be on for 10 minutes
+    assert hass.states.get("sensor.sensor0").state == "0.17"
+    assert hass.states.get("sensor.sensor1").state == "16.7"
+    assert hass.states.get("sensor.sensor2").state == "1"
+
+    time += timedelta(minutes=5)  # 01:10
+
+    with freeze_time(time):
+        async_fire_time_changed(hass, time)
+        await hass.async_block_till_done()
+
+    # Sliding window will continue to erase the initial on period, so now it will only be on for 5 minutes
+    assert hass.states.get("sensor.sensor0").state == "0.08"
+    assert hass.states.get("sensor.sensor1").state == "8.3"
+    assert hass.states.get("sensor.sensor2").state == "1"
+
+    time += timedelta(minutes=10)  # 01:20
+
+    with freeze_time(time):
+        async_fire_time_changed(hass, time)
+        await hass.async_block_till_done()
+
+    assert hass.states.get("sensor.sensor0").state == "0.0"
+    assert hass.states.get("sensor.sensor1").state == "0.0"
+    assert hass.states.get("sensor.sensor2").state == "0"
+
+
 async def test_does_not_work_into_the_future(
     recorder_mock: Recorder, hass: HomeAssistant
 ) -> None:
@@ -1366,10 +1495,6 @@ async def test_measure_from_end_going_backwards(
 
     past_next_update = start_time + timedelta(minutes=30)
     with (
-        patch(
-            "homeassistant.components.recorder.history.state_changes_during_period",
-            _fake_states,
-        ),
         freeze_time(past_next_update),
     ):
         async_fire_time_changed(hass, past_next_update)
@@ -1526,29 +1651,10 @@ async def test_state_change_during_window_rollover(
 
     assert hass.states.get("sensor.sensor1").state == "11.98"
 
-    # One minute has passed and the time has now rolled over into a new day, resetting the recorder window. The sensor will then query the database for updates,
+    # One minute has passed and the time has now rolled over into a new day, resetting the recorder window.
-    # and will see that the sensor is ON starting from midnight.
+    # The sensor will be ON since midnight.
     t3 = t2 + timedelta(minutes=1)
-
+    with freeze_time(t3):
-    def _fake_states_t3(*args, **kwargs):
-        return {
-            "binary_sensor.state": [
-                ha.State(
-                    "binary_sensor.state",
-                    "on",
-                    last_changed=t3.replace(hour=0, minute=0, second=0, microsecond=0),
-                    last_updated=t3.replace(hour=0, minute=0, second=0, microsecond=0),
-                ),
-            ]
-        }
-
-    with (
-        patch(
-            "homeassistant.components.recorder.history.state_changes_during_period",
-            _fake_states_t3,
-        ),
-        freeze_time(t3),
-    ):
         # The sensor turns off around this time, before the sensor does its normal polled update.
         hass.states.async_set("binary_sensor.state", "off")
         await hass.async_block_till_done(wait_background_tasks=True)