From 8699e69ae5d41311c2de8edfc9babd5dbf9fdd8b Mon Sep 17 00:00:00 2001 From: karwosts <32912880+karwosts@users.noreply.github.com> Date: Sun, 20 Apr 2025 11:08:28 -0700 Subject: [PATCH] Optimize sliding window history_stats to not re-query the database every interval (#143279) Co-authored-by: J. Nick Koston --- .../components/history_stats/data.py | 43 ++++- tests/components/history_stats/test_sensor.py | 158 +++++++++++++++--- 2 files changed, 166 insertions(+), 35 deletions(-) diff --git a/homeassistant/components/history_stats/data.py b/homeassistant/components/history_stats/data.py index a69abe26f6c..756a6b3ce9d 100644 --- a/homeassistant/components/history_stats/data.py +++ b/homeassistant/components/history_stats/data.py @@ -54,7 +54,7 @@ class HistoryStats: self._period = (MIN_TIME_UTC, MIN_TIME_UTC) self._state: HistoryStatsState = HistoryStatsState(None, None, self._period) self._history_current_period: list[HistoryState] = [] - self._previous_run_before_start = False + self._has_recorder_data = False self._entity_states = set(entity_states) self._duration = duration self._start = start @@ -88,20 +88,20 @@ class HistoryStats: if current_period_start_timestamp > now_timestamp: # History cannot tell the future self._history_current_period = [] - self._previous_run_before_start = True + self._has_recorder_data = False self._state = HistoryStatsState(None, None, self._period) return self._state # # We avoid querying the database if the below did NOT happen: # - # - The previous run happened before the start time - # - The start time changed - # - The period shrank in size + # - No previous run occurred (uninitialized) + # - The start time moved back in time + # - The end time moved back in time # - The previous period ended before now # if ( - not self._previous_run_before_start - and current_period_start_timestamp == previous_period_start_timestamp + self._has_recorder_data + and current_period_start_timestamp >= previous_period_start_timestamp and ( current_period_end_timestamp == previous_period_end_timestamp or ( @@ -110,6 +110,12 @@ class HistoryStats: ) ) ): + start_changed = ( + current_period_start_timestamp != previous_period_start_timestamp + ) + if start_changed: + self._prune_history_cache(current_period_start_timestamp) + new_data = False if event and (new_state := event.data["new_state"]) is not None: if ( @@ -121,7 +127,11 @@ class HistoryStats: HistoryState(new_state.state, new_state.last_changed_timestamp) ) new_data = True - if not new_data and current_period_end_timestamp < now_timestamp: + if ( + not new_data + and current_period_end_timestamp < now_timestamp + and not start_changed + ): # If period has not changed and current time after the period end... # Don't compute anything as the value cannot have changed return self._state @@ -139,7 +149,7 @@ class HistoryStats: HistoryState(new_state.state, new_state.last_changed_timestamp) ) - self._previous_run_before_start = False + self._has_recorder_data = True seconds_matched, match_count = self._async_compute_seconds_and_changes( now_timestamp, @@ -223,3 +233,18 @@ class HistoryStats: # Save value in seconds seconds_matched = elapsed return seconds_matched, match_count + + def _prune_history_cache(self, start_timestamp: float) -> None: + """Remove unnecessary old data from the history state cache from previous runs. + + Update the timestamp of the last record from before the start to the current start time. + """ + trim_count = 0 + for i, history_state in enumerate(self._history_current_period): + if history_state.last_changed >= start_timestamp: + break + history_state.last_changed = start_timestamp + if i > 0: + trim_count += 1 + if trim_count: # Don't slice if no data was removed + self._history_current_period = self._history_current_period[trim_count:] diff --git a/tests/components/history_stats/test_sensor.py b/tests/components/history_stats/test_sensor.py index e2dba1b9355..ee426cf3048 100644 --- a/tests/components/history_stats/test_sensor.py +++ b/tests/components/history_stats/test_sensor.py @@ -969,6 +969,135 @@ async def test_async_start_from_history_and_switch_to_watching_state_changes_mul assert hass.states.get("sensor.sensor4").state == "87.5" +async def test_start_from_history_then_watch_state_changes_sliding( + recorder_mock: Recorder, + hass: HomeAssistant, +) -> None: + """Test we startup from history and switch to watching state changes. + + With a sliding window, history_stats does not requery the recorder. + """ + await hass.config.async_set_time_zone("UTC") + utcnow = dt_util.utcnow() + start_time = utcnow.replace(hour=0, minute=0, second=0, microsecond=0) + time = start_time + + def _fake_states(*args, **kwargs): + return { + "binary_sensor.state": [ + ha.State( + "binary_sensor.state", + "off", + last_changed=start_time - timedelta(hours=1), + last_updated=start_time - timedelta(hours=1), + ), + ] + } + + with ( + patch( + "homeassistant.components.recorder.history.state_changes_during_period", + _fake_states, + ), + freeze_time(start_time), + ): + await async_setup_component( + hass, + "sensor", + { + "sensor": [ + { + "platform": "history_stats", + "entity_id": "binary_sensor.state", + "name": f"sensor{i}", + "state": "on", + "end": "{{ utcnow() }}", + "duration": {"hours": 1}, + "type": sensor_type, + } + for i, sensor_type in enumerate(["time", "ratio", "count"]) + ] + }, + ) + await hass.async_block_till_done() + + for i in range(3): + await async_update_entity(hass, f"sensor.sensor{i}") + await hass.async_block_till_done() + + assert hass.states.get("sensor.sensor0").state == "0.0" + assert hass.states.get("sensor.sensor1").state == "0.0" + assert hass.states.get("sensor.sensor2").state == "0" + + with freeze_time(time): + hass.states.async_set("binary_sensor.state", "on") + await hass.async_block_till_done() + async_fire_time_changed(hass, time) + await hass.async_block_till_done() + + assert hass.states.get("sensor.sensor0").state == "0.0" + assert hass.states.get("sensor.sensor1").state == "0.0" + assert hass.states.get("sensor.sensor2").state == "1" + + # After sensor has been on for 15 minutes, check state + time += timedelta(minutes=15) # 00:15 + with freeze_time(time): + async_fire_time_changed(hass, time) + await hass.async_block_till_done() + + assert hass.states.get("sensor.sensor0").state == "0.25" + assert hass.states.get("sensor.sensor1").state == "25.0" + assert hass.states.get("sensor.sensor2").state == "1" + + with freeze_time(time): + hass.states.async_set("binary_sensor.state", "off") + await hass.async_block_till_done() + async_fire_time_changed(hass, time) + await hass.async_block_till_done() + + time += timedelta(minutes=30) # 00:45 + + with freeze_time(time): + async_fire_time_changed(hass, time) + await hass.async_block_till_done() + + assert hass.states.get("sensor.sensor0").state == "0.25" + assert hass.states.get("sensor.sensor1").state == "25.0" + assert hass.states.get("sensor.sensor2").state == "1" + + time += timedelta(minutes=20) # 01:05 + + with freeze_time(time): + async_fire_time_changed(hass, time) + await hass.async_block_till_done() + + # Sliding window will have started to erase the initial on period, so now it will only be on for 10 minutes + assert hass.states.get("sensor.sensor0").state == "0.17" + assert hass.states.get("sensor.sensor1").state == "16.7" + assert hass.states.get("sensor.sensor2").state == "1" + + time += timedelta(minutes=5) # 01:10 + + with freeze_time(time): + async_fire_time_changed(hass, time) + await hass.async_block_till_done() + + # Sliding window will continue to erase the initial on period, so now it will only be on for 5 minutes + assert hass.states.get("sensor.sensor0").state == "0.08" + assert hass.states.get("sensor.sensor1").state == "8.3" + assert hass.states.get("sensor.sensor2").state == "1" + + time += timedelta(minutes=10) # 01:20 + + with freeze_time(time): + async_fire_time_changed(hass, time) + await hass.async_block_till_done() + + assert hass.states.get("sensor.sensor0").state == "0.0" + assert hass.states.get("sensor.sensor1").state == "0.0" + assert hass.states.get("sensor.sensor2").state == "0" + + async def test_does_not_work_into_the_future( recorder_mock: Recorder, hass: HomeAssistant ) -> None: @@ -1366,10 +1495,6 @@ async def test_measure_from_end_going_backwards( past_next_update = start_time + timedelta(minutes=30) with ( - patch( - "homeassistant.components.recorder.history.state_changes_during_period", - _fake_states, - ), freeze_time(past_next_update), ): async_fire_time_changed(hass, past_next_update) @@ -1526,29 +1651,10 @@ async def test_state_change_during_window_rollover( assert hass.states.get("sensor.sensor1").state == "11.98" - # One minute has passed and the time has now rolled over into a new day, resetting the recorder window. The sensor will then query the database for updates, - # and will see that the sensor is ON starting from midnight. + # One minute has passed and the time has now rolled over into a new day, resetting the recorder window. + # The sensor will be ON since midnight. t3 = t2 + timedelta(minutes=1) - - def _fake_states_t3(*args, **kwargs): - return { - "binary_sensor.state": [ - ha.State( - "binary_sensor.state", - "on", - last_changed=t3.replace(hour=0, minute=0, second=0, microsecond=0), - last_updated=t3.replace(hour=0, minute=0, second=0, microsecond=0), - ), - ] - } - - with ( - patch( - "homeassistant.components.recorder.history.state_changes_during_period", - _fake_states_t3, - ), - freeze_time(t3), - ): + with freeze_time(t3): # The sensor turns off around this time, before the sensor does its normal polled update. hass.states.async_set("binary_sensor.state", "off") await hass.async_block_till_done(wait_background_tasks=True)