Add 'max_sub_interval' option to derivative sensor (#125870)

* Add 'max_sub_interval' option to derivative sensor

* add strings

* little coverage

* improve test accuracy

* reimplement at dev head

* string

* handle unavailable

* simplify

* Add self to codeowner

* fix on remove

* Update homeassistant/components/derivative/sensor.py

Co-authored-by: Erik Montnemery <erik@montnemery.com>

* Fix parenthesis

* sort strings

---------

Co-authored-by: Erik Montnemery <erik@montnemery.com>

CODEOWNERS

@@ -331,8 +331,8 @@ build.json @home-assistant/supervisor
 /tests/components/demo/ @home-assistant/core
 /homeassistant/components/denonavr/ @ol-iver @starkillerOG
 /tests/components/denonavr/ @ol-iver @starkillerOG
-/homeassistant/components/derivative/ @afaucogney
+/homeassistant/components/derivative/ @afaucogney @karwosts
-/tests/components/derivative/ @afaucogney
+/tests/components/derivative/ @afaucogney @karwosts
 /homeassistant/components/devialet/ @fwestenberg
 /tests/components/devialet/ @fwestenberg
 /homeassistant/components/device_automation/ @home-assistant/core

homeassistant/components/derivative/config_flow.py

@@ -26,6 +26,7 @@ from homeassistant.helpers.schema_config_entry_flow import (
 )
 
 from .const import (
+    CONF_MAX_SUB_INTERVAL,
     CONF_ROUND_DIGITS,
     CONF_TIME_WINDOW,
     CONF_UNIT_PREFIX,
@@ -104,6 +105,9 @@ async def _get_options_dict(handler: SchemaCommonFlowHandler | None) -> dict:
                 options=TIME_UNITS, translation_key="time_unit"
             ),
         ),
+        vol.Optional(CONF_MAX_SUB_INTERVAL): selector.DurationSelector(
+            selector.DurationSelectorConfig(allow_negative=False)
+        ),
     }
 
 

homeassistant/components/derivative/const.py

@@ -7,3 +7,4 @@ CONF_TIME_WINDOW = "time_window"
 CONF_UNIT = "unit"
 CONF_UNIT_PREFIX = "unit_prefix"
 CONF_UNIT_TIME = "unit_time"
+CONF_MAX_SUB_INTERVAL = "max_sub_interval"

homeassistant/components/derivative/manifest.json

@@ -2,7 +2,7 @@
   "domain": "derivative",
   "name": "Derivative",
   "after_dependencies": ["counter"],
-  "codeowners": ["@afaucogney"],
+  "codeowners": ["@afaucogney", "@karwosts"],
   "config_flow": true,
   "documentation": "https://www.home-assistant.io/integrations/derivative",
   "integration_type": "helper",

homeassistant/components/derivative/sensor.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from datetime import datetime, timedelta
-from decimal import Decimal, DecimalException
+from decimal import Decimal, DecimalException, InvalidOperation
 import logging
 
 import voluptuous as vol
@@ -25,6 +25,7 @@ from homeassistant.const import (
     UnitOfTime,
 )
 from homeassistant.core import (
+    CALLBACK_TYPE,
     Event,
     EventStateChangedData,
     EventStateReportedData,
@@ -40,12 +41,14 @@ from homeassistant.helpers.entity_platform import (
     AddEntitiesCallback,
 )
 from homeassistant.helpers.event import (
+    async_call_later,
     async_track_state_change_event,
     async_track_state_report_event,
 )
 from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType
 
 from .const import (
+    CONF_MAX_SUB_INTERVAL,
     CONF_ROUND_DIGITS,
     CONF_TIME_WINDOW,
     CONF_UNIT,
@@ -89,10 +92,20 @@ PLATFORM_SCHEMA = SENSOR_PLATFORM_SCHEMA.extend(
         vol.Optional(CONF_UNIT_TIME, default=UnitOfTime.HOURS): vol.In(UNIT_TIME),
         vol.Optional(CONF_UNIT): cv.string,
         vol.Optional(CONF_TIME_WINDOW, default=DEFAULT_TIME_WINDOW): cv.time_period,
+        vol.Optional(CONF_MAX_SUB_INTERVAL): cv.positive_time_period,
     }
 )
 
 
+def _is_decimal_state(state: str) -> bool:
+    try:
+        Decimal(state)
+    except (InvalidOperation, TypeError):
+        return False
+    else:
+        return True
+
+
 async def async_setup_entry(
     hass: HomeAssistant,
     config_entry: ConfigEntry,
@@ -114,6 +127,11 @@ async def async_setup_entry(
         # Before we had support for optional selectors, "none" was used for selecting nothing
         unit_prefix = None
 
+    if max_sub_interval_dict := config_entry.options.get(CONF_MAX_SUB_INTERVAL, None):
+        max_sub_interval = cv.time_period(max_sub_interval_dict)
+    else:
+        max_sub_interval = None
+
     derivative_sensor = DerivativeSensor(
         name=config_entry.title,
         round_digits=int(config_entry.options[CONF_ROUND_DIGITS]),
@@ -124,6 +142,7 @@ async def async_setup_entry(
         unit_prefix=unit_prefix,
         unit_time=config_entry.options[CONF_UNIT_TIME],
         device_info=device_info,
+        max_sub_interval=max_sub_interval,
     )
 
     async_add_entities([derivative_sensor])
@@ -145,6 +164,7 @@ async def async_setup_platform(
         unit_prefix=config[CONF_UNIT_PREFIX],
         unit_time=config[CONF_UNIT_TIME],
         unique_id=None,
+        max_sub_interval=config.get(CONF_MAX_SUB_INTERVAL),
     )
 
     async_add_entities([derivative])
@@ -166,6 +186,7 @@ class DerivativeSensor(RestoreSensor, SensorEntity):
         unit_of_measurement: str | None,
         unit_prefix: str | None,
         unit_time: UnitOfTime,
+        max_sub_interval: timedelta | None,
         unique_id: str | None,
         device_info: DeviceInfo | None = None,
     ) -> None:
@@ -192,6 +213,34 @@ class DerivativeSensor(RestoreSensor, SensorEntity):
         self._unit_prefix = UNIT_PREFIXES[unit_prefix]
         self._unit_time = UNIT_TIME[unit_time]
         self._time_window = time_window.total_seconds()
+        self._max_sub_interval: timedelta | None = (
+            None  # disable time based derivative
+            if max_sub_interval is None or max_sub_interval.total_seconds() == 0
+            else max_sub_interval
+        )
+        self._cancel_max_sub_interval_exceeded_callback: CALLBACK_TYPE = (
+            lambda *args: None
+        )
+
+    def _calc_derivative_from_state_list(self, current_time: datetime) -> Decimal:
+        def calculate_weight(start: datetime, end: datetime, now: datetime) -> float:
+            window_start = now - timedelta(seconds=self._time_window)
+            return (end - max(start, window_start)).total_seconds() / self._time_window
+
+        derivative = Decimal("0.00")
+        for start, end, value in self._state_list:
+            weight = calculate_weight(start, end, current_time)
+            derivative = derivative + (value * Decimal(weight))
+
+        return derivative
+
+    def _prune_state_list(self, current_time: datetime) -> None:
+        # filter out all derivatives older than `time_window` from our window list
+        self._state_list = [
+            (time_start, time_end, state)
+            for time_start, time_end, state in self._state_list
+            if (current_time - time_end).total_seconds() < self._time_window
+        ]
 
     async def async_added_to_hass(self) -> None:
         """Handle entity which will be added."""
@@ -209,13 +258,52 @@ class DerivativeSensor(RestoreSensor, SensorEntity):
             except SyntaxError as err:
                 _LOGGER.warning("Could not restore last state: %s", err)
 
+        def schedule_max_sub_interval_exceeded(source_state: State | None) -> None:
+            """Schedule calculation using the source state and max_sub_interval.
+
+            The callback reference is stored for possible cancellation if the source state
+            reports a change before max_sub_interval has passed.
+            If the callback is executed, meaning there was no state change reported, the
+            source_state is assumed constant and calculation is done using its value.
+            """
+            if (
+                self._max_sub_interval is not None
+                and source_state is not None
+                and (_is_decimal_state(source_state.state))
+            ):
+
+                @callback
+                def _calc_derivative_on_max_sub_interval_exceeded_callback(
+                    now: datetime,
+                ) -> None:
+                    """Calculate derivative based on time and reschedule."""
+
+                    self._prune_state_list(now)
+                    derivative = self._calc_derivative_from_state_list(now)
+                    self._attr_native_value = round(derivative, self._round_digits)
+
+                    self.async_write_ha_state()
+
+                    # If derivative is now zero, don't schedule another timeout callback, as it will have no effect
+                    if derivative != 0:
+                        schedule_max_sub_interval_exceeded(source_state)
+
+                self._cancel_max_sub_interval_exceeded_callback = async_call_later(
+                    self.hass,
+                    self._max_sub_interval,
+                    _calc_derivative_on_max_sub_interval_exceeded_callback,
+                )
+
         @callback
         def on_state_reported(event: Event[EventStateReportedData]) -> None:
             """Handle constant sensor state."""
+            self._cancel_max_sub_interval_exceeded_callback()
+            new_state = event.data["new_state"]
             if self._attr_native_value == Decimal(0):
                 # If the derivative is zero, and the source sensor hasn't
                 # changed state, then we know it will still be zero.
                 return
+            schedule_max_sub_interval_exceeded(new_state)
             new_state = event.data["new_state"]
             if new_state is not None:
                 calc_derivative(
@@ -225,7 +313,9 @@ class DerivativeSensor(RestoreSensor, SensorEntity):
         @callback
         def on_state_changed(event: Event[EventStateChangedData]) -> None:
             """Handle changed sensor state."""
+            self._cancel_max_sub_interval_exceeded_callback()
             new_state = event.data["new_state"]
+            schedule_max_sub_interval_exceeded(new_state)
             old_state = event.data["old_state"]
             if new_state is not None and old_state is not None:
                 calc_derivative(new_state, old_state.state, old_state.last_reported)
@@ -312,6 +402,16 @@ class DerivativeSensor(RestoreSensor, SensorEntity):
             self._attr_native_value = round(derivative, self._round_digits)
             self.async_write_ha_state()
 
+        if self._max_sub_interval is not None:
+            source_state = self.hass.states.get(self._sensor_source_id)
+            schedule_max_sub_interval_exceeded(source_state)
+
+            @callback
+            def on_removed() -> None:
+                self._cancel_max_sub_interval_exceeded_callback()
+
+            self.async_on_remove(on_removed)
+
         self.async_on_remove(
             async_track_state_change_event(
                 self.hass, self._sensor_source_id, on_state_changed

homeassistant/components/derivative/strings.json

@@ -6,6 +6,7 @@
         "title": "Create Derivative sensor",
         "description": "Create a sensor that estimates the derivative of a sensor.",
         "data": {
+          "max_sub_interval": "Max sub-interval",
           "name": "[%key:common::config_flow::data::name%]",
           "round": "Precision",
           "source": "Input sensor",
@@ -14,6 +15,7 @@
           "unit_time": "Time unit"
         },
         "data_description": {
+          "max_sub_interval": "If defined, derivative automatically recalculates if the source has not updated for this duration.",
           "round": "Controls the number of decimal digits in the output.",
           "time_window": "If set, the sensor's value is a time-weighted moving average of derivatives within this window.",
           "unit_prefix": "The output will be scaled according to the selected metric prefix and time unit of the derivative."
@@ -25,6 +27,7 @@
     "step": {
       "init": {
         "data": {
+          "max_sub_interval": "[%key:component::derivative::config::step::user::data::max_sub_interval%]",
           "name": "[%key:common::config_flow::data::name%]",
           "round": "[%key:component::derivative::config::step::user::data::round%]",
           "source": "[%key:component::derivative::config::step::user::data::source%]",
@@ -33,6 +36,7 @@
           "unit_time": "[%key:component::derivative::config::step::user::data::unit_time%]"
         },
         "data_description": {
+          "max_sub_interval": "[%key:component::derivative::config::step::user::data_description::max_sub_interval%]",
           "round": "[%key:component::derivative::config::step::user::data_description::round%]",
           "time_window": "[%key:component::derivative::config::step::user::data_description::time_window%]",
           "unit_prefix": "[%key:component::derivative::config::step::user::data_description::unit_prefix%]"

tests/components/derivative/test_config_flow.py

@@ -36,6 +36,7 @@ async def test_config_flow(hass: HomeAssistant, platform) -> None:
                 "source": input_sensor_entity_id,
                 "time_window": {"seconds": 0},
                 "unit_time": "min",
+                "max_sub_interval": {"minutes": 1},
             },
         )
         await hass.async_block_till_done()
@@ -49,6 +50,7 @@ async def test_config_flow(hass: HomeAssistant, platform) -> None:
         "source": "sensor.input",
         "time_window": {"seconds": 0.0},
         "unit_time": "min",
+        "max_sub_interval": {"minutes": 1.0},
     }
     assert len(mock_setup_entry.mock_calls) == 1
 
@@ -60,6 +62,7 @@ async def test_config_flow(hass: HomeAssistant, platform) -> None:
         "source": "sensor.input",
         "time_window": {"seconds": 0.0},
         "unit_time": "min",
+        "max_sub_interval": {"minutes": 1.0},
     }
     assert config_entry.title == "My derivative"
 
@@ -78,6 +81,7 @@ async def test_options(hass: HomeAssistant, platform) -> None:
             "time_window": {"seconds": 0.0},
             "unit_prefix": "k",
             "unit_time": "min",
+            "max_sub_interval": {"seconds": 30},
         },
         title="My derivative",
     )

tests/components/derivative/test_sensor.py

@@ -9,13 +9,13 @@ from freezegun import freeze_time
 
 from homeassistant.components.derivative.const import DOMAIN
 from homeassistant.components.sensor import ATTR_STATE_CLASS, SensorStateClass
-from homeassistant.const import UnitOfPower, UnitOfTime
+from homeassistant.const import STATE_UNAVAILABLE, UnitOfPower, UnitOfTime
 from homeassistant.core import HomeAssistant, State
 from homeassistant.helpers import device_registry as dr, entity_registry as er
 from homeassistant.setup import async_setup_component
 from homeassistant.util import dt as dt_util
 
-from tests.common import MockConfigEntry
+from tests.common import MockConfigEntry, async_fire_time_changed
 
 
 async def test_state(hass: HomeAssistant) -> None:
@@ -371,6 +371,177 @@ async def test_double_signal_after_delay(hass: HomeAssistant) -> None:
             previous = derivative
 
 
+async def test_sub_intervals_instantaneous(hass: HomeAssistant) -> None:
+    """Test derivative sensor state."""
+    # We simulate the following situation:
+    # Value changes from 0 to 10 in 5 seconds (derivative = 2)
+    # The max_sub_interval is 20 seconds
+    # After max_sub_interval elapses, derivative should change to 0
+    # Value changes to 0, 35 seconds after changing to 10 (derivative = -10/35 = -0.29)
+    # State goes unavailable, derivative stops changing after that.
+    # State goes back to 0, derivative returns to 0 after a max_sub_interval
+
+    max_sub_interval = 20
+
+    config, entity_id = await _setup_sensor(
+        hass,
+        {
+            "unit_time": UnitOfTime.SECONDS,
+            "round": 2,
+            "max_sub_interval": {"seconds": max_sub_interval},
+        },
+    )
+
+    base = dt_util.utcnow()
+    with freeze_time(base) as freezer:
+        freezer.move_to(base)
+        hass.states.async_set(entity_id, 0, {}, force_update=True)
+        await hass.async_block_till_done()
+
+        now = base + timedelta(seconds=5)
+        freezer.move_to(now)
+        hass.states.async_set(entity_id, 10, {}, force_update=True)
+        await hass.async_block_till_done()
+
+        state = hass.states.get("sensor.power")
+        derivative = round(float(state.state), config["sensor"]["round"])
+        assert derivative == 2
+
+        # No change yet as sub_interval not elapsed
+        now += timedelta(seconds=15)
+        async_fire_time_changed(hass, now)
+        await hass.async_block_till_done()
+
+        state = hass.states.get("sensor.power")
+        derivative = round(float(state.state), config["sensor"]["round"])
+        assert derivative == 2
+
+        # After 5 more seconds the sub_interval should fire and derivative should be 0
+        now += timedelta(seconds=10)
+        async_fire_time_changed(hass, now)
+        await hass.async_block_till_done()
+
+        state = hass.states.get("sensor.power")
+        derivative = round(float(state.state), config["sensor"]["round"])
+        assert derivative == 0
+
+        now += timedelta(seconds=10)
+        freezer.move_to(now)
+        hass.states.async_set(entity_id, 0, {}, force_update=True)
+        await hass.async_block_till_done()
+
+        state = hass.states.get("sensor.power")
+        derivative = round(float(state.state), config["sensor"]["round"])
+        assert derivative == -0.29
+
+        now += timedelta(seconds=10)
+        freezer.move_to(now)
+        hass.states.async_set(entity_id, STATE_UNAVAILABLE, {}, force_update=True)
+        await hass.async_block_till_done()
+
+        state = hass.states.get("sensor.power")
+        derivative = round(float(state.state), config["sensor"]["round"])
+        assert derivative == -0.29
+
+        now += timedelta(seconds=60)
+        async_fire_time_changed(hass, now)
+        await hass.async_block_till_done()
+
+        state = hass.states.get("sensor.power")
+        derivative = round(float(state.state), config["sensor"]["round"])
+        assert derivative == -0.29
+
+        now += timedelta(seconds=10)
+        freezer.move_to(now)
+        hass.states.async_set(entity_id, 0, {}, force_update=True)
+        await hass.async_block_till_done()
+
+        state = hass.states.get("sensor.power")
+        derivative = round(float(state.state), config["sensor"]["round"])
+        assert derivative == -0.29
+
+        now += timedelta(seconds=max_sub_interval + 1)
+        async_fire_time_changed(hass, now)
+        await hass.async_block_till_done()
+
+        state = hass.states.get("sensor.power")
+        derivative = round(float(state.state), config["sensor"]["round"])
+        assert derivative == 0
+
+
+async def test_sub_intervals_with_time_window(hass: HomeAssistant) -> None:
+    """Test derivative sensor state."""
+    # We simulate the following situation:
+    # The value rises by 1 every second for 1 minute, then pauses
+    # The time window is 30 seconds
+    # The max_sub_interval is 5 seconds
+    # After the value stops increasing, the derivative should slowly trend back to 0
+
+    values = []
+    for value in range(60):
+        values += [value]
+    time_window = 30
+    max_sub_interval = 5
+    times = values
+
+    config, entity_id = await _setup_sensor(
+        hass,
+        {
+            "time_window": {"seconds": time_window},
+            "unit_time": UnitOfTime.SECONDS,
+            "round": 2,
+            "max_sub_interval": {"seconds": max_sub_interval},
+        },
+    )
+
+    base = dt_util.utcnow()
+    with freeze_time(base) as freezer:
+        last_state_change = None
+        for time, value in zip(times, values, strict=False):
+            now = base + timedelta(seconds=time)
+            freezer.move_to(now)
+            hass.states.async_set(entity_id, value, {}, force_update=True)
+            last_state_change = now
+            await hass.async_block_till_done()
+
+            if time_window < time:
+                state = hass.states.get("sensor.power")
+                derivative = round(float(state.state), config["sensor"]["round"])
+                # Test that the error is never more than
+                # (time_window_in_minutes / true_derivative * 100) = 1% + ε
+                assert abs(1 - derivative) <= 0.01 + 1e-6
+
+        for time in range(60):
+            now = last_state_change + timedelta(seconds=time)
+            freezer.move_to(now)
+
+            async_fire_time_changed(hass, now)
+            await hass.async_block_till_done()
+
+            state = hass.states.get("sensor.power")
+            derivative = round(float(state.state), config["sensor"]["round"])
+
+            def calc_expected(elapsed_seconds: int, calculation_delay: int = 0):
+                last_sub_interval = (
+                    elapsed_seconds // max_sub_interval
+                ) * max_sub_interval
+                return (
+                    0
+                    if (last_sub_interval >= time_window)
+                    else (
+                        (time_window - last_sub_interval - calculation_delay)
+                        / time_window
+                    )
+                )
+
+            rounding_err = 0.01 + 1e-6
+            expect_max = calc_expected(time) + rounding_err
+            # Allow one second of slop for internal delays
+            expect_min = calc_expected(time, 1) - rounding_err
+
+            assert expect_min <= derivative <= expect_max, f"Failed at time {time}"
+
+
 async def test_prefix(hass: HomeAssistant) -> None:
     """Test derivative sensor state using a power source."""
     config = {