Warn if total_increasing sensor has negative states (#56564)

homeassistant/components/sensor/recorder.py

@@ -47,6 +47,7 @@ from homeassistant.const import (
     VOLUME_CUBIC_METERS,
 )
 from homeassistant.core import HomeAssistant, State
+from homeassistant.exceptions import HomeAssistantError
 from homeassistant.helpers.entity import entity_sources
 import homeassistant.util.dt as dt_util
 import homeassistant.util.pressure as pressure_util
@@ -120,6 +121,8 @@ UNIT_CONVERSIONS: dict[str, dict[str, Callable]] = {
 # Keep track of entities for which a warning about decreasing value has been logged
 SEEN_DIP = "sensor_seen_total_increasing_dip"
 WARN_DIP = "sensor_warn_total_increasing_dip"
+# Keep track of entities for which a warning about negative value has been logged
+WARN_NEGATIVE = "sensor_warn_total_increasing_negative"
 # Keep track of entities for which a warning about unsupported unit has been logged
 WARN_UNSUPPORTED_UNIT = "sensor_warn_unsupported_unit"
 WARN_UNSTABLE_UNIT = "sensor_warn_unstable_unit"
@@ -256,6 +259,24 @@ def _normalize_states(
     return DEVICE_CLASS_UNITS[device_class], fstates
 
 
+def _suggest_report_issue(hass: HomeAssistant, entity_id: str) -> str:
+    """Suggest to report an issue."""
+    domain = entity_sources(hass).get(entity_id, {}).get("domain")
+    custom_component = entity_sources(hass).get(entity_id, {}).get("custom_component")
+    report_issue = ""
+    if custom_component:
+        report_issue = "report it to the custom component author."
+    else:
+        report_issue = (
+            "create a bug report at "
+            "https://github.com/home-assistant/core/issues?q=is%3Aopen+is%3Aissue"
+        )
+        if domain:
+            report_issue += f"+label%3A%22integration%3A+{domain}%22"
+
+    return report_issue
+
+
 def warn_dip(hass: HomeAssistant, entity_id: str) -> None:
     """Log a warning once if a sensor with state_class_total has a decreasing value.
 
@@ -277,11 +298,26 @@ def warn_dip(hass: HomeAssistant, entity_id: str) -> None:
             return
         _LOGGER.warning(
             "Entity %s %shas state class total_increasing, but its state is "
-            "not strictly increasing. Please create a bug report at %s",
+            "not strictly increasing. Please %s",
             entity_id,
             f"from integration {domain} " if domain else "",
-            "https://github.com/home-assistant/core/issues?q=is%3Aopen+is%3Aissue"
+            _suggest_report_issue(hass, entity_id),
-            "+label%3A%22integration%3A+recorder%22",
+        )
+
+
+def warn_negative(hass: HomeAssistant, entity_id: str) -> None:
+    """Log a warning once if a sensor with state_class_total has a negative value."""
+    if WARN_NEGATIVE not in hass.data:
+        hass.data[WARN_NEGATIVE] = set()
+    if entity_id not in hass.data[WARN_NEGATIVE]:
+        hass.data[WARN_NEGATIVE].add(entity_id)
+        domain = entity_sources(hass).get(entity_id, {}).get("domain")
+        _LOGGER.warning(
+            "Entity %s %shas state class total_increasing, but its state is "
+            "negative. Please %s",
+            entity_id,
+            f"from integration {domain} " if domain else "",
+            _suggest_report_issue(hass, entity_id),
         )
 
 
@@ -295,6 +331,10 @@ def reset_detected(
     if 0.9 * previous_state <= state < previous_state:
         warn_dip(hass, entity_id)
 
+    if state < 0:
+        warn_negative(hass, entity_id)
+        raise HomeAssistantError
+
     return state < 0.9 * previous_state
 
 
@@ -473,17 +513,20 @@ def compile_statistics(  # noqa: C901
                         entity_id,
                         fstate,
                     )
-                elif state_class == STATE_CLASS_TOTAL_INCREASING and (
+                elif state_class == STATE_CLASS_TOTAL_INCREASING:
-                    old_state is None
+                    try:
-                    or reset_detected(hass, entity_id, fstate, new_state)
+                        if old_state is None or reset_detected(
-                ):
+                            hass, entity_id, fstate, new_state
-                    reset = True
+                        ):
-                    _LOGGER.info(
+                            reset = True
-                        "Detected new cycle for %s, value dropped from %s to %s",
+                            _LOGGER.info(
-                        entity_id,
+                                "Detected new cycle for %s, value dropped from %s to %s",
-                        new_state,
+                                entity_id,
-                        fstate,
+                                new_state,
-                    )
+                                fstate,
+                            )
+                    except HomeAssistantError:
+                        continue
 
                 if reset:
                     # The sensor has been reset, update the sum

homeassistant/helpers/entity.py

@@ -734,7 +734,10 @@ class Entity(ABC):
         Not to be extended by integrations.
         """
         if self.platform:
-            info = {"domain": self.platform.platform_name}
+            info = {
+                "domain": self.platform.platform_name,
+                "custom_component": "custom_components" in type(self).__module__,
+            }
 
             if self.platform.config_entry:
                 info["source"] = SOURCE_CONFIG_ENTRY

tests/components/sensor/test_recorder.py

@@ -8,6 +8,7 @@ from unittest.mock import patch
 import pytest
 from pytest import approx
 
+from homeassistant import loader
 from homeassistant.components.recorder import history
 from homeassistant.components.recorder.const import DATA_INSTANCE
 from homeassistant.components.recorder.models import process_timestamp_to_utc_isoformat
@@ -609,6 +610,114 @@ def test_compile_hourly_sum_statistics_nan_inf_state(
     assert "Error while processing event StatisticsTask" not in caplog.text
 
 
+@pytest.mark.parametrize(
+    "entity_id,warning_1,warning_2",
+    [
+        (
+            "sensor.test1",
+            "",
+            "bug report at https://github.com/home-assistant/core/issues?q=is%3Aopen+is%3Aissue",
+        ),
+        (
+            "sensor.today_energy",
+            "from integration demo ",
+            "bug report at https://github.com/home-assistant/core/issues?q=is%3Aopen+is%3Aissue+label%3A%22integration%3A+demo%22",
+        ),
+        (
+            "sensor.custom_sensor",
+            "from integration test ",
+            "report it to the custom component author",
+        ),
+    ],
+)
+@pytest.mark.parametrize("state_class", ["total_increasing"])
+@pytest.mark.parametrize(
+    "device_class,unit,native_unit,factor",
+    [
+        ("energy", "kWh", "kWh", 1),
+    ],
+)
+def test_compile_hourly_sum_statistics_negative_state(
+    hass_recorder,
+    caplog,
+    entity_id,
+    warning_1,
+    warning_2,
+    state_class,
+    device_class,
+    unit,
+    native_unit,
+    factor,
+):
+    """Test compiling hourly statistics with negative states."""
+    zero = dt_util.utcnow()
+    hass = hass_recorder()
+    hass.data.pop(loader.DATA_CUSTOM_COMPONENTS)
+    recorder = hass.data[DATA_INSTANCE]
+
+    platform = getattr(hass.components, "test.sensor")
+    platform.init(empty=True)
+    mocksensor = platform.MockSensor(name="custom_sensor")
+    mocksensor._attr_should_poll = False
+    platform.ENTITIES["custom_sensor"] = mocksensor
+
+    setup_component(
+        hass, "sensor", {"sensor": [{"platform": "demo"}, {"platform": "test"}]}
+    )
+    hass.block_till_done()
+    attributes = {
+        "device_class": device_class,
+        "state_class": state_class,
+        "unit_of_measurement": unit,
+    }
+    seq = [15, 16, 15, 16, 20, -20, 20, 10]
+
+    states = {entity_id: []}
+    if state := hass.states.get(entity_id):
+        states[entity_id].append(state)
+    one = zero
+    for i in range(len(seq)):
+        one = one + timedelta(seconds=5)
+        _states = record_meter_state(hass, one, entity_id, attributes, seq[i : i + 1])
+        states[entity_id].extend(_states[entity_id])
+
+    hist = history.get_significant_states(
+        hass,
+        zero - timedelta.resolution,
+        one + timedelta.resolution,
+        significant_changes_only=False,
+    )
+    assert dict(states)[entity_id] == dict(hist)[entity_id]
+
+    recorder.do_adhoc_statistics(start=zero)
+    wait_recording_done(hass)
+    statistic_ids = list_statistic_ids(hass)
+    assert {
+        "statistic_id": entity_id,
+        "unit_of_measurement": native_unit,
+    } in statistic_ids
+    stats = statistics_during_period(hass, zero, period="5minute")
+    assert stats[entity_id] == [
+        {
+            "statistic_id": entity_id,
+            "start": process_timestamp_to_utc_isoformat(zero),
+            "end": process_timestamp_to_utc_isoformat(zero + timedelta(minutes=5)),
+            "max": None,
+            "mean": None,
+            "min": None,
+            "last_reset": None,
+            "state": approx(factor * seq[7]),
+            "sum": approx(factor * 15),  # (15 - 10) + (10 - 0)
+        },
+    ]
+    assert "Error while processing event StatisticsTask" not in caplog.text
+    assert (
+        f"Entity {entity_id} {warning_1}has state class total_increasing, but its state is negative"
+        in caplog.text
+    )
+    assert warning_2 in caplog.text
+
+
 @pytest.mark.parametrize(
     "device_class,unit,native_unit,factor",
     [
@@ -823,16 +932,14 @@ def test_compile_hourly_sum_statistics_total_increasing_small_dip(
     assert (
         "Entity sensor.test1 has state class total_increasing, but its state is not "
         "strictly increasing. Please create a bug report at https://github.com/"
-        "home-assistant/core/issues?q=is%3Aopen+is%3Aissue+label%3A%22integration%3A"
+        "home-assistant/core/issues?q=is%3Aopen+is%3Aissue"
-        "+recorder%22"
     ) not in caplog.text
     recorder.do_adhoc_statistics(start=period2)
     wait_recording_done(hass)
     assert (
         "Entity sensor.test1 has state class total_increasing, but its state is not "
         "strictly increasing. Please create a bug report at https://github.com/"
-        "home-assistant/core/issues?q=is%3Aopen+is%3Aissue+label%3A%22integration%3A"
+        "home-assistant/core/issues?q=is%3Aopen+is%3Aissue"
-        "+recorder%22"
     ) in caplog.text
     statistic_ids = list_statistic_ids(hass)
     assert statistic_ids == [

tests/components/websocket_api/test_commands.py

@@ -922,12 +922,14 @@ async def test_entity_source_admin(hass, websocket_client, hass_admin_user):
     assert msg["success"]
     assert msg["result"] == {
         "test_domain.entity_1": {
-            "source": entity.SOURCE_PLATFORM_CONFIG,
+            "custom_component": False,
             "domain": "test_platform",
+            "source": entity.SOURCE_PLATFORM_CONFIG,
         },
         "test_domain.entity_2": {
-            "source": entity.SOURCE_PLATFORM_CONFIG,
+            "custom_component": False,
             "domain": "test_platform",
+            "source": entity.SOURCE_PLATFORM_CONFIG,
         },
     }
 
@@ -942,8 +944,9 @@ async def test_entity_source_admin(hass, websocket_client, hass_admin_user):
     assert msg["success"]
     assert msg["result"] == {
         "test_domain.entity_2": {
-            "source": entity.SOURCE_PLATFORM_CONFIG,
+            "custom_component": False,
             "domain": "test_platform",
+            "source": entity.SOURCE_PLATFORM_CONFIG,
         },
     }
 
@@ -962,12 +965,14 @@ async def test_entity_source_admin(hass, websocket_client, hass_admin_user):
     assert msg["success"]
     assert msg["result"] == {
         "test_domain.entity_1": {
-            "source": entity.SOURCE_PLATFORM_CONFIG,
+            "custom_component": False,
             "domain": "test_platform",
+            "source": entity.SOURCE_PLATFORM_CONFIG,
         },
         "test_domain.entity_2": {
-            "source": entity.SOURCE_PLATFORM_CONFIG,
+            "custom_component": False,
             "domain": "test_platform",
+            "source": entity.SOURCE_PLATFORM_CONFIG,
         },
     }
 
@@ -1001,8 +1006,9 @@ async def test_entity_source_admin(hass, websocket_client, hass_admin_user):
     assert msg["success"]
     assert msg["result"] == {
         "test_domain.entity_2": {
-            "source": entity.SOURCE_PLATFORM_CONFIG,
+            "custom_component": False,
             "domain": "test_platform",
+            "source": entity.SOURCE_PLATFORM_CONFIG,
         },
     }
 

tests/helpers/test_entity.py

@@ -707,13 +707,15 @@ async def test_setup_source(hass):
 
     assert entity.entity_sources(hass) == {
         "test_domain.platform_config_source": {
-            "source": entity.SOURCE_PLATFORM_CONFIG,
+            "custom_component": False,
             "domain": "test_platform",
+            "source": entity.SOURCE_PLATFORM_CONFIG,
         },
         "test_domain.config_entry_source": {
-            "source": entity.SOURCE_CONFIG_ENTRY,
             "config_entry": platform.config_entry.entry_id,
+            "custom_component": False,
             "domain": "test_platform",
+            "source": entity.SOURCE_CONFIG_ENTRY,
         },
     }