diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py index 8a1de979641..d68694800cd 100644 --- a/homeassistant/components/recorder/db_schema.py +++ b/homeassistant/components/recorder/db_schema.py @@ -25,6 +25,7 @@ from sqlalchemy import ( SmallInteger, String, Text, + case, type_coerce, ) from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite @@ -821,3 +822,11 @@ ENTITY_ID_IN_EVENT: ColumnElement = EVENT_DATA_JSON["entity_id"] OLD_ENTITY_ID_IN_EVENT: ColumnElement = OLD_FORMAT_EVENT_DATA_JSON["entity_id"] DEVICE_ID_IN_EVENT: ColumnElement = EVENT_DATA_JSON["device_id"] OLD_STATE = aliased(States, name="old_state") + +SHARED_ATTR_OR_LEGACY_ATTRIBUTES = case( + (StateAttributes.shared_attrs.is_(None), States.attributes), + else_=StateAttributes.shared_attrs, +).label("attributes") +SHARED_DATA_OR_LEGACY_EVENT_DATA = case( + (EventData.shared_data.is_(None), Events.event_data), else_=EventData.shared_data +).label("event_data") diff --git a/homeassistant/components/recorder/history/modern.py b/homeassistant/components/recorder/history/modern.py index b5ac7c571ac..5322074c205 100644 --- a/homeassistant/components/recorder/history/modern.py +++ b/homeassistant/components/recorder/history/modern.py @@ -28,7 +28,7 @@ from homeassistant.core import HomeAssistant, State, split_entity_id import homeassistant.util.dt as dt_util from ... import recorder -from ..db_schema import StateAttributes, States +from ..db_schema import SHARED_ATTR_OR_LEGACY_ATTRIBUTES, StateAttributes, States from ..filters import Filters from ..models import ( LazyState, @@ -70,7 +70,7 @@ def _stmt_and_join_attributes( if include_last_changed: _select = _select.add_columns(States.last_changed_ts) if not no_attributes: - _select = _select.add_columns(States.attributes, StateAttributes.shared_attrs) + _select = _select.add_columns(SHARED_ATTR_OR_LEGACY_ATTRIBUTES) return _select @@ -87,7 +87,7 @@ def _stmt_and_join_attributes_for_start_state( literal(value=None).label("last_changed_ts").cast(CASTABLE_DOUBLE_TYPE) ) if not no_attributes: - _select = _select.add_columns(States.attributes, StateAttributes.shared_attrs) + _select = _select.add_columns(SHARED_ATTR_OR_LEGACY_ATTRIBUTES) return _select @@ -104,7 +104,7 @@ def _select_from_subquery( base_select = base_select.add_columns(subquery.c.last_changed_ts) if no_attributes: return base_select - return base_select.add_columns(subquery.c.attributes, subquery.c.shared_attrs) + return base_select.add_columns(subquery.c.attributes) def get_significant_states( diff --git a/homeassistant/components/recorder/models/legacy.py b/homeassistant/components/recorder/models/legacy.py index 8a093472afb..b29b9250302 100644 --- a/homeassistant/components/recorder/models/legacy.py +++ b/homeassistant/components/recorder/models/legacy.py @@ -15,7 +15,7 @@ from homeassistant.const import ( from homeassistant.core import Context, State import homeassistant.util.dt as dt_util -from .state_attributes import decode_attributes_from_row +from .state_attributes import decode_attributes_from_source from .time import ( process_datetime_to_timestamp, process_timestamp, @@ -57,7 +57,9 @@ class LegacyLazyStatePreSchema31(State): def attributes(self) -> dict[str, Any]: """State attributes.""" if self._attributes is None: - self._attributes = decode_attributes_from_row(self._row, self.attr_cache) + self._attributes = decode_attributes_from_row_legacy( + self._row, self.attr_cache + ) return self._attributes @attributes.setter @@ -147,7 +149,7 @@ def legacy_row_to_compressed_state_pre_schema_31( """Convert a database row to a compressed state before schema 31.""" comp_state = { COMPRESSED_STATE_STATE: row.state, - COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache), + COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row_legacy(row, attr_cache), } if start_time: comp_state[COMPRESSED_STATE_LAST_UPDATED] = start_time.timestamp() @@ -202,7 +204,9 @@ class LegacyLazyState(State): def attributes(self) -> dict[str, Any]: """State attributes.""" if self._attributes is None: - self._attributes = decode_attributes_from_row(self._row, self.attr_cache) + self._attributes = decode_attributes_from_row_legacy( + self._row, self.attr_cache + ) return self._attributes @attributes.setter @@ -273,7 +277,7 @@ def legacy_row_to_compressed_state( """Convert a database row to a compressed state schema 31 and later.""" comp_state = { COMPRESSED_STATE_STATE: row.state, - COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache), + COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row_legacy(row, attr_cache), } if start_time: comp_state[COMPRESSED_STATE_LAST_UPDATED] = dt_util.utc_to_timestamp(start_time) @@ -285,3 +289,13 @@ def legacy_row_to_compressed_state( ) and row_last_updated_ts != row_last_changed_ts: comp_state[COMPRESSED_STATE_LAST_CHANGED] = row_last_changed_ts return comp_state + + +def decode_attributes_from_row_legacy( + row: Row, attr_cache: dict[str, dict[str, Any]] +) -> dict[str, Any]: + """Decode attributes from a database row.""" + return decode_attributes_from_source( + getattr(row, "shared_attrs", None) or getattr(row, "attributes", None), + attr_cache, + ) diff --git a/homeassistant/components/recorder/models/state.py b/homeassistant/components/recorder/models/state.py index 9d0d24c43fe..84ae6ae4462 100644 --- a/homeassistant/components/recorder/models/state.py +++ b/homeassistant/components/recorder/models/state.py @@ -16,7 +16,7 @@ from homeassistant.const import ( from homeassistant.core import Context, State import homeassistant.util.dt as dt_util -from .state_attributes import decode_attributes_from_row +from .state_attributes import decode_attributes_from_source from .time import process_timestamp # pylint: disable=invalid-name @@ -70,7 +70,9 @@ class LazyState(State): def attributes(self) -> dict[str, Any]: """State attributes.""" if self._attributes is None: - self._attributes = decode_attributes_from_row(self._row, self.attr_cache) + self._attributes = decode_attributes_from_source( + getattr(self._row, "attributes", None), self.attr_cache + ) return self._attributes @attributes.setter @@ -144,10 +146,12 @@ def row_to_compressed_state( state: str, last_updated_ts: float | None, ) -> dict[str, Any]: - """Convert a database row to a compressed state schema 31 and later.""" + """Convert a database row to a compressed state schema 41 and later.""" comp_state: dict[str, Any] = { COMPRESSED_STATE_STATE: state, - COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache), + COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_source( + getattr(row, "attributes", None), attr_cache + ), } row_last_updated_ts: float = last_updated_ts or start_time_ts # type: ignore[assignment] comp_state[COMPRESSED_STATE_LAST_UPDATED] = row_last_updated_ts diff --git a/homeassistant/components/recorder/models/state_attributes.py b/homeassistant/components/recorder/models/state_attributes.py index 3ed109afa07..c9cc110e1e0 100644 --- a/homeassistant/components/recorder/models/state_attributes.py +++ b/homeassistant/components/recorder/models/state_attributes.py @@ -5,21 +5,16 @@ from __future__ import annotations import logging from typing import Any -from sqlalchemy.engine.row import Row - from homeassistant.util.json import json_loads_object EMPTY_JSON_OBJECT = "{}" _LOGGER = logging.getLogger(__name__) -def decode_attributes_from_row( - row: Row, attr_cache: dict[str, dict[str, Any]] +def decode_attributes_from_source( + source: Any, attr_cache: dict[str, dict[str, Any]] ) -> dict[str, Any]: - """Decode attributes from a database row.""" - source: str | None = getattr(row, "shared_attrs", None) or getattr( - row, "attributes", None - ) + """Decode attributes from a row source.""" if not source or source == EMPTY_JSON_OBJECT: return {} if (attributes := attr_cache.get(source)) is not None: diff --git a/tests/components/recorder/test_history.py b/tests/components/recorder/test_history.py index 209bbf06361..b9c44f486b3 100644 --- a/tests/components/recorder/test_history.py +++ b/tests/components/recorder/test_history.py @@ -23,8 +23,11 @@ from homeassistant.components.recorder.db_schema import ( ) from homeassistant.components.recorder.filters import Filters from homeassistant.components.recorder.history import legacy -from homeassistant.components.recorder.models import LazyState, process_timestamp -from homeassistant.components.recorder.models.legacy import LegacyLazyStatePreSchema31 +from homeassistant.components.recorder.models import process_timestamp +from homeassistant.components.recorder.models.legacy import ( + LegacyLazyState, + LegacyLazyStatePreSchema31, +) from homeassistant.components.recorder.util import session_scope import homeassistant.core as ha from homeassistant.core import HomeAssistant, State @@ -60,13 +63,11 @@ async def _async_get_states( return [ LegacyLazyStatePreSchema31(row, attr_cache, None) if pre_31_schema - else LazyState( + else LegacyLazyState( row, attr_cache, None, row.entity_id, - row.state, - getattr(row, "last_updated_ts", None), ) for row in legacy._get_rows_with_session( hass, @@ -903,6 +904,7 @@ async def test_state_changes_during_period_query_during_migration_to_schema_25( conn.commit() with patch.object(instance, "schema_version", 24): + instance.states_meta_manager.active = False no_attributes = True hist = history.state_changes_during_period( hass, @@ -944,9 +946,8 @@ async def test_get_states_query_during_migration_to_schema_25( point = start + timedelta(seconds=1) end = point + timedelta(seconds=1) entity_id = "light.test" - await recorder.get_instance(hass).async_add_executor_job( - _add_db_entries, hass, point, [entity_id] - ) + await instance.async_add_executor_job(_add_db_entries, hass, point, [entity_id]) + assert instance.states_meta_manager.active no_attributes = True hist = await _async_get_states(hass, end, [entity_id], no_attributes=no_attributes) @@ -964,6 +965,7 @@ async def test_get_states_query_during_migration_to_schema_25( conn.commit() with patch.object(instance, "schema_version", 24): + instance.states_meta_manager.active = False no_attributes = True hist = await _async_get_states( hass, end, [entity_id], no_attributes=no_attributes @@ -998,9 +1000,8 @@ async def test_get_states_query_during_migration_to_schema_25_multiple_entities( entity_id_2 = "switch.test" entity_ids = [entity_id_1, entity_id_2] - await recorder.get_instance(hass).async_add_executor_job( - _add_db_entries, hass, point, entity_ids - ) + await instance.async_add_executor_job(_add_db_entries, hass, point, entity_ids) + assert instance.states_meta_manager.active no_attributes = True hist = await _async_get_states(hass, end, entity_ids, no_attributes=no_attributes) @@ -1018,6 +1019,7 @@ async def test_get_states_query_during_migration_to_schema_25_multiple_entities( conn.commit() with patch.object(instance, "schema_version", 24): + instance.states_meta_manager.active = False no_attributes = True hist = await _async_get_states( hass, end, entity_ids, no_attributes=no_attributes diff --git a/tests/components/recorder/test_models.py b/tests/components/recorder/test_models.py index 9dc2d53125f..f47f1d3e78b 100644 --- a/tests/components/recorder/test_models.py +++ b/tests/components/recorder/test_models.py @@ -273,14 +273,13 @@ async def test_lazy_state_handles_include_json( assert "Error converting row to state attributes" in caplog.text -async def test_lazy_state_prefers_shared_attrs_over_attrs( +async def test_lazy_state_can_decode_attributes( caplog: pytest.LogCaptureFixture, ) -> None: - """Test that the LazyState prefers shared_attrs over attributes.""" + """Test that the LazyState prefers can decode attributes.""" row = PropertyMock( entity_id="sensor.invalid", - shared_attrs='{"shared":true}', - attributes='{"shared":false}', + attributes='{"shared":true}', ) assert LazyState(row, {}, None, row.entity_id, "", 1).attributes == {"shared": True} @@ -293,7 +292,7 @@ async def test_lazy_state_handles_different_last_updated_and_last_changed( row = PropertyMock( entity_id="sensor.valid", state="off", - shared_attrs='{"shared":true}', + attributes='{"shared":true}', last_updated_ts=now.timestamp(), last_changed_ts=(now - timedelta(seconds=60)).timestamp(), ) @@ -324,7 +323,7 @@ async def test_lazy_state_handles_same_last_updated_and_last_changed( row = PropertyMock( entity_id="sensor.valid", state="off", - shared_attrs='{"shared":true}', + attributes='{"shared":true}', last_updated_ts=now.timestamp(), last_changed_ts=now.timestamp(), )