jeans/core
jeans/core
1
0
Fork 0
mirror of https://github.com/home-assistant/core.git synced 2025-07-25 22:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reduce number of columns when selecting attributes for history (#91717)

Browse Source
This commit is contained in:
J. Nick Koston 2023-04-22 07:21:08 -05:00 committed by GitHub
parent 6e628d2f06
commit 34b824a27b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 61 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
homeassistant/components/recorder/db_schema.py
View File

@ -25,6 +25,7 @@ from sqlalchemy import (
SmallInteger, SmallInteger,
String, String,
Text, Text,
case,
type_coerce, type_coerce,
) )
from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite
@ -821,3 +822,11 @@ ENTITY_ID_IN_EVENT: ColumnElement = EVENT_DATA_JSON["entity_id"]
OLD_ENTITY_ID_IN_EVENT: ColumnElement = OLD_FORMAT_EVENT_DATA_JSON["entity_id"] OLD_ENTITY_ID_IN_EVENT: ColumnElement = OLD_FORMAT_EVENT_DATA_JSON["entity_id"]
DEVICE_ID_IN_EVENT: ColumnElement = EVENT_DATA_JSON["device_id"] DEVICE_ID_IN_EVENT: ColumnElement = EVENT_DATA_JSON["device_id"]
OLD_STATE = aliased(States, name="old_state") OLD_STATE = aliased(States, name="old_state")
SHARED_ATTR_OR_LEGACY_ATTRIBUTES = case(
(StateAttributes.shared_attrs.is_(None), States.attributes),
else_=StateAttributes.shared_attrs,
).label("attributes")
SHARED_DATA_OR_LEGACY_EVENT_DATA = case(
(EventData.shared_data.is_(None), Events.event_data), else_=EventData.shared_data
).label("event_data")

8
homeassistant/components/recorder/history/modern.py
View File

@ -28,7 +28,7 @@ from homeassistant.core import HomeAssistant, State, split_entity_id
import homeassistant.util.dt as dt_util import homeassistant.util.dt as dt_util
from ... import recorder from ... import recorder
from ..db_schema import StateAttributes, States from ..db_schema import SHARED_ATTR_OR_LEGACY_ATTRIBUTES, StateAttributes, States
from ..filters import Filters from ..filters import Filters
from ..models import ( from ..models import (
LazyState, LazyState,
@ -70,7 +70,7 @@ def _stmt_and_join_attributes(
if include_last_changed: if include_last_changed:
_select = _select.add_columns(States.last_changed_ts) _select = _select.add_columns(States.last_changed_ts)
if not no_attributes: if not no_attributes:
_select = _select.add_columns(States.attributes, StateAttributes.shared_attrs) _select = _select.add_columns(SHARED_ATTR_OR_LEGACY_ATTRIBUTES)
return _select return _select
@ -87,7 +87,7 @@ def _stmt_and_join_attributes_for_start_state(
literal(value=None).label("last_changed_ts").cast(CASTABLE_DOUBLE_TYPE) literal(value=None).label("last_changed_ts").cast(CASTABLE_DOUBLE_TYPE)
) )
if not no_attributes: if not no_attributes:
_select = _select.add_columns(States.attributes, StateAttributes.shared_attrs) _select = _select.add_columns(SHARED_ATTR_OR_LEGACY_ATTRIBUTES)
return _select return _select
@ -104,7 +104,7 @@ def _select_from_subquery(
base_select = base_select.add_columns(subquery.c.last_changed_ts) base_select = base_select.add_columns(subquery.c.last_changed_ts)
if no_attributes: if no_attributes:
return base_select return base_select
return base_select.add_columns(subquery.c.attributes, subquery.c.shared_attrs) return base_select.add_columns(subquery.c.attributes)
def get_significant_states( def get_significant_states(

24
homeassistant/components/recorder/models/legacy.py
View File

@ -15,7 +15,7 @@ from homeassistant.const import (
from homeassistant.core import Context, State from homeassistant.core import Context, State
import homeassistant.util.dt as dt_util import homeassistant.util.dt as dt_util
from .state_attributes import decode_attributes_from_row from .state_attributes import decode_attributes_from_source
from .time import ( from .time import (
process_datetime_to_timestamp, process_datetime_to_timestamp,
process_timestamp, process_timestamp,
@ -57,7 +57,9 @@ class LegacyLazyStatePreSchema31(State):
def attributes(self) -> dict[str, Any]: def attributes(self) -> dict[str, Any]:
"""State attributes.""" """State attributes."""
if self._attributes is None: if self._attributes is None:
self._attributes = decode_attributes_from_row(self._row, self.attr_cache) self._attributes = decode_attributes_from_row_legacy(
self._row, self.attr_cache
)
return self._attributes return self._attributes
@attributes.setter @attributes.setter
@ -147,7 +149,7 @@ def legacy_row_to_compressed_state_pre_schema_31(
"""Convert a database row to a compressed state before schema 31.""" """Convert a database row to a compressed state before schema 31."""
comp_state = { comp_state = {
COMPRESSED_STATE_STATE: row.state, COMPRESSED_STATE_STATE: row.state,
COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache), COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row_legacy(row, attr_cache),
} }
if start_time: if start_time:
comp_state[COMPRESSED_STATE_LAST_UPDATED] = start_time.timestamp() comp_state[COMPRESSED_STATE_LAST_UPDATED] = start_time.timestamp()
@ -202,7 +204,9 @@ class LegacyLazyState(State):
def attributes(self) -> dict[str, Any]: def attributes(self) -> dict[str, Any]:
"""State attributes.""" """State attributes."""
if self._attributes is None: if self._attributes is None:
self._attributes = decode_attributes_from_row(self._row, self.attr_cache) self._attributes = decode_attributes_from_row_legacy(
self._row, self.attr_cache
)
return self._attributes return self._attributes
@attributes.setter @attributes.setter
@ -273,7 +277,7 @@ def legacy_row_to_compressed_state(
"""Convert a database row to a compressed state schema 31 and later.""" """Convert a database row to a compressed state schema 31 and later."""
comp_state = { comp_state = {
COMPRESSED_STATE_STATE: row.state, COMPRESSED_STATE_STATE: row.state,
COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache), COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row_legacy(row, attr_cache),
} }
if start_time: if start_time:
comp_state[COMPRESSED_STATE_LAST_UPDATED] = dt_util.utc_to_timestamp(start_time) comp_state[COMPRESSED_STATE_LAST_UPDATED] = dt_util.utc_to_timestamp(start_time)
@ -285,3 +289,13 @@ def legacy_row_to_compressed_state(
) and row_last_updated_ts != row_last_changed_ts: ) and row_last_updated_ts != row_last_changed_ts:
comp_state[COMPRESSED_STATE_LAST_CHANGED] = row_last_changed_ts comp_state[COMPRESSED_STATE_LAST_CHANGED] = row_last_changed_ts
return comp_state return comp_state
def decode_attributes_from_row_legacy(
row: Row, attr_cache: dict[str, dict[str, Any]]
) -> dict[str, Any]:
"""Decode attributes from a database row."""
return decode_attributes_from_source(
getattr(row, "shared_attrs", None) or getattr(row, "attributes", None),
attr_cache,
)

12
homeassistant/components/recorder/models/state.py
View File

@ -16,7 +16,7 @@ from homeassistant.const import (
from homeassistant.core import Context, State from homeassistant.core import Context, State
import homeassistant.util.dt as dt_util import homeassistant.util.dt as dt_util
from .state_attributes import decode_attributes_from_row from .state_attributes import decode_attributes_from_source
from .time import process_timestamp from .time import process_timestamp
# pylint: disable=invalid-name # pylint: disable=invalid-name
@ -70,7 +70,9 @@ class LazyState(State):
def attributes(self) -> dict[str, Any]: def attributes(self) -> dict[str, Any]:
"""State attributes.""" """State attributes."""
if self._attributes is None: if self._attributes is None:
self._attributes = decode_attributes_from_row(self._row, self.attr_cache) self._attributes = decode_attributes_from_source(
getattr(self._row, "attributes", None), self.attr_cache
)
return self._attributes return self._attributes
@attributes.setter @attributes.setter
@ -144,10 +146,12 @@ def row_to_compressed_state(
state: str, state: str,
last_updated_ts: float | None, last_updated_ts: float | None,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Convert a database row to a compressed state schema 31 and later.""" """Convert a database row to a compressed state schema 41 and later."""
comp_state: dict[str, Any] = { comp_state: dict[str, Any] = {
COMPRESSED_STATE_STATE: state, COMPRESSED_STATE_STATE: state,
COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache), COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_source(
getattr(row, "attributes", None), attr_cache
),
} }
row_last_updated_ts: float = last_updated_ts or start_time_ts # type: ignore[assignment] row_last_updated_ts: float = last_updated_ts or start_time_ts # type: ignore[assignment]
comp_state[COMPRESSED_STATE_LAST_UPDATED] = row_last_updated_ts comp_state[COMPRESSED_STATE_LAST_UPDATED] = row_last_updated_ts

11
homeassistant/components/recorder/models/state_attributes.py
View File

@ -5,21 +5,16 @@ from __future__ import annotations
import logging import logging
from typing import Any from typing import Any
from sqlalchemy.engine.row import Row
from homeassistant.util.json import json_loads_object from homeassistant.util.json import json_loads_object
EMPTY_JSON_OBJECT = "{}" EMPTY_JSON_OBJECT = "{}"
_LOGGER = logging.getLogger(__name__) _LOGGER = logging.getLogger(__name__)
def decode_attributes_from_row( def decode_attributes_from_source(
row: Row, attr_cache: dict[str, dict[str, Any]] source: Any, attr_cache: dict[str, dict[str, Any]]
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Decode attributes from a database row.""" """Decode attributes from a row source."""
source: str | None = getattr(row, "shared_attrs", None) or getattr(
row, "attributes", None
)
if not source or source == EMPTY_JSON_OBJECT: if not source or source == EMPTY_JSON_OBJECT:
return {} return {}
if (attributes := attr_cache.get(source)) is not None: if (attributes := attr_cache.get(source)) is not None:

24
tests/components/recorder/test_history.py
View File

@ -23,8 +23,11 @@ from homeassistant.components.recorder.db_schema import (
) )
from homeassistant.components.recorder.filters import Filters from homeassistant.components.recorder.filters import Filters
from homeassistant.components.recorder.history import legacy from homeassistant.components.recorder.history import legacy
from homeassistant.components.recorder.models import LazyState, process_timestamp from homeassistant.components.recorder.models import process_timestamp
from homeassistant.components.recorder.models.legacy import LegacyLazyStatePreSchema31 from homeassistant.components.recorder.models.legacy import (
LegacyLazyState,
LegacyLazyStatePreSchema31,
)
from homeassistant.components.recorder.util import session_scope from homeassistant.components.recorder.util import session_scope
import homeassistant.core as ha import homeassistant.core as ha
from homeassistant.core import HomeAssistant, State from homeassistant.core import HomeAssistant, State
@ -60,13 +63,11 @@ async def _async_get_states(
return [ return [
LegacyLazyStatePreSchema31(row, attr_cache, None) LegacyLazyStatePreSchema31(row, attr_cache, None)
if pre_31_schema if pre_31_schema
else LazyState( else LegacyLazyState(
row, row,
attr_cache, attr_cache,
None, None,
row.entity_id, row.entity_id,
row.state,
getattr(row, "last_updated_ts", None),
) )
for row in legacy._get_rows_with_session( for row in legacy._get_rows_with_session(
hass, hass,
@ -903,6 +904,7 @@ async def test_state_changes_during_period_query_during_migration_to_schema_25(
conn.commit() conn.commit()
with patch.object(instance, "schema_version", 24): with patch.object(instance, "schema_version", 24):
instance.states_meta_manager.active = False
no_attributes = True no_attributes = True
hist = history.state_changes_during_period( hist = history.state_changes_during_period(
hass, hass,
@ -944,9 +946,8 @@ async def test_get_states_query_during_migration_to_schema_25(
point = start + timedelta(seconds=1) point = start + timedelta(seconds=1)
end = point + timedelta(seconds=1) end = point + timedelta(seconds=1)
entity_id = "light.test" entity_id = "light.test"
await recorder.get_instance(hass).async_add_executor_job( await instance.async_add_executor_job(_add_db_entries, hass, point, [entity_id])
_add_db_entries, hass, point, [entity_id] assert instance.states_meta_manager.active
)
no_attributes = True no_attributes = True
hist = await _async_get_states(hass, end, [entity_id], no_attributes=no_attributes) hist = await _async_get_states(hass, end, [entity_id], no_attributes=no_attributes)
@ -964,6 +965,7 @@ async def test_get_states_query_during_migration_to_schema_25(
conn.commit() conn.commit()
with patch.object(instance, "schema_version", 24): with patch.object(instance, "schema_version", 24):
instance.states_meta_manager.active = False
no_attributes = True no_attributes = True
hist = await _async_get_states( hist = await _async_get_states(
hass, end, [entity_id], no_attributes=no_attributes hass, end, [entity_id], no_attributes=no_attributes
@ -998,9 +1000,8 @@ async def test_get_states_query_during_migration_to_schema_25_multiple_entities(
entity_id_2 = "switch.test" entity_id_2 = "switch.test"
entity_ids = [entity_id_1, entity_id_2] entity_ids = [entity_id_1, entity_id_2]
await recorder.get_instance(hass).async_add_executor_job( await instance.async_add_executor_job(_add_db_entries, hass, point, entity_ids)
_add_db_entries, hass, point, entity_ids assert instance.states_meta_manager.active
)
no_attributes = True no_attributes = True
hist = await _async_get_states(hass, end, entity_ids, no_attributes=no_attributes) hist = await _async_get_states(hass, end, entity_ids, no_attributes=no_attributes)
@ -1018,6 +1019,7 @@ async def test_get_states_query_during_migration_to_schema_25_multiple_entities(
conn.commit() conn.commit()
with patch.object(instance, "schema_version", 24): with patch.object(instance, "schema_version", 24):
instance.states_meta_manager.active = False
no_attributes = True no_attributes = True
hist = await _async_get_states( hist = await _async_get_states(
hass, end, entity_ids, no_attributes=no_attributes hass, end, entity_ids, no_attributes=no_attributes

11
tests/components/recorder/test_models.py
View File

@ -273,14 +273,13 @@ async def test_lazy_state_handles_include_json(
assert "Error converting row to state attributes" in caplog.text assert "Error converting row to state attributes" in caplog.text
async def test_lazy_state_prefers_shared_attrs_over_attrs( async def test_lazy_state_can_decode_attributes(
caplog: pytest.LogCaptureFixture, caplog: pytest.LogCaptureFixture,
) -> None: ) -> None:
"""Test that the LazyState prefers shared_attrs over attributes.""" """Test that the LazyState prefers can decode attributes."""
row = PropertyMock( row = PropertyMock(
entity_id="sensor.invalid", entity_id="sensor.invalid",
shared_attrs='{"shared":true}', attributes='{"shared":true}',
attributes='{"shared":false}',
) )
assert LazyState(row, {}, None, row.entity_id, "", 1).attributes == {"shared": True} assert LazyState(row, {}, None, row.entity_id, "", 1).attributes == {"shared": True}
@ -293,7 +292,7 @@ async def test_lazy_state_handles_different_last_updated_and_last_changed(
row = PropertyMock( row = PropertyMock(
entity_id="sensor.valid", entity_id="sensor.valid",
state="off", state="off",
shared_attrs='{"shared":true}', attributes='{"shared":true}',
last_updated_ts=now.timestamp(), last_updated_ts=now.timestamp(),
last_changed_ts=(now - timedelta(seconds=60)).timestamp(), last_changed_ts=(now - timedelta(seconds=60)).timestamp(),
) )
@ -324,7 +323,7 @@ async def test_lazy_state_handles_same_last_updated_and_last_changed(
row = PropertyMock( row = PropertyMock(
entity_id="sensor.valid", entity_id="sensor.valid",
state="off", state="off",
shared_attrs='{"shared":true}', attributes='{"shared":true}',
last_updated_ts=now.timestamp(), last_updated_ts=now.timestamp(),
last_changed_ts=now.timestamp(), last_changed_ts=now.timestamp(),
) )