Cache serialization of config entry storage (#127435)

homeassistant/config_entries.py

@@ -57,7 +57,7 @@ from .helpers.event import (
     async_call_later,
 )
 from .helpers.frame import report
-from .helpers.json import json_bytes, json_fragment
+from .helpers.json import json_bytes, json_bytes_sorted, json_fragment
 from .helpers.typing import UNDEFINED, ConfigType, DiscoveryInfoType, UndefinedType
 from .loader import async_suggest_report_issue
 from .setup import (
@@ -247,14 +247,13 @@ type UpdateListenerType = Callable[
     [HomeAssistant, ConfigEntry], Coroutine[Any, Any, None]
 ]
 
-FROZEN_CONFIG_ENTRY_ATTRS = {
+STATE_KEYS = {
-    "entry_id",
-    "domain",
     "state",
     "reason",
     "error_reason_translation_key",
     "error_reason_translation_placeholders",
 }
+FROZEN_CONFIG_ENTRY_ATTRS = {"entry_id", "domain", *STATE_KEYS}
 UPDATE_ENTRY_CONFIG_ENTRY_ATTRS = {
     "unique_id",
     "title",
@@ -447,7 +446,8 @@ class ConfigEntry(Generic[_DataT]):
             raise AttributeError(f"{key} cannot be changed")
 
         super().__setattr__(key, value)
-        self.clear_cache()
+        self.clear_state_cache()
+        self.clear_storage_cache()
 
     @property
     def supports_options(self) -> bool:
@@ -473,13 +473,13 @@ class ConfigEntry(Generic[_DataT]):
             )
         return self._supports_reconfigure or False
 
-    def clear_cache(self) -> None:
+    def clear_state_cache(self) -> None:
-        """Clear cached properties."""
+        """Clear cached properties that are included in as_json_fragment."""
         self.__dict__.pop("as_json_fragment", None)
 
     @cached_property
     def as_json_fragment(self) -> json_fragment:
-        """Return JSON fragment of a config entry."""
+        """Return JSON fragment of a config entry that is used for the API."""
         json_repr = {
             "created_at": self.created_at.timestamp(),
             "entry_id": self.entry_id,
@@ -501,6 +501,15 @@ class ConfigEntry(Generic[_DataT]):
         }
         return json_fragment(json_bytes(json_repr))
 
+    def clear_storage_cache(self) -> None:
+        """Clear cached properties that are included in as_storage_fragment."""
+        self.__dict__.pop("as_storage_fragment", None)
+
+    @cached_property
+    def as_storage_fragment(self) -> json_fragment:
+        """Return a storage fragment for this entry."""
+        return json_fragment(json_bytes_sorted(self.as_dict()))
+
     async def async_setup(
         self,
         hass: HomeAssistant,
@@ -833,7 +842,8 @@ class ConfigEntry(Generic[_DataT]):
         """Invoke remove callback on component."""
         old_modified_at = self.modified_at
         object.__setattr__(self, "modified_at", utcnow())
-        self.clear_cache()
+        self.clear_state_cache()
+        self.clear_storage_cache()
 
         if self.source == SOURCE_IGNORE:
             return
@@ -890,7 +900,10 @@ class ConfigEntry(Generic[_DataT]):
             "error_reason_translation_placeholders",
             error_reason_translation_placeholders,
         )
-        self.clear_cache()
+        self.clear_state_cache()
+        # Storage cache is not cleared here because the state is not stored
+        # in storage and we do not want to clear the cache on every state change
+        # since state changes are frequent.
         async_dispatcher_send_internal(
             hass, SIGNAL_CONFIG_ENTRY_CHANGED, ConfigEntryChange.UPDATED, self
         )
@@ -1663,7 +1676,8 @@ class ConfigEntryItems(UserDict[str, ConfigEntry]):
         self._unindex_entry(entry_id)
         object.__setattr__(entry, "unique_id", new_unique_id)
         self._index_entry(entry)
-        entry.clear_cache()
+        entry.clear_state_cache()
+        entry.clear_storage_cache()
 
     def get_entries_for_domain(self, domain: str) -> list[ConfigEntry]:
         """Get entries for a domain."""
@@ -2138,7 +2152,8 @@ class ConfigEntries:
             )
 
         self._async_schedule_save()
-        entry.clear_cache()
+        entry.clear_state_cache()
+        entry.clear_storage_cache()
         self._async_dispatch(ConfigEntryChange.UPDATED, entry)
         return True
 
@@ -2321,7 +2336,10 @@ class ConfigEntries:
     @callback
     def _data_to_save(self) -> dict[str, list[dict[str, Any]]]:
         """Return data to save."""
-        return {"entries": [entry.as_dict() for entry in self._entries.values()]}
+        # typing does not know that the storage fragment will serialize to a dict
+        return {
+            "entries": [entry.as_storage_fragment for entry in self._entries.values()]  # type: ignore[misc]
+        }
 
     async def async_wait_component(self, entry: ConfigEntry) -> bool:
         """Wait for an entry's component to load and return if the entry is loaded.

homeassistant/helpers/json.py

@@ -162,13 +162,17 @@ def json_dumps(data: Any) -> str:
     return json_bytes(data).decode("utf-8")
 
 
+json_bytes_sorted = partial(
+    orjson.dumps,
+    option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS,
+    default=json_encoder_default,
+)
+"""Dump json bytes with keys sorted."""
+
+
 def json_dumps_sorted(data: Any) -> str:
     """Dump json string with keys sorted."""
-    return orjson.dumps(
+    return json_bytes_sorted(data).decode("utf-8")
-        data,
-        option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS,
-        default=json_encoder_default,
-    ).decode("utf-8")
 
 
 JSON_DUMP: Final = json_dumps

tests/helpers/test_json.py

@@ -18,6 +18,7 @@ from homeassistant.helpers.json import (
     ExtendedJSONEncoder,
     JSONEncoder as DefaultHASSJSONEncoder,
     find_paths_unserializable_data,
+    json_bytes_sorted,
     json_bytes_strip_null,
     json_dumps,
     json_dumps_sorted,
@@ -107,6 +108,14 @@ def test_json_dumps_sorted() -> None:
     )
 
 
+def test_json_bytes_sorted() -> None:
+    """Test the json bytes sorted function."""
+    data = {"c": 3, "a": 1, "b": 2}
+    assert json_bytes_sorted(data) == json.dumps(
+        data, sort_keys=True, separators=(",", ":")
+    ).encode("utf-8")
+
+
 def test_json_dumps_float_subclass() -> None:
     """Test the json dumps a float subclass."""
 

tests/test_config_entries.py

@@ -40,11 +40,13 @@ from homeassistant.exceptions import (
 from homeassistant.helpers import entity_registry as er, issue_registry as ir
 from homeassistant.helpers.discovery_flow import DiscoveryKey
 from homeassistant.helpers.entity_platform import AddEntitiesCallback
+from homeassistant.helpers.json import json_dumps
 from homeassistant.helpers.typing import ConfigType
 from homeassistant.helpers.update_coordinator import DataUpdateCoordinator
 from homeassistant.setup import async_set_domains_to_be_loaded, async_setup_component
 from homeassistant.util.async_ import create_eager_task
 import homeassistant.util.dt as dt_util
+from homeassistant.util.json import json_loads
 
 from .common import (
     MockConfigEntry,
@@ -6590,3 +6592,48 @@ async def test_reauth_helper_alignment(
     # Ensure context and init data are aligned
     assert helper_flow_context == reauth_flow_context
     assert helper_flow_init_data == reauth_flow_init_data
+
+
+def test_state_not_stored_in_storage() -> None:
+    """Test that state is not stored in storage.
+
+    Verify we don't start accidentally storing state in storage.
+    """
+    entry = MockConfigEntry(domain="test")
+    loaded = json_loads(json_dumps(entry.as_storage_fragment))
+    for key in config_entries.STATE_KEYS:
+        assert key not in loaded
+
+
+def test_storage_cache_is_cleared_on_entry_update(hass: HomeAssistant) -> None:
+    """Test that the storage cache is cleared when an entry is updated."""
+    entry = MockConfigEntry(domain="test")
+    entry.add_to_hass(hass)
+    _ = entry.as_storage_fragment
+    hass.config_entries.async_update_entry(entry, data={"new": "data"})
+    loaded = json_loads(json_dumps(entry.as_storage_fragment))
+    assert "new" in loaded["data"]
+
+
+async def test_storage_cache_is_cleared_on_entry_disable(hass: HomeAssistant) -> None:
+    """Test that the storage cache is cleared when an entry is disabled."""
+    entry = MockConfigEntry(domain="test")
+    entry.add_to_hass(hass)
+    _ = entry.as_storage_fragment
+    await hass.config_entries.async_set_disabled_by(
+        entry.entry_id, config_entries.ConfigEntryDisabler.USER
+    )
+    loaded = json_loads(json_dumps(entry.as_storage_fragment))
+    assert loaded["disabled_by"] == "user"
+
+
+async def test_state_cache_is_cleared_on_entry_disable(hass: HomeAssistant) -> None:
+    """Test that the state cache is cleared when an entry is disabled."""
+    entry = MockConfigEntry(domain="test")
+    entry.add_to_hass(hass)
+    _ = entry.as_storage_fragment
+    await hass.config_entries.async_set_disabled_by(
+        entry.entry_id, config_entries.ConfigEntryDisabler.USER
+    )
+    loaded = json_loads(json_dumps(entry.as_json_fragment))
+    assert loaded["disabled_by"] == "user"