diff --git a/homeassistant/components/cloud/google_config.py b/homeassistant/components/cloud/google_config.py
index 42f25f43ae1..10601bf4784 100644
--- a/homeassistant/components/cloud/google_config.py
+++ b/homeassistant/components/cloud/google_config.py
@@ -23,7 +23,6 @@ from homeassistant.components.homeassistant.exposed_entities import (
 from homeassistant.components.sensor import SensorDeviceClass
 from homeassistant.const import CLOUD_NEVER_EXPOSED_ENTITIES
 from homeassistant.core import (
-    CALLBACK_TYPE,
     CoreState,
     Event,
     HomeAssistant,
@@ -145,7 +144,6 @@ class CloudGoogleConfig(AbstractConfig):
         self._prefs = prefs
         self._cloud = cloud
         self._sync_entities_lock = asyncio.Lock()
-        self._on_deinitialize: list[CALLBACK_TYPE] = []
 
     @property
     def enabled(self) -> bool:
@@ -283,13 +281,6 @@ class CloudGoogleConfig(AbstractConfig):
             )
         )
 
-    @callback
-    def async_deinitialize(self) -> None:
-        """Remove listeners."""
-        _LOGGER.debug("async_deinitialize")
-        while self._on_deinitialize:
-            self._on_deinitialize.pop()()
-
     def should_expose(self, state: State) -> bool:
         """If a state object should be exposed."""
         return self._should_expose_entity_id(state.entity_id)
diff --git a/homeassistant/components/google_assistant/helpers.py b/homeassistant/components/google_assistant/helpers.py
index d75ebb49509..84cbdb6211e 100644
--- a/homeassistant/components/google_assistant/helpers.py
+++ b/homeassistant/components/google_assistant/helpers.py
@@ -105,6 +105,7 @@ class AbstractConfig(ABC):
         self._local_last_active: datetime | None = None
         self._local_sdk_version_warn = False
         self.is_supported_cache: dict[str, tuple[int | None, bool]] = {}
+        self._on_deinitialize: list[CALLBACK_TYPE] = []
 
     async def async_initialize(self) -> None:
         """Perform async initialization of config."""
@@ -118,7 +119,14 @@ class AbstractConfig(ABC):
             """Sync entities to Google."""
             await self.async_sync_entities_all()
 
-        start.async_at_start(self.hass, sync_google)
+        self._on_deinitialize.append(start.async_at_start(self.hass, sync_google))
+
+    @callback
+    def async_deinitialize(self) -> None:
+        """Remove listeners."""
+        _LOGGER.debug("async_deinitialize")
+        while self._on_deinitialize:
+            self._on_deinitialize.pop()()
 
     @property
     def enabled(self):