diff --git a/homeassistant/helpers/device_registry.py b/homeassistant/helpers/device_registry.py
index 4cc7ad83c32..0e6a1367b2b 100644
--- a/homeassistant/helpers/device_registry.py
+++ b/homeassistant/helpers/device_registry.py
@@ -1191,6 +1191,7 @@ def async_setup_cleanup(hass: HomeAssistant, dev_reg: DeviceRegistry) -> None:
             entity_registry.EVENT_ENTITY_REGISTRY_UPDATED,
             _async_entity_registry_changed,
             event_filter=entity_registry_changed_filter,
+            run_immediately=True,
         )
         return
 
@@ -1200,10 +1201,13 @@ def async_setup_cleanup(hass: HomeAssistant, dev_reg: DeviceRegistry) -> None:
             entity_registry.EVENT_ENTITY_REGISTRY_UPDATED,
             _async_entity_registry_changed,
             event_filter=entity_registry_changed_filter,
+            run_immediately=True,
         )
         await debounced_cleanup.async_call()
 
-    hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STARTED, startup_clean)
+    hass.bus.async_listen_once(
+        EVENT_HOMEASSISTANT_STARTED, startup_clean, run_immediately=True
+    )
 
     @callback
     def _on_homeassistant_stop(event: Event) -> None:
diff --git a/homeassistant/helpers/entity_registry.py b/homeassistant/helpers/entity_registry.py
index 277a93ef4b4..9d0d8b8191c 100644
--- a/homeassistant/helpers/entity_registry.py
+++ b/homeassistant/helpers/entity_registry.py
@@ -1476,7 +1476,9 @@ def _async_setup_cleanup(hass: HomeAssistant, registry: EntityRegistry) -> None:
         """Cancel cleanup."""
         cancel()
 
-    hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, _on_homeassistant_stop)
+    hass.bus.async_listen_once(
+        EVENT_HOMEASSISTANT_STOP, _on_homeassistant_stop, run_immediately=True
+    )
 
 
 @callback
@@ -1502,6 +1504,7 @@ def _async_setup_entity_restore(hass: HomeAssistant, registry: EntityRegistry) -
         EVENT_ENTITY_REGISTRY_UPDATED,
         cleanup_restored_states,
         event_filter=cleanup_restored_states_filter,
+        run_immediately=True,
     )
 
     if hass.is_running:
diff --git a/tests/components/dlna_dmr/test_media_player.py b/tests/components/dlna_dmr/test_media_player.py
index 4eb4780add3..6b64ef6f347 100644
--- a/tests/components/dlna_dmr/test_media_player.py
+++ b/tests/components/dlna_dmr/test_media_player.py
@@ -266,9 +266,8 @@ async def test_setup_entry_no_options(
     domain_data_mock.async_release_event_notifier.assert_awaited_once()
     dmr_device_mock.async_unsubscribe_services.assert_awaited_once()
     assert dmr_device_mock.on_event is None
-    mock_state = hass.states.get(mock_entity_id)
-    assert mock_state is not None
-    assert mock_state.state == ha_const.STATE_UNAVAILABLE
+    # Entity should be removed by the cleanup
+    assert hass.states.get(mock_entity_id) is None
 
 
 @pytest.mark.parametrize(
@@ -345,9 +344,8 @@ async def test_setup_entry_with_options(
     domain_data_mock.async_release_event_notifier.assert_awaited_once()
     dmr_device_mock.async_unsubscribe_services.assert_awaited_once()
     assert dmr_device_mock.on_event is None
-    mock_state = hass.states.get(mock_entity_id)
-    assert mock_state is not None
-    assert mock_state.state == ha_const.STATE_UNAVAILABLE
+    # Entity should be removed by the cleanup
+    assert hass.states.get(mock_entity_id) is None
 
 
 async def test_setup_entry_mac_address(
@@ -1384,10 +1382,8 @@ async def test_unavailable_device(
     # Check event notifiers are not released
     domain_data_mock.async_release_event_notifier.assert_not_called()
 
-    # Confirm the entity is still unavailable
-    mock_state = hass.states.get(mock_entity_id)
-    assert mock_state is not None
-    assert mock_state.state == ha_const.STATE_UNAVAILABLE
+    # Entity should be removed by the cleanup
+    assert hass.states.get(mock_entity_id) is None
 
 
 @pytest.mark.parametrize(
@@ -1477,9 +1473,8 @@ async def test_become_available(
     domain_data_mock.async_release_event_notifier.assert_awaited_once()
     dmr_device_mock.async_unsubscribe_services.assert_awaited_once()
     assert dmr_device_mock.on_event is None
-    mock_state = hass.states.get(mock_entity_id)
-    assert mock_state is not None
-    assert mock_state.state == ha_const.STATE_UNAVAILABLE
+    # Entity should be removed by the cleanup
+    assert hass.states.get(mock_entity_id) is None
 
 
 @pytest.mark.parametrize(