diff --git a/homeassistant/components/conversation/chat_log.py b/homeassistant/components/conversation/chat_log.py
index 53e248d0a98..ad7a9d0ce9e 100644
--- a/homeassistant/components/conversation/chat_log.py
+++ b/homeassistant/components/conversation/chat_log.py
@@ -43,13 +43,6 @@ def async_get_chat_log(
     else:
         history = ChatLog(hass, session.conversation_id)
 
-        @callback
-        def do_cleanup() -> None:
-            """Handle cleanup."""
-            all_history.pop(session.conversation_id)
-
-        session.async_on_cleanup(do_cleanup)
-
     if user_input is not None:
         history.async_add_user_content(UserContent(content=user_input.text))
 
@@ -63,6 +56,15 @@ def async_get_chat_log(
         )
         return
 
+    if session.conversation_id not in all_history:
+
+        @callback
+        def do_cleanup() -> None:
+            """Handle cleanup."""
+            all_history.pop(session.conversation_id)
+
+        session.async_on_cleanup(do_cleanup)
+
     all_history[session.conversation_id] = history