diff --git a/homeassistant/components/esphome/manager.py b/homeassistant/components/esphome/manager.py
index fb3e0a1e79a..c7433080c84 100644
--- a/homeassistant/components/esphome/manager.py
+++ b/homeassistant/components/esphome/manager.py
@@ -343,7 +343,6 @@ class ESPHomeManager:
             ),
             "esphome.voice_assistant_udp_server.run_pipeline",
         )
-        self.entry_data.async_set_assist_pipeline_state(True)
 
         return port
 
diff --git a/homeassistant/components/esphome/voice_assistant.py b/homeassistant/components/esphome/voice_assistant.py
index a9397eda935..c501d756e54 100644
--- a/homeassistant/components/esphome/voice_assistant.py
+++ b/homeassistant/components/esphome/voice_assistant.py
@@ -71,6 +71,7 @@ class VoiceAssistantUDPServer(asyncio.DatagramProtocol):
         self.hass = hass
 
         assert entry_data.device_info is not None
+        self.entry_data = entry_data
         self.device_info = entry_data.device_info
 
         self.queue: asyncio.Queue[bytes] = asyncio.Queue()
@@ -158,7 +159,9 @@ class VoiceAssistantUDPServer(asyncio.DatagramProtocol):
 
         data_to_send = None
         error = False
-        if event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_END:
+        if event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_START:
+            self.entry_data.async_set_assist_pipeline_state(True)
+        elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_END:
             assert event.data is not None
             data_to_send = {"text": event.data["stt_output"]["text"]}
         elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_INTENT_END: