diff --git a/homeassistant/components/voip/voip.py b/homeassistant/components/voip/voip.py
index 8b96941e00a..8ef03e0ddf0 100644
--- a/homeassistant/components/voip/voip.py
+++ b/homeassistant/components/voip/voip.py
@@ -285,13 +285,13 @@ class PipelineRtpDatagramProtocol(RtpDatagramProtocol):
             chunk = await self._audio_queue.get()
 
         while chunk:
+            chunk_buffer.append(chunk)
+
             segmenter.process(chunk)
             if segmenter.in_command:
+                # Buffer until command starts
                 return True
 
-            # Buffer until command starts
-            chunk_buffer.append(chunk)
-
             async with async_timeout.timeout(self.audio_timeout):
                 chunk = await self._audio_queue.get()