diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp
index 60489d7d78..c74b028c4b 100644
--- a/esphome/components/audio/audio_decoder.cpp
+++ b/esphome/components/audio/audio_decoder.cpp
@@ -171,7 +171,7 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) {
 
     bytes_available_before_processing = this->input_transfer_buffer_->available();
 
-    if ((this->potentially_failed_count_ > 10) && (bytes_read == 0)) {
+    if ((this->potentially_failed_count_ > 0) && (bytes_read == 0)) {
       // Failed to decode in last attempt and there is no new data
 
       if ((this->input_transfer_buffer_->free() == 0) && first_loop_iteration) {