diff --git a/esphome/components/i2s_audio/__init__.py b/esphome/components/i2s_audio/__init__.py
index 291ae4ba95..0d413adb8a 100644
--- a/esphome/components/i2s_audio/__init__.py
+++ b/esphome/components/i2s_audio/__init__.py
@@ -39,6 +39,7 @@ CONF_SECONDARY = "secondary"
 
 CONF_USE_APLL = "use_apll"
 CONF_BITS_PER_CHANNEL = "bits_per_channel"
+CONF_MCLK_MULTIPLE = "mclk_multiple"
 CONF_MONO = "mono"
 CONF_LEFT = "left"
 CONF_RIGHT = "right"
@@ -122,8 +123,25 @@ I2S_SLOT_BIT_WIDTH = {
     32: i2s_slot_bit_width_t.I2S_SLOT_BIT_WIDTH_32BIT,
 }
 
+i2s_mclk_multiple_t = cg.global_ns.enum("i2s_mclk_multiple_t")
+I2S_MCLK_MULTIPLE = {
+    128: i2s_mclk_multiple_t.I2S_MCLK_MULTIPLE_128,
+    256: i2s_mclk_multiple_t.I2S_MCLK_MULTIPLE_256,
+    384: i2s_mclk_multiple_t.I2S_MCLK_MULTIPLE_384,
+    512: i2s_mclk_multiple_t.I2S_MCLK_MULTIPLE_512,
+}
+
 _validate_bits = cv.float_with_unit("bits", "bit")
 
+
+def validate_mclk_divisible_by_3(config):
+    if config[CONF_BITS_PER_SAMPLE] == 24 and config[CONF_MCLK_MULTIPLE] % 3 != 0:
+        raise cv.Invalid(
+            f"{CONF_MCLK_MULTIPLE} must be divisible by 3 when bits per sample is 24"
+        )
+    return config
+
+
 _use_legacy_driver = None
 
 
@@ -155,6 +173,7 @@ def i2s_audio_component_schema(
                 cv.Any(cv.float_with_unit("bits", "bit"), "default"),
                 cv.one_of(*I2S_BITS_PER_CHANNEL),
             ),
+            cv.Optional(CONF_MCLK_MULTIPLE, default=256): cv.one_of(*I2S_MCLK_MULTIPLE),
         }
     )
 
@@ -182,11 +201,10 @@ async def register_i2s_audio_component(var, config):
             slot_mask = CONF_BOTH
         cg.add(var.set_slot_mode(I2S_SLOT_MODE[slot_mode]))
         cg.add(var.set_std_slot_mask(I2S_STD_SLOT_MASK[slot_mask]))
-        cg.add(
-            var.set_slot_bit_width(I2S_SLOT_BIT_WIDTH[config[CONF_BITS_PER_CHANNEL]])
-        )
+        cg.add(var.set_slot_bit_width(I2S_SLOT_BIT_WIDTH[config[CONF_BITS_PER_SAMPLE]]))
     cg.add(var.set_sample_rate(config[CONF_SAMPLE_RATE]))
     cg.add(var.set_use_apll(config[CONF_USE_APLL]))
+    cg.add(var.set_mclk_multiple(I2S_MCLK_MULTIPLE[config[CONF_MCLK_MULTIPLE]]))
 
 
 def validate_use_legacy(value):
diff --git a/esphome/components/i2s_audio/i2s_audio.h b/esphome/components/i2s_audio/i2s_audio.h
index d8050665e9..e839bcd891 100644
--- a/esphome/components/i2s_audio/i2s_audio.h
+++ b/esphome/components/i2s_audio/i2s_audio.h
@@ -31,6 +31,7 @@ class I2SAudioBase : public Parented<I2SAudioComponent> {
 #endif
   void set_sample_rate(uint32_t sample_rate) { this->sample_rate_ = sample_rate; }
   void set_use_apll(uint32_t use_apll) { this->use_apll_ = use_apll; }
+  void set_mclk_multiple(i2s_mclk_multiple_t mclk_multiple) { this->mclk_multiple_ = mclk_multiple; }
 
  protected:
 #ifdef USE_I2S_LEGACY
@@ -46,6 +47,7 @@ class I2SAudioBase : public Parented<I2SAudioComponent> {
 #endif
   uint32_t sample_rate_;
   bool use_apll_;
+  i2s_mclk_multiple_t mclk_multiple_;
 };
 
 class I2SAudioIn : public I2SAudioBase {};
diff --git a/esphome/components/i2s_audio/microphone/__init__.py b/esphome/components/i2s_audio/microphone/__init__.py
index 06eb29986d..1fb4e9df99 100644
--- a/esphome/components/i2s_audio/microphone/__init__.py
+++ b/esphome/components/i2s_audio/microphone/__init__.py
@@ -22,6 +22,7 @@ from .. import (
     i2s_audio_ns,
     register_i2s_audio_component,
     use_legacy,
+    validate_mclk_divisible_by_3,
 )
 
 CODEOWNERS = ["@jesserockz"]
@@ -112,6 +113,7 @@ CONFIG_SCHEMA = cv.All(
     _validate_channel,
     _set_num_channels_from_config,
     _set_stream_limits,
+    validate_mclk_divisible_by_3,
 )
 
 
diff --git a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp
index 78a7f92c2f..72d1e4476c 100644
--- a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp
+++ b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp
@@ -15,10 +15,25 @@
 namespace esphome {
 namespace i2s_audio {
 
-static const size_t BUFFER_SIZE = 512;
+static const UBaseType_t MAX_LISTENERS = 16;
+
+static const uint32_t READ_DURATION_MS = 16;
+
+static const size_t TASK_STACK_SIZE = 4096;
+static const ssize_t TASK_PRIORITY = 23;
 
 static const char *const TAG = "i2s_audio.microphone";
 
+enum MicrophoneEventGroupBits : uint32_t {
+  COMMAND_STOP = (1 << 0),  // stops the microphone task
+  TASK_STARTING = (1 << 10),
+  TASK_RUNNING = (1 << 11),
+  TASK_STOPPING = (1 << 12),
+  TASK_STOPPED = (1 << 13),
+
+  ALL_BITS = 0x00FFFFFF,  // All valid FreeRTOS event group bits
+};
+
 void I2SAudioMicrophone::setup() {
   ESP_LOGCONFIG(TAG, "Setting up I2S Audio Microphone...");
 #ifdef USE_I2S_LEGACY
@@ -41,18 +56,32 @@ void I2SAudioMicrophone::setup() {
       }
     }
   }
+
+  this->active_listeners_semaphore_ = xSemaphoreCreateCounting(MAX_LISTENERS, MAX_LISTENERS);
+  if (this->active_listeners_semaphore_ == nullptr) {
+    ESP_LOGE(TAG, "Failed to create semaphore");
+    this->mark_failed();
+    return;
+  }
+
+  this->event_group_ = xEventGroupCreate();
+  if (this->event_group_ == nullptr) {
+    ESP_LOGE(TAG, "Failed to create event group");
+    this->mark_failed();
+    return;
+  }
 }
 
 void I2SAudioMicrophone::start() {
   if (this->is_failed())
     return;
-  if (this->state_ == microphone::STATE_RUNNING)
-    return;  // Already running
-  this->state_ = microphone::STATE_STARTING;
+
+  xSemaphoreTake(this->active_listeners_semaphore_, 0);
 }
-void I2SAudioMicrophone::start_() {
+
+bool I2SAudioMicrophone::start_driver_() {
   if (!this->parent_->try_lock()) {
-    return;  // Waiting for another i2s to return lock
+    return false;  // Waiting for another i2s to return lock
   }
   esp_err_t err;
 
@@ -94,11 +123,11 @@ void I2SAudioMicrophone::start_() {
       .communication_format = I2S_COMM_FORMAT_STAND_I2S,
       .intr_alloc_flags = ESP_INTR_FLAG_LEVEL1,
       .dma_buf_count = 4,
-      .dma_buf_len = 256,
+      .dma_buf_len = 240,  // Must be divisible by 3 to support 24 bits per sample on old driver and newer variants
       .use_apll = this->use_apll_,
       .tx_desc_auto_clear = false,
       .fixed_mclk = 0,
-      .mclk_multiple = I2S_MCLK_MULTIPLE_256,
+      .mclk_multiple = this->mclk_multiple_,
       .bits_per_chan = this->bits_per_channel_,
   };
 
@@ -109,20 +138,20 @@ void I2SAudioMicrophone::start_() {
     if (err != ESP_OK) {
       ESP_LOGW(TAG, "Error installing I2S driver: %s", esp_err_to_name(err));
       this->status_set_error();
-      return;
+      return false;
     }
 
     err = i2s_set_adc_mode(ADC_UNIT_1, this->adc_channel_);
     if (err != ESP_OK) {
       ESP_LOGW(TAG, "Error setting ADC mode: %s", esp_err_to_name(err));
       this->status_set_error();
-      return;
+      return false;
     }
     err = i2s_adc_enable(this->parent_->get_port());
     if (err != ESP_OK) {
       ESP_LOGW(TAG, "Error enabling ADC: %s", esp_err_to_name(err));
       this->status_set_error();
-      return;
+      return false;
     }
 
   } else
@@ -135,7 +164,7 @@ void I2SAudioMicrophone::start_() {
     if (err != ESP_OK) {
       ESP_LOGW(TAG, "Error installing I2S driver: %s", esp_err_to_name(err));
       this->status_set_error();
-      return;
+      return false;
     }
 
     i2s_pin_config_t pin_config = this->parent_->get_pin_config();
@@ -145,7 +174,7 @@ void I2SAudioMicrophone::start_() {
     if (err != ESP_OK) {
       ESP_LOGW(TAG, "Error setting I2S pin: %s", esp_err_to_name(err));
       this->status_set_error();
-      return;
+      return false;
     }
   }
 #else
@@ -161,7 +190,7 @@ void I2SAudioMicrophone::start_() {
   if (err != ESP_OK) {
     ESP_LOGW(TAG, "Error creating new I2S channel: %s", esp_err_to_name(err));
     this->status_set_error();
-    return;
+    return false;
   }
 
   i2s_clock_src_t clk_src = I2S_CLK_SRC_DEFAULT;
@@ -178,7 +207,7 @@ void I2SAudioMicrophone::start_() {
     i2s_pdm_rx_clk_config_t clk_cfg = {
         .sample_rate_hz = this->sample_rate_,
         .clk_src = clk_src,
-        .mclk_multiple = I2S_MCLK_MULTIPLE_256,
+        .mclk_multiple = this->mclk_multiple_,
         .dn_sample_mode = I2S_PDM_DSR_8S,
     };
 
@@ -216,7 +245,7 @@ void I2SAudioMicrophone::start_() {
     i2s_std_clk_config_t clk_cfg = {
         .sample_rate_hz = this->sample_rate_,
         .clk_src = clk_src,
-        .mclk_multiple = I2S_MCLK_MULTIPLE_256,
+        .mclk_multiple = this->mclk_multiple_,
     };
     i2s_std_slot_config_t std_slot_cfg =
         I2S_STD_PHILIPS_SLOT_DEFAULT_CONFIG((i2s_data_bit_width_t) this->slot_bit_width_, this->slot_mode_);
@@ -236,7 +265,7 @@ void I2SAudioMicrophone::start_() {
   if (err != ESP_OK) {
     ESP_LOGW(TAG, "Error initializing I2S channel: %s", esp_err_to_name(err));
     this->status_set_error();
-    return;
+    return false;
   }
 
   /* Before reading data, start the RX channel first */
@@ -244,28 +273,25 @@ void I2SAudioMicrophone::start_() {
   if (err != ESP_OK) {
     ESP_LOGW(TAG, "Error enabling I2S Microphone: %s", esp_err_to_name(err));
     this->status_set_error();
-    return;
+    return false;
   }
 #endif
 
   this->audio_stream_info_ = audio::AudioStreamInfo(bits_per_sample, channel_count, this->sample_rate_);
 
-  this->state_ = microphone::STATE_RUNNING;
-  this->high_freq_.start();
   this->status_clear_error();
+
+  return true;
 }
 
 void I2SAudioMicrophone::stop() {
   if (this->state_ == microphone::STATE_STOPPED || this->is_failed())
     return;
-  if (this->state_ == microphone::STATE_STARTING) {
-    this->state_ = microphone::STATE_STOPPED;
-    return;
-  }
-  this->state_ = microphone::STATE_STOPPING;
+
+  xSemaphoreGive(this->active_listeners_semaphore_);
 }
 
-void I2SAudioMicrophone::stop_() {
+void I2SAudioMicrophone::stop_driver_() {
   esp_err_t err;
 #ifdef USE_I2S_LEGACY
 #if SOC_I2S_SUPPORTS_ADC
@@ -307,11 +333,51 @@ void I2SAudioMicrophone::stop_() {
   }
 #endif
   this->parent_->unlock();
-  this->state_ = microphone::STATE_STOPPED;
-  this->high_freq_.stop();
   this->status_clear_error();
 }
 
+void I2SAudioMicrophone::mic_task(void *params) {
+  I2SAudioMicrophone *this_microphone = (I2SAudioMicrophone *) params;
+
+  xEventGroupSetBits(this_microphone->event_group_, MicrophoneEventGroupBits::TASK_STARTING);
+
+  uint8_t start_counter = 0;
+  bool started = this_microphone->start_driver_();
+  while (!started && start_counter < 10) {
+    // Attempt to load the driver again in 100 ms. Doesn't slow down main loop since its in a task.
+    vTaskDelay(pdMS_TO_TICKS(100));
+    ++start_counter;
+    started = this_microphone->start_driver_();
+  }
+
+  if (started) {
+    xEventGroupSetBits(this_microphone->event_group_, MicrophoneEventGroupBits::TASK_RUNNING);
+    const size_t bytes_to_read = this_microphone->audio_stream_info_.ms_to_bytes(READ_DURATION_MS);
+    std::vector<uint8_t> samples;
+    samples.reserve(bytes_to_read);
+
+    while (!(xEventGroupGetBits(this_microphone->event_group_) & COMMAND_STOP)) {
+      if (this_microphone->data_callbacks_.size() > 0) {
+        samples.resize(bytes_to_read);
+        size_t bytes_read = this_microphone->read_(samples.data(), bytes_to_read, 2 * pdMS_TO_TICKS(READ_DURATION_MS));
+        samples.resize(bytes_read);
+        this_microphone->data_callbacks_.call(samples);
+      } else {
+        delay(READ_DURATION_MS);
+      }
+    }
+  }
+
+  xEventGroupSetBits(this_microphone->event_group_, MicrophoneEventGroupBits::TASK_STOPPING);
+  this_microphone->stop_driver_();
+
+  xEventGroupSetBits(this_microphone->event_group_, MicrophoneEventGroupBits::TASK_STOPPED);
+  while (true) {
+    // Continuously delay until the loop method delete the task
+    delay(10);
+  }
+}
+
 size_t I2SAudioMicrophone::read_(uint8_t *buf, size_t len, TickType_t ticks_to_wait) {
   size_t bytes_read = 0;
 #ifdef USE_I2S_LEGACY
@@ -345,29 +411,60 @@ size_t I2SAudioMicrophone::read_(uint8_t *buf, size_t len, TickType_t ticks_to_w
   return bytes_read;
 }
 
-void I2SAudioMicrophone::read_() {
-  std::vector<uint8_t> samples;
-  const size_t bytes_to_read = this->audio_stream_info_.ms_to_bytes(32);
-  samples.resize(bytes_to_read);
-  size_t bytes_read = this->read_(samples.data(), bytes_to_read, 0);
-  samples.resize(bytes_read);
-  this->data_callbacks_.call(samples);
-}
-
 void I2SAudioMicrophone::loop() {
+  uint32_t event_group_bits = xEventGroupGetBits(this->event_group_);
+
+  if (event_group_bits & MicrophoneEventGroupBits::TASK_STARTING) {
+    ESP_LOGD(TAG, "Task has started, attempting to setup I2S audio driver");
+    xEventGroupClearBits(this->event_group_, MicrophoneEventGroupBits::TASK_STARTING);
+  }
+
+  if (event_group_bits & MicrophoneEventGroupBits::TASK_RUNNING) {
+    ESP_LOGD(TAG, "Task is running and reading data");
+
+    xEventGroupClearBits(this->event_group_, MicrophoneEventGroupBits::TASK_RUNNING);
+    this->state_ = microphone::STATE_RUNNING;
+  }
+
+  if (event_group_bits & MicrophoneEventGroupBits::TASK_STOPPING) {
+    ESP_LOGD(TAG, "Task is stopping, attempting to unload the I2S audio driver");
+    xEventGroupClearBits(this->event_group_, MicrophoneEventGroupBits::TASK_STOPPING);
+  }
+
+  if ((event_group_bits & MicrophoneEventGroupBits::TASK_STOPPED)) {
+    ESP_LOGD(TAG, "Task is finished, freeing resources");
+    vTaskDelete(this->task_handle_);
+    this->task_handle_ = nullptr;
+    xEventGroupClearBits(this->event_group_, ALL_BITS);
+    this->state_ = microphone::STATE_STOPPED;
+  }
+
+  if ((uxSemaphoreGetCount(this->active_listeners_semaphore_) < MAX_LISTENERS) &&
+      (this->state_ == microphone::STATE_STOPPED)) {
+    this->state_ = microphone::STATE_STARTING;
+  }
+  if ((uxSemaphoreGetCount(this->active_listeners_semaphore_) == MAX_LISTENERS) &&
+      (this->state_ == microphone::STATE_RUNNING)) {
+    this->state_ = microphone::STATE_STOPPING;
+  }
+
   switch (this->state_) {
-    case microphone::STATE_STOPPED:
-      break;
     case microphone::STATE_STARTING:
-      this->start_();
-      break;
-    case microphone::STATE_RUNNING:
-      if (this->data_callbacks_.size() > 0) {
-        this->read_();
+      if ((this->task_handle_ == nullptr) && !this->status_has_error()) {
+        xTaskCreate(I2SAudioMicrophone::mic_task, "mic_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY,
+                    &this->task_handle_);
+
+        if (this->task_handle_ == nullptr) {
+          this->status_momentary_error("Task failed to start, attempting again in 1 second", 1000);
+        }
       }
       break;
+    case microphone::STATE_RUNNING:
+      break;
     case microphone::STATE_STOPPING:
-      this->stop_();
+      xEventGroupSetBits(this->event_group_, MicrophoneEventGroupBits::COMMAND_STOP);
+      break;
+    case microphone::STATE_STOPPED:
       break;
   }
 }
diff --git a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.h b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.h
index 072d312e0f..8e6d83cad3 100644
--- a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.h
+++ b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.h
@@ -7,6 +7,9 @@
 #include "esphome/components/microphone/microphone.h"
 #include "esphome/core/component.h"
 
+#include <freertos/event_groups.h>
+#include <freertos/semphr.h>
+
 namespace esphome {
 namespace i2s_audio {
 
@@ -35,11 +38,18 @@ class I2SAudioMicrophone : public I2SAudioIn, public microphone::Microphone, pub
 #endif
 
  protected:
-  void start_();
-  void stop_();
-  void read_();
+  bool start_driver_();
+  void stop_driver_();
+
   size_t read_(uint8_t *buf, size_t len, TickType_t ticks_to_wait);
 
+  static void mic_task(void *params);
+
+  SemaphoreHandle_t active_listeners_semaphore_{nullptr};
+  EventGroupHandle_t event_group_{nullptr};
+
+  TaskHandle_t task_handle_{nullptr};
+
 #ifdef USE_I2S_LEGACY
   int8_t din_pin_{I2S_PIN_NO_CHANGE};
 #if SOC_I2S_SUPPORTS_ADC
@@ -51,8 +61,6 @@ class I2SAudioMicrophone : public I2SAudioIn, public microphone::Microphone, pub
   i2s_chan_handle_t rx_handle_;
 #endif
   bool pdm_{false};
-
-  HighFrequencyLoopRequester high_freq_;
 };
 
 }  // namespace i2s_audio
diff --git a/esphome/components/i2s_audio/speaker/__init__.py b/esphome/components/i2s_audio/speaker/__init__.py
index 7e41cd3991..bb9f24bf0b 100644
--- a/esphome/components/i2s_audio/speaker/__init__.py
+++ b/esphome/components/i2s_audio/speaker/__init__.py
@@ -27,6 +27,7 @@ from .. import (
     i2s_audio_ns,
     register_i2s_audio_component,
     use_legacy,
+    validate_mclk_divisible_by_3,
 )
 
 AUTO_LOAD = ["audio"]
@@ -155,6 +156,7 @@ CONFIG_SCHEMA = cv.All(
     _validate_esp32_variant,
     _set_num_channels_from_config,
     _set_stream_limits,
+    validate_mclk_divisible_by_3,
 )
 
 
diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
index cb3bbc8cf2..7d247003f7 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
@@ -545,7 +545,7 @@ esp_err_t I2SAudioSpeaker::start_i2s_driver_(audio::AudioStreamInfo &audio_strea
     .use_apll = this->use_apll_,
     .tx_desc_auto_clear = true,
     .fixed_mclk = I2S_PIN_NO_CHANGE,
-    .mclk_multiple = I2S_MCLK_MULTIPLE_256,
+    .mclk_multiple = this->mclk_multiple_,
     .bits_per_chan = this->bits_per_channel_,
 #if SOC_I2S_SUPPORTS_TDM
     .chan_mask = (i2s_channel_t) (I2S_TDM_ACTIVE_CH0 | I2S_TDM_ACTIVE_CH1),
@@ -614,7 +614,7 @@ esp_err_t I2SAudioSpeaker::start_i2s_driver_(audio::AudioStreamInfo &audio_strea
   i2s_std_clk_config_t clk_cfg = {
       .sample_rate_hz = audio_stream_info.get_sample_rate(),
       .clk_src = clk_src,
-      .mclk_multiple = I2S_MCLK_MULTIPLE_256,
+      .mclk_multiple = this->mclk_multiple_,
   };
 
   i2s_slot_mode_t slot_mode = this->slot_mode_;
diff --git a/tests/components/microphone/common.yaml b/tests/components/microphone/common.yaml
index ea79266281..ccadc7aee5 100644
--- a/tests/components/microphone/common.yaml
+++ b/tests/components/microphone/common.yaml
@@ -9,3 +9,4 @@ microphone:
     i2s_din_pin: ${i2s_din_pin}
     adc_type: external
     pdm: false
+    mclk_multiple: 384