Use finished speaking detection in ESPHome/Wyoming (#122962)

homeassistant/components/assist_pipeline/pipeline.py

@@ -505,6 +505,9 @@ class AudioSettings:
     samples_per_chunk: int | None = None
     """Number of samples that will be in each audio chunk (None for no chunking)."""
 
+    silence_seconds: float = 0.5
+    """Seconds of silence after voice command has ended."""
+
     def __post_init__(self) -> None:
         """Verify settings post-initialization."""
         if (self.noise_suppression_level < 0) or (self.noise_suppression_level > 4):
@@ -909,7 +912,9 @@ class PipelineRun:
             # Transcribe audio stream
             stt_vad: VoiceCommandSegmenter | None = None
             if self.audio_settings.is_vad_enabled:
-                stt_vad = VoiceCommandSegmenter()
+                stt_vad = VoiceCommandSegmenter(
+                    silence_seconds=self.audio_settings.silence_seconds
+                )
 
             result = await self.stt_provider.async_process_audio_stream(
                 metadata,

homeassistant/components/assist_pipeline/vad.py

@@ -80,7 +80,7 @@ class VoiceCommandSegmenter:
     speech_seconds: float = 0.3
     """Seconds of speech before voice command has started."""
 
-    silence_seconds: float = 0.5
+    silence_seconds: float = 1.0
     """Seconds of silence after voice command has ended."""
 
     timeout_seconds: float = 15.0

homeassistant/components/esphome/select.py

@@ -83,7 +83,7 @@ class EsphomeAssistPipelineSelect(EsphomeAssistEntity, AssistPipelineSelect):
 
 
 class EsphomeVadSensitivitySelect(EsphomeAssistEntity, VadSensitivitySelect):
-    """VAD sensitivity selector for VoIP devices."""
+    """VAD sensitivity selector for ESPHome devices."""
 
     def __init__(self, hass: HomeAssistant, entry_data: RuntimeEntryData) -> None:
         """Initialize a VAD sensitivity selector."""

homeassistant/components/esphome/voice_assistant.py

@@ -34,6 +34,7 @@ from homeassistant.components.assist_pipeline.error import (
     WakeWordDetectionAborted,
     WakeWordDetectionError,
 )
+from homeassistant.components.assist_pipeline.vad import VadSensitivity
 from homeassistant.components.intent.timers import TimerEventType, TimerInfo
 from homeassistant.components.media_player import async_process_play_media_url
 from homeassistant.core import Context, HomeAssistant, callback
@@ -243,6 +244,11 @@ class VoiceAssistantPipeline:
                     auto_gain_dbfs=audio_settings.auto_gain,
                     volume_multiplier=audio_settings.volume_multiplier,
                     is_vad_enabled=bool(flags & VoiceAssistantCommandFlag.USE_VAD),
+                    silence_seconds=VadSensitivity.to_seconds(
+                        pipeline_select.get_vad_sensitivity(
+                            self.hass, DOMAIN, self.device_info.mac_address
+                        )
+                    ),
                 ),
             )
 

homeassistant/components/wyoming/devices.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from collections.abc import Callable
 from dataclasses import dataclass
 
+from homeassistant.components.assist_pipeline.vad import VadSensitivity
 from homeassistant.core import HomeAssistant, callback
 from homeassistant.helpers import entity_registry as er
 
@@ -23,6 +24,7 @@ class SatelliteDevice:
     noise_suppression_level: int = 0
     auto_gain: int = 0
     volume_multiplier: float = 1.0
+    vad_sensitivity: VadSensitivity = VadSensitivity.DEFAULT
 
     _is_active_listener: Callable[[], None] | None = None
     _is_muted_listener: Callable[[], None] | None = None
@@ -77,6 +79,14 @@ class SatelliteDevice:
             if self._audio_settings_listener is not None:
                 self._audio_settings_listener()
 
+    @callback
+    def set_vad_sensitivity(self, vad_sensitivity: VadSensitivity) -> None:
+        """Set VAD sensitivity."""
+        if vad_sensitivity != self.vad_sensitivity:
+            self.vad_sensitivity = vad_sensitivity
+            if self._audio_settings_listener is not None:
+                self._audio_settings_listener()
+
     @callback
     def set_is_active_listener(self, is_active_listener: Callable[[], None]) -> None:
         """Listen for updates to is_active."""
@@ -140,3 +150,10 @@ class SatelliteDevice:
         return ent_reg.async_get_entity_id(
             "number", DOMAIN, f"{self.satellite_id}-volume_multiplier"
         )
+
+    def get_vad_sensitivity_entity_id(self, hass: HomeAssistant) -> str | None:
+        """Return entity id for VAD sensitivity."""
+        ent_reg = er.async_get(hass)
+        return ent_reg.async_get_entity_id(
+            "select", DOMAIN, f"{self.satellite_id}-vad_sensitivity"
+        )

homeassistant/components/wyoming/satellite.py

@@ -25,6 +25,7 @@ from wyoming.wake import Detect, Detection
 
 from homeassistant.components import assist_pipeline, intent, stt, tts
 from homeassistant.components.assist_pipeline import select as pipeline_select
+from homeassistant.components.assist_pipeline.vad import VadSensitivity
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.core import Context, HomeAssistant, callback
 
@@ -409,6 +410,9 @@ class WyomingSatellite:
                     noise_suppression_level=self.device.noise_suppression_level,
                     auto_gain_dbfs=self.device.auto_gain,
                     volume_multiplier=self.device.volume_multiplier,
+                    silence_seconds=VadSensitivity.to_seconds(
+                        self.device.vad_sensitivity
+                    ),
                 ),
                 device_id=self.device.device_id,
                 wake_word_phrase=wake_word_phrase,

homeassistant/components/wyoming/select.py

@@ -4,7 +4,11 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Final
 
-from homeassistant.components.assist_pipeline.select import AssistPipelineSelect
+from homeassistant.components.assist_pipeline.select import (
+    AssistPipelineSelect,
+    VadSensitivitySelect,
+)
+from homeassistant.components.assist_pipeline.vad import VadSensitivity
 from homeassistant.components.select import SelectEntity, SelectEntityDescription
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.const import EntityCategory
@@ -45,6 +49,7 @@ async def async_setup_entry(
         [
             WyomingSatellitePipelineSelect(hass, device),
             WyomingSatelliteNoiseSuppressionLevelSelect(device),
+            WyomingSatelliteVadSensitivitySelect(hass, device),
         ]
     )
 
@@ -92,3 +97,21 @@ class WyomingSatelliteNoiseSuppressionLevelSelect(
         self._attr_current_option = option
         self.async_write_ha_state()
         self._device.set_noise_suppression_level(_NOISE_SUPPRESSION_LEVEL[option])
+
+
+class WyomingSatelliteVadSensitivitySelect(
+    WyomingSatelliteEntity, VadSensitivitySelect
+):
+    """VAD sensitivity selector for Wyoming satellites."""
+
+    def __init__(self, hass: HomeAssistant, device: SatelliteDevice) -> None:
+        """Initialize a VAD sensitivity selector."""
+        self.device = device
+
+        WyomingSatelliteEntity.__init__(self, device)
+        VadSensitivitySelect.__init__(self, hass, device.satellite_id)
+
+    async def async_select_option(self, option: str) -> None:
+        """Select an option."""
+        await super().async_select_option(option)
+        self.device.set_vad_sensitivity(VadSensitivity(option))

homeassistant/components/wyoming/strings.json

@@ -46,6 +46,14 @@
           "high": "High",
           "max": "Max"
         }
+      },
+      "vad_sensitivity": {
+        "name": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::name%]",
+        "state": {
+          "default": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::default%]",
+          "aggressive": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::aggressive%]",
+          "relaxed": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::relaxed%]"
+        }
       }
     },
     "switch": {

tests/components/wyoming/test_select.py

@@ -5,6 +5,7 @@ from unittest.mock import Mock, patch
 from homeassistant.components import assist_pipeline
 from homeassistant.components.assist_pipeline.pipeline import PipelineData
 from homeassistant.components.assist_pipeline.select import OPTION_PREFERRED
+from homeassistant.components.assist_pipeline.vad import VadSensitivity
 from homeassistant.components.wyoming.devices import SatelliteDevice
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.core import HomeAssistant
@@ -140,3 +141,50 @@ async def test_noise_suppression_level_select(
     )
 
     assert satellite_device.noise_suppression_level == 2
+
+
+async def test_vad_sensitivity_select(
+    hass: HomeAssistant,
+    satellite_config_entry: ConfigEntry,
+    satellite_device: SatelliteDevice,
+) -> None:
+    """Test VAD sensitivity select."""
+    vs_entity_id = satellite_device.get_vad_sensitivity_entity_id(hass)
+    assert vs_entity_id
+
+    state = hass.states.get(vs_entity_id)
+    assert state is not None
+    assert state.state == VadSensitivity.DEFAULT
+    assert satellite_device.vad_sensitivity == VadSensitivity.DEFAULT
+
+    # Change setting
+    with patch.object(satellite_device, "set_vad_sensitivity") as mock_vs_changed:
+        await hass.services.async_call(
+            "select",
+            "select_option",
+            {"entity_id": vs_entity_id, "option": VadSensitivity.AGGRESSIVE.value},
+            blocking=True,
+        )
+
+        state = hass.states.get(vs_entity_id)
+        assert state is not None
+        assert state.state == VadSensitivity.AGGRESSIVE.value
+
+        # set function should have been called
+        mock_vs_changed.assert_called_once_with(VadSensitivity.AGGRESSIVE)
+
+    # test restore
+    satellite_device = await reload_satellite(hass, satellite_config_entry.entry_id)
+
+    state = hass.states.get(vs_entity_id)
+    assert state is not None
+    assert state.state == VadSensitivity.AGGRESSIVE.value
+
+    await hass.services.async_call(
+        "select",
+        "select_option",
+        {"entity_id": vs_entity_id, "option": VadSensitivity.RELAXED.value},
+        blocking=True,
+    )
+
+    assert satellite_device.vad_sensitivity == VadSensitivity.RELAXED