Use finished speaking detection in ESPHome/Wyoming (#122962)

This commit is contained in:
Michael Hansen 2024-07-31 13:39:03 -05:00 committed by GitHub
parent 8a4206da99
commit d5388452d4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 115 additions and 4 deletions

View File

@ -505,6 +505,9 @@ class AudioSettings:
samples_per_chunk: int | None = None
"""Number of samples that will be in each audio chunk (None for no chunking)."""
silence_seconds: float = 0.5
"""Seconds of silence after voice command has ended."""
def __post_init__(self) -> None:
"""Verify settings post-initialization."""
if (self.noise_suppression_level < 0) or (self.noise_suppression_level > 4):
@ -909,7 +912,9 @@ class PipelineRun:
# Transcribe audio stream
stt_vad: VoiceCommandSegmenter | None = None
if self.audio_settings.is_vad_enabled:
stt_vad = VoiceCommandSegmenter()
stt_vad = VoiceCommandSegmenter(
silence_seconds=self.audio_settings.silence_seconds
)
result = await self.stt_provider.async_process_audio_stream(
metadata,

View File

@ -80,7 +80,7 @@ class VoiceCommandSegmenter:
speech_seconds: float = 0.3
"""Seconds of speech before voice command has started."""
silence_seconds: float = 0.5
silence_seconds: float = 1.0
"""Seconds of silence after voice command has ended."""
timeout_seconds: float = 15.0

View File

@ -83,7 +83,7 @@ class EsphomeAssistPipelineSelect(EsphomeAssistEntity, AssistPipelineSelect):
class EsphomeVadSensitivitySelect(EsphomeAssistEntity, VadSensitivitySelect):
"""VAD sensitivity selector for VoIP devices."""
"""VAD sensitivity selector for ESPHome devices."""
def __init__(self, hass: HomeAssistant, entry_data: RuntimeEntryData) -> None:
"""Initialize a VAD sensitivity selector."""

View File

@ -34,6 +34,7 @@ from homeassistant.components.assist_pipeline.error import (
WakeWordDetectionAborted,
WakeWordDetectionError,
)
from homeassistant.components.assist_pipeline.vad import VadSensitivity
from homeassistant.components.intent.timers import TimerEventType, TimerInfo
from homeassistant.components.media_player import async_process_play_media_url
from homeassistant.core import Context, HomeAssistant, callback
@ -243,6 +244,11 @@ class VoiceAssistantPipeline:
auto_gain_dbfs=audio_settings.auto_gain,
volume_multiplier=audio_settings.volume_multiplier,
is_vad_enabled=bool(flags & VoiceAssistantCommandFlag.USE_VAD),
silence_seconds=VadSensitivity.to_seconds(
pipeline_select.get_vad_sensitivity(
self.hass, DOMAIN, self.device_info.mac_address
)
),
),
)

View File

@ -5,6 +5,7 @@ from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from homeassistant.components.assist_pipeline.vad import VadSensitivity
from homeassistant.core import HomeAssistant, callback
from homeassistant.helpers import entity_registry as er
@ -23,6 +24,7 @@ class SatelliteDevice:
noise_suppression_level: int = 0
auto_gain: int = 0
volume_multiplier: float = 1.0
vad_sensitivity: VadSensitivity = VadSensitivity.DEFAULT
_is_active_listener: Callable[[], None] | None = None
_is_muted_listener: Callable[[], None] | None = None
@ -77,6 +79,14 @@ class SatelliteDevice:
if self._audio_settings_listener is not None:
self._audio_settings_listener()
@callback
def set_vad_sensitivity(self, vad_sensitivity: VadSensitivity) -> None:
"""Set VAD sensitivity."""
if vad_sensitivity != self.vad_sensitivity:
self.vad_sensitivity = vad_sensitivity
if self._audio_settings_listener is not None:
self._audio_settings_listener()
@callback
def set_is_active_listener(self, is_active_listener: Callable[[], None]) -> None:
"""Listen for updates to is_active."""
@ -140,3 +150,10 @@ class SatelliteDevice:
return ent_reg.async_get_entity_id(
"number", DOMAIN, f"{self.satellite_id}-volume_multiplier"
)
def get_vad_sensitivity_entity_id(self, hass: HomeAssistant) -> str | None:
"""Return entity id for VAD sensitivity."""
ent_reg = er.async_get(hass)
return ent_reg.async_get_entity_id(
"select", DOMAIN, f"{self.satellite_id}-vad_sensitivity"
)

View File

@ -25,6 +25,7 @@ from wyoming.wake import Detect, Detection
from homeassistant.components import assist_pipeline, intent, stt, tts
from homeassistant.components.assist_pipeline import select as pipeline_select
from homeassistant.components.assist_pipeline.vad import VadSensitivity
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import Context, HomeAssistant, callback
@ -409,6 +410,9 @@ class WyomingSatellite:
noise_suppression_level=self.device.noise_suppression_level,
auto_gain_dbfs=self.device.auto_gain,
volume_multiplier=self.device.volume_multiplier,
silence_seconds=VadSensitivity.to_seconds(
self.device.vad_sensitivity
),
),
device_id=self.device.device_id,
wake_word_phrase=wake_word_phrase,

View File

@ -4,7 +4,11 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Final
from homeassistant.components.assist_pipeline.select import AssistPipelineSelect
from homeassistant.components.assist_pipeline.select import (
AssistPipelineSelect,
VadSensitivitySelect,
)
from homeassistant.components.assist_pipeline.vad import VadSensitivity
from homeassistant.components.select import SelectEntity, SelectEntityDescription
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import EntityCategory
@ -45,6 +49,7 @@ async def async_setup_entry(
[
WyomingSatellitePipelineSelect(hass, device),
WyomingSatelliteNoiseSuppressionLevelSelect(device),
WyomingSatelliteVadSensitivitySelect(hass, device),
]
)
@ -92,3 +97,21 @@ class WyomingSatelliteNoiseSuppressionLevelSelect(
self._attr_current_option = option
self.async_write_ha_state()
self._device.set_noise_suppression_level(_NOISE_SUPPRESSION_LEVEL[option])
class WyomingSatelliteVadSensitivitySelect(
WyomingSatelliteEntity, VadSensitivitySelect
):
"""VAD sensitivity selector for Wyoming satellites."""
def __init__(self, hass: HomeAssistant, device: SatelliteDevice) -> None:
"""Initialize a VAD sensitivity selector."""
self.device = device
WyomingSatelliteEntity.__init__(self, device)
VadSensitivitySelect.__init__(self, hass, device.satellite_id)
async def async_select_option(self, option: str) -> None:
"""Select an option."""
await super().async_select_option(option)
self.device.set_vad_sensitivity(VadSensitivity(option))

View File

@ -46,6 +46,14 @@
"high": "High",
"max": "Max"
}
},
"vad_sensitivity": {
"name": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::name%]",
"state": {
"default": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::default%]",
"aggressive": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::aggressive%]",
"relaxed": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::relaxed%]"
}
}
},
"switch": {

View File

@ -5,6 +5,7 @@ from unittest.mock import Mock, patch
from homeassistant.components import assist_pipeline
from homeassistant.components.assist_pipeline.pipeline import PipelineData
from homeassistant.components.assist_pipeline.select import OPTION_PREFERRED
from homeassistant.components.assist_pipeline.vad import VadSensitivity
from homeassistant.components.wyoming.devices import SatelliteDevice
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import HomeAssistant
@ -140,3 +141,50 @@ async def test_noise_suppression_level_select(
)
assert satellite_device.noise_suppression_level == 2
async def test_vad_sensitivity_select(
hass: HomeAssistant,
satellite_config_entry: ConfigEntry,
satellite_device: SatelliteDevice,
) -> None:
"""Test VAD sensitivity select."""
vs_entity_id = satellite_device.get_vad_sensitivity_entity_id(hass)
assert vs_entity_id
state = hass.states.get(vs_entity_id)
assert state is not None
assert state.state == VadSensitivity.DEFAULT
assert satellite_device.vad_sensitivity == VadSensitivity.DEFAULT
# Change setting
with patch.object(satellite_device, "set_vad_sensitivity") as mock_vs_changed:
await hass.services.async_call(
"select",
"select_option",
{"entity_id": vs_entity_id, "option": VadSensitivity.AGGRESSIVE.value},
blocking=True,
)
state = hass.states.get(vs_entity_id)
assert state is not None
assert state.state == VadSensitivity.AGGRESSIVE.value
# set function should have been called
mock_vs_changed.assert_called_once_with(VadSensitivity.AGGRESSIVE)
# test restore
satellite_device = await reload_satellite(hass, satellite_config_entry.entry_id)
state = hass.states.get(vs_entity_id)
assert state is not None
assert state.state == VadSensitivity.AGGRESSIVE.value
await hass.services.async_call(
"select",
"select_option",
{"entity_id": vs_entity_id, "option": VadSensitivity.RELAXED.value},
blocking=True,
)
assert satellite_device.vad_sensitivity == VadSensitivity.RELAXED