From 1aa0dbdaf570517fbc5aac55f6faecc07b611159 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Mon, 26 Aug 2024 15:10:43 +0200 Subject: [PATCH] Revert "Revert "Use speex for noise suppression and auto gain"" (#124637) Revert "Revert "Use speex for noise suppression and auto gain" (#124620)" This reverts commit 302ffe5e56488f2f139686d375e5c82339304c87. --- .../assist_pipeline/audio_enhancer.py | 30 +++++++++++++++++-- .../components/assist_pipeline/manifest.json | 2 +- .../components/assist_pipeline/pipeline.py | 4 +-- homeassistant/components/voip/voip.py | 4 +-- homeassistant/package_constraints.txt | 1 + requirements_all.txt | 3 ++ requirements_test_all.txt | 3 ++ 7 files changed, 39 insertions(+), 8 deletions(-) diff --git a/homeassistant/components/assist_pipeline/audio_enhancer.py b/homeassistant/components/assist_pipeline/audio_enhancer.py index c9c60f421b1..ff2b122187a 100644 --- a/homeassistant/components/assist_pipeline/audio_enhancer.py +++ b/homeassistant/components/assist_pipeline/audio_enhancer.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import logging from pymicro_vad import MicroVad +from pyspeex_noise import AudioProcessor from .const import BYTES_PER_CHUNK @@ -41,8 +42,8 @@ class AudioEnhancer(ABC): """Enhance chunk of PCM audio @ 16Khz with 16-bit mono samples.""" -class MicroVadEnhancer(AudioEnhancer): - """Audio enhancer that just runs microVAD.""" +class MicroVadSpeexEnhancer(AudioEnhancer): + """Audio enhancer that runs microVAD and speex.""" def __init__( self, auto_gain: int, noise_suppression: int, is_vad_enabled: bool @@ -50,6 +51,24 @@ class MicroVadEnhancer(AudioEnhancer): """Initialize audio enhancer.""" super().__init__(auto_gain, noise_suppression, is_vad_enabled) + self.audio_processor: AudioProcessor | None = None + + # Scale from 0-4 + self.noise_suppression = noise_suppression * -15 + + # Scale from 0-31 + self.auto_gain = auto_gain * 300 + + if (self.auto_gain != 0) or (self.noise_suppression != 0): + self.audio_processor = AudioProcessor( + self.auto_gain, self.noise_suppression + ) + _LOGGER.debug( + "Initialized speex with auto_gain=%s, noise_suppression=%s", + self.auto_gain, + self.noise_suppression, + ) + self.vad: MicroVad | None = None self.threshold = 0.5 @@ -61,12 +80,17 @@ class MicroVadEnhancer(AudioEnhancer): """Enhance 10ms chunk of PCM audio @ 16Khz with 16-bit mono samples.""" is_speech: bool | None = None + assert len(audio) == BYTES_PER_CHUNK + if self.vad is not None: # Run VAD - assert len(audio) == BYTES_PER_CHUNK speech_prob = self.vad.Process10ms(audio) is_speech = speech_prob > self.threshold + if self.audio_processor is not None: + # Run noise suppression and auto gain + audio = self.audio_processor.Process10ms(audio).audio + return EnhancedAudioChunk( audio=audio, timestamp_ms=timestamp_ms, is_speech=is_speech ) diff --git a/homeassistant/components/assist_pipeline/manifest.json b/homeassistant/components/assist_pipeline/manifest.json index 00950b138fd..c22b7391d33 100644 --- a/homeassistant/components/assist_pipeline/manifest.json +++ b/homeassistant/components/assist_pipeline/manifest.json @@ -7,5 +7,5 @@ "integration_type": "system", "iot_class": "local_push", "quality_scale": "internal", - "requirements": ["pymicro-vad==1.0.1"] + "requirements": ["pymicro-vad==1.0.1", "pyspeex-noise==1.0.0"] } diff --git a/homeassistant/components/assist_pipeline/pipeline.py b/homeassistant/components/assist_pipeline/pipeline.py index 9fada934ca1..342f811c99b 100644 --- a/homeassistant/components/assist_pipeline/pipeline.py +++ b/homeassistant/components/assist_pipeline/pipeline.py @@ -49,7 +49,7 @@ from homeassistant.util import ( ) from homeassistant.util.limited_size_dict import LimitedSizeDict -from .audio_enhancer import AudioEnhancer, EnhancedAudioChunk, MicroVadEnhancer +from .audio_enhancer import AudioEnhancer, EnhancedAudioChunk, MicroVadSpeexEnhancer from .const import ( BYTES_PER_CHUNK, CONF_DEBUG_RECORDING_DIR, @@ -589,7 +589,7 @@ class PipelineRun: # Initialize with audio settings if self.audio_settings.needs_processor and (self.audio_enhancer is None): # Default audio enhancer - self.audio_enhancer = MicroVadEnhancer( + self.audio_enhancer = MicroVadSpeexEnhancer( self.audio_settings.auto_gain_dbfs, self.audio_settings.noise_suppression_level, self.audio_settings.is_vad_enabled, diff --git a/homeassistant/components/voip/voip.py b/homeassistant/components/voip/voip.py index 161e938a3b6..be1e58b6eec 100644 --- a/homeassistant/components/voip/voip.py +++ b/homeassistant/components/voip/voip.py @@ -33,7 +33,7 @@ from homeassistant.components.assist_pipeline import ( ) from homeassistant.components.assist_pipeline.audio_enhancer import ( AudioEnhancer, - MicroVadEnhancer, + MicroVadSpeexEnhancer, ) from homeassistant.components.assist_pipeline.vad import ( AudioBuffer, @@ -235,7 +235,7 @@ class PipelineRtpDatagramProtocol(RtpDatagramProtocol): try: # Wait for speech before starting pipeline segmenter = VoiceCommandSegmenter(silence_seconds=self.silence_seconds) - audio_enhancer = MicroVadEnhancer(0, 0, True) + audio_enhancer = MicroVadSpeexEnhancer(0, 0, True) chunk_buffer: deque[bytes] = deque( maxlen=self.buffered_chunks_before_speech, ) diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 56187150c45..767804e5136 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -49,6 +49,7 @@ pymicro-vad==1.0.1 PyNaCl==1.5.0 pyOpenSSL==24.2.1 pyserial==3.5 +pyspeex-noise==1.0.0 python-slugify==8.0.4 PyTurboJPEG==1.7.1 pyudev==0.24.1 diff --git a/requirements_all.txt b/requirements_all.txt index 98dd06ad428..630e9180798 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2228,6 +2228,9 @@ pysoma==0.0.12 # homeassistant.components.spc pyspcwebgw==0.7.0 +# homeassistant.components.assist_pipeline +pyspeex-noise==1.0.0 + # homeassistant.components.squeezebox pysqueezebox==0.7.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index da09d09e12a..476bae2570e 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1782,6 +1782,9 @@ pysoma==0.0.12 # homeassistant.components.spc pyspcwebgw==0.7.0 +# homeassistant.components.assist_pipeline +pyspeex-noise==1.0.0 + # homeassistant.components.squeezebox pysqueezebox==0.7.1