Improve tts playback in voice command dialog (#16288)

src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts

@@ -65,6 +65,8 @@ export class HaVoiceCommandDialog extends LitElement {
 
   private _audioBuffer?: Int16Array[];
 
+  private _audio?: HTMLAudioElement;
+
   private _stt_binary_handler_id?: number | null;
 
   public async showDialog(): Promise<void> {
@@ -81,11 +83,20 @@ export class HaVoiceCommandDialog extends LitElement {
 
   public async closeDialog(): Promise<void> {
     this._opened = false;
+    this._pipeline = undefined;
     this._agentInfo = undefined;
     this._conversation = undefined;
     this._conversationId = null;
     this._audioRecorder?.close();
     this._audioRecorder = undefined;
+    if (this._audio) {
+      this._audio.pause();
+      this._audio.removeEventListener("ended", this._unloadAudio);
+      this._audio.removeEventListener("pause", this._unloadAudio);
+      this._audio.removeEventListener("canplaythrough", this._playAudio);
+      this._audio.removeEventListener("error", this._audioError);
+      this._audio = undefined;
+    }
     fireEvent(this, "dialog-closed", { dialog: this.localName });
   }
 
@@ -198,7 +209,10 @@ export class HaVoiceCommandDialog extends LitElement {
   }
 
   protected willUpdate(changedProperties: PropertyValues): void {
-    if (!this.hasUpdated || changedProperties.has("_pipelineId")) {
+    if (
+      changedProperties.has("_pipelineId") ||
+      (changedProperties.has("_opened") && this._opened === true)
+    ) {
       this._getPipeline();
     }
   }
@@ -249,6 +263,7 @@ export class HaVoiceCommandDialog extends LitElement {
   }
 
   private async _processText(text: string) {
+    this._audio?.pause();
     this._addMessage({ who: "user", text });
     const message: Message = {
       who: "hass",
@@ -300,6 +315,7 @@ export class HaVoiceCommandDialog extends LitElement {
   }
 
   private async _startListening() {
+    this._audio?.pause();
     if (!this._audioRecorder) {
       this._audioRecorder = new AudioRecorder((audio) => {
         if (this._audioBuffer) {
@@ -362,8 +378,15 @@ export class HaVoiceCommandDialog extends LitElement {
 
           if (event.type === "tts-end") {
             const url = event.data.tts_output.url;
-            const audio = new Audio(url);
-            audio.play();
+            if (!this._audio) {
+              this._audio = new Audio(url);
+              this._audio.addEventListener("ended", this._unloadAudio);
+              this._audio.addEventListener("pause", this._unloadAudio);
+              this._audio.addEventListener("canplaythrough", this._playAudio);
+              this._audio.addEventListener("error", this._audioError);
+            } else {
+              this._audio.src = url;
+            }
           }
 
           if (event.type === "run-end") {
@@ -432,6 +455,19 @@ export class HaVoiceCommandDialog extends LitElement {
     this.hass.connection.socket!.send(data);
   }
 
+  private _playAudio = () => {
+    this._audio?.play();
+  };
+
+  private _audioError = () => {
+    showAlertDialog(this, { title: "Error playing audio." });
+    this._audio?.removeAttribute("src");
+  };
+
+  private _unloadAudio = () => {
+    this._audio?.removeAttribute("src");
+  };
+
   private _scrollMessagesBottom() {
     const scrollContainer = this._scrollContainer;
     if (!scrollContainer) {

src/util/audio-recorder.ts

@@ -58,11 +58,15 @@ export class AudioRecorder {
   public close() {
     this._active = false;
     this._stream?.getTracks()[0].stop();
+    if (this._recorder) {
+      this._recorder.port.onmessage = null;
+    }
+    this._source?.disconnect();
     this._context?.close();
     this._stream = undefined;
-    this._context = undefined;
     this._source = undefined;
     this._recorder = undefined;
+    this._context = undefined;
   }
 
   private async _createContext() {