Extract assist-chat out of voice-command-dialog (#23184)

src/components/ha-assist-chat.ts

@@ -0,0 +1,639 @@
+import type { CSSResultGroup, PropertyValues, TemplateResult } from "lit";
+import { css, LitElement, html, nothing } from "lit";
+import { mdiAlertCircle, mdiMicrophone, mdiSend } from "@mdi/js";
+import { customElement, property, query, state } from "lit/decorators";
+import { classMap } from "lit/directives/class-map";
+import type { HomeAssistant } from "../types";
+import {
+  runAssistPipeline,
+  type AssistPipeline,
+} from "../data/assist_pipeline";
+import { supportsFeature } from "../common/entity/supports-feature";
+import { ConversationEntityFeature } from "../data/conversation";
+import { AudioRecorder } from "../util/audio-recorder";
+import "./ha-alert";
+import "./ha-textfield";
+import type { HaTextField } from "./ha-textfield";
+import { documentationUrl } from "../util/documentation-url";
+import { showAlertDialog } from "../dialogs/generic/show-dialog-box";
+
+interface AssistMessage {
+  who: string;
+  text?: string | TemplateResult;
+  error?: boolean;
+}
+
+@customElement("ha-assist-chat")
+export class HaAssistChat extends LitElement {
+  @property({ attribute: false }) public hass!: HomeAssistant;
+
+  @property({ attribute: false }) public pipeline?: AssistPipeline;
+
+  @property({ type: Boolean, attribute: false })
+  public startListening?: boolean;
+
+  @query("#message-input") private _messageInput!: HaTextField;
+
+  @query("#scroll-container") private _scrollContainer!: HTMLDivElement;
+
+  @state() private _conversation: AssistMessage[] = [];
+
+  @state() private _showSendButton = false;
+
+  @state() private _processing = false;
+
+  private _conversationId: string | null = null;
+
+  private _audioRecorder?: AudioRecorder;
+
+  private _audioBuffer?: Int16Array[];
+
+  private _audio?: HTMLAudioElement;
+
+  private _stt_binary_handler_id?: number | null;
+
+  protected willUpdate(changedProperties: PropertyValues): void {
+    if (!this.hasUpdated || changedProperties.has("pipeline")) {
+      this._conversation = [
+        {
+          who: "hass",
+          text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"),
+        },
+      ];
+    }
+  }
+
+  protected firstUpdated(changedProperties: PropertyValues): void {
+    super.firstUpdated(changedProperties);
+    if (
+      this.startListening &&
+      this.pipeline &&
+      this.pipeline.stt_engine &&
+      AudioRecorder.isSupported
+    ) {
+      this._toggleListening();
+    }
+    setTimeout(() => this._messageInput.focus(), 0);
+  }
+
+  protected updated(changedProps: PropertyValues) {
+    super.updated(changedProps);
+    if (changedProps.has("_conversation")) {
+      this._scrollMessagesBottom();
+    }
+  }
+
+  public disconnectedCallback() {
+    super.disconnectedCallback();
+    this._audioRecorder?.close();
+    this._audioRecorder = undefined;
+    this._audio?.pause();
+    this._conversation = [];
+    this._conversationId = null;
+  }
+
+  protected render(): TemplateResult {
+    const controlHA = !this.pipeline
+      ? false
+      : this.pipeline.prefer_local_intents ||
+        (this.hass.states[this.pipeline.conversation_engine]
+          ? supportsFeature(
+              this.hass.states[this.pipeline.conversation_engine],
+              ConversationEntityFeature.CONTROL
+            )
+          : true);
+    const supportsMicrophone = AudioRecorder.isSupported;
+    const supportsSTT = this.pipeline?.stt_engine;
+
+    return html`
+      ${controlHA
+        ? nothing
+        : html`
+            <ha-alert>
+              ${this.hass.localize(
+                "ui.dialogs.voice_command.conversation_no_control"
+              )}
+            </ha-alert>
+          `}
+      <div class="messages">
+        <div class="messages-container" id="scroll-container">
+          ${this._conversation!.map(
+            // New lines matter for messages
+            // prettier-ignore
+            (message) => html`
+                <div class="message ${classMap({ error: !!message.error, [message.who]: true })}">${message.text}</div>
+              `
+          )}
+        </div>
+      </div>
+      <div class="input" slot="primaryAction">
+        <ha-textfield
+          id="message-input"
+          @keyup=${this._handleKeyUp}
+          @input=${this._handleInput}
+          .label=${this.hass.localize(`ui.dialogs.voice_command.input_label`)}
+          .iconTrailing=${true}
+        >
+          <div slot="trailingIcon">
+            ${this._showSendButton || !supportsSTT
+              ? html`
+                  <ha-icon-button
+                    class="listening-icon"
+                    .path=${mdiSend}
+                    @click=${this._handleSendMessage}
+                    .disabled=${this._processing}
+                    .label=${this.hass.localize(
+                      "ui.dialogs.voice_command.send_text"
+                    )}
+                  >
+                  </ha-icon-button>
+                `
+              : html`
+                  ${this._audioRecorder?.active
+                    ? html`
+                        <div class="bouncer">
+                          <div class="double-bounce1"></div>
+                          <div class="double-bounce2"></div>
+                        </div>
+                      `
+                    : nothing}
+
+                  <div class="listening-icon">
+                    <ha-icon-button
+                      .path=${mdiMicrophone}
+                      @click=${this._handleListeningButton}
+                      .disabled=${this._processing}
+                      .label=${this.hass.localize(
+                        "ui.dialogs.voice_command.start_listening"
+                      )}
+                    >
+                    </ha-icon-button>
+                    ${!supportsMicrophone
+                      ? html`
+                          <ha-svg-icon
+                            .path=${mdiAlertCircle}
+                            class="unsupported"
+                          ></ha-svg-icon>
+                        `
+                      : null}
+                  </div>
+                `}
+          </div>
+        </ha-textfield>
+      </div>
+    `;
+  }
+
+  private _scrollMessagesBottom() {
+    const scrollContainer = this._scrollContainer;
+    if (!scrollContainer) {
+      return;
+    }
+    scrollContainer.scrollTo(0, scrollContainer.scrollHeight);
+  }
+
+  private _handleKeyUp(ev: KeyboardEvent) {
+    const input = ev.target as HaTextField;
+    if (!this._processing && ev.key === "Enter" && input.value) {
+      this._processText(input.value);
+      input.value = "";
+      this._showSendButton = false;
+    }
+  }
+
+  private _handleInput(ev: InputEvent) {
+    const value = (ev.target as HaTextField).value;
+    if (value && !this._showSendButton) {
+      this._showSendButton = true;
+    } else if (!value && this._showSendButton) {
+      this._showSendButton = false;
+    }
+  }
+
+  private _handleSendMessage() {
+    if (this._messageInput.value) {
+      this._processText(this._messageInput.value.trim());
+      this._messageInput.value = "";
+      this._showSendButton = false;
+    }
+  }
+
+  private _handleListeningButton(ev) {
+    ev.stopPropagation();
+    ev.preventDefault();
+    this._toggleListening();
+  }
+
+  private async _toggleListening() {
+    const supportsMicrophone = AudioRecorder.isSupported;
+    if (!supportsMicrophone) {
+      this._showNotSupportedMessage();
+      return;
+    }
+    if (!this._audioRecorder?.active) {
+      this._startListening();
+    } else {
+      this._stopListening();
+    }
+  }
+
+  private _addMessage(message: AssistMessage) {
+    this._conversation = [...this._conversation!, message];
+  }
+
+  private async _showNotSupportedMessage() {
+    this._addMessage({
+      who: "hass",
+      text:
+        // New lines matter for messages
+        // prettier-ignore
+        html`${this.hass.localize(
+          "ui.dialogs.voice_command.not_supported_microphone_browser"
+        )}
+
+        ${this.hass.localize(
+          "ui.dialogs.voice_command.not_supported_microphone_documentation",
+          {
+            documentation_link: html`<a
+                target="_blank"
+                rel="noopener noreferrer"
+                href=${documentationUrl(
+                  this.hass,
+                  "/docs/configuration/securing/#remote-access"
+                )}
+              >${this.hass.localize(
+                  "ui.dialogs.voice_command.not_supported_microphone_documentation_link"
+                )}</a>`,
+          }
+        )}`,
+    });
+  }
+
+  private async _startListening() {
+    this._processing = true;
+    this._audio?.pause();
+    if (!this._audioRecorder) {
+      this._audioRecorder = new AudioRecorder((audio) => {
+        if (this._audioBuffer) {
+          this._audioBuffer.push(audio);
+        } else {
+          this._sendAudioChunk(audio);
+        }
+      });
+    }
+    this._stt_binary_handler_id = undefined;
+    this._audioBuffer = [];
+    const userMessage: AssistMessage = {
+      who: "user",
+      text: "…",
+    };
+    await this._audioRecorder.start();
+
+    this._addMessage(userMessage);
+    this.requestUpdate("_audioRecorder");
+
+    const hassMessage: AssistMessage = {
+      who: "hass",
+      text: "…",
+    };
+    // To make sure the answer is placed at the right user text, we add it before we process it
+    try {
+      const unsub = await runAssistPipeline(
+        this.hass,
+        (event) => {
+          if (event.type === "run-start") {
+            this._stt_binary_handler_id =
+              event.data.runner_data.stt_binary_handler_id;
+          }
+
+          // When we start STT stage, the WS has a binary handler
+          if (event.type === "stt-start" && this._audioBuffer) {
+            // Send the buffer over the WS to the STT engine.
+            for (const buffer of this._audioBuffer) {
+              this._sendAudioChunk(buffer);
+            }
+            this._audioBuffer = undefined;
+          }
+
+          // Stop recording if the server is done with STT stage
+          if (event.type === "stt-end") {
+            this._stt_binary_handler_id = undefined;
+            this._stopListening();
+            userMessage.text = event.data.stt_output.text;
+            this.requestUpdate("_conversation");
+            // To make sure the answer is placed at the right user text, we add it before we process it
+            this._addMessage(hassMessage);
+          }
+
+          if (event.type === "intent-end") {
+            this._conversationId = event.data.intent_output.conversation_id;
+            const plain = event.data.intent_output.response.speech?.plain;
+            if (plain) {
+              hassMessage.text = plain.speech;
+            }
+            this.requestUpdate("_conversation");
+          }
+
+          if (event.type === "tts-end") {
+            const url = event.data.tts_output.url;
+            this._audio = new Audio(url);
+            this._audio.play();
+            this._audio.addEventListener("ended", this._unloadAudio);
+            this._audio.addEventListener("pause", this._unloadAudio);
+            this._audio.addEventListener("canplaythrough", this._playAudio);
+            this._audio.addEventListener("error", this._audioError);
+          }
+
+          if (event.type === "run-end") {
+            this._stt_binary_handler_id = undefined;
+            unsub();
+          }
+
+          if (event.type === "error") {
+            this._stt_binary_handler_id = undefined;
+            if (userMessage.text === "…") {
+              userMessage.text = event.data.message;
+              userMessage.error = true;
+            } else {
+              hassMessage.text = event.data.message;
+              hassMessage.error = true;
+            }
+            this._stopListening();
+            this.requestUpdate("_conversation");
+            unsub();
+          }
+        },
+        {
+          start_stage: "stt",
+          end_stage: this.pipeline?.tts_engine ? "tts" : "intent",
+          input: { sample_rate: this._audioRecorder.sampleRate! },
+          pipeline: this.pipeline?.id,
+          conversation_id: this._conversationId,
+        }
+      );
+    } catch (err: any) {
+      await showAlertDialog(this, {
+        title: "Error starting pipeline",
+        text: err.message || err,
+      });
+      this._stopListening();
+    } finally {
+      this._processing = false;
+    }
+  }
+
+  private _stopListening() {
+    this._audioRecorder?.stop();
+    this.requestUpdate("_audioRecorder");
+    // We're currently STTing, so finish audio
+    if (this._stt_binary_handler_id) {
+      if (this._audioBuffer) {
+        for (const chunk of this._audioBuffer) {
+          this._sendAudioChunk(chunk);
+        }
+      }
+      // Send empty message to indicate we're done streaming.
+      this._sendAudioChunk(new Int16Array());
+      this._stt_binary_handler_id = undefined;
+    }
+    this._audioBuffer = undefined;
+  }
+
+  private _sendAudioChunk(chunk: Int16Array) {
+    this.hass.connection.socket!.binaryType = "arraybuffer";
+
+    // eslint-disable-next-line eqeqeq
+    if (this._stt_binary_handler_id == undefined) {
+      return;
+    }
+    // Turn into 8 bit so we can prefix our handler ID.
+    const data = new Uint8Array(1 + chunk.length * 2);
+    data[0] = this._stt_binary_handler_id;
+    data.set(new Uint8Array(chunk.buffer), 1);
+
+    this.hass.connection.socket!.send(data);
+  }
+
+  private _playAudio = () => {
+    this._audio?.play();
+  };
+
+  private _audioError = () => {
+    showAlertDialog(this, { title: "Error playing audio." });
+    this._audio?.removeAttribute("src");
+  };
+
+  private _unloadAudio = () => {
+    this._audio?.removeAttribute("src");
+    this._audio = undefined;
+  };
+
+  private async _processText(text: string) {
+    this._processing = true;
+    this._audio?.pause();
+    this._addMessage({ who: "user", text });
+    const message: AssistMessage = {
+      who: "hass",
+      text: "…",
+    };
+    // To make sure the answer is placed at the right user text, we add it before we process it
+    this._addMessage(message);
+    try {
+      const unsub = await runAssistPipeline(
+        this.hass,
+        (event) => {
+          if (event.type === "intent-end") {
+            this._conversationId = event.data.intent_output.conversation_id;
+            const plain = event.data.intent_output.response.speech?.plain;
+            if (plain) {
+              message.text = plain.speech;
+            }
+            this.requestUpdate("_conversation");
+            unsub();
+          }
+          if (event.type === "error") {
+            message.text = event.data.message;
+            message.error = true;
+            this.requestUpdate("_conversation");
+            unsub();
+          }
+        },
+        {
+          start_stage: "intent",
+          input: { text },
+          end_stage: "intent",
+          pipeline: this.pipeline?.id,
+          conversation_id: this._conversationId,
+        }
+      );
+    } catch {
+      message.text = this.hass.localize("ui.dialogs.voice_command.error");
+      message.error = true;
+      this.requestUpdate("_conversation");
+    } finally {
+      this._processing = false;
+    }
+  }
+
+  static get styles(): CSSResultGroup {
+    return css`
+      :host {
+        flex: 1;
+        display: flex;
+        flex-direction: column;
+        min-height: var(--ha-assist-chat-min-height, 415px);
+      }
+      ha-textfield {
+        display: block;
+        margin: 0 24px 16px;
+      }
+      .messages {
+        flex: 1;
+        display: block;
+        box-sizing: border-box;
+        position: relative;
+      }
+      .messages-container {
+        position: absolute;
+        bottom: 0px;
+        right: 0px;
+        left: 0px;
+        padding: 24px;
+        box-sizing: border-box;
+        overflow-y: auto;
+        max-height: 100%;
+      }
+      .message {
+        white-space: pre-line;
+        font-size: 18px;
+        clear: both;
+        margin: 8px 0;
+        padding: 8px;
+        border-radius: 15px;
+      }
+
+      @media all and (max-width: 450px), all and (max-height: 500px) {
+        .message {
+          font-size: 16px;
+        }
+      }
+
+      .message p {
+        margin: 0;
+      }
+      .message p:not(:last-child) {
+        margin-bottom: 8px;
+      }
+
+      .message.user {
+        margin-left: 24px;
+        margin-inline-start: 24px;
+        margin-inline-end: initial;
+        float: var(--float-end);
+        text-align: right;
+        border-bottom-right-radius: 0px;
+        background-color: var(--primary-color);
+        color: var(--text-primary-color);
+        direction: var(--direction);
+      }
+
+      .message.hass {
+        margin-right: 24px;
+        margin-inline-end: 24px;
+        margin-inline-start: initial;
+        float: var(--float-start);
+        border-bottom-left-radius: 0px;
+        background-color: var(--secondary-background-color);
+
+        color: var(--primary-text-color);
+        direction: var(--direction);
+      }
+
+      .message.user a {
+        color: var(--text-primary-color);
+      }
+
+      .message.hass a {
+        color: var(--primary-text-color);
+      }
+
+      .message.error {
+        background-color: var(--error-color);
+        color: var(--text-primary-color);
+      }
+
+      .bouncer {
+        width: 48px;
+        height: 48px;
+        position: absolute;
+      }
+      .double-bounce1,
+      .double-bounce2 {
+        width: 48px;
+        height: 48px;
+        border-radius: 50%;
+        background-color: var(--primary-color);
+        opacity: 0.2;
+        position: absolute;
+        top: 0;
+        left: 0;
+        -webkit-animation: sk-bounce 2s infinite ease-in-out;
+        animation: sk-bounce 2s infinite ease-in-out;
+      }
+      .double-bounce2 {
+        -webkit-animation-delay: -1s;
+        animation-delay: -1s;
+      }
+      @-webkit-keyframes sk-bounce {
+        0%,
+        100% {
+          -webkit-transform: scale(0);
+        }
+        50% {
+          -webkit-transform: scale(1);
+        }
+      }
+      @keyframes sk-bounce {
+        0%,
+        100% {
+          transform: scale(0);
+          -webkit-transform: scale(0);
+        }
+        50% {
+          transform: scale(1);
+          -webkit-transform: scale(1);
+        }
+      }
+
+      .listening-icon {
+        position: relative;
+        color: var(--secondary-text-color);
+        margin-right: -24px;
+        margin-inline-end: -24px;
+        margin-inline-start: initial;
+        direction: var(--direction);
+        transform: scaleX(var(--scale-direction));
+      }
+
+      .listening-icon[active] {
+        color: var(--primary-color);
+      }
+
+      .unsupported {
+        color: var(--error-color);
+        position: absolute;
+        --mdc-icon-size: 16px;
+        right: 5px;
+        inset-inline-end: 5px;
+        inset-inline-start: initial;
+        top: 0px;
+      }
+    `;
+  }
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "ha-assist-chat": HaAssistChat;
+  }
+}

src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts

@@ -1,16 +1,13 @@
 import "@material/mwc-button/mwc-button";
 import {
-  mdiAlertCircle,
   mdiChevronDown,
   mdiClose,
   mdiHelpCircleOutline,
-  mdiMicrophone,
-  mdiSend,
   mdiStar,
 } from "@mdi/js";
-import type { CSSResultGroup, PropertyValues, TemplateResult } from "lit";
+import type { CSSResultGroup, PropertyValues } from "lit";
 import { css, html, LitElement, nothing } from "lit";
-import { customElement, property, query, state } from "lit/decorators";
+import { customElement, property, state } from "lit/decorators";
 import { storage } from "../../common/decorators/storage";
 import { fireEvent } from "../../common/dom/fire_event";
 import { stopPropagation } from "../../common/dom/stop_propagation";
@@ -20,36 +17,23 @@ import "../../components/ha-dialog";
 import "../../components/ha-dialog-header";
 import "../../components/ha-icon-button";
 import "../../components/ha-list-item";
-import "../../components/ha-textfield";
 import "../../components/ha-alert";
-import type { HaTextField } from "../../components/ha-textfield";
+import "../../components/ha-assist-chat";
+import "../../components/ha-circular-progress";
 import type { AssistPipeline } from "../../data/assist_pipeline";
 import {
   getAssistPipeline,
   listAssistPipelines,
-  runAssistPipeline,
 } from "../../data/assist_pipeline";
 import { haStyleDialog } from "../../resources/styles";
 import type { HomeAssistant } from "../../types";
-import { AudioRecorder } from "../../util/audio-recorder";
 import { documentationUrl } from "../../util/documentation-url";
-import { showAlertDialog } from "../generic/show-dialog-box";
 import type { VoiceCommandDialogParams } from "./show-ha-voice-command-dialog";
-import { supportsFeature } from "../../common/entity/supports-feature";
-import { ConversationEntityFeature } from "../../data/conversation";
-
-interface Message {
-  who: string;
-  text?: string | TemplateResult;
-  error?: boolean;
-}
 
 @customElement("ha-voice-command-dialog")
 export class HaVoiceCommandDialog extends LitElement {
   @property({ attribute: false }) public hass!: HomeAssistant;
 
-  @state() private _conversation?: Message[];
-
   @state() private _opened = false;
 
   @storage({
@@ -61,69 +45,34 @@ export class HaVoiceCommandDialog extends LitElement {
 
   @state() private _pipeline?: AssistPipeline;
 
-  @state() private _showSendButton = false;
-
   @state() private _pipelines?: AssistPipeline[];
 
   @state() private _preferredPipeline?: string;
 
-  @query("#scroll-container") private _scrollContainer!: HTMLDivElement;
+  @state() private _errorLoadAssist?: "not_found" | "unknown";
 
-  @query("#message-input") private _messageInput!: HaTextField;
+  private _startListening = false;
-
-  private _conversationId: string | null = null;
-
-  private _audioRecorder?: AudioRecorder;
-
-  private _audioBuffer?: Int16Array[];
-
-  private _audio?: HTMLAudioElement;
-
-  private _stt_binary_handler_id?: number | null;
-
-  private _pipelinePromise?: Promise<AssistPipeline>;
 
   public async showDialog(
     params: Required<VoiceCommandDialogParams>
   ): Promise<void> {
-    if (params.pipeline_id === "last_used") {
+    if (
-      // Do not set pipeline id (retrieve from storage)
+      params.pipeline_id === "preferred" ||
-    } else if (params.pipeline_id === "preferred") {
+      (params.pipeline_id === "last_used" && !this._pipelineId)
+    ) {
       await this._loadPipelines();
       this._pipelineId = this._preferredPipeline;
-    } else {
+    } else if (!["last_used", "preferred"].includes(params.pipeline_id)) {
       this._pipelineId = params.pipeline_id;
     }
 
-    this._conversation = [
+    this._startListening = params.start_listening;
-      {
-        who: "hass",
-        text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"),
-      },
-    ];
     this._opened = true;
-    await this.updateComplete;
-    this._scrollMessagesBottom();
-
-    await this._pipelinePromise;
-    if (
-      params?.start_listening &&
-      this._pipeline?.stt_engine &&
-      AudioRecorder.isSupported
-    ) {
-      this._toggleListening();
-    }
   }
 
   public async closeDialog(): Promise<void> {
     this._opened = false;
-    this._pipeline = undefined;
     this._pipelines = undefined;
-    this._conversation = undefined;
-    this._conversationId = null;
-    this._audioRecorder?.close();
-    this._audioRecorder = undefined;
-    this._audio?.pause();
     fireEvent(this, "dialog-closed", { dialog: this.localName });
   }
 
@@ -132,24 +81,13 @@ export class HaVoiceCommandDialog extends LitElement {
       return nothing;
     }
 
-    const controlHA = !this._pipeline
-      ? false
-      : this._pipeline.prefer_local_intents ||
-        (this.hass.states[this._pipeline.conversation_engine]
-          ? supportsFeature(
-              this.hass.states[this._pipeline.conversation_engine],
-              ConversationEntityFeature.CONTROL
-            )
-          : true);
-    const supportsMicrophone = AudioRecorder.isSupported;
-    const supportsSTT = this._pipeline?.stt_engine;
-
     return html`
       <ha-dialog
         open
         @closed=${this.closeDialog}
         .heading=${this.hass.localize("ui.dialogs.voice_command.title")}
         flexContent
+        hideactions
       >
         <ha-dialog-header slot="heading">
           <ha-icon-button
@@ -173,26 +111,34 @@ export class HaVoiceCommandDialog extends LitElement {
                   .path=${mdiChevronDown}
                 ></ha-svg-icon>
               </ha-button>
-              ${this._pipelines?.map(
+              ${!this._pipelines
-                (pipeline) =>
+                ? html`<div class="pipelines-loading">
-                  html`<ha-list-item
+                    <ha-circular-progress
-                    ?selected=${pipeline.id === this._pipelineId ||
+                      indeterminate
-                    (!this._pipelineId &&
+                      size="small"
-                      pipeline.id === this._preferredPipeline)}
+                    ></ha-circular-progress>
-                    .pipeline=${pipeline.id}
+                  </div>`
-                    @click=${this._selectPipeline}
+                : this._pipelines?.map(
-                    .hasMeta=${pipeline.id === this._preferredPipeline}
+                    (pipeline) =>
-                  >
+                      html`<ha-list-item
-                    ${pipeline.name}${pipeline.id === this._preferredPipeline
+                        ?selected=${pipeline.id === this._pipelineId ||
-                      ? html`
+                        (!this._pipelineId &&
-                          <ha-svg-icon
+                          pipeline.id === this._preferredPipeline)}
-                            slot="meta"
+                        .pipeline=${pipeline.id}
-                            .path=${mdiStar}
+                        @click=${this._selectPipeline}
-                          ></ha-svg-icon>
+                        .hasMeta=${pipeline.id === this._preferredPipeline}
-                        `
+                      >
-                      : nothing}
+                        ${pipeline.name}${pipeline.id ===
-                  </ha-list-item>`
+                        this._preferredPipeline
-              )}
+                          ? html`
+                              <ha-svg-icon
+                                slot="meta"
+                                .path=${mdiStar}
+                              ></ha-svg-icon>
+                            `
+                          : nothing}
+                      </ha-list-item>`
+                  )}
               ${this.hass.user?.is_admin
                 ? html`<li divider role="separator"></li>
                     <a href="/config/voice-assistants/assistants"
@@ -217,80 +163,29 @@ export class HaVoiceCommandDialog extends LitElement {
             ></ha-icon-button>
           </a>
         </ha-dialog-header>
-        ${controlHA
-          ? nothing
-          : html`
-              <ha-alert>
-                ${this.hass.localize(
-                  "ui.dialogs.voice_command.conversation_no_control"
-                )}
-              </ha-alert>
-            `}
-        <div class="messages">
-          <div class="messages-container" id="scroll-container">
-            ${this._conversation!.map(
-              // New lines matter for messages
-              // prettier-ignore
-              (message) => html`
-                <div class=${this._computeMessageClasses(message)}>${message.text}</div>
-              `
-            )}
-          </div>
-        </div>
-        <div class="input" slot="primaryAction">
-          <ha-textfield
-            id="message-input"
-            @keyup=${this._handleKeyUp}
-            @input=${this._handleInput}
-            .label=${this.hass.localize(`ui.dialogs.voice_command.input_label`)}
-            dialogInitialFocus
-            iconTrailing
-          >
-            <span slot="trailingIcon">
-              ${this._showSendButton || !supportsSTT
-                ? html`
-                    <ha-icon-button
-                      class="listening-icon"
-                      .path=${mdiSend}
-                      @click=${this._handleSendMessage}
-                      .label=${this.hass.localize(
-                        "ui.dialogs.voice_command.send_text"
-                      )}
-                    >
-                    </ha-icon-button>
-                  `
-                : html`
-                    ${this._audioRecorder?.active
-                      ? html`
-                          <div class="bouncer">
-                            <div class="double-bounce1"></div>
-                            <div class="double-bounce2"></div>
-                          </div>
-                        `
-                      : nothing}
 
-                    <div class="listening-icon">
+        ${this._pipeline
-                      <ha-icon-button
+          ? html`
-                        .path=${mdiMicrophone}
+              <ha-assist-chat
-                        @click=${this._handleListeningButton}
+                .hass=${this.hass}
-                        .label=${this.hass.localize(
+                .pipeline=${this._pipeline}
-                          "ui.dialogs.voice_command.start_listening"
+                .startListening=${this._startListening}
-                        )}
+              >
-                      >
+              </ha-assist-chat>
-                      </ha-icon-button>
+            `
-                      ${!supportsMicrophone
+          : html`<div class="pipelines-loading">
-                        ? html`
+              <ha-circular-progress
-                            <ha-svg-icon
+                indeterminate
-                              .path=${mdiAlertCircle}
+                size="large"
-                              class="unsupported"
+              ></ha-circular-progress>
-                            ></ha-svg-icon>
+            </div>`}
-                          `
+        ${this._errorLoadAssist
-                        : null}
+          ? html`<ha-alert alert-type="error">
-                    </div>
+              ${this.hass.localize(
-                  `}
+                `ui.dialogs.voice_command.${this._errorLoadAssist}_error_load_assist`
-            </span>
+              )}
-          </ha-textfield>
+            </ha-alert>`
-        </div>
+          : nothing}
       </ha-dialog>
     `;
   }
@@ -298,23 +193,14 @@ export class HaVoiceCommandDialog extends LitElement {
   protected willUpdate(changedProperties: PropertyValues): void {
     if (
       changedProperties.has("_pipelineId") ||
-      (changedProperties.has("_opened") && this._opened === true)
+      (changedProperties.has("_opened") &&
+        this._opened === true &&
+        this._pipelineId)
     ) {
       this._getPipeline();
     }
   }
 
-  private async _getPipeline() {
-    try {
-      this._pipelinePromise = getAssistPipeline(this.hass, this._pipelineId);
-      this._pipeline = await this._pipelinePromise;
-    } catch (e: any) {
-      if (e.code === "not_found") {
-        this._pipelineId = undefined;
-      }
-    }
-  }
-
   private async _loadPipelines() {
     if (this._pipelines) {
       return;
@@ -328,343 +214,28 @@ export class HaVoiceCommandDialog extends LitElement {
 
   private async _selectPipeline(ev: CustomEvent) {
     this._pipelineId = (ev.currentTarget as any).pipeline;
-    this._conversation = [
-      {
-        who: "hass",
-        text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"),
-      },
-    ];
     await this.updateComplete;
-    this._scrollMessagesBottom();
   }
 
-  protected updated(changedProps: PropertyValues) {
+  private async _getPipeline() {
-    super.updated(changedProps);
-    if (changedProps.has("_conversation") || changedProps.has("results")) {
-      this._scrollMessagesBottom();
-    }
-  }
-
-  private _addMessage(message: Message) {
-    this._conversation = [...this._conversation!, message];
-  }
-
-  private _handleKeyUp(ev: KeyboardEvent) {
-    const input = ev.target as HaTextField;
-    if (ev.key === "Enter" && input.value) {
-      this._processText(input.value);
-      input.value = "";
-      this._showSendButton = false;
-    }
-  }
-
-  private _handleInput(ev: InputEvent) {
-    const value = (ev.target as HaTextField).value;
-    if (value && !this._showSendButton) {
-      this._showSendButton = true;
-    } else if (!value && this._showSendButton) {
-      this._showSendButton = false;
-    }
-  }
-
-  private _handleSendMessage() {
-    if (this._messageInput.value) {
-      this._processText(this._messageInput.value.trim());
-      this._messageInput.value = "";
-      this._showSendButton = false;
-    }
-  }
-
-  private async _processText(text: string) {
-    this._audio?.pause();
-    this._addMessage({ who: "user", text });
-    const message: Message = {
-      who: "hass",
-      text: "…",
-    };
-    // To make sure the answer is placed at the right user text, we add it before we process it
-    this._addMessage(message);
     try {
-      const unsub = await runAssistPipeline(
+      this._pipeline = await getAssistPipeline(this.hass, this._pipelineId);
-        this.hass,
+    } catch (e: any) {
-        (event) => {
+      if (e.code === "not_found") {
-          if (event.type === "intent-end") {
+        this._errorLoadAssist = "not_found";
-            this._conversationId = event.data.intent_output.conversation_id;
+      } else {
-            const plain = event.data.intent_output.response.speech?.plain;
+        this._errorLoadAssist = "unknown";
-            if (plain) {
+        // eslint-disable-next-line no-console
-              message.text = plain.speech;
+        console.error(e);
-            }
-            this.requestUpdate("_conversation");
-            unsub();
-          }
-          if (event.type === "error") {
-            message.text = event.data.message;
-            message.error = true;
-            this.requestUpdate("_conversation");
-            unsub();
-          }
-        },
-        {
-          start_stage: "intent",
-          input: { text },
-          end_stage: "intent",
-          pipeline: this._pipeline?.id,
-          conversation_id: this._conversationId,
-        }
-      );
-    } catch {
-      message.text = this.hass.localize("ui.dialogs.voice_command.error");
-      message.error = true;
-      this.requestUpdate("_conversation");
-    }
-  }
-
-  private _handleListeningButton(ev) {
-    ev.stopPropagation();
-    ev.preventDefault();
-    this._toggleListening();
-  }
-
-  private _toggleListening() {
-    const supportsMicrophone = AudioRecorder.isSupported;
-    if (!supportsMicrophone) {
-      this._showNotSupportedMessage();
-      return;
-    }
-    if (!this._audioRecorder?.active) {
-      this._startListening();
-    } else {
-      this._stopListening();
-    }
-  }
-
-  private async _showNotSupportedMessage() {
-    this._addMessage({
-      who: "hass",
-      text:
-        // New lines matter for messages
-        // prettier-ignore
-        html`${this.hass.localize(
-          "ui.dialogs.voice_command.not_supported_microphone_browser"
-        )}
-
-        ${this.hass.localize(
-          "ui.dialogs.voice_command.not_supported_microphone_documentation",
-          {
-            documentation_link: html`<a
-                target="_blank"
-                rel="noopener noreferrer"
-                href=${documentationUrl(
-                  this.hass,
-                  "/docs/configuration/securing/#remote-access"
-                )}
-              >${this.hass.localize(
-                  "ui.dialogs.voice_command.not_supported_microphone_documentation_link"
-                )}</a>`,
-          }
-        )}`,
-    });
-  }
-
-  private async _startListening() {
-    this._audio?.pause();
-    if (!this._audioRecorder) {
-      this._audioRecorder = new AudioRecorder((audio) => {
-        if (this._audioBuffer) {
-          this._audioBuffer.push(audio);
-        } else {
-          this._sendAudioChunk(audio);
-        }
-      });
-    }
-    this._stt_binary_handler_id = undefined;
-    this._audioBuffer = [];
-    const userMessage: Message = {
-      who: "user",
-      text: "…",
-    };
-    await this._audioRecorder.start();
-
-    this._addMessage(userMessage);
-    this.requestUpdate("_audioRecorder");
-
-    const hassMessage: Message = {
-      who: "hass",
-      text: "…",
-    };
-    // To make sure the answer is placed at the right user text, we add it before we process it
-    try {
-      const unsub = await runAssistPipeline(
-        this.hass,
-        (event) => {
-          if (event.type === "run-start") {
-            this._stt_binary_handler_id =
-              event.data.runner_data.stt_binary_handler_id;
-          }
-
-          // When we start STT stage, the WS has a binary handler
-          if (event.type === "stt-start" && this._audioBuffer) {
-            // Send the buffer over the WS to the STT engine.
-            for (const buffer of this._audioBuffer) {
-              this._sendAudioChunk(buffer);
-            }
-            this._audioBuffer = undefined;
-          }
-
-          // Stop recording if the server is done with STT stage
-          if (event.type === "stt-end") {
-            this._stt_binary_handler_id = undefined;
-            this._stopListening();
-            userMessage.text = event.data.stt_output.text;
-            this.requestUpdate("_conversation");
-            // To make sure the answer is placed at the right user text, we add it before we process it
-            this._addMessage(hassMessage);
-          }
-
-          if (event.type === "intent-end") {
-            this._conversationId = event.data.intent_output.conversation_id;
-            const plain = event.data.intent_output.response.speech?.plain;
-            if (plain) {
-              hassMessage.text = plain.speech;
-            }
-            this.requestUpdate("_conversation");
-          }
-
-          if (event.type === "tts-end") {
-            const url = event.data.tts_output.url;
-            this._audio = new Audio(url);
-            this._audio.play();
-            this._audio.addEventListener("ended", this._unloadAudio);
-            this._audio.addEventListener("pause", this._unloadAudio);
-            this._audio.addEventListener("canplaythrough", this._playAudio);
-            this._audio.addEventListener("error", this._audioError);
-          }
-
-          if (event.type === "run-end") {
-            this._stt_binary_handler_id = undefined;
-            unsub();
-          }
-
-          if (event.type === "error") {
-            this._stt_binary_handler_id = undefined;
-            if (userMessage.text === "…") {
-              userMessage.text = event.data.message;
-              userMessage.error = true;
-            } else {
-              hassMessage.text = event.data.message;
-              hassMessage.error = true;
-            }
-            this._stopListening();
-            this.requestUpdate("_conversation");
-            unsub();
-          }
-        },
-        {
-          start_stage: "stt",
-          end_stage: this._pipeline?.tts_engine ? "tts" : "intent",
-          input: { sample_rate: this._audioRecorder.sampleRate! },
-          pipeline: this._pipeline?.id,
-          conversation_id: this._conversationId,
-        }
-      );
-    } catch (err: any) {
-      await showAlertDialog(this, {
-        title: "Error starting pipeline",
-        text: err.message || err,
-      });
-      this._stopListening();
-    }
-  }
-
-  private _stopListening() {
-    this._audioRecorder?.stop();
-    this.requestUpdate("_audioRecorder");
-    // We're currently STTing, so finish audio
-    if (this._stt_binary_handler_id) {
-      if (this._audioBuffer) {
-        for (const chunk of this._audioBuffer) {
-          this._sendAudioChunk(chunk);
-        }
       }
-      // Send empty message to indicate we're done streaming.
-      this._sendAudioChunk(new Int16Array());
-      this._stt_binary_handler_id = undefined;
     }
-    this._audioBuffer = undefined;
-  }
-
-  private _sendAudioChunk(chunk: Int16Array) {
-    this.hass.connection.socket!.binaryType = "arraybuffer";
-
-    // eslint-disable-next-line eqeqeq
-    if (this._stt_binary_handler_id == undefined) {
-      return;
-    }
-    // Turn into 8 bit so we can prefix our handler ID.
-    const data = new Uint8Array(1 + chunk.length * 2);
-    data[0] = this._stt_binary_handler_id;
-    data.set(new Uint8Array(chunk.buffer), 1);
-
-    this.hass.connection.socket!.send(data);
-  }
-
-  private _playAudio = () => {
-    this._audio?.play();
-  };
-
-  private _audioError = () => {
-    showAlertDialog(this, { title: "Error playing audio." });
-    this._audio?.removeAttribute("src");
-  };
-
-  private _unloadAudio = () => {
-    this._audio?.removeAttribute("src");
-    this._audio = undefined;
-  };
-
-  private _scrollMessagesBottom() {
-    const scrollContainer = this._scrollContainer;
-    if (!scrollContainer) {
-      return;
-    }
-    scrollContainer.scrollTo(0, 99999);
-  }
-
-  private _computeMessageClasses(message: Message) {
-    return `message ${message.who} ${message.error ? " error" : ""}`;
   }
 
   static get styles(): CSSResultGroup {
     return [
       haStyleDialog,
       css`
-        .listening-icon {
-          position: relative;
-          color: var(--secondary-text-color);
-          margin-right: -24px;
-          margin-inline-end: -24px;
-          margin-inline-start: initial;
-          direction: var(--direction);
-          transform: scaleX(var(--scale-direction));
-        }
-
-        .listening-icon[active] {
-          color: var(--primary-color);
-        }
-
-        .unsupported {
-          color: var(--error-color);
-          position: absolute;
-          --mdc-icon-size: 16px;
-          right: 5px;
-          inset-inline-end: 5px;
-          inset-inline-start: initial;
-          top: 0px;
-        }
-
         ha-dialog {
-          --primary-action-button-flex: 1;
-          --secondary-action-button-flex: 0;
           --mdc-dialog-max-width: 500px;
           --mdc-dialog-max-height: 500px;
           --dialog-content-padding: 0;
@@ -722,157 +293,10 @@ export class HaVoiceCommandDialog extends LitElement {
         ha-button-menu a {
           text-decoration: none;
         }
-        ha-textfield {
+
-          display: block;
+        .pipelines-loading {
-          overflow: hidden;
-        }
-        a.button {
-          text-decoration: none;
-        }
-        a.button > mwc-button {
-          width: 100%;
-        }
-        .side-by-side {
           display: flex;
-          margin: 8px 0;
+          justify-content: center;
-        }
-        .side-by-side > * {
-          flex: 1 0;
-          padding: 4px;
-        }
-        .messages {
-          display: block;
-          height: 400px;
-          box-sizing: border-box;
-          position: relative;
-        }
-        @media all and (max-width: 450px), all and (max-height: 500px) {
-          ha-dialog {
-            --mdc-dialog-max-width: 100%;
-          }
-          .messages {
-            height: 100%;
-            flex: 1;
-          }
-        }
-        .messages-container {
-          position: absolute;
-          bottom: 0px;
-          right: 0px;
-          left: 0px;
-          padding: 24px;
-          box-sizing: border-box;
-          overflow-y: auto;
-          max-height: 100%;
-        }
-        .message {
-          white-space: pre-line;
-          font-size: 18px;
-          clear: both;
-          margin: 8px 0;
-          padding: 8px;
-          border-radius: 15px;
-        }
-        .message p {
-          margin: 0;
-        }
-        .message p:not(:last-child) {
-          margin-bottom: 8px;
-        }
-
-        .message.user {
-          margin-left: 24px;
-          margin-inline-start: 24px;
-          margin-inline-end: initial;
-          float: var(--float-end);
-          text-align: right;
-          border-bottom-right-radius: 0px;
-          background-color: var(--primary-color);
-          color: var(--text-primary-color);
-          direction: var(--direction);
-        }
-
-        .message.hass {
-          margin-right: 24px;
-          margin-inline-end: 24px;
-          margin-inline-start: initial;
-          float: var(--float-start);
-          border-bottom-left-radius: 0px;
-          background-color: var(--secondary-background-color);
-          color: var(--primary-text-color);
-          direction: var(--direction);
-        }
-
-        .message.user a {
-          color: var(--text-primary-color);
-        }
-
-        .message.hass a {
-          color: var(--primary-text-color);
-        }
-
-        .message img {
-          width: 100%;
-          border-radius: 10px;
-        }
-
-        .message.error {
-          background-color: var(--error-color);
-          color: var(--text-primary-color);
-        }
-
-        .input {
-          margin-left: 0;
-          margin-right: 0;
-        }
-
-        .bouncer {
-          width: 48px;
-          height: 48px;
-          position: absolute;
-        }
-        .double-bounce1,
-        .double-bounce2 {
-          width: 48px;
-          height: 48px;
-          border-radius: 50%;
-          background-color: var(--primary-color);
-          opacity: 0.2;
-          position: absolute;
-          top: 0;
-          left: 0;
-          -webkit-animation: sk-bounce 2s infinite ease-in-out;
-          animation: sk-bounce 2s infinite ease-in-out;
-        }
-        .double-bounce2 {
-          -webkit-animation-delay: -1s;
-          animation-delay: -1s;
-        }
-        @-webkit-keyframes sk-bounce {
-          0%,
-          100% {
-            -webkit-transform: scale(0);
-          }
-          50% {
-            -webkit-transform: scale(1);
-          }
-        }
-        @keyframes sk-bounce {
-          0%,
-          100% {
-            transform: scale(0);
-            -webkit-transform: scale(0);
-          }
-          50% {
-            transform: scale(1);
-            -webkit-transform: scale(1);
-          }
-        }
-
-        @media all and (max-width: 450px), all and (max-height: 500px) {
-          .message {
-            font-size: 16px;
-          }
         }
       `,
     ];

src/translations/en.json

@@ -1215,7 +1215,9 @@
         "manage_assistants": "Manage assistants",
         "not_supported_microphone_browser": "Your connection to Home Assistant is not secured using HTTPS. This causes browsers to block Home Assistant from accessing the microphone.",
         "not_supported_microphone_documentation": "Use the Home Assistant app or visit {documentation_link} to learn how to use a secure URL",
-        "not_supported_microphone_documentation_link": "the documentation"
+        "not_supported_microphone_documentation_link": "the documentation",
+        "unknown_error_load_assist": "Loading the assist pipeline failed",
+        "not_found_error_load_assist": "Cannot find the assist pipeline"
       },
       "generic": {
         "cancel": "Cancel",