From 84157c8ea5a1a8619790e43604fd478f1028cf4f Mon Sep 17 00:00:00 2001 From: Wendelin <12148533+wendevlin@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:23:48 +0100 Subject: [PATCH] Extract assist-chat out of voice-command-dialog (#23184) --- src/components/ha-assist-chat.ts | 639 +++++++++++++++ .../ha-voice-command-dialog.ts | 732 ++---------------- src/translations/en.json | 4 +- 3 files changed, 720 insertions(+), 655 deletions(-) create mode 100644 src/components/ha-assist-chat.ts diff --git a/src/components/ha-assist-chat.ts b/src/components/ha-assist-chat.ts new file mode 100644 index 0000000000..21bad3eefa --- /dev/null +++ b/src/components/ha-assist-chat.ts @@ -0,0 +1,639 @@ +import type { CSSResultGroup, PropertyValues, TemplateResult } from "lit"; +import { css, LitElement, html, nothing } from "lit"; +import { mdiAlertCircle, mdiMicrophone, mdiSend } from "@mdi/js"; +import { customElement, property, query, state } from "lit/decorators"; +import { classMap } from "lit/directives/class-map"; +import type { HomeAssistant } from "../types"; +import { + runAssistPipeline, + type AssistPipeline, +} from "../data/assist_pipeline"; +import { supportsFeature } from "../common/entity/supports-feature"; +import { ConversationEntityFeature } from "../data/conversation"; +import { AudioRecorder } from "../util/audio-recorder"; +import "./ha-alert"; +import "./ha-textfield"; +import type { HaTextField } from "./ha-textfield"; +import { documentationUrl } from "../util/documentation-url"; +import { showAlertDialog } from "../dialogs/generic/show-dialog-box"; + +interface AssistMessage { + who: string; + text?: string | TemplateResult; + error?: boolean; +} + +@customElement("ha-assist-chat") +export class HaAssistChat extends LitElement { + @property({ attribute: false }) public hass!: HomeAssistant; + + @property({ attribute: false }) public pipeline?: AssistPipeline; + + @property({ type: Boolean, attribute: false }) + public startListening?: boolean; + + @query("#message-input") private _messageInput!: HaTextField; + + @query("#scroll-container") private _scrollContainer!: HTMLDivElement; + + @state() private _conversation: AssistMessage[] = []; + + @state() private _showSendButton = false; + + @state() private _processing = false; + + private _conversationId: string | null = null; + + private _audioRecorder?: AudioRecorder; + + private _audioBuffer?: Int16Array[]; + + private _audio?: HTMLAudioElement; + + private _stt_binary_handler_id?: number | null; + + protected willUpdate(changedProperties: PropertyValues): void { + if (!this.hasUpdated || changedProperties.has("pipeline")) { + this._conversation = [ + { + who: "hass", + text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"), + }, + ]; + } + } + + protected firstUpdated(changedProperties: PropertyValues): void { + super.firstUpdated(changedProperties); + if ( + this.startListening && + this.pipeline && + this.pipeline.stt_engine && + AudioRecorder.isSupported + ) { + this._toggleListening(); + } + setTimeout(() => this._messageInput.focus(), 0); + } + + protected updated(changedProps: PropertyValues) { + super.updated(changedProps); + if (changedProps.has("_conversation")) { + this._scrollMessagesBottom(); + } + } + + public disconnectedCallback() { + super.disconnectedCallback(); + this._audioRecorder?.close(); + this._audioRecorder = undefined; + this._audio?.pause(); + this._conversation = []; + this._conversationId = null; + } + + protected render(): TemplateResult { + const controlHA = !this.pipeline + ? false + : this.pipeline.prefer_local_intents || + (this.hass.states[this.pipeline.conversation_engine] + ? supportsFeature( + this.hass.states[this.pipeline.conversation_engine], + ConversationEntityFeature.CONTROL + ) + : true); + const supportsMicrophone = AudioRecorder.isSupported; + const supportsSTT = this.pipeline?.stt_engine; + + return html` + ${controlHA + ? nothing + : html` + + ${this.hass.localize( + "ui.dialogs.voice_command.conversation_no_control" + )} + + `} +
+
+ ${this._conversation!.map( + // New lines matter for messages + // prettier-ignore + (message) => html` +
${message.text}
+ ` + )} +
+
+
+ +
+ ${this._showSendButton || !supportsSTT + ? html` + + + ` + : html` + ${this._audioRecorder?.active + ? html` +
+
+
+
+ ` + : nothing} + +
+ + + ${!supportsMicrophone + ? html` + + ` + : null} +
+ `} +
+ +
+ `; + } + + private _scrollMessagesBottom() { + const scrollContainer = this._scrollContainer; + if (!scrollContainer) { + return; + } + scrollContainer.scrollTo(0, scrollContainer.scrollHeight); + } + + private _handleKeyUp(ev: KeyboardEvent) { + const input = ev.target as HaTextField; + if (!this._processing && ev.key === "Enter" && input.value) { + this._processText(input.value); + input.value = ""; + this._showSendButton = false; + } + } + + private _handleInput(ev: InputEvent) { + const value = (ev.target as HaTextField).value; + if (value && !this._showSendButton) { + this._showSendButton = true; + } else if (!value && this._showSendButton) { + this._showSendButton = false; + } + } + + private _handleSendMessage() { + if (this._messageInput.value) { + this._processText(this._messageInput.value.trim()); + this._messageInput.value = ""; + this._showSendButton = false; + } + } + + private _handleListeningButton(ev) { + ev.stopPropagation(); + ev.preventDefault(); + this._toggleListening(); + } + + private async _toggleListening() { + const supportsMicrophone = AudioRecorder.isSupported; + if (!supportsMicrophone) { + this._showNotSupportedMessage(); + return; + } + if (!this._audioRecorder?.active) { + this._startListening(); + } else { + this._stopListening(); + } + } + + private _addMessage(message: AssistMessage) { + this._conversation = [...this._conversation!, message]; + } + + private async _showNotSupportedMessage() { + this._addMessage({ + who: "hass", + text: + // New lines matter for messages + // prettier-ignore + html`${this.hass.localize( + "ui.dialogs.voice_command.not_supported_microphone_browser" + )} + + ${this.hass.localize( + "ui.dialogs.voice_command.not_supported_microphone_documentation", + { + documentation_link: html`${this.hass.localize( + "ui.dialogs.voice_command.not_supported_microphone_documentation_link" + )}`, + } + )}`, + }); + } + + private async _startListening() { + this._processing = true; + this._audio?.pause(); + if (!this._audioRecorder) { + this._audioRecorder = new AudioRecorder((audio) => { + if (this._audioBuffer) { + this._audioBuffer.push(audio); + } else { + this._sendAudioChunk(audio); + } + }); + } + this._stt_binary_handler_id = undefined; + this._audioBuffer = []; + const userMessage: AssistMessage = { + who: "user", + text: "…", + }; + await this._audioRecorder.start(); + + this._addMessage(userMessage); + this.requestUpdate("_audioRecorder"); + + const hassMessage: AssistMessage = { + who: "hass", + text: "…", + }; + // To make sure the answer is placed at the right user text, we add it before we process it + try { + const unsub = await runAssistPipeline( + this.hass, + (event) => { + if (event.type === "run-start") { + this._stt_binary_handler_id = + event.data.runner_data.stt_binary_handler_id; + } + + // When we start STT stage, the WS has a binary handler + if (event.type === "stt-start" && this._audioBuffer) { + // Send the buffer over the WS to the STT engine. + for (const buffer of this._audioBuffer) { + this._sendAudioChunk(buffer); + } + this._audioBuffer = undefined; + } + + // Stop recording if the server is done with STT stage + if (event.type === "stt-end") { + this._stt_binary_handler_id = undefined; + this._stopListening(); + userMessage.text = event.data.stt_output.text; + this.requestUpdate("_conversation"); + // To make sure the answer is placed at the right user text, we add it before we process it + this._addMessage(hassMessage); + } + + if (event.type === "intent-end") { + this._conversationId = event.data.intent_output.conversation_id; + const plain = event.data.intent_output.response.speech?.plain; + if (plain) { + hassMessage.text = plain.speech; + } + this.requestUpdate("_conversation"); + } + + if (event.type === "tts-end") { + const url = event.data.tts_output.url; + this._audio = new Audio(url); + this._audio.play(); + this._audio.addEventListener("ended", this._unloadAudio); + this._audio.addEventListener("pause", this._unloadAudio); + this._audio.addEventListener("canplaythrough", this._playAudio); + this._audio.addEventListener("error", this._audioError); + } + + if (event.type === "run-end") { + this._stt_binary_handler_id = undefined; + unsub(); + } + + if (event.type === "error") { + this._stt_binary_handler_id = undefined; + if (userMessage.text === "…") { + userMessage.text = event.data.message; + userMessage.error = true; + } else { + hassMessage.text = event.data.message; + hassMessage.error = true; + } + this._stopListening(); + this.requestUpdate("_conversation"); + unsub(); + } + }, + { + start_stage: "stt", + end_stage: this.pipeline?.tts_engine ? "tts" : "intent", + input: { sample_rate: this._audioRecorder.sampleRate! }, + pipeline: this.pipeline?.id, + conversation_id: this._conversationId, + } + ); + } catch (err: any) { + await showAlertDialog(this, { + title: "Error starting pipeline", + text: err.message || err, + }); + this._stopListening(); + } finally { + this._processing = false; + } + } + + private _stopListening() { + this._audioRecorder?.stop(); + this.requestUpdate("_audioRecorder"); + // We're currently STTing, so finish audio + if (this._stt_binary_handler_id) { + if (this._audioBuffer) { + for (const chunk of this._audioBuffer) { + this._sendAudioChunk(chunk); + } + } + // Send empty message to indicate we're done streaming. + this._sendAudioChunk(new Int16Array()); + this._stt_binary_handler_id = undefined; + } + this._audioBuffer = undefined; + } + + private _sendAudioChunk(chunk: Int16Array) { + this.hass.connection.socket!.binaryType = "arraybuffer"; + + // eslint-disable-next-line eqeqeq + if (this._stt_binary_handler_id == undefined) { + return; + } + // Turn into 8 bit so we can prefix our handler ID. + const data = new Uint8Array(1 + chunk.length * 2); + data[0] = this._stt_binary_handler_id; + data.set(new Uint8Array(chunk.buffer), 1); + + this.hass.connection.socket!.send(data); + } + + private _playAudio = () => { + this._audio?.play(); + }; + + private _audioError = () => { + showAlertDialog(this, { title: "Error playing audio." }); + this._audio?.removeAttribute("src"); + }; + + private _unloadAudio = () => { + this._audio?.removeAttribute("src"); + this._audio = undefined; + }; + + private async _processText(text: string) { + this._processing = true; + this._audio?.pause(); + this._addMessage({ who: "user", text }); + const message: AssistMessage = { + who: "hass", + text: "…", + }; + // To make sure the answer is placed at the right user text, we add it before we process it + this._addMessage(message); + try { + const unsub = await runAssistPipeline( + this.hass, + (event) => { + if (event.type === "intent-end") { + this._conversationId = event.data.intent_output.conversation_id; + const plain = event.data.intent_output.response.speech?.plain; + if (plain) { + message.text = plain.speech; + } + this.requestUpdate("_conversation"); + unsub(); + } + if (event.type === "error") { + message.text = event.data.message; + message.error = true; + this.requestUpdate("_conversation"); + unsub(); + } + }, + { + start_stage: "intent", + input: { text }, + end_stage: "intent", + pipeline: this.pipeline?.id, + conversation_id: this._conversationId, + } + ); + } catch { + message.text = this.hass.localize("ui.dialogs.voice_command.error"); + message.error = true; + this.requestUpdate("_conversation"); + } finally { + this._processing = false; + } + } + + static get styles(): CSSResultGroup { + return css` + :host { + flex: 1; + display: flex; + flex-direction: column; + min-height: var(--ha-assist-chat-min-height, 415px); + } + ha-textfield { + display: block; + margin: 0 24px 16px; + } + .messages { + flex: 1; + display: block; + box-sizing: border-box; + position: relative; + } + .messages-container { + position: absolute; + bottom: 0px; + right: 0px; + left: 0px; + padding: 24px; + box-sizing: border-box; + overflow-y: auto; + max-height: 100%; + } + .message { + white-space: pre-line; + font-size: 18px; + clear: both; + margin: 8px 0; + padding: 8px; + border-radius: 15px; + } + + @media all and (max-width: 450px), all and (max-height: 500px) { + .message { + font-size: 16px; + } + } + + .message p { + margin: 0; + } + .message p:not(:last-child) { + margin-bottom: 8px; + } + + .message.user { + margin-left: 24px; + margin-inline-start: 24px; + margin-inline-end: initial; + float: var(--float-end); + text-align: right; + border-bottom-right-radius: 0px; + background-color: var(--primary-color); + color: var(--text-primary-color); + direction: var(--direction); + } + + .message.hass { + margin-right: 24px; + margin-inline-end: 24px; + margin-inline-start: initial; + float: var(--float-start); + border-bottom-left-radius: 0px; + background-color: var(--secondary-background-color); + + color: var(--primary-text-color); + direction: var(--direction); + } + + .message.user a { + color: var(--text-primary-color); + } + + .message.hass a { + color: var(--primary-text-color); + } + + .message.error { + background-color: var(--error-color); + color: var(--text-primary-color); + } + + .bouncer { + width: 48px; + height: 48px; + position: absolute; + } + .double-bounce1, + .double-bounce2 { + width: 48px; + height: 48px; + border-radius: 50%; + background-color: var(--primary-color); + opacity: 0.2; + position: absolute; + top: 0; + left: 0; + -webkit-animation: sk-bounce 2s infinite ease-in-out; + animation: sk-bounce 2s infinite ease-in-out; + } + .double-bounce2 { + -webkit-animation-delay: -1s; + animation-delay: -1s; + } + @-webkit-keyframes sk-bounce { + 0%, + 100% { + -webkit-transform: scale(0); + } + 50% { + -webkit-transform: scale(1); + } + } + @keyframes sk-bounce { + 0%, + 100% { + transform: scale(0); + -webkit-transform: scale(0); + } + 50% { + transform: scale(1); + -webkit-transform: scale(1); + } + } + + .listening-icon { + position: relative; + color: var(--secondary-text-color); + margin-right: -24px; + margin-inline-end: -24px; + margin-inline-start: initial; + direction: var(--direction); + transform: scaleX(var(--scale-direction)); + } + + .listening-icon[active] { + color: var(--primary-color); + } + + .unsupported { + color: var(--error-color); + position: absolute; + --mdc-icon-size: 16px; + right: 5px; + inset-inline-end: 5px; + inset-inline-start: initial; + top: 0px; + } + `; + } +} + +declare global { + interface HTMLElementTagNameMap { + "ha-assist-chat": HaAssistChat; + } +} diff --git a/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts b/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts index 42c311bef2..1bfa890e95 100644 --- a/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts +++ b/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts @@ -1,16 +1,13 @@ import "@material/mwc-button/mwc-button"; import { - mdiAlertCircle, mdiChevronDown, mdiClose, mdiHelpCircleOutline, - mdiMicrophone, - mdiSend, mdiStar, } from "@mdi/js"; -import type { CSSResultGroup, PropertyValues, TemplateResult } from "lit"; +import type { CSSResultGroup, PropertyValues } from "lit"; import { css, html, LitElement, nothing } from "lit"; -import { customElement, property, query, state } from "lit/decorators"; +import { customElement, property, state } from "lit/decorators"; import { storage } from "../../common/decorators/storage"; import { fireEvent } from "../../common/dom/fire_event"; import { stopPropagation } from "../../common/dom/stop_propagation"; @@ -20,36 +17,23 @@ import "../../components/ha-dialog"; import "../../components/ha-dialog-header"; import "../../components/ha-icon-button"; import "../../components/ha-list-item"; -import "../../components/ha-textfield"; import "../../components/ha-alert"; -import type { HaTextField } from "../../components/ha-textfield"; +import "../../components/ha-assist-chat"; +import "../../components/ha-circular-progress"; import type { AssistPipeline } from "../../data/assist_pipeline"; import { getAssistPipeline, listAssistPipelines, - runAssistPipeline, } from "../../data/assist_pipeline"; import { haStyleDialog } from "../../resources/styles"; import type { HomeAssistant } from "../../types"; -import { AudioRecorder } from "../../util/audio-recorder"; import { documentationUrl } from "../../util/documentation-url"; -import { showAlertDialog } from "../generic/show-dialog-box"; import type { VoiceCommandDialogParams } from "./show-ha-voice-command-dialog"; -import { supportsFeature } from "../../common/entity/supports-feature"; -import { ConversationEntityFeature } from "../../data/conversation"; - -interface Message { - who: string; - text?: string | TemplateResult; - error?: boolean; -} @customElement("ha-voice-command-dialog") export class HaVoiceCommandDialog extends LitElement { @property({ attribute: false }) public hass!: HomeAssistant; - @state() private _conversation?: Message[]; - @state() private _opened = false; @storage({ @@ -61,69 +45,34 @@ export class HaVoiceCommandDialog extends LitElement { @state() private _pipeline?: AssistPipeline; - @state() private _showSendButton = false; - @state() private _pipelines?: AssistPipeline[]; @state() private _preferredPipeline?: string; - @query("#scroll-container") private _scrollContainer!: HTMLDivElement; + @state() private _errorLoadAssist?: "not_found" | "unknown"; - @query("#message-input") private _messageInput!: HaTextField; - - private _conversationId: string | null = null; - - private _audioRecorder?: AudioRecorder; - - private _audioBuffer?: Int16Array[]; - - private _audio?: HTMLAudioElement; - - private _stt_binary_handler_id?: number | null; - - private _pipelinePromise?: Promise; + private _startListening = false; public async showDialog( params: Required ): Promise { - if (params.pipeline_id === "last_used") { - // Do not set pipeline id (retrieve from storage) - } else if (params.pipeline_id === "preferred") { + if ( + params.pipeline_id === "preferred" || + (params.pipeline_id === "last_used" && !this._pipelineId) + ) { await this._loadPipelines(); this._pipelineId = this._preferredPipeline; - } else { + } else if (!["last_used", "preferred"].includes(params.pipeline_id)) { this._pipelineId = params.pipeline_id; } - this._conversation = [ - { - who: "hass", - text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"), - }, - ]; + this._startListening = params.start_listening; this._opened = true; - await this.updateComplete; - this._scrollMessagesBottom(); - - await this._pipelinePromise; - if ( - params?.start_listening && - this._pipeline?.stt_engine && - AudioRecorder.isSupported - ) { - this._toggleListening(); - } } public async closeDialog(): Promise { this._opened = false; - this._pipeline = undefined; this._pipelines = undefined; - this._conversation = undefined; - this._conversationId = null; - this._audioRecorder?.close(); - this._audioRecorder = undefined; - this._audio?.pause(); fireEvent(this, "dialog-closed", { dialog: this.localName }); } @@ -132,24 +81,13 @@ export class HaVoiceCommandDialog extends LitElement { return nothing; } - const controlHA = !this._pipeline - ? false - : this._pipeline.prefer_local_intents || - (this.hass.states[this._pipeline.conversation_engine] - ? supportsFeature( - this.hass.states[this._pipeline.conversation_engine], - ConversationEntityFeature.CONTROL - ) - : true); - const supportsMicrophone = AudioRecorder.isSupported; - const supportsSTT = this._pipeline?.stt_engine; - return html` - ${this._pipelines?.map( - (pipeline) => - html` - ${pipeline.name}${pipeline.id === this._preferredPipeline - ? html` - - ` - : nothing} - ` - )} + ${!this._pipelines + ? html`
+ +
` + : this._pipelines?.map( + (pipeline) => + html` + ${pipeline.name}${pipeline.id === + this._preferredPipeline + ? html` + + ` + : nothing} + ` + )} ${this.hass.user?.is_admin ? html`
  •      - ${controlHA - ? nothing - : html` - - ${this.hass.localize( - "ui.dialogs.voice_command.conversation_no_control" - )} - - `} -
    -
    - ${this._conversation!.map( - // New lines matter for messages - // prettier-ignore - (message) => html` -
    ${message.text}
    - ` - )} -
    -
    -
    - - - ${this._showSendButton || !supportsSTT - ? html` - - - ` - : html` - ${this._audioRecorder?.active - ? html` -
    -
    -
    -
    - ` - : nothing} -
    - - - ${!supportsMicrophone - ? html` - - ` - : null} -
    - `} -     -     -
    + ${this._pipeline + ? html` + + + ` + : html`
    + +
    `} + ${this._errorLoadAssist + ? html` + ${this.hass.localize( + `ui.dialogs.voice_command.${this._errorLoadAssist}_error_load_assist` + )} + ` + : nothing}     `; } @@ -298,23 +193,14 @@ export class HaVoiceCommandDialog extends LitElement { protected willUpdate(changedProperties: PropertyValues): void { if ( changedProperties.has("_pipelineId") || - (changedProperties.has("_opened") && this._opened === true) + (changedProperties.has("_opened") && + this._opened === true && + this._pipelineId) ) { this._getPipeline(); } } - private async _getPipeline() { - try { - this._pipelinePromise = getAssistPipeline(this.hass, this._pipelineId); - this._pipeline = await this._pipelinePromise; - } catch (e: any) { - if (e.code === "not_found") { - this._pipelineId = undefined; - } - } - } - private async _loadPipelines() { if (this._pipelines) { return; @@ -328,343 +214,28 @@ export class HaVoiceCommandDialog extends LitElement { private async _selectPipeline(ev: CustomEvent) { this._pipelineId = (ev.currentTarget as any).pipeline; - this._conversation = [ - { - who: "hass", - text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"), - }, - ]; await this.updateComplete; - this._scrollMessagesBottom(); } - protected updated(changedProps: PropertyValues) { - super.updated(changedProps); - if (changedProps.has("_conversation") || changedProps.has("results")) { - this._scrollMessagesBottom(); - } - } - - private _addMessage(message: Message) { - this._conversation = [...this._conversation!, message]; - } - - private _handleKeyUp(ev: KeyboardEvent) { - const input = ev.target as HaTextField; - if (ev.key === "Enter" && input.value) { - this._processText(input.value); - input.value = ""; - this._showSendButton = false; - } - } - - private _handleInput(ev: InputEvent) { - const value = (ev.target as HaTextField).value; - if (value && !this._showSendButton) { - this._showSendButton = true; - } else if (!value && this._showSendButton) { - this._showSendButton = false; - } - } - - private _handleSendMessage() { - if (this._messageInput.value) { - this._processText(this._messageInput.value.trim()); - this._messageInput.value = ""; - this._showSendButton = false; - } - } - - private async _processText(text: string) { - this._audio?.pause(); - this._addMessage({ who: "user", text }); - const message: Message = { - who: "hass", - text: "…", - }; - // To make sure the answer is placed at the right user text, we add it before we process it - this._addMessage(message); + private async _getPipeline() { try { - const unsub = await runAssistPipeline( - this.hass, - (event) => { - if (event.type === "intent-end") { - this._conversationId = event.data.intent_output.conversation_id; - const plain = event.data.intent_output.response.speech?.plain; - if (plain) { - message.text = plain.speech; - } - this.requestUpdate("_conversation"); - unsub(); - } - if (event.type === "error") { - message.text = event.data.message; - message.error = true; - this.requestUpdate("_conversation"); - unsub(); - } - }, - { - start_stage: "intent", - input: { text }, - end_stage: "intent", - pipeline: this._pipeline?.id, - conversation_id: this._conversationId, - } - ); - } catch { - message.text = this.hass.localize("ui.dialogs.voice_command.error"); - message.error = true; - this.requestUpdate("_conversation"); - } - } - - private _handleListeningButton(ev) { - ev.stopPropagation(); - ev.preventDefault(); - this._toggleListening(); - } - - private _toggleListening() { - const supportsMicrophone = AudioRecorder.isSupported; - if (!supportsMicrophone) { - this._showNotSupportedMessage(); - return; - } - if (!this._audioRecorder?.active) { - this._startListening(); - } else { - this._stopListening(); - } - } - - private async _showNotSupportedMessage() { - this._addMessage({ - who: "hass", - text: - // New lines matter for messages - // prettier-ignore - html`${this.hass.localize( - "ui.dialogs.voice_command.not_supported_microphone_browser" - )} - - ${this.hass.localize( - "ui.dialogs.voice_command.not_supported_microphone_documentation", - { - documentation_link: html`${this.hass.localize( - "ui.dialogs.voice_command.not_supported_microphone_documentation_link" - )}`, - } - )}`, - }); - } - - private async _startListening() { - this._audio?.pause(); - if (!this._audioRecorder) { - this._audioRecorder = new AudioRecorder((audio) => { - if (this._audioBuffer) { - this._audioBuffer.push(audio); - } else { - this._sendAudioChunk(audio); - } - }); - } - this._stt_binary_handler_id = undefined; - this._audioBuffer = []; - const userMessage: Message = { - who: "user", - text: "…", - }; - await this._audioRecorder.start(); - - this._addMessage(userMessage); - this.requestUpdate("_audioRecorder"); - - const hassMessage: Message = { - who: "hass", - text: "…", - }; - // To make sure the answer is placed at the right user text, we add it before we process it - try { - const unsub = await runAssistPipeline( - this.hass, - (event) => { - if (event.type === "run-start") { - this._stt_binary_handler_id = - event.data.runner_data.stt_binary_handler_id; - } - - // When we start STT stage, the WS has a binary handler - if (event.type === "stt-start" && this._audioBuffer) { - // Send the buffer over the WS to the STT engine. - for (const buffer of this._audioBuffer) { - this._sendAudioChunk(buffer); - } - this._audioBuffer = undefined; - } - - // Stop recording if the server is done with STT stage - if (event.type === "stt-end") { - this._stt_binary_handler_id = undefined; - this._stopListening(); - userMessage.text = event.data.stt_output.text; - this.requestUpdate("_conversation"); - // To make sure the answer is placed at the right user text, we add it before we process it - this._addMessage(hassMessage); - } - - if (event.type === "intent-end") { - this._conversationId = event.data.intent_output.conversation_id; - const plain = event.data.intent_output.response.speech?.plain; - if (plain) { - hassMessage.text = plain.speech; - } - this.requestUpdate("_conversation"); - } - - if (event.type === "tts-end") { - const url = event.data.tts_output.url; - this._audio = new Audio(url); - this._audio.play(); - this._audio.addEventListener("ended", this._unloadAudio); - this._audio.addEventListener("pause", this._unloadAudio); - this._audio.addEventListener("canplaythrough", this._playAudio); - this._audio.addEventListener("error", this._audioError); - } - - if (event.type === "run-end") { - this._stt_binary_handler_id = undefined; - unsub(); - } - - if (event.type === "error") { - this._stt_binary_handler_id = undefined; - if (userMessage.text === "…") { - userMessage.text = event.data.message; - userMessage.error = true; - } else { - hassMessage.text = event.data.message; - hassMessage.error = true; - } - this._stopListening(); - this.requestUpdate("_conversation"); - unsub(); - } - }, - { - start_stage: "stt", - end_stage: this._pipeline?.tts_engine ? "tts" : "intent", - input: { sample_rate: this._audioRecorder.sampleRate! }, - pipeline: this._pipeline?.id, - conversation_id: this._conversationId, - } - ); - } catch (err: any) { - await showAlertDialog(this, { - title: "Error starting pipeline", - text: err.message || err, - }); - this._stopListening(); - } - } - - private _stopListening() { - this._audioRecorder?.stop(); - this.requestUpdate("_audioRecorder"); - // We're currently STTing, so finish audio - if (this._stt_binary_handler_id) { - if (this._audioBuffer) { - for (const chunk of this._audioBuffer) { - this._sendAudioChunk(chunk); - } + this._pipeline = await getAssistPipeline(this.hass, this._pipelineId); + } catch (e: any) { + if (e.code === "not_found") { + this._errorLoadAssist = "not_found"; + } else { + this._errorLoadAssist = "unknown"; + // eslint-disable-next-line no-console + console.error(e); } - // Send empty message to indicate we're done streaming. - this._sendAudioChunk(new Int16Array()); - this._stt_binary_handler_id = undefined; } - this._audioBuffer = undefined; - } - - private _sendAudioChunk(chunk: Int16Array) { - this.hass.connection.socket!.binaryType = "arraybuffer"; - - // eslint-disable-next-line eqeqeq - if (this._stt_binary_handler_id == undefined) { - return; - } - // Turn into 8 bit so we can prefix our handler ID. - const data = new Uint8Array(1 + chunk.length * 2); - data[0] = this._stt_binary_handler_id; - data.set(new Uint8Array(chunk.buffer), 1); - - this.hass.connection.socket!.send(data); - } - - private _playAudio = () => { - this._audio?.play(); - }; - - private _audioError = () => { - showAlertDialog(this, { title: "Error playing audio." }); - this._audio?.removeAttribute("src"); - }; - - private _unloadAudio = () => { - this._audio?.removeAttribute("src"); - this._audio = undefined; - }; - - private _scrollMessagesBottom() { - const scrollContainer = this._scrollContainer; - if (!scrollContainer) { - return; - } - scrollContainer.scrollTo(0, 99999); - } - - private _computeMessageClasses(message: Message) { - return `message ${message.who} ${message.error ? " error" : ""}`; } static get styles(): CSSResultGroup { return [ haStyleDialog, css` - .listening-icon { - position: relative; - color: var(--secondary-text-color); - margin-right: -24px; - margin-inline-end: -24px; - margin-inline-start: initial; - direction: var(--direction); - transform: scaleX(var(--scale-direction)); - } - - .listening-icon[active] { - color: var(--primary-color); - } - - .unsupported { - color: var(--error-color); - position: absolute; - --mdc-icon-size: 16px; - right: 5px; - inset-inline-end: 5px; - inset-inline-start: initial; - top: 0px; - } - ha-dialog { - --primary-action-button-flex: 1; - --secondary-action-button-flex: 0; --mdc-dialog-max-width: 500px; --mdc-dialog-max-height: 500px; --dialog-content-padding: 0; @@ -722,157 +293,10 @@ export class HaVoiceCommandDialog extends LitElement { ha-button-menu a { text-decoration: none; } - ha-textfield { - display: block; - overflow: hidden; - } - a.button { - text-decoration: none; - } - a.button > mwc-button { - width: 100%; - } - .side-by-side { + + .pipelines-loading { display: flex; - margin: 8px 0; - } - .side-by-side > * { - flex: 1 0; - padding: 4px; - } - .messages { - display: block; - height: 400px; - box-sizing: border-box; - position: relative; - } - @media all and (max-width: 450px), all and (max-height: 500px) { - ha-dialog { - --mdc-dialog-max-width: 100%; - } - .messages { - height: 100%; - flex: 1; - } - } - .messages-container { - position: absolute; - bottom: 0px; - right: 0px; - left: 0px; - padding: 24px; - box-sizing: border-box; - overflow-y: auto; - max-height: 100%; - } - .message { - white-space: pre-line; - font-size: 18px; - clear: both; - margin: 8px 0; - padding: 8px; - border-radius: 15px; - } - .message p { - margin: 0; - } - .message p:not(:last-child) { - margin-bottom: 8px; - } - - .message.user { - margin-left: 24px; - margin-inline-start: 24px; - margin-inline-end: initial; - float: var(--float-end); - text-align: right; - border-bottom-right-radius: 0px; - background-color: var(--primary-color); - color: var(--text-primary-color); - direction: var(--direction); - } - - .message.hass { - margin-right: 24px; - margin-inline-end: 24px; - margin-inline-start: initial; - float: var(--float-start); - border-bottom-left-radius: 0px; - background-color: var(--secondary-background-color); - color: var(--primary-text-color); - direction: var(--direction); - } - - .message.user a { - color: var(--text-primary-color); - } - - .message.hass a { - color: var(--primary-text-color); - } - - .message img { - width: 100%; - border-radius: 10px; - } - - .message.error { - background-color: var(--error-color); - color: var(--text-primary-color); - } - - .input { - margin-left: 0; - margin-right: 0; - } - - .bouncer { - width: 48px; - height: 48px; - position: absolute; - } - .double-bounce1, - .double-bounce2 { - width: 48px; - height: 48px; - border-radius: 50%; - background-color: var(--primary-color); - opacity: 0.2; - position: absolute; - top: 0; - left: 0; - -webkit-animation: sk-bounce 2s infinite ease-in-out; - animation: sk-bounce 2s infinite ease-in-out; - } - .double-bounce2 { - -webkit-animation-delay: -1s; - animation-delay: -1s; - } - @-webkit-keyframes sk-bounce { - 0%, - 100% { - -webkit-transform: scale(0); - } - 50% { - -webkit-transform: scale(1); - } - } - @keyframes sk-bounce { - 0%, - 100% { - transform: scale(0); - -webkit-transform: scale(0); - } - 50% { - transform: scale(1); - -webkit-transform: scale(1); - } - } - - @media all and (max-width: 450px), all and (max-height: 500px) { - .message { - font-size: 16px; - } + justify-content: center; } `, ]; diff --git a/src/translations/en.json b/src/translations/en.json index 89611846f3..9f685eca89 100644 --- a/src/translations/en.json +++ b/src/translations/en.json @@ -1215,7 +1215,9 @@ "manage_assistants": "Manage assistants", "not_supported_microphone_browser": "Your connection to Home Assistant is not secured using HTTPS. This causes browsers to block Home Assistant from accessing the microphone.", "not_supported_microphone_documentation": "Use the Home Assistant app or visit {documentation_link} to learn how to use a secure URL", - "not_supported_microphone_documentation_link": "the documentation" + "not_supported_microphone_documentation_link": "the documentation", + "unknown_error_load_assist": "Loading the assist pipeline failed", + "not_found_error_load_assist": "Cannot find the assist pipeline" }, "generic": { "cancel": "Cancel",