From 5f0497a3b87203a7ad33377e0d3736495eb5362b Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Fri, 2 Aug 2024 08:57:32 +0000 Subject: [PATCH] Break out assist-chat from voice command dialog --- .../voice-command-dialog/assist-chat.ts | 628 ++++++++++++++++++ .../ha-voice-command-dialog.ts | 601 +---------------- 2 files changed, 638 insertions(+), 591 deletions(-) create mode 100644 src/dialogs/voice-command-dialog/assist-chat.ts diff --git a/src/dialogs/voice-command-dialog/assist-chat.ts b/src/dialogs/voice-command-dialog/assist-chat.ts new file mode 100644 index 0000000000..7bb5df429c --- /dev/null +++ b/src/dialogs/voice-command-dialog/assist-chat.ts @@ -0,0 +1,628 @@ +import { mdiAlertCircle, mdiMicrophone, mdiSend } from "@mdi/js"; +import { + css, + CSSResultGroup, + html, + LitElement, + nothing, + PropertyValues, + TemplateResult, +} from "lit"; +import { customElement, property, query, state } from "lit/decorators"; +import "../../components/ha-icon-button"; +import "../../components/ha-textfield"; +import type { HaTextField } from "../../components/ha-textfield"; +import { + AssistPipeline, + getAssistPipeline, + runAssistPipeline, +} from "../../data/assist_pipeline"; +import type { HomeAssistant } from "../../types"; +import { AudioRecorder } from "../../util/audio-recorder"; +import { documentationUrl } from "../../util/documentation-url"; +import { showAlertDialog } from "../generic/show-dialog-box"; + +interface Message { + who: string; + text?: string | TemplateResult; + error?: boolean; +} + +@customElement("assist-chat") +export class HaAssistChat extends LitElement { + @property({ attribute: false }) public hass!: HomeAssistant; + + @property({ attribute: "pipeline-id" }) public pipelineId!: string; + + @state() private _conversation?: Message[]; + + @state() private _pipeline?: AssistPipeline; + + @state() private _showSendButton = false; + + @query("#scroll-container") private _scrollContainer!: HTMLDivElement; + + @query("#message-input") private _messageInput!: HaTextField; + + private _conversationId: string | null = null; + + private _audioRecorder?: AudioRecorder; + + private _audioBuffer?: Int16Array[]; + + private _audio?: HTMLAudioElement; + + private _stt_binary_handler_id?: number | null; + + protected render() { + const supportsMicrophone = AudioRecorder.isSupported; + const supportsSTT = this._pipeline?.stt_engine; + + return html` +
+
+ ${this._conversation!.map( + // New lines matter for messages + // prettier-ignore + (message) => html` +
${message.text}
+ ` + )} +
+
+
+ + + ${this._showSendButton || !supportsSTT + ? html` + + + ` + : html` + ${this._audioRecorder?.active + ? html` +
+
+
+
+ ` + : nothing} + +
+ + + ${!supportsMicrophone + ? html` + + ` + : null} +
+ `} + + +
+ `; + } + + protected willUpdate(changedProperties: PropertyValues): void { + if (changedProperties.has("pipelineId")) { + this._getPipeline(); + this._conversation = [ + { + who: "hass", + text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"), + }, + ]; + } + } + + private async _getPipeline() { + try { + this._pipeline = await getAssistPipeline(this.hass, this.pipelineId); + } catch (e: any) { + // Pipeline doesn't exist, we won't be able to check + // if it supports STT. We gracefully handle this. + } + } + + protected updated(changedProps: PropertyValues) { + super.updated(changedProps); + if (changedProps.has("_conversation") || changedProps.has("results")) { + this._scrollMessagesBottom(); + } + } + + private _addMessage(message: Message) { + this._conversation = [...this._conversation!, message]; + } + + private _handleKeyUp(ev: KeyboardEvent) { + const input = ev.target as HaTextField; + if (ev.key === "Enter" && input.value) { + this._processText(input.value); + input.value = ""; + this._showSendButton = false; + } + } + + private _handleInput(ev: InputEvent) { + const value = (ev.target as HaTextField).value; + if (value && !this._showSendButton) { + this._showSendButton = true; + } else if (!value && this._showSendButton) { + this._showSendButton = false; + } + } + + private _handleSendMessage() { + if (this._messageInput.value) { + this._processText(this._messageInput.value.trim()); + this._messageInput.value = ""; + this._showSendButton = false; + } + } + + private async _processText(text: string) { + this._audio?.pause(); + this._addMessage({ who: "user", text }); + const message: Message = { + who: "hass", + text: "…", + }; + // To make sure the answer is placed at the right user text, we add it before we process it + this._addMessage(message); + try { + const unsub = await runAssistPipeline( + this.hass, + (event) => { + if (event.type === "intent-end") { + this._conversationId = event.data.intent_output.conversation_id; + const plain = event.data.intent_output.response.speech?.plain; + if (plain) { + message.text = plain.speech; + } + this.requestUpdate("_conversation"); + unsub(); + } + if (event.type === "error") { + message.text = event.data.message; + message.error = true; + this.requestUpdate("_conversation"); + unsub(); + } + }, + { + start_stage: "intent", + input: { text }, + end_stage: "intent", + pipeline: this.pipelineId, + conversation_id: this._conversationId, + } + ); + } catch { + message.text = this.hass.localize("ui.dialogs.voice_command.error"); + message.error = true; + this.requestUpdate("_conversation"); + } + } + + private _handleListeningButton(ev) { + ev.stopPropagation(); + ev.preventDefault(); + this.toggleListening(); + } + + public toggleListening() { + const supportsMicrophone = AudioRecorder.isSupported; + if (!supportsMicrophone) { + this._showNotSupportedMessage(); + return; + } + if (!this._audioRecorder?.active) { + this._startListening(); + } else { + this.stopListening(); + } + } + + private async _showNotSupportedMessage() { + this._addMessage({ + who: "hass", + text: + // New lines matter for messages + // prettier-ignore + html`${this.hass.localize( + "ui.dialogs.voice_command.not_supported_microphone_browser" + )} + + ${this.hass.localize( + "ui.dialogs.voice_command.not_supported_microphone_documentation", + { + documentation_link: html`${this.hass.localize( + "ui.dialogs.voice_command.not_supported_microphone_documentation_link" + )}`, + } + )}`, + }); + } + + private async _startListening() { + this._audio?.pause(); + if (!this._audioRecorder) { + this._audioRecorder = new AudioRecorder((audio) => { + if (this._audioBuffer) { + this._audioBuffer.push(audio); + } else { + this._sendAudioChunk(audio); + } + }); + } + this._stt_binary_handler_id = undefined; + this._audioBuffer = []; + const userMessage: Message = { + who: "user", + text: "…", + }; + this._audioRecorder.start().then(() => { + this._addMessage(userMessage); + this.requestUpdate("_audioRecorder"); + }); + const hassMessage: Message = { + who: "hass", + text: "…", + }; + // To make sure the answer is placed at the right user text, we add it before we process it + try { + const unsub = await runAssistPipeline( + this.hass, + (event) => { + if (event.type === "run-start") { + this._stt_binary_handler_id = + event.data.runner_data.stt_binary_handler_id; + } + + // When we start STT stage, the WS has a binary handler + if (event.type === "stt-start" && this._audioBuffer) { + // Send the buffer over the WS to the STT engine. + for (const buffer of this._audioBuffer) { + this._sendAudioChunk(buffer); + } + this._audioBuffer = undefined; + } + + // Stop recording if the server is done with STT stage + if (event.type === "stt-end") { + this._stt_binary_handler_id = undefined; + this.stopListening(); + userMessage.text = event.data.stt_output.text; + this.requestUpdate("_conversation"); + // To make sure the answer is placed at the right user text, we add it before we process it + this._addMessage(hassMessage); + } + + if (event.type === "intent-end") { + this._conversationId = event.data.intent_output.conversation_id; + const plain = event.data.intent_output.response.speech?.plain; + if (plain) { + hassMessage.text = plain.speech; + } + this.requestUpdate("_conversation"); + } + + if (event.type === "tts-end") { + const url = event.data.tts_output.url; + this._audio = new Audio(url); + this._audio.play(); + this._audio.addEventListener("ended", this._unloadAudio); + this._audio.addEventListener("pause", this._unloadAudio); + this._audio.addEventListener("canplaythrough", this._playAudio); + this._audio.addEventListener("error", this._audioError); + } + + if (event.type === "run-end") { + this._stt_binary_handler_id = undefined; + unsub(); + } + + if (event.type === "error") { + this._stt_binary_handler_id = undefined; + if (userMessage.text === "…") { + userMessage.text = event.data.message; + userMessage.error = true; + } else { + hassMessage.text = event.data.message; + hassMessage.error = true; + } + this.stopListening(); + this.requestUpdate("_conversation"); + unsub(); + } + }, + { + start_stage: "stt", + end_stage: this._pipeline?.tts_engine ? "tts" : "intent", + input: { sample_rate: this._audioRecorder.sampleRate! }, + pipeline: this._pipeline?.id, + conversation_id: this._conversationId, + } + ); + } catch (err: any) { + await showAlertDialog(this, { + title: "Error starting pipeline", + text: err.message || err, + }); + this.stopListening(); + } + } + + public stopListening() { + this._audioRecorder?.stop(); + this.requestUpdate("_audioRecorder"); + // We're currently STTing, so finish audio + if (this._stt_binary_handler_id) { + if (this._audioBuffer) { + for (const chunk of this._audioBuffer) { + this._sendAudioChunk(chunk); + } + } + // Send empty message to indicate we're done streaming. + this._sendAudioChunk(new Int16Array()); + this._stt_binary_handler_id = undefined; + } + this._audioBuffer = undefined; + } + + private _sendAudioChunk(chunk: Int16Array) { + this.hass.connection.socket!.binaryType = "arraybuffer"; + + // eslint-disable-next-line eqeqeq + if (this._stt_binary_handler_id == undefined) { + return; + } + // Turn into 8 bit so we can prefix our handler ID. + const data = new Uint8Array(1 + chunk.length * 2); + data[0] = this._stt_binary_handler_id; + data.set(new Uint8Array(chunk.buffer), 1); + + this.hass.connection.socket!.send(data); + } + + private _playAudio = () => { + this._audio?.play(); + }; + + private _audioError = () => { + showAlertDialog(this, { title: "Error playing audio." }); + this._audio?.removeAttribute("src"); + }; + + private _unloadAudio = () => { + this._audio?.removeAttribute("src"); + this._audio = undefined; + }; + + private _scrollMessagesBottom() { + const scrollContainer = this._scrollContainer; + if (!scrollContainer) { + return; + } + scrollContainer.scrollTo(0, 99999); + } + + private _computeMessageClasses(message: Message) { + return `message ${message.who} ${message.error ? " error" : ""}`; + } + + static get styles(): CSSResultGroup { + return css` + .listening-icon { + position: relative; + color: var(--secondary-text-color); + margin-right: -24px; + margin-inline-end: -24px; + margin-inline-start: initial; + direction: var(--direction); + } + + .listening-icon[active] { + color: var(--primary-color); + } + + .unsupported { + color: var(--error-color); + position: absolute; + --mdc-icon-size: 16px; + right: 5px; + inset-inline-end: 5px; + inset-inline-start: initial; + top: 0px; + } + + ha-textfield { + display: block; + overflow: hidden; + } + a.button { + text-decoration: none; + } + .side-by-side { + display: flex; + margin: 8px 0; + } + .side-by-side > * { + flex: 1 0; + padding: 4px; + } + .messages { + display: block; + height: 400px; + box-sizing: border-box; + position: relative; + } + @media all and (max-width: 450px), all and (max-height: 500px) { + ha-dialog { + --mdc-dialog-max-width: 100%; + } + .messages { + height: 100%; + flex: 1; + } + } + .messages-container { + position: absolute; + bottom: 0px; + right: 0px; + left: 0px; + padding: 24px; + box-sizing: border-box; + overflow-y: auto; + max-height: 100%; + } + .message { + white-space: pre-line; + font-size: 18px; + clear: both; + margin: 8px 0; + padding: 8px; + border-radius: 15px; + } + .message p { + margin: 0; + } + .message p:not(:last-child) { + margin-bottom: 8px; + } + + .message.user { + margin-left: 24px; + margin-inline-start: 24px; + margin-inline-end: initial; + float: var(--float-end); + text-align: right; + border-bottom-right-radius: 0px; + background-color: var(--primary-color); + color: var(--text-primary-color); + direction: var(--direction); + } + + .message.hass { + margin-right: 24px; + margin-inline-end: 24px; + margin-inline-start: initial; + float: var(--float-start); + border-bottom-left-radius: 0px; + background-color: var(--secondary-background-color); + color: var(--primary-text-color); + direction: var(--direction); + } + + .message.user a { + color: var(--text-primary-color); + } + + .message.hass a { + color: var(--primary-text-color); + } + + .message img { + width: 100%; + border-radius: 10px; + } + + .message.error { + background-color: var(--error-color); + color: var(--text-primary-color); + } + + .input { + margin-left: 0; + margin-right: 0; + } + + .bouncer { + width: 48px; + height: 48px; + position: absolute; + } + .double-bounce1, + .double-bounce2 { + width: 48px; + height: 48px; + border-radius: 50%; + background-color: var(--primary-color); + opacity: 0.2; + position: absolute; + top: 0; + left: 0; + -webkit-animation: sk-bounce 2s infinite ease-in-out; + animation: sk-bounce 2s infinite ease-in-out; + } + .double-bounce2 { + -webkit-animation-delay: -1s; + animation-delay: -1s; + } + @-webkit-keyframes sk-bounce { + 0%, + 100% { + -webkit-transform: scale(0); + } + 50% { + -webkit-transform: scale(1); + } + } + @keyframes sk-bounce { + 0%, + 100% { + transform: scale(0); + -webkit-transform: scale(0); + } + 50% { + transform: scale(1); + -webkit-transform: scale(1); + } + } + + @media all and (max-width: 450px), all and (max-height: 500px) { + .message { + font-size: 16px; + } + } + `; + } +} + +declare global { + interface HTMLElementTagNameMap { + "assist-chat": HaAssistChat; + } +} diff --git a/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts b/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts index 7df82bdc0f..6f5bee84c0 100644 --- a/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts +++ b/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts @@ -1,11 +1,8 @@ import "@material/mwc-button/mwc-button"; import { - mdiAlertCircle, mdiChevronDown, mdiClose, mdiHelpCircleOutline, - mdiMicrophone, - mdiSend, mdiStar, } from "@mdi/js"; import { @@ -15,7 +12,6 @@ import { LitElement, nothing, PropertyValues, - TemplateResult, } from "lit"; import { customElement, property, query, state } from "lit/decorators"; import { storage } from "../../common/decorators/storage"; @@ -27,35 +23,25 @@ import "../../components/ha-dialog"; import "../../components/ha-dialog-header"; import "../../components/ha-icon-button"; import "../../components/ha-list-item"; -import "../../components/ha-textfield"; -import type { HaTextField } from "../../components/ha-textfield"; import { AssistPipeline, getAssistPipeline, listAssistPipelines, - runAssistPipeline, } from "../../data/assist_pipeline"; import { haStyleDialog } from "../../resources/styles"; import type { HomeAssistant } from "../../types"; import { AudioRecorder } from "../../util/audio-recorder"; import { documentationUrl } from "../../util/documentation-url"; -import { showAlertDialog } from "../generic/show-dialog-box"; import { VoiceCommandDialogParams } from "./show-ha-voice-command-dialog"; import { supportsFeature } from "../../common/entity/supports-feature"; import { ConversationEntityFeature } from "../../data/conversation"; - -interface Message { - who: string; - text?: string | TemplateResult; - error?: boolean; -} +import "./assist-chat"; +import type { HaAssistChat } from "./assist-chat"; @customElement("ha-voice-command-dialog") export class HaVoiceCommandDialog extends LitElement { @property({ attribute: false }) public hass!: HomeAssistant; - @state() private _conversation?: Message[]; - @state() private _opened = false; @storage({ @@ -67,25 +53,11 @@ export class HaVoiceCommandDialog extends LitElement { @state() private _pipeline?: AssistPipeline; - @state() private _showSendButton = false; - @state() private _pipelines?: AssistPipeline[]; @state() private _preferredPipeline?: string; - @query("#scroll-container") private _scrollContainer!: HTMLDivElement; - - @query("#message-input") private _messageInput!: HaTextField; - - private _conversationId: string | null = null; - - private _audioRecorder?: AudioRecorder; - - private _audioBuffer?: Int16Array[]; - - private _audio?: HTMLAudioElement; - - private _stt_binary_handler_id?: number | null; + @query("assist-chat") private _assistChat!: HaAssistChat; private _pipelinePromise?: Promise; @@ -101,15 +73,8 @@ export class HaVoiceCommandDialog extends LitElement { this._pipelineId = params.pipeline_id; } - this._conversation = [ - { - who: "hass", - text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"), - }, - ]; this._opened = true; await this.updateComplete; - this._scrollMessagesBottom(); await this._pipelinePromise; if ( @@ -117,7 +82,7 @@ export class HaVoiceCommandDialog extends LitElement { this._pipeline?.stt_engine && AudioRecorder.isSupported ) { - this._toggleListening(); + this._assistChat.toggleListening(); } } @@ -125,11 +90,7 @@ export class HaVoiceCommandDialog extends LitElement { this._opened = false; this._pipeline = undefined; this._pipelines = undefined; - this._conversation = undefined; - this._conversationId = null; - this._audioRecorder?.close(); - this._audioRecorder = undefined; - this._audio?.pause(); + this._assistChat.stopListening(); fireEvent(this, "dialog-closed", { dialog: this.localName }); } @@ -144,15 +105,13 @@ export class HaVoiceCommandDialog extends LitElement { this.hass.states[this._pipeline?.conversation_engine], ConversationEntityFeature.CONTROL ); - const supportsMicrophone = AudioRecorder.isSupported; - const supportsSTT = this._pipeline?.stt_engine; return html` `} -
-
- ${this._conversation!.map( - // New lines matter for messages - // prettier-ignore - (message) => html` -
${message.text}
- ` - )} -
-
-
- - - ${this._showSendButton || !supportsSTT - ? html` - - - ` - : html` - ${this._audioRecorder?.active - ? html` -
-
-
-
- ` - : nothing} - -
- - - ${!supportsMicrophone - ? html` - - ` - : null} -
- `} - - -
+ `; } @@ -331,339 +229,12 @@ export class HaVoiceCommandDialog extends LitElement { private async _selectPipeline(ev: CustomEvent) { this._pipelineId = (ev.currentTarget as any).pipeline; - this._conversation = [ - { - who: "hass", - text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"), - }, - ]; - await this.updateComplete; - this._scrollMessagesBottom(); - } - - protected updated(changedProps: PropertyValues) { - super.updated(changedProps); - if (changedProps.has("_conversation") || changedProps.has("results")) { - this._scrollMessagesBottom(); - } - } - - private _addMessage(message: Message) { - this._conversation = [...this._conversation!, message]; - } - - private _handleKeyUp(ev: KeyboardEvent) { - const input = ev.target as HaTextField; - if (ev.key === "Enter" && input.value) { - this._processText(input.value); - input.value = ""; - this._showSendButton = false; - } - } - - private _handleInput(ev: InputEvent) { - const value = (ev.target as HaTextField).value; - if (value && !this._showSendButton) { - this._showSendButton = true; - } else if (!value && this._showSendButton) { - this._showSendButton = false; - } - } - - private _handleSendMessage() { - if (this._messageInput.value) { - this._processText(this._messageInput.value.trim()); - this._messageInput.value = ""; - this._showSendButton = false; - } - } - - private async _processText(text: string) { - this._audio?.pause(); - this._addMessage({ who: "user", text }); - const message: Message = { - who: "hass", - text: "…", - }; - // To make sure the answer is placed at the right user text, we add it before we process it - this._addMessage(message); - try { - const unsub = await runAssistPipeline( - this.hass, - (event) => { - if (event.type === "intent-end") { - this._conversationId = event.data.intent_output.conversation_id; - const plain = event.data.intent_output.response.speech?.plain; - if (plain) { - message.text = plain.speech; - } - this.requestUpdate("_conversation"); - unsub(); - } - if (event.type === "error") { - message.text = event.data.message; - message.error = true; - this.requestUpdate("_conversation"); - unsub(); - } - }, - { - start_stage: "intent", - input: { text }, - end_stage: "intent", - pipeline: this._pipeline?.id, - conversation_id: this._conversationId, - } - ); - } catch { - message.text = this.hass.localize("ui.dialogs.voice_command.error"); - message.error = true; - this.requestUpdate("_conversation"); - } - } - - private _handleListeningButton(ev) { - ev.stopPropagation(); - ev.preventDefault(); - this._toggleListening(); - } - - private _toggleListening() { - const supportsMicrophone = AudioRecorder.isSupported; - if (!supportsMicrophone) { - this._showNotSupportedMessage(); - return; - } - if (!this._audioRecorder?.active) { - this._startListening(); - } else { - this._stopListening(); - } - } - - private async _showNotSupportedMessage() { - this._addMessage({ - who: "hass", - text: - // New lines matter for messages - // prettier-ignore - html`${this.hass.localize( - "ui.dialogs.voice_command.not_supported_microphone_browser" - )} - - ${this.hass.localize( - "ui.dialogs.voice_command.not_supported_microphone_documentation", - { - documentation_link: html`${this.hass.localize( - "ui.dialogs.voice_command.not_supported_microphone_documentation_link" - )}`, - } - )}`, - }); - } - - private async _startListening() { - this._audio?.pause(); - if (!this._audioRecorder) { - this._audioRecorder = new AudioRecorder((audio) => { - if (this._audioBuffer) { - this._audioBuffer.push(audio); - } else { - this._sendAudioChunk(audio); - } - }); - } - this._stt_binary_handler_id = undefined; - this._audioBuffer = []; - const userMessage: Message = { - who: "user", - text: "…", - }; - await this._audioRecorder.start(); - - this._addMessage(userMessage); - this.requestUpdate("_audioRecorder"); - - const hassMessage: Message = { - who: "hass", - text: "…", - }; - // To make sure the answer is placed at the right user text, we add it before we process it - try { - const unsub = await runAssistPipeline( - this.hass, - (event) => { - if (event.type === "run-start") { - this._stt_binary_handler_id = - event.data.runner_data.stt_binary_handler_id; - } - - // When we start STT stage, the WS has a binary handler - if (event.type === "stt-start" && this._audioBuffer) { - // Send the buffer over the WS to the STT engine. - for (const buffer of this._audioBuffer) { - this._sendAudioChunk(buffer); - } - this._audioBuffer = undefined; - } - - // Stop recording if the server is done with STT stage - if (event.type === "stt-end") { - this._stt_binary_handler_id = undefined; - this._stopListening(); - userMessage.text = event.data.stt_output.text; - this.requestUpdate("_conversation"); - // To make sure the answer is placed at the right user text, we add it before we process it - this._addMessage(hassMessage); - } - - if (event.type === "intent-end") { - this._conversationId = event.data.intent_output.conversation_id; - const plain = event.data.intent_output.response.speech?.plain; - if (plain) { - hassMessage.text = plain.speech; - } - this.requestUpdate("_conversation"); - } - - if (event.type === "tts-end") { - const url = event.data.tts_output.url; - this._audio = new Audio(url); - this._audio.play(); - this._audio.addEventListener("ended", this._unloadAudio); - this._audio.addEventListener("pause", this._unloadAudio); - this._audio.addEventListener("canplaythrough", this._playAudio); - this._audio.addEventListener("error", this._audioError); - } - - if (event.type === "run-end") { - this._stt_binary_handler_id = undefined; - unsub(); - } - - if (event.type === "error") { - this._stt_binary_handler_id = undefined; - if (userMessage.text === "…") { - userMessage.text = event.data.message; - userMessage.error = true; - } else { - hassMessage.text = event.data.message; - hassMessage.error = true; - } - this._stopListening(); - this.requestUpdate("_conversation"); - unsub(); - } - }, - { - start_stage: "stt", - end_stage: this._pipeline?.tts_engine ? "tts" : "intent", - input: { sample_rate: this._audioRecorder.sampleRate! }, - pipeline: this._pipeline?.id, - conversation_id: this._conversationId, - } - ); - } catch (err: any) { - await showAlertDialog(this, { - title: "Error starting pipeline", - text: err.message || err, - }); - this._stopListening(); - } - } - - private _stopListening() { - this._audioRecorder?.stop(); - this.requestUpdate("_audioRecorder"); - // We're currently STTing, so finish audio - if (this._stt_binary_handler_id) { - if (this._audioBuffer) { - for (const chunk of this._audioBuffer) { - this._sendAudioChunk(chunk); - } - } - // Send empty message to indicate we're done streaming. - this._sendAudioChunk(new Int16Array()); - this._stt_binary_handler_id = undefined; - } - this._audioBuffer = undefined; - } - - private _sendAudioChunk(chunk: Int16Array) { - this.hass.connection.socket!.binaryType = "arraybuffer"; - - // eslint-disable-next-line eqeqeq - if (this._stt_binary_handler_id == undefined) { - return; - } - // Turn into 8 bit so we can prefix our handler ID. - const data = new Uint8Array(1 + chunk.length * 2); - data[0] = this._stt_binary_handler_id; - data.set(new Uint8Array(chunk.buffer), 1); - - this.hass.connection.socket!.send(data); - } - - private _playAudio = () => { - this._audio?.play(); - }; - - private _audioError = () => { - showAlertDialog(this, { title: "Error playing audio." }); - this._audio?.removeAttribute("src"); - }; - - private _unloadAudio = () => { - this._audio?.removeAttribute("src"); - this._audio = undefined; - }; - - private _scrollMessagesBottom() { - const scrollContainer = this._scrollContainer; - if (!scrollContainer) { - return; - } - scrollContainer.scrollTo(0, 99999); - } - - private _computeMessageClasses(message: Message) { - return `message ${message.who} ${message.error ? " error" : ""}`; } static get styles(): CSSResultGroup { return [ haStyleDialog, css` - .listening-icon { - position: relative; - color: var(--secondary-text-color); - margin-right: -24px; - margin-inline-end: -24px; - margin-inline-start: initial; - direction: var(--direction); - } - - .listening-icon[active] { - color: var(--primary-color); - } - - .unsupported { - color: var(--error-color); - position: absolute; - --mdc-icon-size: 16px; - right: 5px; - inset-inline-end: 5px; - inset-inline-start: initial; - top: 0px; - } - ha-dialog { --primary-action-button-flex: 1; --secondary-action-button-flex: 0; @@ -724,158 +295,6 @@ export class HaVoiceCommandDialog extends LitElement { ha-button-menu a { text-decoration: none; } - ha-textfield { - display: block; - overflow: hidden; - } - a.button { - text-decoration: none; - } - a.button > mwc-button { - width: 100%; - } - .side-by-side { - display: flex; - margin: 8px 0; - } - .side-by-side > * { - flex: 1 0; - padding: 4px; - } - .messages { - display: block; - height: 400px; - box-sizing: border-box; - position: relative; - } - @media all and (max-width: 450px), all and (max-height: 500px) { - ha-dialog { - --mdc-dialog-max-width: 100%; - } - .messages { - height: 100%; - flex: 1; - } - } - .messages-container { - position: absolute; - bottom: 0px; - right: 0px; - left: 0px; - padding: 24px; - box-sizing: border-box; - overflow-y: auto; - max-height: 100%; - } - .message { - white-space: pre-line; - font-size: 18px; - clear: both; - margin: 8px 0; - padding: 8px; - border-radius: 15px; - } - .message p { - margin: 0; - } - .message p:not(:last-child) { - margin-bottom: 8px; - } - - .message.user { - margin-left: 24px; - margin-inline-start: 24px; - margin-inline-end: initial; - float: var(--float-end); - text-align: right; - border-bottom-right-radius: 0px; - background-color: var(--primary-color); - color: var(--text-primary-color); - direction: var(--direction); - } - - .message.hass { - margin-right: 24px; - margin-inline-end: 24px; - margin-inline-start: initial; - float: var(--float-start); - border-bottom-left-radius: 0px; - background-color: var(--secondary-background-color); - color: var(--primary-text-color); - direction: var(--direction); - } - - .message.user a { - color: var(--text-primary-color); - } - - .message.hass a { - color: var(--primary-text-color); - } - - .message img { - width: 100%; - border-radius: 10px; - } - - .message.error { - background-color: var(--error-color); - color: var(--text-primary-color); - } - - .input { - margin-left: 0; - margin-right: 0; - } - - .bouncer { - width: 48px; - height: 48px; - position: absolute; - } - .double-bounce1, - .double-bounce2 { - width: 48px; - height: 48px; - border-radius: 50%; - background-color: var(--primary-color); - opacity: 0.2; - position: absolute; - top: 0; - left: 0; - -webkit-animation: sk-bounce 2s infinite ease-in-out; - animation: sk-bounce 2s infinite ease-in-out; - } - .double-bounce2 { - -webkit-animation-delay: -1s; - animation-delay: -1s; - } - @-webkit-keyframes sk-bounce { - 0%, - 100% { - -webkit-transform: scale(0); - } - 50% { - -webkit-transform: scale(1); - } - } - @keyframes sk-bounce { - 0%, - 100% { - transform: scale(0); - -webkit-transform: scale(0); - } - 50% { - transform: scale(1); - -webkit-transform: scale(1); - } - } - - @media all and (max-width: 450px), all and (max-height: 500px) { - .message { - font-size: 16px; - } - } `, ]; }