Extract assist-chat out of voice-command-dialog (#23184)

This commit is contained in:
Wendelin 2024-12-10 12:23:48 +01:00 committed by GitHub
parent 8f19c0abb0
commit 84157c8ea5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 720 additions and 655 deletions

View File

@ -0,0 +1,639 @@
import type { CSSResultGroup, PropertyValues, TemplateResult } from "lit";
import { css, LitElement, html, nothing } from "lit";
import { mdiAlertCircle, mdiMicrophone, mdiSend } from "@mdi/js";
import { customElement, property, query, state } from "lit/decorators";
import { classMap } from "lit/directives/class-map";
import type { HomeAssistant } from "../types";
import {
runAssistPipeline,
type AssistPipeline,
} from "../data/assist_pipeline";
import { supportsFeature } from "../common/entity/supports-feature";
import { ConversationEntityFeature } from "../data/conversation";
import { AudioRecorder } from "../util/audio-recorder";
import "./ha-alert";
import "./ha-textfield";
import type { HaTextField } from "./ha-textfield";
import { documentationUrl } from "../util/documentation-url";
import { showAlertDialog } from "../dialogs/generic/show-dialog-box";
interface AssistMessage {
who: string;
text?: string | TemplateResult;
error?: boolean;
}
@customElement("ha-assist-chat")
export class HaAssistChat extends LitElement {
@property({ attribute: false }) public hass!: HomeAssistant;
@property({ attribute: false }) public pipeline?: AssistPipeline;
@property({ type: Boolean, attribute: false })
public startListening?: boolean;
@query("#message-input") private _messageInput!: HaTextField;
@query("#scroll-container") private _scrollContainer!: HTMLDivElement;
@state() private _conversation: AssistMessage[] = [];
@state() private _showSendButton = false;
@state() private _processing = false;
private _conversationId: string | null = null;
private _audioRecorder?: AudioRecorder;
private _audioBuffer?: Int16Array[];
private _audio?: HTMLAudioElement;
private _stt_binary_handler_id?: number | null;
protected willUpdate(changedProperties: PropertyValues): void {
if (!this.hasUpdated || changedProperties.has("pipeline")) {
this._conversation = [
{
who: "hass",
text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"),
},
];
}
}
protected firstUpdated(changedProperties: PropertyValues): void {
super.firstUpdated(changedProperties);
if (
this.startListening &&
this.pipeline &&
this.pipeline.stt_engine &&
AudioRecorder.isSupported
) {
this._toggleListening();
}
setTimeout(() => this._messageInput.focus(), 0);
}
protected updated(changedProps: PropertyValues) {
super.updated(changedProps);
if (changedProps.has("_conversation")) {
this._scrollMessagesBottom();
}
}
public disconnectedCallback() {
super.disconnectedCallback();
this._audioRecorder?.close();
this._audioRecorder = undefined;
this._audio?.pause();
this._conversation = [];
this._conversationId = null;
}
protected render(): TemplateResult {
const controlHA = !this.pipeline
? false
: this.pipeline.prefer_local_intents ||
(this.hass.states[this.pipeline.conversation_engine]
? supportsFeature(
this.hass.states[this.pipeline.conversation_engine],
ConversationEntityFeature.CONTROL
)
: true);
const supportsMicrophone = AudioRecorder.isSupported;
const supportsSTT = this.pipeline?.stt_engine;
return html`
${controlHA
? nothing
: html`
<ha-alert>
${this.hass.localize(
"ui.dialogs.voice_command.conversation_no_control"
)}
</ha-alert>
`}
<div class="messages">
<div class="messages-container" id="scroll-container">
${this._conversation!.map(
// New lines matter for messages
// prettier-ignore
(message) => html`
<div class="message ${classMap({ error: !!message.error, [message.who]: true })}">${message.text}</div>
`
)}
</div>
</div>
<div class="input" slot="primaryAction">
<ha-textfield
id="message-input"
@keyup=${this._handleKeyUp}
@input=${this._handleInput}
.label=${this.hass.localize(`ui.dialogs.voice_command.input_label`)}
.iconTrailing=${true}
>
<div slot="trailingIcon">
${this._showSendButton || !supportsSTT
? html`
<ha-icon-button
class="listening-icon"
.path=${mdiSend}
@click=${this._handleSendMessage}
.disabled=${this._processing}
.label=${this.hass.localize(
"ui.dialogs.voice_command.send_text"
)}
>
</ha-icon-button>
`
: html`
${this._audioRecorder?.active
? html`
<div class="bouncer">
<div class="double-bounce1"></div>
<div class="double-bounce2"></div>
</div>
`
: nothing}
<div class="listening-icon">
<ha-icon-button
.path=${mdiMicrophone}
@click=${this._handleListeningButton}
.disabled=${this._processing}
.label=${this.hass.localize(
"ui.dialogs.voice_command.start_listening"
)}
>
</ha-icon-button>
${!supportsMicrophone
? html`
<ha-svg-icon
.path=${mdiAlertCircle}
class="unsupported"
></ha-svg-icon>
`
: null}
</div>
`}
</div>
</ha-textfield>
</div>
`;
}
private _scrollMessagesBottom() {
const scrollContainer = this._scrollContainer;
if (!scrollContainer) {
return;
}
scrollContainer.scrollTo(0, scrollContainer.scrollHeight);
}
private _handleKeyUp(ev: KeyboardEvent) {
const input = ev.target as HaTextField;
if (!this._processing && ev.key === "Enter" && input.value) {
this._processText(input.value);
input.value = "";
this._showSendButton = false;
}
}
private _handleInput(ev: InputEvent) {
const value = (ev.target as HaTextField).value;
if (value && !this._showSendButton) {
this._showSendButton = true;
} else if (!value && this._showSendButton) {
this._showSendButton = false;
}
}
private _handleSendMessage() {
if (this._messageInput.value) {
this._processText(this._messageInput.value.trim());
this._messageInput.value = "";
this._showSendButton = false;
}
}
private _handleListeningButton(ev) {
ev.stopPropagation();
ev.preventDefault();
this._toggleListening();
}
private async _toggleListening() {
const supportsMicrophone = AudioRecorder.isSupported;
if (!supportsMicrophone) {
this._showNotSupportedMessage();
return;
}
if (!this._audioRecorder?.active) {
this._startListening();
} else {
this._stopListening();
}
}
private _addMessage(message: AssistMessage) {
this._conversation = [...this._conversation!, message];
}
private async _showNotSupportedMessage() {
this._addMessage({
who: "hass",
text:
// New lines matter for messages
// prettier-ignore
html`${this.hass.localize(
"ui.dialogs.voice_command.not_supported_microphone_browser"
)}
${this.hass.localize(
"ui.dialogs.voice_command.not_supported_microphone_documentation",
{
documentation_link: html`<a
target="_blank"
rel="noopener noreferrer"
href=${documentationUrl(
this.hass,
"/docs/configuration/securing/#remote-access"
)}
>${this.hass.localize(
"ui.dialogs.voice_command.not_supported_microphone_documentation_link"
)}</a>`,
}
)}`,
});
}
private async _startListening() {
this._processing = true;
this._audio?.pause();
if (!this._audioRecorder) {
this._audioRecorder = new AudioRecorder((audio) => {
if (this._audioBuffer) {
this._audioBuffer.push(audio);
} else {
this._sendAudioChunk(audio);
}
});
}
this._stt_binary_handler_id = undefined;
this._audioBuffer = [];
const userMessage: AssistMessage = {
who: "user",
text: "…",
};
await this._audioRecorder.start();
this._addMessage(userMessage);
this.requestUpdate("_audioRecorder");
const hassMessage: AssistMessage = {
who: "hass",
text: "…",
};
// To make sure the answer is placed at the right user text, we add it before we process it
try {
const unsub = await runAssistPipeline(
this.hass,
(event) => {
if (event.type === "run-start") {
this._stt_binary_handler_id =
event.data.runner_data.stt_binary_handler_id;
}
// When we start STT stage, the WS has a binary handler
if (event.type === "stt-start" && this._audioBuffer) {
// Send the buffer over the WS to the STT engine.
for (const buffer of this._audioBuffer) {
this._sendAudioChunk(buffer);
}
this._audioBuffer = undefined;
}
// Stop recording if the server is done with STT stage
if (event.type === "stt-end") {
this._stt_binary_handler_id = undefined;
this._stopListening();
userMessage.text = event.data.stt_output.text;
this.requestUpdate("_conversation");
// To make sure the answer is placed at the right user text, we add it before we process it
this._addMessage(hassMessage);
}
if (event.type === "intent-end") {
this._conversationId = event.data.intent_output.conversation_id;
const plain = event.data.intent_output.response.speech?.plain;
if (plain) {
hassMessage.text = plain.speech;
}
this.requestUpdate("_conversation");
}
if (event.type === "tts-end") {
const url = event.data.tts_output.url;
this._audio = new Audio(url);
this._audio.play();
this._audio.addEventListener("ended", this._unloadAudio);
this._audio.addEventListener("pause", this._unloadAudio);
this._audio.addEventListener("canplaythrough", this._playAudio);
this._audio.addEventListener("error", this._audioError);
}
if (event.type === "run-end") {
this._stt_binary_handler_id = undefined;
unsub();
}
if (event.type === "error") {
this._stt_binary_handler_id = undefined;
if (userMessage.text === "…") {
userMessage.text = event.data.message;
userMessage.error = true;
} else {
hassMessage.text = event.data.message;
hassMessage.error = true;
}
this._stopListening();
this.requestUpdate("_conversation");
unsub();
}
},
{
start_stage: "stt",
end_stage: this.pipeline?.tts_engine ? "tts" : "intent",
input: { sample_rate: this._audioRecorder.sampleRate! },
pipeline: this.pipeline?.id,
conversation_id: this._conversationId,
}
);
} catch (err: any) {
await showAlertDialog(this, {
title: "Error starting pipeline",
text: err.message || err,
});
this._stopListening();
} finally {
this._processing = false;
}
}
private _stopListening() {
this._audioRecorder?.stop();
this.requestUpdate("_audioRecorder");
// We're currently STTing, so finish audio
if (this._stt_binary_handler_id) {
if (this._audioBuffer) {
for (const chunk of this._audioBuffer) {
this._sendAudioChunk(chunk);
}
}
// Send empty message to indicate we're done streaming.
this._sendAudioChunk(new Int16Array());
this._stt_binary_handler_id = undefined;
}
this._audioBuffer = undefined;
}
private _sendAudioChunk(chunk: Int16Array) {
this.hass.connection.socket!.binaryType = "arraybuffer";
// eslint-disable-next-line eqeqeq
if (this._stt_binary_handler_id == undefined) {
return;
}
// Turn into 8 bit so we can prefix our handler ID.
const data = new Uint8Array(1 + chunk.length * 2);
data[0] = this._stt_binary_handler_id;
data.set(new Uint8Array(chunk.buffer), 1);
this.hass.connection.socket!.send(data);
}
private _playAudio = () => {
this._audio?.play();
};
private _audioError = () => {
showAlertDialog(this, { title: "Error playing audio." });
this._audio?.removeAttribute("src");
};
private _unloadAudio = () => {
this._audio?.removeAttribute("src");
this._audio = undefined;
};
private async _processText(text: string) {
this._processing = true;
this._audio?.pause();
this._addMessage({ who: "user", text });
const message: AssistMessage = {
who: "hass",
text: "…",
};
// To make sure the answer is placed at the right user text, we add it before we process it
this._addMessage(message);
try {
const unsub = await runAssistPipeline(
this.hass,
(event) => {
if (event.type === "intent-end") {
this._conversationId = event.data.intent_output.conversation_id;
const plain = event.data.intent_output.response.speech?.plain;
if (plain) {
message.text = plain.speech;
}
this.requestUpdate("_conversation");
unsub();
}
if (event.type === "error") {
message.text = event.data.message;
message.error = true;
this.requestUpdate("_conversation");
unsub();
}
},
{
start_stage: "intent",
input: { text },
end_stage: "intent",
pipeline: this.pipeline?.id,
conversation_id: this._conversationId,
}
);
} catch {
message.text = this.hass.localize("ui.dialogs.voice_command.error");
message.error = true;
this.requestUpdate("_conversation");
} finally {
this._processing = false;
}
}
static get styles(): CSSResultGroup {
return css`
:host {
flex: 1;
display: flex;
flex-direction: column;
min-height: var(--ha-assist-chat-min-height, 415px);
}
ha-textfield {
display: block;
margin: 0 24px 16px;
}
.messages {
flex: 1;
display: block;
box-sizing: border-box;
position: relative;
}
.messages-container {
position: absolute;
bottom: 0px;
right: 0px;
left: 0px;
padding: 24px;
box-sizing: border-box;
overflow-y: auto;
max-height: 100%;
}
.message {
white-space: pre-line;
font-size: 18px;
clear: both;
margin: 8px 0;
padding: 8px;
border-radius: 15px;
}
@media all and (max-width: 450px), all and (max-height: 500px) {
.message {
font-size: 16px;
}
}
.message p {
margin: 0;
}
.message p:not(:last-child) {
margin-bottom: 8px;
}
.message.user {
margin-left: 24px;
margin-inline-start: 24px;
margin-inline-end: initial;
float: var(--float-end);
text-align: right;
border-bottom-right-radius: 0px;
background-color: var(--primary-color);
color: var(--text-primary-color);
direction: var(--direction);
}
.message.hass {
margin-right: 24px;
margin-inline-end: 24px;
margin-inline-start: initial;
float: var(--float-start);
border-bottom-left-radius: 0px;
background-color: var(--secondary-background-color);
color: var(--primary-text-color);
direction: var(--direction);
}
.message.user a {
color: var(--text-primary-color);
}
.message.hass a {
color: var(--primary-text-color);
}
.message.error {
background-color: var(--error-color);
color: var(--text-primary-color);
}
.bouncer {
width: 48px;
height: 48px;
position: absolute;
}
.double-bounce1,
.double-bounce2 {
width: 48px;
height: 48px;
border-radius: 50%;
background-color: var(--primary-color);
opacity: 0.2;
position: absolute;
top: 0;
left: 0;
-webkit-animation: sk-bounce 2s infinite ease-in-out;
animation: sk-bounce 2s infinite ease-in-out;
}
.double-bounce2 {
-webkit-animation-delay: -1s;
animation-delay: -1s;
}
@-webkit-keyframes sk-bounce {
0%,
100% {
-webkit-transform: scale(0);
}
50% {
-webkit-transform: scale(1);
}
}
@keyframes sk-bounce {
0%,
100% {
transform: scale(0);
-webkit-transform: scale(0);
}
50% {
transform: scale(1);
-webkit-transform: scale(1);
}
}
.listening-icon {
position: relative;
color: var(--secondary-text-color);
margin-right: -24px;
margin-inline-end: -24px;
margin-inline-start: initial;
direction: var(--direction);
transform: scaleX(var(--scale-direction));
}
.listening-icon[active] {
color: var(--primary-color);
}
.unsupported {
color: var(--error-color);
position: absolute;
--mdc-icon-size: 16px;
right: 5px;
inset-inline-end: 5px;
inset-inline-start: initial;
top: 0px;
}
`;
}
}
declare global {
interface HTMLElementTagNameMap {
"ha-assist-chat": HaAssistChat;
}
}

View File

@ -1,16 +1,13 @@
import "@material/mwc-button/mwc-button";
import {
mdiAlertCircle,
mdiChevronDown,
mdiClose,
mdiHelpCircleOutline,
mdiMicrophone,
mdiSend,
mdiStar,
} from "@mdi/js";
import type { CSSResultGroup, PropertyValues, TemplateResult } from "lit";
import type { CSSResultGroup, PropertyValues } from "lit";
import { css, html, LitElement, nothing } from "lit";
import { customElement, property, query, state } from "lit/decorators";
import { customElement, property, state } from "lit/decorators";
import { storage } from "../../common/decorators/storage";
import { fireEvent } from "../../common/dom/fire_event";
import { stopPropagation } from "../../common/dom/stop_propagation";
@ -20,36 +17,23 @@ import "../../components/ha-dialog";
import "../../components/ha-dialog-header";
import "../../components/ha-icon-button";
import "../../components/ha-list-item";
import "../../components/ha-textfield";
import "../../components/ha-alert";
import type { HaTextField } from "../../components/ha-textfield";
import "../../components/ha-assist-chat";
import "../../components/ha-circular-progress";
import type { AssistPipeline } from "../../data/assist_pipeline";
import {
getAssistPipeline,
listAssistPipelines,
runAssistPipeline,
} from "../../data/assist_pipeline";
import { haStyleDialog } from "../../resources/styles";
import type { HomeAssistant } from "../../types";
import { AudioRecorder } from "../../util/audio-recorder";
import { documentationUrl } from "../../util/documentation-url";
import { showAlertDialog } from "../generic/show-dialog-box";
import type { VoiceCommandDialogParams } from "./show-ha-voice-command-dialog";
import { supportsFeature } from "../../common/entity/supports-feature";
import { ConversationEntityFeature } from "../../data/conversation";
interface Message {
who: string;
text?: string | TemplateResult;
error?: boolean;
}
@customElement("ha-voice-command-dialog")
export class HaVoiceCommandDialog extends LitElement {
@property({ attribute: false }) public hass!: HomeAssistant;
@state() private _conversation?: Message[];
@state() private _opened = false;
@storage({
@ -61,69 +45,34 @@ export class HaVoiceCommandDialog extends LitElement {
@state() private _pipeline?: AssistPipeline;
@state() private _showSendButton = false;
@state() private _pipelines?: AssistPipeline[];
@state() private _preferredPipeline?: string;
@query("#scroll-container") private _scrollContainer!: HTMLDivElement;
@state() private _errorLoadAssist?: "not_found" | "unknown";
@query("#message-input") private _messageInput!: HaTextField;
private _conversationId: string | null = null;
private _audioRecorder?: AudioRecorder;
private _audioBuffer?: Int16Array[];
private _audio?: HTMLAudioElement;
private _stt_binary_handler_id?: number | null;
private _pipelinePromise?: Promise<AssistPipeline>;
private _startListening = false;
public async showDialog(
params: Required<VoiceCommandDialogParams>
): Promise<void> {
if (params.pipeline_id === "last_used") {
// Do not set pipeline id (retrieve from storage)
} else if (params.pipeline_id === "preferred") {
if (
params.pipeline_id === "preferred" ||
(params.pipeline_id === "last_used" && !this._pipelineId)
) {
await this._loadPipelines();
this._pipelineId = this._preferredPipeline;
} else {
} else if (!["last_used", "preferred"].includes(params.pipeline_id)) {
this._pipelineId = params.pipeline_id;
}
this._conversation = [
{
who: "hass",
text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"),
},
];
this._startListening = params.start_listening;
this._opened = true;
await this.updateComplete;
this._scrollMessagesBottom();
await this._pipelinePromise;
if (
params?.start_listening &&
this._pipeline?.stt_engine &&
AudioRecorder.isSupported
) {
this._toggleListening();
}
}
public async closeDialog(): Promise<void> {
this._opened = false;
this._pipeline = undefined;
this._pipelines = undefined;
this._conversation = undefined;
this._conversationId = null;
this._audioRecorder?.close();
this._audioRecorder = undefined;
this._audio?.pause();
fireEvent(this, "dialog-closed", { dialog: this.localName });
}
@ -132,24 +81,13 @@ export class HaVoiceCommandDialog extends LitElement {
return nothing;
}
const controlHA = !this._pipeline
? false
: this._pipeline.prefer_local_intents ||
(this.hass.states[this._pipeline.conversation_engine]
? supportsFeature(
this.hass.states[this._pipeline.conversation_engine],
ConversationEntityFeature.CONTROL
)
: true);
const supportsMicrophone = AudioRecorder.isSupported;
const supportsSTT = this._pipeline?.stt_engine;
return html`
<ha-dialog
open
@closed=${this.closeDialog}
.heading=${this.hass.localize("ui.dialogs.voice_command.title")}
flexContent
hideactions
>
<ha-dialog-header slot="heading">
<ha-icon-button
@ -173,26 +111,34 @@ export class HaVoiceCommandDialog extends LitElement {
.path=${mdiChevronDown}
></ha-svg-icon>
</ha-button>
${this._pipelines?.map(
(pipeline) =>
html`<ha-list-item
?selected=${pipeline.id === this._pipelineId ||
(!this._pipelineId &&
pipeline.id === this._preferredPipeline)}
.pipeline=${pipeline.id}
@click=${this._selectPipeline}
.hasMeta=${pipeline.id === this._preferredPipeline}
>
${pipeline.name}${pipeline.id === this._preferredPipeline
? html`
<ha-svg-icon
slot="meta"
.path=${mdiStar}
></ha-svg-icon>
`
: nothing}
</ha-list-item>`
)}
${!this._pipelines
? html`<div class="pipelines-loading">
<ha-circular-progress
indeterminate
size="small"
></ha-circular-progress>
</div>`
: this._pipelines?.map(
(pipeline) =>
html`<ha-list-item
?selected=${pipeline.id === this._pipelineId ||
(!this._pipelineId &&
pipeline.id === this._preferredPipeline)}
.pipeline=${pipeline.id}
@click=${this._selectPipeline}
.hasMeta=${pipeline.id === this._preferredPipeline}
>
${pipeline.name}${pipeline.id ===
this._preferredPipeline
? html`
<ha-svg-icon
slot="meta"
.path=${mdiStar}
></ha-svg-icon>
`
: nothing}
</ha-list-item>`
)}
${this.hass.user?.is_admin
? html`<li divider role="separator"></li>
<a href="/config/voice-assistants/assistants"
@ -217,80 +163,29 @@ export class HaVoiceCommandDialog extends LitElement {
></ha-icon-button>
</a>
</ha-dialog-header>
${controlHA
? nothing
: html`
<ha-alert>
${this.hass.localize(
"ui.dialogs.voice_command.conversation_no_control"
)}
</ha-alert>
`}
<div class="messages">
<div class="messages-container" id="scroll-container">
${this._conversation!.map(
// New lines matter for messages
// prettier-ignore
(message) => html`
<div class=${this._computeMessageClasses(message)}>${message.text}</div>
`
)}
</div>
</div>
<div class="input" slot="primaryAction">
<ha-textfield
id="message-input"
@keyup=${this._handleKeyUp}
@input=${this._handleInput}
.label=${this.hass.localize(`ui.dialogs.voice_command.input_label`)}
dialogInitialFocus
iconTrailing
>
<span slot="trailingIcon">
${this._showSendButton || !supportsSTT
? html`
<ha-icon-button
class="listening-icon"
.path=${mdiSend}
@click=${this._handleSendMessage}
.label=${this.hass.localize(
"ui.dialogs.voice_command.send_text"
)}
>
</ha-icon-button>
`
: html`
${this._audioRecorder?.active
? html`
<div class="bouncer">
<div class="double-bounce1"></div>
<div class="double-bounce2"></div>
</div>
`
: nothing}
<div class="listening-icon">
<ha-icon-button
.path=${mdiMicrophone}
@click=${this._handleListeningButton}
.label=${this.hass.localize(
"ui.dialogs.voice_command.start_listening"
)}
>
</ha-icon-button>
${!supportsMicrophone
? html`
<ha-svg-icon
.path=${mdiAlertCircle}
class="unsupported"
></ha-svg-icon>
`
: null}
</div>
`}
</span>
</ha-textfield>
</div>
${this._pipeline
? html`
<ha-assist-chat
.hass=${this.hass}
.pipeline=${this._pipeline}
.startListening=${this._startListening}
>
</ha-assist-chat>
`
: html`<div class="pipelines-loading">
<ha-circular-progress
indeterminate
size="large"
></ha-circular-progress>
</div>`}
${this._errorLoadAssist
? html`<ha-alert alert-type="error">
${this.hass.localize(
`ui.dialogs.voice_command.${this._errorLoadAssist}_error_load_assist`
)}
</ha-alert>`
: nothing}
</ha-dialog>
`;
}
@ -298,23 +193,14 @@ export class HaVoiceCommandDialog extends LitElement {
protected willUpdate(changedProperties: PropertyValues): void {
if (
changedProperties.has("_pipelineId") ||
(changedProperties.has("_opened") && this._opened === true)
(changedProperties.has("_opened") &&
this._opened === true &&
this._pipelineId)
) {
this._getPipeline();
}
}
private async _getPipeline() {
try {
this._pipelinePromise = getAssistPipeline(this.hass, this._pipelineId);
this._pipeline = await this._pipelinePromise;
} catch (e: any) {
if (e.code === "not_found") {
this._pipelineId = undefined;
}
}
}
private async _loadPipelines() {
if (this._pipelines) {
return;
@ -328,343 +214,28 @@ export class HaVoiceCommandDialog extends LitElement {
private async _selectPipeline(ev: CustomEvent) {
this._pipelineId = (ev.currentTarget as any).pipeline;
this._conversation = [
{
who: "hass",
text: this.hass.localize("ui.dialogs.voice_command.how_can_i_help"),
},
];
await this.updateComplete;
this._scrollMessagesBottom();
}
protected updated(changedProps: PropertyValues) {
super.updated(changedProps);
if (changedProps.has("_conversation") || changedProps.has("results")) {
this._scrollMessagesBottom();
}
}
private _addMessage(message: Message) {
this._conversation = [...this._conversation!, message];
}
private _handleKeyUp(ev: KeyboardEvent) {
const input = ev.target as HaTextField;
if (ev.key === "Enter" && input.value) {
this._processText(input.value);
input.value = "";
this._showSendButton = false;
}
}
private _handleInput(ev: InputEvent) {
const value = (ev.target as HaTextField).value;
if (value && !this._showSendButton) {
this._showSendButton = true;
} else if (!value && this._showSendButton) {
this._showSendButton = false;
}
}
private _handleSendMessage() {
if (this._messageInput.value) {
this._processText(this._messageInput.value.trim());
this._messageInput.value = "";
this._showSendButton = false;
}
}
private async _processText(text: string) {
this._audio?.pause();
this._addMessage({ who: "user", text });
const message: Message = {
who: "hass",
text: "…",
};
// To make sure the answer is placed at the right user text, we add it before we process it
this._addMessage(message);
private async _getPipeline() {
try {
const unsub = await runAssistPipeline(
this.hass,
(event) => {
if (event.type === "intent-end") {
this._conversationId = event.data.intent_output.conversation_id;
const plain = event.data.intent_output.response.speech?.plain;
if (plain) {
message.text = plain.speech;
}
this.requestUpdate("_conversation");
unsub();
}
if (event.type === "error") {
message.text = event.data.message;
message.error = true;
this.requestUpdate("_conversation");
unsub();
}
},
{
start_stage: "intent",
input: { text },
end_stage: "intent",
pipeline: this._pipeline?.id,
conversation_id: this._conversationId,
}
);
} catch {
message.text = this.hass.localize("ui.dialogs.voice_command.error");
message.error = true;
this.requestUpdate("_conversation");
}
}
private _handleListeningButton(ev) {
ev.stopPropagation();
ev.preventDefault();
this._toggleListening();
}
private _toggleListening() {
const supportsMicrophone = AudioRecorder.isSupported;
if (!supportsMicrophone) {
this._showNotSupportedMessage();
return;
}
if (!this._audioRecorder?.active) {
this._startListening();
} else {
this._stopListening();
}
}
private async _showNotSupportedMessage() {
this._addMessage({
who: "hass",
text:
// New lines matter for messages
// prettier-ignore
html`${this.hass.localize(
"ui.dialogs.voice_command.not_supported_microphone_browser"
)}
${this.hass.localize(
"ui.dialogs.voice_command.not_supported_microphone_documentation",
{
documentation_link: html`<a
target="_blank"
rel="noopener noreferrer"
href=${documentationUrl(
this.hass,
"/docs/configuration/securing/#remote-access"
)}
>${this.hass.localize(
"ui.dialogs.voice_command.not_supported_microphone_documentation_link"
)}</a>`,
}
)}`,
});
}
private async _startListening() {
this._audio?.pause();
if (!this._audioRecorder) {
this._audioRecorder = new AudioRecorder((audio) => {
if (this._audioBuffer) {
this._audioBuffer.push(audio);
} else {
this._sendAudioChunk(audio);
}
});
}
this._stt_binary_handler_id = undefined;
this._audioBuffer = [];
const userMessage: Message = {
who: "user",
text: "…",
};
await this._audioRecorder.start();
this._addMessage(userMessage);
this.requestUpdate("_audioRecorder");
const hassMessage: Message = {
who: "hass",
text: "…",
};
// To make sure the answer is placed at the right user text, we add it before we process it
try {
const unsub = await runAssistPipeline(
this.hass,
(event) => {
if (event.type === "run-start") {
this._stt_binary_handler_id =
event.data.runner_data.stt_binary_handler_id;
}
// When we start STT stage, the WS has a binary handler
if (event.type === "stt-start" && this._audioBuffer) {
// Send the buffer over the WS to the STT engine.
for (const buffer of this._audioBuffer) {
this._sendAudioChunk(buffer);
}
this._audioBuffer = undefined;
}
// Stop recording if the server is done with STT stage
if (event.type === "stt-end") {
this._stt_binary_handler_id = undefined;
this._stopListening();
userMessage.text = event.data.stt_output.text;
this.requestUpdate("_conversation");
// To make sure the answer is placed at the right user text, we add it before we process it
this._addMessage(hassMessage);
}
if (event.type === "intent-end") {
this._conversationId = event.data.intent_output.conversation_id;
const plain = event.data.intent_output.response.speech?.plain;
if (plain) {
hassMessage.text = plain.speech;
}
this.requestUpdate("_conversation");
}
if (event.type === "tts-end") {
const url = event.data.tts_output.url;
this._audio = new Audio(url);
this._audio.play();
this._audio.addEventListener("ended", this._unloadAudio);
this._audio.addEventListener("pause", this._unloadAudio);
this._audio.addEventListener("canplaythrough", this._playAudio);
this._audio.addEventListener("error", this._audioError);
}
if (event.type === "run-end") {
this._stt_binary_handler_id = undefined;
unsub();
}
if (event.type === "error") {
this._stt_binary_handler_id = undefined;
if (userMessage.text === "…") {
userMessage.text = event.data.message;
userMessage.error = true;
} else {
hassMessage.text = event.data.message;
hassMessage.error = true;
}
this._stopListening();
this.requestUpdate("_conversation");
unsub();
}
},
{
start_stage: "stt",
end_stage: this._pipeline?.tts_engine ? "tts" : "intent",
input: { sample_rate: this._audioRecorder.sampleRate! },
pipeline: this._pipeline?.id,
conversation_id: this._conversationId,
}
);
} catch (err: any) {
await showAlertDialog(this, {
title: "Error starting pipeline",
text: err.message || err,
});
this._stopListening();
}
}
private _stopListening() {
this._audioRecorder?.stop();
this.requestUpdate("_audioRecorder");
// We're currently STTing, so finish audio
if (this._stt_binary_handler_id) {
if (this._audioBuffer) {
for (const chunk of this._audioBuffer) {
this._sendAudioChunk(chunk);
}
this._pipeline = await getAssistPipeline(this.hass, this._pipelineId);
} catch (e: any) {
if (e.code === "not_found") {
this._errorLoadAssist = "not_found";
} else {
this._errorLoadAssist = "unknown";
// eslint-disable-next-line no-console
console.error(e);
}
// Send empty message to indicate we're done streaming.
this._sendAudioChunk(new Int16Array());
this._stt_binary_handler_id = undefined;
}
this._audioBuffer = undefined;
}
private _sendAudioChunk(chunk: Int16Array) {
this.hass.connection.socket!.binaryType = "arraybuffer";
// eslint-disable-next-line eqeqeq
if (this._stt_binary_handler_id == undefined) {
return;
}
// Turn into 8 bit so we can prefix our handler ID.
const data = new Uint8Array(1 + chunk.length * 2);
data[0] = this._stt_binary_handler_id;
data.set(new Uint8Array(chunk.buffer), 1);
this.hass.connection.socket!.send(data);
}
private _playAudio = () => {
this._audio?.play();
};
private _audioError = () => {
showAlertDialog(this, { title: "Error playing audio." });
this._audio?.removeAttribute("src");
};
private _unloadAudio = () => {
this._audio?.removeAttribute("src");
this._audio = undefined;
};
private _scrollMessagesBottom() {
const scrollContainer = this._scrollContainer;
if (!scrollContainer) {
return;
}
scrollContainer.scrollTo(0, 99999);
}
private _computeMessageClasses(message: Message) {
return `message ${message.who} ${message.error ? " error" : ""}`;
}
static get styles(): CSSResultGroup {
return [
haStyleDialog,
css`
.listening-icon {
position: relative;
color: var(--secondary-text-color);
margin-right: -24px;
margin-inline-end: -24px;
margin-inline-start: initial;
direction: var(--direction);
transform: scaleX(var(--scale-direction));
}
.listening-icon[active] {
color: var(--primary-color);
}
.unsupported {
color: var(--error-color);
position: absolute;
--mdc-icon-size: 16px;
right: 5px;
inset-inline-end: 5px;
inset-inline-start: initial;
top: 0px;
}
ha-dialog {
--primary-action-button-flex: 1;
--secondary-action-button-flex: 0;
--mdc-dialog-max-width: 500px;
--mdc-dialog-max-height: 500px;
--dialog-content-padding: 0;
@ -722,157 +293,10 @@ export class HaVoiceCommandDialog extends LitElement {
ha-button-menu a {
text-decoration: none;
}
ha-textfield {
display: block;
overflow: hidden;
}
a.button {
text-decoration: none;
}
a.button > mwc-button {
width: 100%;
}
.side-by-side {
.pipelines-loading {
display: flex;
margin: 8px 0;
}
.side-by-side > * {
flex: 1 0;
padding: 4px;
}
.messages {
display: block;
height: 400px;
box-sizing: border-box;
position: relative;
}
@media all and (max-width: 450px), all and (max-height: 500px) {
ha-dialog {
--mdc-dialog-max-width: 100%;
}
.messages {
height: 100%;
flex: 1;
}
}
.messages-container {
position: absolute;
bottom: 0px;
right: 0px;
left: 0px;
padding: 24px;
box-sizing: border-box;
overflow-y: auto;
max-height: 100%;
}
.message {
white-space: pre-line;
font-size: 18px;
clear: both;
margin: 8px 0;
padding: 8px;
border-radius: 15px;
}
.message p {
margin: 0;
}
.message p:not(:last-child) {
margin-bottom: 8px;
}
.message.user {
margin-left: 24px;
margin-inline-start: 24px;
margin-inline-end: initial;
float: var(--float-end);
text-align: right;
border-bottom-right-radius: 0px;
background-color: var(--primary-color);
color: var(--text-primary-color);
direction: var(--direction);
}
.message.hass {
margin-right: 24px;
margin-inline-end: 24px;
margin-inline-start: initial;
float: var(--float-start);
border-bottom-left-radius: 0px;
background-color: var(--secondary-background-color);
color: var(--primary-text-color);
direction: var(--direction);
}
.message.user a {
color: var(--text-primary-color);
}
.message.hass a {
color: var(--primary-text-color);
}
.message img {
width: 100%;
border-radius: 10px;
}
.message.error {
background-color: var(--error-color);
color: var(--text-primary-color);
}
.input {
margin-left: 0;
margin-right: 0;
}
.bouncer {
width: 48px;
height: 48px;
position: absolute;
}
.double-bounce1,
.double-bounce2 {
width: 48px;
height: 48px;
border-radius: 50%;
background-color: var(--primary-color);
opacity: 0.2;
position: absolute;
top: 0;
left: 0;
-webkit-animation: sk-bounce 2s infinite ease-in-out;
animation: sk-bounce 2s infinite ease-in-out;
}
.double-bounce2 {
-webkit-animation-delay: -1s;
animation-delay: -1s;
}
@-webkit-keyframes sk-bounce {
0%,
100% {
-webkit-transform: scale(0);
}
50% {
-webkit-transform: scale(1);
}
}
@keyframes sk-bounce {
0%,
100% {
transform: scale(0);
-webkit-transform: scale(0);
}
50% {
transform: scale(1);
-webkit-transform: scale(1);
}
}
@media all and (max-width: 450px), all and (max-height: 500px) {
.message {
font-size: 16px;
}
justify-content: center;
}
`,
];

View File

@ -1215,7 +1215,9 @@
"manage_assistants": "Manage assistants",
"not_supported_microphone_browser": "Your connection to Home Assistant is not secured using HTTPS. This causes browsers to block Home Assistant from accessing the microphone.",
"not_supported_microphone_documentation": "Use the Home Assistant app or visit {documentation_link} to learn how to use a secure URL",
"not_supported_microphone_documentation_link": "the documentation"
"not_supported_microphone_documentation_link": "the documentation",
"unknown_error_load_assist": "Loading the assist pipeline failed",
"not_found_error_load_assist": "Cannot find the assist pipeline"
},
"generic": {
"cancel": "Cancel",