diff --git a/src/data/voice_assistant.ts b/src/data/voice_assistant.ts index 0de01e1db0..c332e19267 100644 --- a/src/data/voice_assistant.ts +++ b/src/data/voice_assistant.ts @@ -14,6 +14,7 @@ interface PipelineRunStartEvent extends PipelineEventBase { language: string; runner_data: { stt_binary_handler_id: number | null; + timeout: number; }; }; } @@ -40,7 +41,7 @@ interface PipelineSTTStartEvent extends PipelineEventBase { interface PipelineSTTEndEvent extends PipelineEventBase { type: "stt-end"; data: { - text: string; + stt_output: { text: string }; }; } diff --git a/src/panels/config/integrations/integration-panels/voice_assistant/assist/assist-pipeline-debug.ts b/src/panels/config/integrations/integration-panels/voice_assistant/assist/assist-pipeline-debug.ts index 7e95f9b590..c0dcd3d5af 100644 --- a/src/panels/config/integrations/integration-panels/voice_assistant/assist/assist-pipeline-debug.ts +++ b/src/panels/config/integrations/integration-panels/voice_assistant/assist/assist-pipeline-debug.ts @@ -1,317 +1,131 @@ import { css, html, LitElement, PropertyValues, TemplateResult } from "lit"; -import { customElement, property, state } from "lit/decorators"; -import "../../../../../../components/ha-card"; -import "../../../../../../components/ha-alert"; +import { customElement, property, query, state } from "lit/decorators"; import "../../../../../../components/ha-button"; -import "../../../../../../components/ha-circular-progress"; -import "../../../../../../components/ha-expansion-panel"; -import "../../../../../../components/ha-textfield"; import { PipelineRun, runPipelineFromText, } from "../../../../../../data/voice_assistant"; import "../../../../../../layouts/hass-subpage"; -import { SubscribeMixin } from "../../../../../../mixins/subscribe-mixin"; +import "../../../../../../components/ha-formfield"; +import "../../../../../../components/ha-checkbox"; import { haStyle } from "../../../../../../resources/styles"; import type { HomeAssistant } from "../../../../../../types"; -import { formatNumber } from "../../../../../../common/number/format_number"; import { showPromptDialog } from "../../../../../../dialogs/generic/show-dialog-box"; - -const RUN_DATA = { - pipeline: "Pipeline", - language: "Language", -}; - -const STT_DATA = { - engine: "Engine", -}; - -const INTENT_DATA = { - engine: "Engine", - intent_input: "Input", -}; - -const TTS_DATA = { - engine: "Engine", - tts_input: "Input", -}; - -const STAGES: Record = { - ready: 0, - stt: 1, - intent: 2, - tts: 3, - done: 4, - error: 5, -}; - -const hasStage = (run: PipelineRun, stage: PipelineRun["stage"]) => - STAGES[run.init_options.start_stage] <= STAGES[stage] && - STAGES[stage] <= STAGES[run.init_options.end_stage]; - -const maybeRenderError = ( - run: PipelineRun, - stage: string, - lastRunStage: string -) => { - if (run.stage !== "error" || lastRunStage !== stage) { - return ""; - } - - return html` - ${run.error!.message} (${run.error!.code}) - `; -}; - -const renderProgress = ( - hass: HomeAssistant, - pipelineRun: PipelineRun, - stage: PipelineRun["stage"] -) => { - const startEvent = pipelineRun.events.find( - (ev) => ev.type === `${stage}-start` - ); - const finishEvent = pipelineRun.events.find( - (ev) => ev.type === `${stage}-end` - ); - - if (!startEvent) { - return ""; - } - - if (pipelineRun.stage === "error") { - return html`❌`; - } - - if (!finishEvent) { - return html``; - } - - const duration = - new Date(finishEvent.timestamp).getTime() - - new Date(startEvent.timestamp).getTime(); - const durationString = formatNumber(duration / 1000, hass.locale, { - maximumFractionDigits: 2, - }); - return html`${durationString}s ✅`; -}; - -const renderData = (data: Record, keys: Record) => - Object.entries(keys).map( - ([key, label]) => - html` -
-
${label}
-
${data[key]}
-
- ` - ); - -const dataMinusKeysRender = ( - data: Record, - keys: Record -) => { - const result = {}; - let render = false; - for (const key in data) { - if (key in keys) { - continue; - } - render = true; - result[key] = data[key]; - } - return render ? html`
${JSON.stringify(result, null, 2)}
` : ""; -}; +import "./assist-render-pipeline-run"; +import type { HaCheckbox } from "../../../../../../components/ha-checkbox"; +import type { HaTextField } from "../../../../../../components/ha-textfield"; +import "../../../../../../components/ha-textfield"; @customElement("assist-pipeline-debug") -export class AssistPipelineDebug extends SubscribeMixin(LitElement) { +export class AssistPipelineDebug extends LitElement { @property({ attribute: false }) public hass!: HomeAssistant; @property({ type: Boolean }) public narrow!: boolean; - @state() private _pipelineRun?: PipelineRun; + @state() private _pipelineRuns: PipelineRun[] = []; @state() private _stopRecording?: () => void; + @query("#continue-conversation") + private _continueConversationCheckbox!: HaCheckbox; + + @query("#continue-conversation-text") + private _continueConversationTextField?: HaTextField; + private _audioBuffer?: Int16Array[]; - protected render(): TemplateResult { - const lastRunStage: string = this._pipelineRun - ? ["tts", "intent", "stt"].find( - (stage) => this._pipelineRun![stage] !== undefined - ) || "ready" - : "ready"; + @state() private _finished = false; + protected render(): TemplateResult { return html` + ${this._pipelineRuns.length > 0 + ? html` + + Clear + + ` + : ""} +
- - Run Text Pipeline - - - Run Audio Pipeline - + ${this._pipelineRuns.length === 0 + ? html` + + Run Text Pipeline + + + Run Audio Pipeline + + ` + : this._pipelineRuns[0].init_options.start_stage === "intent" + ? html` + + + Send + + ` + : html` + + + + `}
- ${this._pipelineRun - ? html` - -
-
-
Run
-
${this._pipelineRun.stage}
-
- - ${renderData(this._pipelineRun.run, RUN_DATA)} -
- - - ${maybeRenderError(this._pipelineRun, "ready", lastRunStage)} - ${hasStage(this._pipelineRun, "stt") - ? html` - -
-
- Speech-to-Text - ${renderProgress( - this.hass, - this._pipelineRun, - "stt" - )} -
- ${this._pipelineRun.stt - ? html` -
- ${renderData(this._pipelineRun.stt, STT_DATA)} - ${dataMinusKeysRender( - this._pipelineRun.stt, - STT_DATA - )} -
- ` - : ""} -
- ${this._pipelineRun.stage === "stt" && - this._stopRecording - ? html` -
- - Stop Recording - -
- ` - : ""} - - ` - : ""} - ${maybeRenderError(this._pipelineRun, "stt", lastRunStage)} - ${hasStage(this._pipelineRun, "intent") - ? html` - -
-
- Natural Language Processing - ${renderProgress( - this.hass, - this._pipelineRun, - "intent" - )} -
- ${this._pipelineRun.intent - ? html` -
- ${renderData( - this._pipelineRun.intent, - INTENT_DATA - )} - ${dataMinusKeysRender( - this._pipelineRun.intent, - INTENT_DATA - )} -
- ` - : ""} -
- - ` - : ""} - ${maybeRenderError(this._pipelineRun, "intent", lastRunStage)} - ${hasStage(this._pipelineRun, "tts") - ? html` - -
-
- Text-to-Speech - ${renderProgress( - this.hass, - this._pipelineRun, - "tts" - )} -
- ${this._pipelineRun.tts - ? html` -
- ${renderData(this._pipelineRun.tts, TTS_DATA)} -
- ` - : ""} -
- ${this._pipelineRun?.tts?.tts_output - ? html` -
- - Play Audio - -
- ` - : ""} - - ` - : ""} - ${maybeRenderError(this._pipelineRun, "tts", lastRunStage)} - - - Raw - 
${JSON.stringify(this._pipelineRun, null, 2)}
- - - ` - : ""} + ${this._pipelineRuns.map((run) => + run === null + ? "" + : html` + + ` + )}
`; } - protected updated(changedProperties: PropertyValues): void { - super.updated(changedProperties); + protected willUpdate(changedProperties: PropertyValues): void { + super.willUpdate(changedProperties); if ( - !changedProperties.has("_pipelineRun") || - !this._pipelineRun || - this._pipelineRun.init_options.start_stage !== "stt" + !changedProperties.has("_pipelineRuns") || + this._pipelineRuns.length === 0 ) { return; } - if (this._pipelineRun.stage === "stt" && this._audioBuffer) { + const currentRun = this._pipelineRuns[0]; + + if (currentRun.init_options.start_stage !== "stt") { + if (["error", "done"].includes(currentRun.stage)) { + this._finished = true; + } + return; + } + + if (currentRun.stage === "stt" && this._audioBuffer) { // Send the buffer over the WS to the STT engine. for (const buffer of this._audioBuffer) { this._sendAudioChunk(buffer); @@ -319,31 +133,70 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) { this._audioBuffer = undefined; } - if (this._pipelineRun.stage !== "stt" && this._stopRecording) { + if (currentRun.stage !== "stt" && this._stopRecording) { this._stopRecording(); } + + if (currentRun.stage === "done") { + const url = currentRun.tts!.tts_output!.url; + const audio = new Audio(url); + audio.addEventListener("ended", () => { + if (this._continueConversationCheckbox.checked) { + this._runAudioPipeline(); + } else { + this._finished = true; + } + }); + audio.play(); + } else if (currentRun.stage === "error") { + this._finished = true; + } + } + + private get conversationId(): string | null { + return this._pipelineRuns.length === 0 + ? null + : this._pipelineRuns[0].intent?.intent_output?.conversation_id || null; } private async _runTextPipeline() { - const text = await showPromptDialog(this, { - title: "Input text", - confirmText: "Run", - }); + const textfield = this._continueConversationTextField; + + let text: string | null; + + if (textfield) { + text = textfield.value; + } else { + text = await showPromptDialog(this, { + title: "Input text", + confirmText: "Run", + }); + } if (!text) { return; } - this._pipelineRun = undefined; + let added = false; runPipelineFromText( this.hass, (run) => { - this._pipelineRun = run; + if (textfield && ["done", "error"].includes(run.stage)) { + textfield.value = ""; + } + + if (added) { + this._pipelineRuns = [run, ...this._pipelineRuns.slice(1)]; + } else { + this._pipelineRuns = [run, ...this._pipelineRuns]; + added = true; + } }, { start_stage: "intent", end_stage: "intent", input: { text }, + conversation_id: this.conversationId, } ); } @@ -375,21 +228,28 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) { this._audioBuffer.push(e.data); return; } - if (this._pipelineRun?.stage !== "stt") { + if (this._pipelineRuns[0].stage !== "stt") { return; } this._sendAudioChunk(e.data); }; - this._pipelineRun = undefined; + this._finished = false; + let added = false; runPipelineFromText( this.hass, (run) => { - this._pipelineRun = run; + if (added) { + this._pipelineRuns = [run, ...this._pipelineRuns.slice(1)]; + } else { + this._pipelineRuns = [run, ...this._pipelineRuns]; + added = true; + } }, { start_stage: "stt", end_stage: "tts", + conversation_id: this.conversationId, } ); } @@ -397,16 +257,20 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) { private _sendAudioChunk(chunk: Int16Array) { // Turn into 8 bit so we can prefix our handler ID. const data = new Uint8Array(1 + chunk.length * 2); - data[0] = this._pipelineRun!.run.runner_data.stt_binary_handler_id!; + data[0] = this._pipelineRuns[0].run.runner_data.stt_binary_handler_id!; data.set(new Uint8Array(chunk.buffer), 1); this.hass.connection.socket!.send(data); } - private _playTTS(): void { - const url = this._pipelineRun!.tts!.tts_output!.url; - const audio = new Audio(url); - audio.play(); + private _handleContinueKeyDown(ev) { + if (ev.keyCode === 13) { + this._runTextPipeline(); + } + } + + private _clearConversation() { + this._pipelineRuns = []; } static styles = [ @@ -419,32 +283,19 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) { direction: ltr; } .start-row { - text-align: center; - } - .start-row ha-button { - margin: 16px; - } - ha-card, - ha-alert { - display: block; - margin-bottom: 16px; - } - .run-pipeline-card ha-textfield { - display: block; - } - .row { display: flex; - justify-content: space-between; + justify-content: space-around; + align-items: center; + margin: 0 16px 16px; } - pre { - margin: 0; + .start-row ha-textfield { + flex: 1; } - ha-expansion-panel { - padding-left: 8px; + assist-render-pipeline-run { + padding-top: 16px; } - .heading { - font-weight: 500; - margin-bottom: 16px; + assist-render-pipeline-run + assist-render-pipeline-run { + border-top: 3px solid black; } `, ]; diff --git a/src/panels/config/integrations/integration-panels/voice_assistant/assist/assist-render-pipeline-run.ts b/src/panels/config/integrations/integration-panels/voice_assistant/assist/assist-render-pipeline-run.ts new file mode 100644 index 0000000000..f9940e0cb2 --- /dev/null +++ b/src/panels/config/integrations/integration-panels/voice_assistant/assist/assist-render-pipeline-run.ts @@ -0,0 +1,336 @@ +import { css, html, LitElement, TemplateResult } from "lit"; +import { customElement, property } from "lit/decorators"; +import "../../../../../../components/ha-card"; +import "../../../../../../components/ha-alert"; +import "../../../../../../components/ha-button"; +import "../../../../../../components/ha-circular-progress"; +import "../../../../../../components/ha-expansion-panel"; +import type { PipelineRun } from "../../../../../../data/voice_assistant"; +import type { HomeAssistant } from "../../../../../../types"; +import { formatNumber } from "../../../../../../common/number/format_number"; + +const RUN_DATA = { + pipeline: "Pipeline", + language: "Language", +}; + +const STT_DATA = { + engine: "Engine", +}; + +const INTENT_DATA = { + engine: "Engine", + intent_input: "Input", +}; + +const TTS_DATA = { + engine: "Engine", + tts_input: "Input", +}; + +const STAGES: Record = { + ready: 0, + stt: 1, + intent: 2, + tts: 3, + done: 4, + error: 5, +}; + +const hasStage = (run: PipelineRun, stage: PipelineRun["stage"]) => + STAGES[run.init_options.start_stage] <= STAGES[stage] && + STAGES[stage] <= STAGES[run.init_options.end_stage]; + +const maybeRenderError = ( + run: PipelineRun, + stage: string, + lastRunStage: string +) => { + if (run.stage !== "error" || lastRunStage !== stage) { + return ""; + } + + return html` + ${run.error!.message} (${run.error!.code}) + `; +}; + +const renderProgress = ( + hass: HomeAssistant, + pipelineRun: PipelineRun, + stage: PipelineRun["stage"] +) => { + const startEvent = pipelineRun.events.find( + (ev) => ev.type === `${stage}-start` + ); + const finishEvent = pipelineRun.events.find( + (ev) => ev.type === `${stage}-end` + ); + + if (!startEvent) { + return ""; + } + + if (pipelineRun.stage === "error") { + return html`❌`; + } + + if (!finishEvent) { + return html``; + } + + const duration = + new Date(finishEvent.timestamp).getTime() - + new Date(startEvent.timestamp).getTime(); + const durationString = formatNumber(duration / 1000, hass.locale, { + maximumFractionDigits: 2, + }); + return html`${durationString}s ✅`; +}; + +const renderData = (data: Record, keys: Record) => + Object.entries(keys).map( + ([key, label]) => + html` +
+
${label}
+
${data[key]}
+
+ ` + ); + +const dataMinusKeysRender = ( + data: Record, + keys: Record +) => { + const result = {}; + let render = false; + for (const key in data) { + if (key in keys) { + continue; + } + render = true; + result[key] = data[key]; + } + return render ? html`
${JSON.stringify(result, null, 2)}
` : ""; +}; + +@customElement("assist-render-pipeline-run") +export class AssistPipelineDebug extends LitElement { + @property({ attribute: false }) public hass!: HomeAssistant; + + @property() private pipelineRun!: PipelineRun; + + protected render(): TemplateResult { + const lastRunStage: string = this.pipelineRun + ? ["tts", "intent", "stt"].find( + (stage) => this.pipelineRun![stage] !== undefined + ) || "ready" + : "ready"; + + const messages: Array<{ from: string; text: string }> = []; + + const userMessage = + this.pipelineRun.init_options.input?.text || + this.pipelineRun?.stt?.stt_output?.text; + + if (userMessage) { + messages.push({ + from: "user", + text: userMessage, + }); + } + + if ( + this.pipelineRun?.intent?.intent_output?.response?.speech?.plain?.speech + ) { + messages.push({ + from: "hass", + text: this.pipelineRun.intent.intent_output.response.speech.plain + .speech, + }); + } + + return html` + +
+
+
Run
+
${this.pipelineRun.stage}
+
+ + ${renderData(this.pipelineRun.run, RUN_DATA)} + ${messages.length > 0 + ? html` +
+ ${messages.map( + ({ from, text }) => html` +
${text}
+ ` + )} +
+
+ ` + : ""} +
+ + + ${maybeRenderError(this.pipelineRun, "ready", lastRunStage)} + ${hasStage(this.pipelineRun, "stt") + ? html` + +
+
+ Speech-to-Text + ${renderProgress(this.hass, this.pipelineRun, "stt")} +
+ ${this.pipelineRun.stt + ? html` +
+ ${renderData(this.pipelineRun.stt, STT_DATA)} + ${dataMinusKeysRender(this.pipelineRun.stt, STT_DATA)} +
+ ` + : ""} +
+ + ` + : ""} + ${maybeRenderError(this.pipelineRun, "stt", lastRunStage)} + ${hasStage(this.pipelineRun, "intent") + ? html` + +
+
+ Natural Language Processing + ${renderProgress(this.hass, this.pipelineRun, "intent")} +
+ ${this.pipelineRun.intent + ? html` +
+ ${renderData(this.pipelineRun.intent, INTENT_DATA)} + ${dataMinusKeysRender( + this.pipelineRun.intent, + INTENT_DATA + )} +
+ ` + : ""} +
+ + ` + : ""} + ${maybeRenderError(this.pipelineRun, "intent", lastRunStage)} + ${hasStage(this.pipelineRun, "tts") + ? html` + +
+
+ Text-to-Speech + ${renderProgress(this.hass, this.pipelineRun, "tts")} +
+ ${this.pipelineRun.tts + ? html` +
+ ${renderData(this.pipelineRun.tts, TTS_DATA)} +
+ ` + : ""} +
+ ${this.pipelineRun?.tts?.tts_output + ? html` +
+ + Play Audio + +
+ ` + : ""} + + ` + : ""} + ${maybeRenderError(this.pipelineRun, "tts", lastRunStage)} + + + Raw + 
${JSON.stringify(this.pipelineRun, null, 2)}
+ + + `; + } + + private _playTTS(): void { + const url = this.pipelineRun!.tts!.tts_output!.url; + const audio = new Audio(url); + audio.play(); + } + + static styles = css` + :host { + display: block; + } + ha-card, + ha-alert { + display: block; + margin-bottom: 16px; + } + .row { + display: flex; + justify-content: space-between; + } + pre { + margin: 0; + } + ha-expansion-panel { + padding-left: 8px; + } + .heading { + font-weight: 500; + margin-bottom: 16px; + } + + .messages { + margin-top: 8px; + } + + .message { + font-size: 18px; + margin: 8px 0; + padding: 8px; + border-radius: 15px; + clear: both; + } + + .message.user { + margin-left: 24px; + margin-inline-start: 24px; + margin-inline-end: initial; + float: var(--float-end); + text-align: right; + border-bottom-right-radius: 0px; + background-color: var(--light-primary-color); + color: var(--text-light-primary-color, var(--primary-text-color)); + direction: var(--direction); + } + + .message.hass { + margin-right: 24px; + margin-inline-end: 24px; + margin-inline-start: initial; + float: var(--float-start); + border-bottom-left-radius: 0px; + background-color: var(--primary-color); + color: var(--text-primary-color); + direction: var(--direction); + } + `; +} + +declare global { + interface HTMLElementTagNameMap { + "assist-render-pipeline-run": AssistPipelineDebug; + } +}