Allow having conversations (#15961)

* Allow having conversations

* Add timeout to run-start data

* willUpdate
This commit is contained in:
Paulus Schoutsen 2023-03-31 15:44:32 -04:00 committed by GitHub
parent 273904a6eb
commit 232f70d44c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 497 additions and 309 deletions

View File

@ -14,6 +14,7 @@ interface PipelineRunStartEvent extends PipelineEventBase {
language: string;
runner_data: {
stt_binary_handler_id: number | null;
timeout: number;
};
};
}
@ -40,7 +41,7 @@ interface PipelineSTTStartEvent extends PipelineEventBase {
interface PipelineSTTEndEvent extends PipelineEventBase {
type: "stt-end";
data: {
text: string;
stt_output: { text: string };
};
}

View File

@ -1,317 +1,131 @@
import { css, html, LitElement, PropertyValues, TemplateResult } from "lit";
import { customElement, property, state } from "lit/decorators";
import "../../../../../../components/ha-card";
import "../../../../../../components/ha-alert";
import { customElement, property, query, state } from "lit/decorators";
import "../../../../../../components/ha-button";
import "../../../../../../components/ha-circular-progress";
import "../../../../../../components/ha-expansion-panel";
import "../../../../../../components/ha-textfield";
import {
PipelineRun,
runPipelineFromText,
} from "../../../../../../data/voice_assistant";
import "../../../../../../layouts/hass-subpage";
import { SubscribeMixin } from "../../../../../../mixins/subscribe-mixin";
import "../../../../../../components/ha-formfield";
import "../../../../../../components/ha-checkbox";
import { haStyle } from "../../../../../../resources/styles";
import type { HomeAssistant } from "../../../../../../types";
import { formatNumber } from "../../../../../../common/number/format_number";
import { showPromptDialog } from "../../../../../../dialogs/generic/show-dialog-box";
const RUN_DATA = {
pipeline: "Pipeline",
language: "Language",
};
const STT_DATA = {
engine: "Engine",
};
const INTENT_DATA = {
engine: "Engine",
intent_input: "Input",
};
const TTS_DATA = {
engine: "Engine",
tts_input: "Input",
};
const STAGES: Record<PipelineRun["stage"], number> = {
ready: 0,
stt: 1,
intent: 2,
tts: 3,
done: 4,
error: 5,
};
const hasStage = (run: PipelineRun, stage: PipelineRun["stage"]) =>
STAGES[run.init_options.start_stage] <= STAGES[stage] &&
STAGES[stage] <= STAGES[run.init_options.end_stage];
const maybeRenderError = (
run: PipelineRun,
stage: string,
lastRunStage: string
) => {
if (run.stage !== "error" || lastRunStage !== stage) {
return "";
}
return html`<ha-alert alert-type="error">
${run.error!.message} (${run.error!.code})
</ha-alert>`;
};
const renderProgress = (
hass: HomeAssistant,
pipelineRun: PipelineRun,
stage: PipelineRun["stage"]
) => {
const startEvent = pipelineRun.events.find(
(ev) => ev.type === `${stage}-start`
);
const finishEvent = pipelineRun.events.find(
(ev) => ev.type === `${stage}-end`
);
if (!startEvent) {
return "";
}
if (pipelineRun.stage === "error") {
return html``;
}
if (!finishEvent) {
return html`<ha-circular-progress
size="tiny"
active
></ha-circular-progress>`;
}
const duration =
new Date(finishEvent.timestamp).getTime() -
new Date(startEvent.timestamp).getTime();
const durationString = formatNumber(duration / 1000, hass.locale, {
maximumFractionDigits: 2,
});
return html`${durationString}s ✅`;
};
const renderData = (data: Record<string, any>, keys: Record<string, string>) =>
Object.entries(keys).map(
([key, label]) =>
html`
<div class="row">
<div>${label}</div>
<div>${data[key]}</div>
</div>
`
);
const dataMinusKeysRender = (
data: Record<string, any>,
keys: Record<string, string>
) => {
const result = {};
let render = false;
for (const key in data) {
if (key in keys) {
continue;
}
render = true;
result[key] = data[key];
}
return render ? html`<pre>${JSON.stringify(result, null, 2)}</pre>` : "";
};
import "./assist-render-pipeline-run";
import type { HaCheckbox } from "../../../../../../components/ha-checkbox";
import type { HaTextField } from "../../../../../../components/ha-textfield";
import "../../../../../../components/ha-textfield";
@customElement("assist-pipeline-debug")
export class AssistPipelineDebug extends SubscribeMixin(LitElement) {
export class AssistPipelineDebug extends LitElement {
@property({ attribute: false }) public hass!: HomeAssistant;
@property({ type: Boolean }) public narrow!: boolean;
@state() private _pipelineRun?: PipelineRun;
@state() private _pipelineRuns: PipelineRun[] = [];
@state() private _stopRecording?: () => void;
@query("#continue-conversation")
private _continueConversationCheckbox!: HaCheckbox;
@query("#continue-conversation-text")
private _continueConversationTextField?: HaTextField;
private _audioBuffer?: Int16Array[];
protected render(): TemplateResult {
const lastRunStage: string = this._pipelineRun
? ["tts", "intent", "stt"].find(
(stage) => this._pipelineRun![stage] !== undefined
) || "ready"
: "ready";
@state() private _finished = false;
protected render(): TemplateResult {
return html`
<hass-subpage
.narrow=${this.narrow}
.hass=${this.hass}
header="Assist Pipeline"
>
${this._pipelineRuns.length > 0
? html`
<ha-button
slot="toolbar-icon"
@click=${this._clearConversation}
.disabled=${!this._finished}
>
Clear
</ha-button>
`
: ""}
<div class="content">
<div class="start-row">
<ha-button
raised
@click=${this._runTextPipeline}
.disabled=${this._pipelineRun &&
!["error", "done"].includes(this._pipelineRun.stage)}
>
Run Text Pipeline
</ha-button>
<ha-button
raised
@click=${this._runAudioPipeline}
.disabled=${this._pipelineRun &&
!["error", "done"].includes(this._pipelineRun.stage)}
>
Run Audio Pipeline
</ha-button>
${this._pipelineRuns.length === 0
? html`
<ha-button raised @click=${this._runTextPipeline}>
Run Text Pipeline
</ha-button>
<ha-button raised @click=${this._runAudioPipeline}>
Run Audio Pipeline
</ha-button>
`
: this._pipelineRuns[0].init_options.start_stage === "intent"
? html`
<ha-textfield
id="continue-conversation-text"
label="Response"
.disabled=${!this._finished}
@keydown=${this._handleContinueKeyDown}
></ha-textfield>
<ha-button
@click=${this._runTextPipeline}
.disabled=${!this._finished}
>
Send
</ha-button>
`
: html`
<ha-formfield label="Continue conversation">
<ha-checkbox
id="continue-conversation"
checked
></ha-checkbox>
</ha-formfield>
`}
</div>
${this._pipelineRun
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<div>Run</div>
<div>${this._pipelineRun.stage}</div>
</div>
${renderData(this._pipelineRun.run, RUN_DATA)}
</div>
</ha-card>
${maybeRenderError(this._pipelineRun, "ready", lastRunStage)}
${hasStage(this._pipelineRun, "stt")
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Speech-to-Text</span>
${renderProgress(
this.hass,
this._pipelineRun,
"stt"
)}
</div>
${this._pipelineRun.stt
? html`
<div class="card-content">
${renderData(this._pipelineRun.stt, STT_DATA)}
${dataMinusKeysRender(
this._pipelineRun.stt,
STT_DATA
)}
</div>
`
: ""}
</div>
${this._pipelineRun.stage === "stt" &&
this._stopRecording
? html`
<div class="card-actions">
<ha-button @click=${this._stopRecording}>
Stop Recording
</ha-button>
</div>
`
: ""}
</ha-card>
`
: ""}
${maybeRenderError(this._pipelineRun, "stt", lastRunStage)}
${hasStage(this._pipelineRun, "intent")
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Natural Language Processing</span>
${renderProgress(
this.hass,
this._pipelineRun,
"intent"
)}
</div>
${this._pipelineRun.intent
? html`
<div class="card-content">
${renderData(
this._pipelineRun.intent,
INTENT_DATA
)}
${dataMinusKeysRender(
this._pipelineRun.intent,
INTENT_DATA
)}
</div>
`
: ""}
</div>
</ha-card>
`
: ""}
${maybeRenderError(this._pipelineRun, "intent", lastRunStage)}
${hasStage(this._pipelineRun, "tts")
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Text-to-Speech</span>
${renderProgress(
this.hass,
this._pipelineRun,
"tts"
)}
</div>
${this._pipelineRun.tts
? html`
<div class="card-content">
${renderData(this._pipelineRun.tts, TTS_DATA)}
</div>
`
: ""}
</div>
${this._pipelineRun?.tts?.tts_output
? html`
<div class="card-actions">
<ha-button @click=${this._playTTS}>
Play Audio
</ha-button>
</div>
`
: ""}
</ha-card>
`
: ""}
${maybeRenderError(this._pipelineRun, "tts", lastRunStage)}
<ha-card>
<ha-expansion-panel>
<span slot="header">Raw</span>
<pre>${JSON.stringify(this._pipelineRun, null, 2)}</pre>
</ha-expansion-panel>
</ha-card>
`
: ""}
${this._pipelineRuns.map((run) =>
run === null
? ""
: html`
<assist-render-pipeline-run
.hass=${this.hass}
.pipelineRun=${run}
></assist-render-pipeline-run>
`
)}
</div>
</hass-subpage>
`;
}
protected updated(changedProperties: PropertyValues): void {
super.updated(changedProperties);
protected willUpdate(changedProperties: PropertyValues): void {
super.willUpdate(changedProperties);
if (
!changedProperties.has("_pipelineRun") ||
!this._pipelineRun ||
this._pipelineRun.init_options.start_stage !== "stt"
!changedProperties.has("_pipelineRuns") ||
this._pipelineRuns.length === 0
) {
return;
}
if (this._pipelineRun.stage === "stt" && this._audioBuffer) {
const currentRun = this._pipelineRuns[0];
if (currentRun.init_options.start_stage !== "stt") {
if (["error", "done"].includes(currentRun.stage)) {
this._finished = true;
}
return;
}
if (currentRun.stage === "stt" && this._audioBuffer) {
// Send the buffer over the WS to the STT engine.
for (const buffer of this._audioBuffer) {
this._sendAudioChunk(buffer);
@ -319,31 +133,70 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) {
this._audioBuffer = undefined;
}
if (this._pipelineRun.stage !== "stt" && this._stopRecording) {
if (currentRun.stage !== "stt" && this._stopRecording) {
this._stopRecording();
}
if (currentRun.stage === "done") {
const url = currentRun.tts!.tts_output!.url;
const audio = new Audio(url);
audio.addEventListener("ended", () => {
if (this._continueConversationCheckbox.checked) {
this._runAudioPipeline();
} else {
this._finished = true;
}
});
audio.play();
} else if (currentRun.stage === "error") {
this._finished = true;
}
}
private get conversationId(): string | null {
return this._pipelineRuns.length === 0
? null
: this._pipelineRuns[0].intent?.intent_output?.conversation_id || null;
}
private async _runTextPipeline() {
const text = await showPromptDialog(this, {
title: "Input text",
confirmText: "Run",
});
const textfield = this._continueConversationTextField;
let text: string | null;
if (textfield) {
text = textfield.value;
} else {
text = await showPromptDialog(this, {
title: "Input text",
confirmText: "Run",
});
}
if (!text) {
return;
}
this._pipelineRun = undefined;
let added = false;
runPipelineFromText(
this.hass,
(run) => {
this._pipelineRun = run;
if (textfield && ["done", "error"].includes(run.stage)) {
textfield.value = "";
}
if (added) {
this._pipelineRuns = [run, ...this._pipelineRuns.slice(1)];
} else {
this._pipelineRuns = [run, ...this._pipelineRuns];
added = true;
}
},
{
start_stage: "intent",
end_stage: "intent",
input: { text },
conversation_id: this.conversationId,
}
);
}
@ -375,21 +228,28 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) {
this._audioBuffer.push(e.data);
return;
}
if (this._pipelineRun?.stage !== "stt") {
if (this._pipelineRuns[0].stage !== "stt") {
return;
}
this._sendAudioChunk(e.data);
};
this._pipelineRun = undefined;
this._finished = false;
let added = false;
runPipelineFromText(
this.hass,
(run) => {
this._pipelineRun = run;
if (added) {
this._pipelineRuns = [run, ...this._pipelineRuns.slice(1)];
} else {
this._pipelineRuns = [run, ...this._pipelineRuns];
added = true;
}
},
{
start_stage: "stt",
end_stage: "tts",
conversation_id: this.conversationId,
}
);
}
@ -397,16 +257,20 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) {
private _sendAudioChunk(chunk: Int16Array) {
// Turn into 8 bit so we can prefix our handler ID.
const data = new Uint8Array(1 + chunk.length * 2);
data[0] = this._pipelineRun!.run.runner_data.stt_binary_handler_id!;
data[0] = this._pipelineRuns[0].run.runner_data.stt_binary_handler_id!;
data.set(new Uint8Array(chunk.buffer), 1);
this.hass.connection.socket!.send(data);
}
private _playTTS(): void {
const url = this._pipelineRun!.tts!.tts_output!.url;
const audio = new Audio(url);
audio.play();
private _handleContinueKeyDown(ev) {
if (ev.keyCode === 13) {
this._runTextPipeline();
}
}
private _clearConversation() {
this._pipelineRuns = [];
}
static styles = [
@ -419,32 +283,19 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) {
direction: ltr;
}
.start-row {
text-align: center;
}
.start-row ha-button {
margin: 16px;
}
ha-card,
ha-alert {
display: block;
margin-bottom: 16px;
}
.run-pipeline-card ha-textfield {
display: block;
}
.row {
display: flex;
justify-content: space-between;
justify-content: space-around;
align-items: center;
margin: 0 16px 16px;
}
pre {
margin: 0;
.start-row ha-textfield {
flex: 1;
}
ha-expansion-panel {
padding-left: 8px;
assist-render-pipeline-run {
padding-top: 16px;
}
.heading {
font-weight: 500;
margin-bottom: 16px;
assist-render-pipeline-run + assist-render-pipeline-run {
border-top: 3px solid black;
}
`,
];

View File

@ -0,0 +1,336 @@
import { css, html, LitElement, TemplateResult } from "lit";
import { customElement, property } from "lit/decorators";
import "../../../../../../components/ha-card";
import "../../../../../../components/ha-alert";
import "../../../../../../components/ha-button";
import "../../../../../../components/ha-circular-progress";
import "../../../../../../components/ha-expansion-panel";
import type { PipelineRun } from "../../../../../../data/voice_assistant";
import type { HomeAssistant } from "../../../../../../types";
import { formatNumber } from "../../../../../../common/number/format_number";
const RUN_DATA = {
pipeline: "Pipeline",
language: "Language",
};
const STT_DATA = {
engine: "Engine",
};
const INTENT_DATA = {
engine: "Engine",
intent_input: "Input",
};
const TTS_DATA = {
engine: "Engine",
tts_input: "Input",
};
const STAGES: Record<PipelineRun["stage"], number> = {
ready: 0,
stt: 1,
intent: 2,
tts: 3,
done: 4,
error: 5,
};
const hasStage = (run: PipelineRun, stage: PipelineRun["stage"]) =>
STAGES[run.init_options.start_stage] <= STAGES[stage] &&
STAGES[stage] <= STAGES[run.init_options.end_stage];
const maybeRenderError = (
run: PipelineRun,
stage: string,
lastRunStage: string
) => {
if (run.stage !== "error" || lastRunStage !== stage) {
return "";
}
return html`<ha-alert alert-type="error">
${run.error!.message} (${run.error!.code})
</ha-alert>`;
};
const renderProgress = (
hass: HomeAssistant,
pipelineRun: PipelineRun,
stage: PipelineRun["stage"]
) => {
const startEvent = pipelineRun.events.find(
(ev) => ev.type === `${stage}-start`
);
const finishEvent = pipelineRun.events.find(
(ev) => ev.type === `${stage}-end`
);
if (!startEvent) {
return "";
}
if (pipelineRun.stage === "error") {
return html``;
}
if (!finishEvent) {
return html`<ha-circular-progress
size="tiny"
active
></ha-circular-progress>`;
}
const duration =
new Date(finishEvent.timestamp).getTime() -
new Date(startEvent.timestamp).getTime();
const durationString = formatNumber(duration / 1000, hass.locale, {
maximumFractionDigits: 2,
});
return html`${durationString}s ✅`;
};
const renderData = (data: Record<string, any>, keys: Record<string, string>) =>
Object.entries(keys).map(
([key, label]) =>
html`
<div class="row">
<div>${label}</div>
<div>${data[key]}</div>
</div>
`
);
const dataMinusKeysRender = (
data: Record<string, any>,
keys: Record<string, string>
) => {
const result = {};
let render = false;
for (const key in data) {
if (key in keys) {
continue;
}
render = true;
result[key] = data[key];
}
return render ? html`<pre>${JSON.stringify(result, null, 2)}</pre>` : "";
};
@customElement("assist-render-pipeline-run")
export class AssistPipelineDebug extends LitElement {
@property({ attribute: false }) public hass!: HomeAssistant;
@property() private pipelineRun!: PipelineRun;
protected render(): TemplateResult {
const lastRunStage: string = this.pipelineRun
? ["tts", "intent", "stt"].find(
(stage) => this.pipelineRun![stage] !== undefined
) || "ready"
: "ready";
const messages: Array<{ from: string; text: string }> = [];
const userMessage =
this.pipelineRun.init_options.input?.text ||
this.pipelineRun?.stt?.stt_output?.text;
if (userMessage) {
messages.push({
from: "user",
text: userMessage,
});
}
if (
this.pipelineRun?.intent?.intent_output?.response?.speech?.plain?.speech
) {
messages.push({
from: "hass",
text: this.pipelineRun.intent.intent_output.response.speech.plain
.speech,
});
}
return html`
<ha-card>
<div class="card-content">
<div class="row heading">
<div>Run</div>
<div>${this.pipelineRun.stage}</div>
</div>
${renderData(this.pipelineRun.run, RUN_DATA)}
${messages.length > 0
? html`
<div class="messages">
${messages.map(
({ from, text }) => html`
<div class=${`message ${from}`}>${text}</div>
`
)}
</div>
<div style="clear:both"></div>
`
: ""}
</div>
</ha-card>
${maybeRenderError(this.pipelineRun, "ready", lastRunStage)}
${hasStage(this.pipelineRun, "stt")
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Speech-to-Text</span>
${renderProgress(this.hass, this.pipelineRun, "stt")}
</div>
${this.pipelineRun.stt
? html`
<div class="card-content">
${renderData(this.pipelineRun.stt, STT_DATA)}
${dataMinusKeysRender(this.pipelineRun.stt, STT_DATA)}
</div>
`
: ""}
</div>
</ha-card>
`
: ""}
${maybeRenderError(this.pipelineRun, "stt", lastRunStage)}
${hasStage(this.pipelineRun, "intent")
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Natural Language Processing</span>
${renderProgress(this.hass, this.pipelineRun, "intent")}
</div>
${this.pipelineRun.intent
? html`
<div class="card-content">
${renderData(this.pipelineRun.intent, INTENT_DATA)}
${dataMinusKeysRender(
this.pipelineRun.intent,
INTENT_DATA
)}
</div>
`
: ""}
</div>
</ha-card>
`
: ""}
${maybeRenderError(this.pipelineRun, "intent", lastRunStage)}
${hasStage(this.pipelineRun, "tts")
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Text-to-Speech</span>
${renderProgress(this.hass, this.pipelineRun, "tts")}
</div>
${this.pipelineRun.tts
? html`
<div class="card-content">
${renderData(this.pipelineRun.tts, TTS_DATA)}
</div>
`
: ""}
</div>
${this.pipelineRun?.tts?.tts_output
? html`
<div class="card-actions">
<ha-button @click=${this._playTTS}>
Play Audio
</ha-button>
</div>
`
: ""}
</ha-card>
`
: ""}
${maybeRenderError(this.pipelineRun, "tts", lastRunStage)}
<ha-card>
<ha-expansion-panel>
<span slot="header">Raw</span>
<pre>${JSON.stringify(this.pipelineRun, null, 2)}</pre>
</ha-expansion-panel>
</ha-card>
`;
}
private _playTTS(): void {
const url = this.pipelineRun!.tts!.tts_output!.url;
const audio = new Audio(url);
audio.play();
}
static styles = css`
:host {
display: block;
}
ha-card,
ha-alert {
display: block;
margin-bottom: 16px;
}
.row {
display: flex;
justify-content: space-between;
}
pre {
margin: 0;
}
ha-expansion-panel {
padding-left: 8px;
}
.heading {
font-weight: 500;
margin-bottom: 16px;
}
.messages {
margin-top: 8px;
}
.message {
font-size: 18px;
margin: 8px 0;
padding: 8px;
border-radius: 15px;
clear: both;
}
.message.user {
margin-left: 24px;
margin-inline-start: 24px;
margin-inline-end: initial;
float: var(--float-end);
text-align: right;
border-bottom-right-radius: 0px;
background-color: var(--light-primary-color);
color: var(--text-light-primary-color, var(--primary-text-color));
direction: var(--direction);
}
.message.hass {
margin-right: 24px;
margin-inline-end: 24px;
margin-inline-start: initial;
float: var(--float-start);
border-bottom-left-radius: 0px;
background-color: var(--primary-color);
color: var(--text-primary-color);
direction: var(--direction);
}
`;
}
declare global {
interface HTMLElementTagNameMap {
"assist-render-pipeline-run": AssistPipelineDebug;
}
}