From f3d55447caeef09f07c539d82630cc372d0ca864 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Sun, 9 Feb 2025 23:36:56 -0500 Subject: [PATCH] Add support for intent-progress assist events (#24143) --- src/components/ha-assist-chat.ts | 98 +++++++++++++++++++++++++++++--- src/data/assist_pipeline.ts | 29 ++++++++++ 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/src/components/ha-assist-chat.ts b/src/components/ha-assist-chat.ts index d9292f57a9..9c206f3b80 100644 --- a/src/components/ha-assist-chat.ts +++ b/src/components/ha-assist-chat.ts @@ -295,10 +295,12 @@ export class HaAssistChat extends LitElement { this._addMessage(userMessage); this.requestUpdate("_audioRecorder"); - const hassMessage: AssistMessage = { + let hassMessage = { who: "hass", text: "…", + error: false, }; + let currentDeltaRole = ""; // To make sure the answer is placed at the right user text, we add it before we process it try { const unsub = await runAssistPipeline( @@ -328,6 +330,43 @@ export class HaAssistChat extends LitElement { this._addMessage(hassMessage); } + if (event.type === "intent-progress") { + const delta = event.data.chat_log_delta; + + // new message + if (delta.role) { + // If currentDeltaRole exists, it means we're receiving our + // second or later message. Let's add it to the chat. + if (currentDeltaRole && delta.role && hassMessage.text !== "…") { + // Remove progress indicator of previous message + hassMessage.text = hassMessage.text.substring( + 0, + hassMessage.text.length - 1 + ); + + hassMessage = { + who: "hass", + text: "…", + error: false, + }; + this._addMessage(hassMessage); + } + currentDeltaRole = delta.role; + } + + if ( + currentDeltaRole === "assistant" && + "content" in delta && + delta.content + ) { + hassMessage.text = + hassMessage.text.substring(0, hassMessage.text.length - 1) + + delta.content + + "…"; + this.requestUpdate("_conversation"); + } + } + if (event.type === "intent-end") { this._conversationId = event.data.intent_output.conversation_id; const plain = event.data.intent_output.response.speech?.plain; @@ -435,28 +474,71 @@ export class HaAssistChat extends LitElement { this._processing = true; this._audio?.pause(); this._addMessage({ who: "user", text }); - const message: AssistMessage = { + let hassMessage = { who: "hass", text: "…", + error: false, }; + let currentDeltaRole = ""; // To make sure the answer is placed at the right user text, we add it before we process it - this._addMessage(message); + this._addMessage(hassMessage); try { const unsub = await runAssistPipeline( this.hass, (event) => { + if (event.type === "intent-progress") { + const delta = event.data.chat_log_delta; + + // new message and previous message has content + if (delta.role) { + // If currentDeltaRole exists, it means we're receiving our + // second or later message. Let's add it to the chat. + if ( + currentDeltaRole && + delta.role === "assistant" && + hassMessage.text !== "…" + ) { + // Remove progress indicator of previous message + hassMessage.text = hassMessage.text.substring( + 0, + hassMessage.text.length - 1 + ); + + hassMessage = { + who: "hass", + text: "…", + error: false, + }; + this._addMessage(hassMessage); + } + currentDeltaRole = delta.role; + } + + if ( + currentDeltaRole === "assistant" && + "content" in delta && + delta.content + ) { + hassMessage.text = + hassMessage.text.substring(0, hassMessage.text.length - 1) + + delta.content + + "…"; + this.requestUpdate("_conversation"); + } + } + if (event.type === "intent-end") { this._conversationId = event.data.intent_output.conversation_id; const plain = event.data.intent_output.response.speech?.plain; if (plain) { - message.text = plain.speech; + hassMessage.text = plain.speech; } this.requestUpdate("_conversation"); unsub(); } if (event.type === "error") { - message.text = event.data.message; - message.error = true; + hassMessage.text = event.data.message; + hassMessage.error = true; this.requestUpdate("_conversation"); unsub(); } @@ -470,8 +552,8 @@ export class HaAssistChat extends LitElement { } ); } catch { - message.text = this.hass.localize("ui.dialogs.voice_command.error"); - message.error = true; + hassMessage.text = this.hass.localize("ui.dialogs.voice_command.error"); + hassMessage.error = true; this.requestUpdate("_conversation"); } finally { this._processing = false; diff --git a/src/data/assist_pipeline.ts b/src/data/assist_pipeline.ts index d8cff2e741..72da3c5e9a 100644 --- a/src/data/assist_pipeline.ts +++ b/src/data/assist_pipeline.ts @@ -108,6 +108,34 @@ interface PipelineIntentStartEvent extends PipelineEventBase { intent_input: string; }; } + +interface ConversationChatLogAssistantDelta { + role: "assistant"; + content: string; + tool_calls: { + id: string; + tool_name: string; + tool_args: Record; + }[]; +} + +interface ConversationChatLogToolResultDelta { + role: "tool_result"; + agent_id: string; + tool_call_id: string; + tool_name: string; + tool_result: unknown; +} +interface PipelineIntentProgressEvent extends PipelineEventBase { + type: "intent-progress"; + data: { + chat_log_delta: + | Partial + // These always come in 1 chunk + | ConversationChatLogToolResultDelta; + }; +} + interface PipelineIntentEndEvent extends PipelineEventBase { type: "intent-end"; data: { @@ -141,6 +169,7 @@ export type PipelineRunEvent = | PipelineSTTStartEvent | PipelineSTTEndEvent | PipelineIntentStartEvent + | PipelineIntentProgressEvent | PipelineIntentEndEvent | PipelineTTSStartEvent | PipelineTTSEndEvent;