Ensure that on runID change, we also resub to new chat log

src/data/assist_pipeline.ts

@@ -214,6 +214,8 @@ export interface PipelineRun {
   stage: "ready" | "wake_word" | "stt" | "intent" | "tts" | "done" | "error";
   run: PipelineRunStartEvent["data"];
   error?: PipelineErrorEvent["data"];
+  started: Date;
+  finished?: Date;
   wake_word?: PipelineWakeWordStartEvent["data"] &
     Partial<PipelineWakeWordEndEvent["data"]> & { done: boolean };
   stt?: PipelineSTTStartEvent["data"] &
@@ -235,6 +237,7 @@ export const processEvent = (
       stage: "ready",
       run: event.data,
       events: [event],
+      started: new Date(event.timestamp),
     };
     return run;
   }
@@ -290,9 +293,14 @@ export const processEvent = (
       tts: { ...run.tts!, ...event.data, done: true },
     };
   } else if (event.type === "run-end") {
-    run = { ...run, stage: "done" };
+    run = { ...run, finished: new Date(event.timestamp), stage: "done" };
   } else if (event.type === "error") {
-    run = { ...run, stage: "error", error: event.data };
+    run = {
+      ...run,
+      finished: new Date(event.timestamp),
+      stage: "error",
+      error: event.data,
+    };
   } else {
     run = { ...run };
   }

src/data/chat_log.ts

@@ -0,0 +1,228 @@
+import type { UnsubscribeFunc } from "home-assistant-js-websocket";
+import type { HomeAssistant } from "../types";
+
+export const enum ChatLogEventType {
+  INITIAL_STATE = "initial_state",
+  CREATED = "created",
+  UPDATED = "updated",
+  DELETED = "deleted",
+  CONTENT_ADDED = "content_added",
+}
+
+export interface ChatLogAttachment {
+  media_content_id: string;
+  mime_type: string;
+  path: string;
+}
+
+export interface ChatLogSystemContent {
+  role: "system";
+  content: string;
+  created: Date;
+}
+
+export interface ChatLogUserContent {
+  role: "user";
+  content: string;
+  created: Date;
+  attachments?: ChatLogAttachment[];
+}
+
+export interface ChatLogAssistantContent {
+  role: "assistant";
+  agent_id: string;
+  created: Date;
+  content?: string;
+  thinking_content?: string;
+  tool_calls?: any[];
+}
+
+export interface ChatLogToolResultContent {
+  role: "tool_result";
+  agent_id: string;
+  tool_call_id: string;
+  tool_name: string;
+  tool_result: any;
+  created: Date;
+}
+
+export type ChatLogContent =
+  | ChatLogSystemContent
+  | ChatLogUserContent
+  | ChatLogAssistantContent
+  | ChatLogToolResultContent;
+
+export interface ChatLog {
+  conversation_id: string;
+  continue_conversation: boolean;
+  content: ChatLogContent[];
+  created: Date;
+}
+
+// Internal wire format types (not exported)
+interface ChatLogSystemContentWire {
+  role: "system";
+  content: string;
+  created: string;
+}
+
+interface ChatLogUserContentWire {
+  role: "user";
+  content: string;
+  created: string;
+  attachments?: ChatLogAttachment[];
+}
+
+interface ChatLogAssistantContentWire {
+  role: "assistant";
+  agent_id: string;
+  created: string;
+  content?: string;
+  thinking_content?: string;
+  tool_calls?: {
+    tool_name: string;
+    tool_args: Record<string, any>;
+    id: string;
+    external: boolean;
+  }[];
+}
+
+interface ChatLogToolResultContentWire {
+  role: "tool_result";
+  agent_id: string;
+  tool_call_id: string;
+  tool_name: string;
+  tool_result: any;
+  created: string;
+}
+
+type ChatLogContentWire =
+  | ChatLogSystemContentWire
+  | ChatLogUserContentWire
+  | ChatLogAssistantContentWire
+  | ChatLogToolResultContentWire;
+
+interface ChatLogWire {
+  conversation_id: string;
+  continue_conversation: boolean;
+  content: ChatLogContentWire[];
+  created: string;
+}
+
+const processContent = (content: ChatLogContentWire): ChatLogContent => ({
+  ...content,
+  created: new Date(content.created),
+});
+
+const processChatLog = (chatLog: ChatLogWire): ChatLog => ({
+  ...chatLog,
+  created: new Date(chatLog.created),
+  content: chatLog.content.map(processContent),
+});
+
+interface ChatLogInitialStateEvent {
+  event_type: ChatLogEventType.INITIAL_STATE;
+  data: ChatLogWire;
+}
+
+interface ChatLogIndexInitialStateEvent {
+  event_type: ChatLogEventType.INITIAL_STATE;
+  data: ChatLogWire[];
+}
+
+interface ChatLogCreatedEvent {
+  conversation_id: string;
+  event_type: ChatLogEventType.CREATED;
+  data: ChatLogWire;
+}
+
+interface ChatLogUpdatedEvent {
+  conversation_id: string;
+  event_type: ChatLogEventType.UPDATED;
+  data: { chat_log: ChatLogWire };
+}
+
+interface ChatLogDeletedEvent {
+  conversation_id: string;
+  event_type: ChatLogEventType.DELETED;
+  data: ChatLogWire;
+}
+
+interface ChatLogContentAddedEvent {
+  conversation_id: string;
+  event_type: ChatLogEventType.CONTENT_ADDED;
+  data: { content: ChatLogContentWire };
+}
+
+type ChatLogSubscriptionEvent =
+  | ChatLogInitialStateEvent
+  | ChatLogUpdatedEvent
+  | ChatLogDeletedEvent
+  | ChatLogContentAddedEvent;
+
+type ChatLogIndexSubscriptionEvent =
+  | ChatLogIndexInitialStateEvent
+  | ChatLogCreatedEvent
+  | ChatLogDeletedEvent;
+
+export const subscribeChatLog = (
+  hass: HomeAssistant,
+  conversationId: string,
+  callback: (chatLog: ChatLog | null) => void
+): Promise<UnsubscribeFunc> => {
+  let chatLog: ChatLog | null = null;
+
+  return hass.connection.subscribeMessage<ChatLogSubscriptionEvent>(
+    (event) => {
+      if (event.event_type === ChatLogEventType.INITIAL_STATE) {
+        chatLog = processChatLog(event.data);
+        callback(chatLog);
+      } else if (event.event_type === ChatLogEventType.CONTENT_ADDED) {
+        if (chatLog) {
+          chatLog = {
+            ...chatLog,
+            content: [...chatLog.content, processContent(event.data.content)],
+          };
+          callback(chatLog);
+        }
+      } else if (event.event_type === ChatLogEventType.UPDATED) {
+        chatLog = processChatLog(event.data.chat_log);
+        callback(chatLog);
+      } else if (event.event_type === ChatLogEventType.DELETED) {
+        chatLog = null;
+        callback(null);
+      }
+    },
+    {
+      type: "conversation/chat_log/subscribe",
+      conversation_id: conversationId,
+    }
+  );
+};
+
+export const subscribeChatLogIndex = (
+  hass: HomeAssistant,
+  callback: (chatLogs: ChatLog[]) => void
+): Promise<UnsubscribeFunc> => {
+  let chatLogs: ChatLog[] = [];
+
+  return hass.connection.subscribeMessage<ChatLogIndexSubscriptionEvent>(
+    (event) => {
+      if (event.event_type === ChatLogEventType.INITIAL_STATE) {
+        chatLogs = event.data.map(processChatLog);
+        callback(chatLogs);
+      } else if (event.event_type === ChatLogEventType.CREATED) {
+        chatLogs = [...chatLogs, processChatLog(event.data)];
+        callback(chatLogs);
+      } else if (event.event_type === ChatLogEventType.DELETED) {
+        chatLogs = chatLogs.filter(
+          (chatLog) => chatLog.conversation_id !== event.conversation_id
+        );
+        callback(chatLogs);
+      }
+    },
+    {
+      type: "conversation/chat_log/subscribe_index",
+    }
+  );
+};

src/panels/config/voice-assistants/debug/assist-pipeline-debug.ts

@@ -6,6 +6,7 @@ import {
 import { LitElement, css, html } from "lit";
 import { customElement, property, state } from "lit/decorators";
 import { repeat } from "lit/directives/repeat";
+import type { UnsubscribeFunc } from "home-assistant-js-websocket";
 import { formatDateTimeWithSeconds } from "../../../../common/datetime/format_date_time";
 import type {
   PipelineRunEvent,
@@ -20,6 +21,8 @@ import "../../../../layouts/hass-subpage";
 import { haStyle } from "../../../../resources/styles";
 import type { HomeAssistant, Route } from "../../../../types";
 import "./assist-render-pipeline-events";
+import type { ChatLog } from "../../../../data/chat_log";
+import { subscribeChatLog } from "../../../../data/chat_log";
 
 @customElement("assist-pipeline-debug")
 export class AssistPipelineDebug extends LitElement {
@@ -37,8 +40,12 @@ export class AssistPipelineDebug extends LitElement {
 
   @state() private _events?: PipelineRunEvent[];
 
+  @state() private _chatLog?: ChatLog;
+
   private _unsubRefreshEventsID?: number;
 
+  private _unsubChatLogUpdates?: Promise<UnsubscribeFunc>;
+
   protected render() {
     return html`<hass-subpage
       .narrow=${this.narrow}
@@ -106,6 +113,7 @@ export class AssistPipelineDebug extends LitElement {
           ? html`<assist-render-pipeline-events
               .hass=${this.hass}
               .events=${this._events}
+              .chatLog=${this._chatLog}
             ></assist-render-pipeline-events>`
           : ""}
       </div>
@@ -120,6 +128,10 @@ export class AssistPipelineDebug extends LitElement {
       clearRefresh = true;
     }
     if (changedProperties.has("_runId")) {
+      if (this._unsubChatLogUpdates) {
+        this._unsubChatLogUpdates.then((unsub) => unsub());
+        this._unsubChatLogUpdates = undefined;
+      }
       this._fetchEvents();
       clearRefresh = true;
     }
@@ -135,6 +147,10 @@ export class AssistPipelineDebug extends LitElement {
       clearTimeout(this._unsubRefreshEventsID);
       this._unsubRefreshEventsID = undefined;
     }
+    if (this._unsubChatLogUpdates) {
+      this._unsubChatLogUpdates.then((unsub) => unsub());
+      this._unsubChatLogUpdates = undefined;
+    }
   }
 
   private async _fetchRuns() {
@@ -181,8 +197,27 @@ export class AssistPipelineDebug extends LitElement {
       });
       return;
     }
+    if (!this._events!.length) {
+      return;
+    }
+    if (!this._unsubChatLogUpdates && this._events[0].type === "run-start") {
+      this._unsubChatLogUpdates = subscribeChatLog(
+        this.hass,
+        this._events[0].data.conversation_id,
+        (chatLog) => {
+          if (chatLog) {
+            this._chatLog = chatLog;
+          } else {
+            this._unsubChatLogUpdates?.then((unsub) => unsub());
+            this._unsubChatLogUpdates = undefined;
+          }
+        }
+      );
+      this._unsubChatLogUpdates.catch(() => {
+        this._unsubChatLogUpdates = undefined;
+      });
+    }
     if (
-      this._events?.length &&
       // If the last event is not a finish run event, the run is still ongoing.
       // Refresh events automatically.
       !["run-end", "error"].includes(this._events[this._events.length - 1].type)

src/panels/config/voice-assistants/debug/assist-pipeline-run-debug.ts

@@ -1,6 +1,7 @@
 import type { TemplateResult } from "lit";
 import { css, html, LitElement } from "lit";
 import { customElement, property, query, state } from "lit/decorators";
+import type { UnsubscribeFunc } from "home-assistant-js-websocket";
 import { extractSearchParam } from "../../../../common/url/search-params";
 import "../../../../components/ha-assist-pipeline-picker";
 import "../../../../components/ha-button";
@@ -24,6 +25,8 @@ import type { HomeAssistant } from "../../../../types";
 import { AudioRecorder } from "../../../../util/audio-recorder";
 import { fileDownload } from "../../../../util/file_download";
 import "./assist-render-pipeline-run";
+import type { ChatLog } from "../../../../data/chat_log";
+import { subscribeChatLog } from "../../../../data/chat_log";
 
 @customElement("assist-pipeline-run-debug")
 export class AssistPipelineRunDebug extends LitElement {
@@ -46,6 +49,13 @@ export class AssistPipelineRunDebug extends LitElement {
   @state() private _pipelineId?: string =
     extractSearchParam("pipeline") || undefined;
 
+  @state() private _chatLog?: ChatLog;
+
+  private _chatLogSubscription: {
+    conversationId: string;
+    unsub: Promise<UnsubscribeFunc>;
+  } | null = null;
+
   protected render(): TemplateResult {
     return html`
       <hass-subpage
@@ -178,6 +188,7 @@ export class AssistPipelineRunDebug extends LitElement {
                   <assist-render-pipeline-run
                     .hass=${this.hass}
                     .pipelineRun=${run}
+                    .chatLog=${this._chatLog}
                   ></assist-render-pipeline-run>
                 `
           )}
@@ -186,6 +197,14 @@ export class AssistPipelineRunDebug extends LitElement {
     `;
   }
 
+  public disconnectedCallback(): void {
+    super.disconnectedCallback();
+    if (this._chatLogSubscription) {
+      this._chatLogSubscription.unsub.then((unsub) => unsub());
+      this._chatLogSubscription = null;
+    }
+  }
+
   private get conversationId(): string | null {
     return this._pipelineRuns.length === 0
       ? null
@@ -408,6 +427,32 @@ export class AssistPipelineRunDebug extends LitElement {
             added = true;
           }
           callback(updatedRun);
+
+          const conversationId = this.conversationId;
+          if (
+            !this._chatLog &&
+            conversationId &&
+            (!this._chatLogSubscription ||
+              this._chatLogSubscription.conversationId !== conversationId)
+          ) {
+            if (this._chatLogSubscription) {
+              this._chatLogSubscription.unsub.then((unsub) => unsub());
+            }
+            this._chatLogSubscription = {
+              conversationId,
+              unsub: subscribeChatLog(this.hass, conversationId, (chatLog) => {
+                if (chatLog) {
+                  this._chatLog = chatLog;
+                } else {
+                  this._chatLogSubscription?.unsub.then((unsub) => unsub());
+                  this._chatLogSubscription = null;
+                }
+              }),
+            };
+            this._chatLogSubscription.unsub.catch(() => {
+              this._chatLogSubscription = null;
+            });
+          }
         },
         {
           ...options,

src/panels/config/voice-assistants/debug/assist-render-pipeline-events.ts

@@ -9,6 +9,7 @@ import type {
 import { processEvent } from "../../../../data/assist_pipeline";
 import type { HomeAssistant } from "../../../../types";
 import "./assist-render-pipeline-run";
+import type { ChatLog } from "../../../../data/chat_log";
 
 @customElement("assist-render-pipeline-events")
 export class AssistPipelineEvents extends LitElement {
@@ -16,6 +17,8 @@ export class AssistPipelineEvents extends LitElement {
 
   @property({ attribute: false }) public events!: PipelineRunEvent[];
 
+  @property({ attribute: false }) public chatLog?: ChatLog;
+
   private _processEvents = memoizeOne(
     (events: PipelineRunEvent[]): PipelineRun | undefined => {
       let run: PipelineRun | undefined;
@@ -46,6 +49,7 @@ export class AssistPipelineEvents extends LitElement {
       <assist-render-pipeline-run
         .hass=${this.hass}
         .pipelineRun=${run}
+        .chatLog=${this.chatLog}
       ></assist-render-pipeline-run>
     `;
   }

src/panels/config/voice-assistants/debug/assist-render-pipeline-run.ts

@@ -1,5 +1,5 @@
 import type { TemplateResult } from "lit";
-import { css, html, LitElement } from "lit";
+import { css, html, LitElement, nothing } from "lit";
 import { customElement, property } from "lit/decorators";
 import "../../../../components/ha-card";
 import "../../../../components/ha-alert";
@@ -11,6 +11,12 @@ import type { HomeAssistant } from "../../../../types";
 import { formatNumber } from "../../../../common/number/format_number";
 import "../../../../components/ha-yaml-editor";
 import { showAlertDialog } from "../../../../dialogs/generic/show-dialog-box";
+import type {
+  ChatLogAssistantContent,
+  ChatLog,
+  ChatLogContent,
+  ChatLogUserContent,
+} from "../../../../data/chat_log";
 
 const RUN_DATA = {
   pipeline: "Pipeline",
@@ -126,7 +132,7 @@ const dataMinusKeysRender = (
     result[key] = data[key];
   }
   return render
-    ? html`<ha-expansion-panel>
+    ? html`<ha-expansion-panel class="yaml-expansion">
         <span slot="header">Raw</span>
         <ha-yaml-editor readOnly autoUpdate .value=${result}></ha-yaml-editor>
       </ha-expansion-panel>`
@@ -139,6 +145,8 @@ export class AssistPipelineDebug extends LitElement {
 
   @property({ attribute: false }) public pipelineRun!: PipelineRun;
 
+  @property({ attribute: false }) public chatLog?: ChatLog;
+
   protected render(): TemplateResult {
     const lastRunStage: string = this.pipelineRun
       ? ["tts", "intent", "stt", "wake_word"].find(
@@ -146,31 +154,47 @@ export class AssistPipelineDebug extends LitElement {
         ) || "ready"
       : "ready";
 
-    const messages: { from: string; text: string }[] = [];
+    let messages: ChatLogContent[];
 
-    const userMessage =
-      (this.pipelineRun.init_options &&
-      "text" in this.pipelineRun.init_options.input
-        ? this.pipelineRun.init_options.input.text
-        : undefined) ||
-      this.pipelineRun?.stt?.stt_output?.text ||
-      this.pipelineRun?.intent?.intent_input;
+    if (this.chatLog) {
+      messages = this.chatLog.content.filter(
+        this.pipelineRun.finished
+          ? (content: ChatLogContent) =>
+              content.role === "system" ||
+              (content.created >= this.pipelineRun.started &&
+                content.created <= this.pipelineRun.finished!)
+          : (content: ChatLogContent) =>
+              content.role === "system" ||
+              content.created >= this.pipelineRun.started
+      );
+    } else {
+      messages = [];
 
-    if (userMessage) {
-      messages.push({
-        from: "user",
-        text: userMessage,
-      });
-    }
+      // We don't have the chat log everywhere yet, just fallback for now.
+      const userMessage =
+        (this.pipelineRun.init_options &&
+        "text" in this.pipelineRun.init_options.input
+          ? this.pipelineRun.init_options.input.text
+          : undefined) ||
+        this.pipelineRun?.stt?.stt_output?.text ||
+        this.pipelineRun?.intent?.intent_input;
 
-    if (
-      this.pipelineRun?.intent?.intent_output?.response?.speech?.plain?.speech
-    ) {
-      messages.push({
-        from: "hass",
-        text: this.pipelineRun.intent.intent_output.response.speech.plain
-          .speech,
-      });
+      if (userMessage) {
+        messages.push({
+          role: "user",
+          content: userMessage,
+        } as ChatLogUserContent);
+      }
+
+      if (
+        this.pipelineRun?.intent?.intent_output?.response?.speech?.plain?.speech
+      ) {
+        messages.push({
+          role: "assistant",
+          content:
+            this.pipelineRun.intent.intent_output.response.speech.plain.speech,
+        } as ChatLogAssistantContent);
+      }
     }
 
     return html`
@@ -185,10 +209,58 @@ export class AssistPipelineDebug extends LitElement {
           ${messages.length > 0
             ? html`
                 <div class="messages">
-                  ${messages.map(
-                    ({ from, text }) => html`
-                      <div class=${`message ${from}`}>${text}</div>
-                    `
+                  ${messages.map((content) =>
+                    content.role === "system" || content.role === "tool_result"
+                      ? html`
+                          <ha-expansion-panel
+                            class="content-expansion ${content.role}"
+                          >
+                            <div slot="header">
+                              ${content.role === "system"
+                                ? "System"
+                                : `Result for ${content.tool_name}`}
+                            </div>
+                            ${content.role === "system"
+                              ? html`<pre>${content.content}</pre>`
+                              : html`
+                                  <ha-yaml-editor
+                                    read-only
+                                    auto-update
+                                    .value=${content}
+                                  ></ha-yaml-editor>
+                                `}
+                          </ha-expansion-panel>
+                        `
+                      : html`
+                          ${content.content
+                            ? html`
+                                <div class=${`message ${content.role}`}>
+                                  ${content.content}
+                                </div>
+                              `
+                            : nothing}
+                          ${content.role === "assistant" &&
+                          content.tool_calls?.length
+                            ? html`
+                                <ha-expansion-panel
+                                  class="content-expansion assistant"
+                                >
+                                  <span slot="header">
+                                    Call
+                                    ${content.tool_calls.length === 1
+                                      ? content.tool_calls[0].tool_name
+                                      : `${content.tool_calls.length} tools`}
+                                  </span>
+
+                                  <ha-yaml-editor
+                                    read-only
+                                    auto-update
+                                    .value=${content.tool_calls}
+                                  ></ha-yaml-editor>
+                                </ha-expansion-panel>
+                              `
+                            : nothing}
+                        `
                   )}
                 </div>
                 <div style="clear:both"></div>
@@ -360,7 +432,7 @@ export class AssistPipelineDebug extends LitElement {
         : ""}
       ${maybeRenderError(this.pipelineRun, "tts", lastRunStage)}
       <ha-card>
-        <ha-expansion-panel>
+        <ha-expansion-panel class="yaml-expansion">
           <span slot="header">Raw</span>
           <ha-yaml-editor
             read-only
@@ -399,12 +471,12 @@ export class AssistPipelineDebug extends LitElement {
     .row > div:last-child {
       text-align: right;
     }
-    ha-expansion-panel {
+    .yaml-expansion {
       padding-left: 8px;
       padding-inline-start: 8px;
       padding-inline-end: initial;
     }
-    .card-content ha-expansion-panel {
+    .card-content .yaml-expansion {
       padding-left: 0px;
       padding-inline-start: 0px;
       padding-inline-end: initial;
@@ -420,27 +492,59 @@ export class AssistPipelineDebug extends LitElement {
       margin-top: 8px;
     }
 
+    .content-expansion {
+      margin: 8px 0;
+      border-radius: var(--ha-border-radius-xl);
+      clear: both;
+      padding: 0 8px;
+      --input-fill-color: none;
+      max-width: calc(100% - 24px);
+      --expansion-panel-summary-padding: 0px;
+      --expansion-panel-content-padding: 0px;
+    }
+
+    .content-expansion *[slot="header"] {
+      font-weight: var(--ha-font-weight-normal);
+    }
+
+    .system {
+      background-color: var(--success-color);
+    }
+
     .message {
+      padding: 8px;
+    }
+
+    .message,
+    .content-expansion {
       font-size: var(--ha-font-size-l);
       margin: 8px 0;
-      padding: 8px;
       border-radius: var(--ha-border-radius-xl);
       clear: both;
     }
 
-    .message.user {
+    .messages pre {
+      white-space: pre-wrap;
+    }
+
+    .user,
+    .tool_result {
       margin-left: 24px;
       margin-inline-start: 24px;
       margin-inline-end: initial;
       float: var(--float-end);
-      text-align: right;
       border-bottom-right-radius: 0px;
       background-color: var(--light-primary-color);
       color: var(--text-light-primary-color, var(--primary-text-color));
       direction: var(--direction);
     }
 
-    .message.hass {
+    .message.user,
+    .content-expansion div[slot="header"] {
+      text-align: right;
+    }
+
+    .assistant {
       margin-right: 24px;
       margin-inline-end: 24px;
       margin-inline-start: initial;