Display historic pipeline events in assist-render-pipeline-run (#16225)

* Display historic pipeline events in `assist-render-pipeline-run`

* Add debug pipeline page

* Update assist-pipeline-run-debug.ts

* dont show alert

* Update assist-pipeline-debug.ts

* simplify

* link to run debug pipeline

src/data/assist_pipeline.ts

@@ -20,6 +20,11 @@ export interface AssistPipelineMutableParams {
   tts_engine: string;
 }
 
+export interface assistRunListing {
+  pipeline_run_id: string;
+  timestamp: string;
+}
+
 interface PipelineEventBase {
   timestamp: string;
 }
@@ -90,7 +95,7 @@ interface PipelineTTSEndEvent extends PipelineEventBase {
   };
 }
 
-type PipelineRunEvent =
+export type PipelineRunEvent =
   | PipelineRunStartEvent
   | PipelineRunEndEvent
   | PipelineErrorEvent
@@ -117,7 +122,7 @@ export type PipelineRunOptions = (
 };
 
 export interface PipelineRun {
-  init_options: PipelineRunOptions;
+  init_options?: PipelineRunOptions;
   events: PipelineRunEvent[];
   stage: "ready" | "stt" | "intent" | "tts" | "done" | "error";
   run: PipelineRunStartEvent["data"];
@@ -130,6 +135,73 @@ export interface PipelineRun {
     Partial<PipelineTTSEndEvent["data"]> & { done: boolean };
 }
 
+export const processEvent = (
+  run: PipelineRun | undefined,
+  event: PipelineRunEvent,
+  options?: PipelineRunOptions
+): PipelineRun | undefined => {
+  if (event.type === "run-start") {
+    run = {
+      init_options: options,
+      stage: "ready",
+      run: event.data,
+      events: [event],
+    };
+    return run;
+  }
+
+  if (!run) {
+    // eslint-disable-next-line no-console
+    console.warn("Received unexpected event before receiving session", event);
+    return undefined;
+  }
+
+  if (event.type === "stt-start") {
+    run = {
+      ...run,
+      stage: "stt",
+      stt: { ...event.data, done: false },
+    };
+  } else if (event.type === "stt-end") {
+    run = {
+      ...run,
+      stt: { ...run.stt!, ...event.data, done: true },
+    };
+  } else if (event.type === "intent-start") {
+    run = {
+      ...run,
+      stage: "intent",
+      intent: { ...event.data, done: false },
+    };
+  } else if (event.type === "intent-end") {
+    run = {
+      ...run,
+      intent: { ...run.intent!, ...event.data, done: true },
+    };
+  } else if (event.type === "tts-start") {
+    run = {
+      ...run,
+      stage: "tts",
+      tts: { ...event.data, done: false },
+    };
+  } else if (event.type === "tts-end") {
+    run = {
+      ...run,
+      tts: { ...run.tts!, ...event.data, done: true },
+    };
+  } else if (event.type === "run-end") {
+    run = { ...run, stage: "done" };
+  } else if (event.type === "error") {
+    run = { ...run, stage: "error", error: event.data };
+  } else {
+    run = { ...run };
+  }
+
+  run.events = [...run.events, event];
+
+  return run;
+};
+
 export const runAssistPipeline = (
   hass: HomeAssistant,
   callback: (event: PipelineRun) => void,
@@ -139,76 +211,15 @@ export const runAssistPipeline = (
 
   const unsubProm = hass.connection.subscribeMessage<PipelineRunEvent>(
     (updateEvent) => {
-      if (updateEvent.type === "run-start") {
+      run = processEvent(run, updateEvent, options);
-        run = {
+
-          init_options: options,
+      if (updateEvent.type === "run-end" || updateEvent.type === "error") {
-          stage: "ready",
+        unsubProm.then((unsub) => unsub());
-          run: updateEvent.data,
+      }
-          error: undefined,
+
-          stt: undefined,
+      if (run) {
-          intent: undefined,
-          tts: undefined,
-          events: [updateEvent],
-        };
         callback(run);
-        return;
       }
-
-      if (!run) {
-        // eslint-disable-next-line no-console
-        console.warn(
-          "Received unexpected event before receiving session",
-          updateEvent
-        );
-        return;
-      }
-
-      if (updateEvent.type === "stt-start") {
-        run = {
-          ...run,
-          stage: "stt",
-          stt: { ...updateEvent.data, done: false },
-        };
-      } else if (updateEvent.type === "stt-end") {
-        run = {
-          ...run,
-          stt: { ...run.stt!, ...updateEvent.data, done: true },
-        };
-      } else if (updateEvent.type === "intent-start") {
-        run = {
-          ...run,
-          stage: "intent",
-          intent: { ...updateEvent.data, done: false },
-        };
-      } else if (updateEvent.type === "intent-end") {
-        run = {
-          ...run,
-          intent: { ...run.intent!, ...updateEvent.data, done: true },
-        };
-      } else if (updateEvent.type === "tts-start") {
-        run = {
-          ...run,
-          stage: "tts",
-          tts: { ...updateEvent.data, done: false },
-        };
-      } else if (updateEvent.type === "tts-end") {
-        run = {
-          ...run,
-          tts: { ...run.tts!, ...updateEvent.data, done: true },
-        };
-      } else if (updateEvent.type === "run-end") {
-        run = { ...run, stage: "done" };
-        unsubProm.then((unsub) => unsub());
-      } else if (updateEvent.type === "error") {
-        run = { ...run, stage: "error", error: updateEvent.data };
-        unsubProm.then((unsub) => unsub());
-      } else {
-        run = { ...run };
-      }
-
-      run.events = [...run.events, updateEvent];
-
-      callback(run);
     },
     {
       ...options,
@@ -224,7 +235,7 @@ export const listAssistPipelineRuns = (
   pipeline_id: string
 ) =>
   hass.callWS<{
-    pipeline_runs: string[];
+    pipeline_runs: assistRunListing[];
   }>({
     type: "assist_pipeline/pipeline_debug/list",
     pipeline_id,

src/panels/config/voice-assistants/debug/assist-debug.ts

@@ -0,0 +1,35 @@
+import { html, LitElement } from "lit";
+import { customElement, property } from "lit/decorators";
+import { HomeAssistant, Route } from "../../../../types";
+import "./assist-pipeline-debug";
+import "./assist-pipeline-run-debug";
+
+@customElement("assist-debug")
+export class AssistDebug extends LitElement {
+  @property({ attribute: false }) public hass!: HomeAssistant;
+
+  @property({ type: Boolean }) public narrow!: boolean;
+
+  @property({ attribute: false }) public route!: Route;
+
+  protected render() {
+    const pipelineId = this.route.path.substring(1);
+    if (pipelineId) {
+      return html`<assist-pipeline-debug
+        .hass=${this.hass}
+        .narrow=${this.narrow}
+        .pipelineId=${pipelineId}
+      ></assist-pipeline-debug>`;
+    }
+    return html`<assist-pipeline-run-debug
+      .hass=${this.hass}
+      .narrow=${this.narrow}
+    ></assist-pipeline-run-debug>`;
+  }
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "assist-debug": AssistDebug;
+  }
+}

src/panels/config/voice-assistants/debug/assist-pipeline-debug.ts

@@ -0,0 +1,200 @@
+import {
+  mdiMicrophoneMessage,
+  mdiRayEndArrow,
+  mdiRayStartArrow,
+} from "@mdi/js";
+import { css, html, LitElement } from "lit";
+import { customElement, property, state } from "lit/decorators";
+import { repeat } from "lit/directives/repeat";
+import { formatDateTimeWithSeconds } from "../../../../common/datetime/format_date_time";
+import {
+  listAssistPipelineRuns,
+  getAssistPipelineRun,
+  PipelineRunEvent,
+  assistRunListing,
+} from "../../../../data/assist_pipeline";
+import { showAlertDialog } from "../../../../dialogs/generic/show-dialog-box";
+import "../../../../layouts/hass-subpage";
+import { haStyle } from "../../../../resources/styles";
+import { HomeAssistant, Route } from "../../../../types";
+import "./assist-render-pipeline-events";
+
+@customElement("assist-pipeline-debug")
+export class AssistPipelineDebug extends LitElement {
+  @property({ attribute: false }) public hass!: HomeAssistant;
+
+  @property({ type: Boolean }) public narrow!: boolean;
+
+  @property({ attribute: false }) public route!: Route;
+
+  @property() public pipelineId!: string;
+
+  @state() private _runId?: string;
+
+  @state() private _runs?: assistRunListing[];
+
+  @state() private _events?: PipelineRunEvent[];
+
+  protected render() {
+    return html`<hass-subpage
+      .narrow=${this.narrow}
+      .hass=${this.hass}
+      header="Debug Assistant"
+    >
+      <a
+        href="/config/voice-assistants/debug?pipeline=${this.pipelineId}"
+        slot="toolbar-icon"
+        ><ha-icon-button .path=${mdiMicrophoneMessage}></ha-icon-button
+      ></a>
+      <div class="toolbar">
+        ${this._runs?.length
+          ? html`
+              <ha-icon-button
+                .disabled=${this._runs[this._runs.length - 1]
+                  .pipeline_run_id === this._runId}
+                label="Older run"
+                @click=${this._pickOlderRun}
+                .path=${mdiRayEndArrow}
+              ></ha-icon-button>
+              <select .value=${this._runId} @change=${this._pickRun}>
+                ${repeat(
+                  this._runs,
+                  (run) => run.pipeline_run_id,
+                  (run) =>
+                    html`<option value=${run.pipeline_run_id}>
+                      ${formatDateTimeWithSeconds(
+                        new Date(run.timestamp),
+                        this.hass.locale
+                      )}
+                    </option>`
+                )}
+              </select>
+              <ha-icon-button
+                .disabled=${this._runs[0].pipeline_run_id === this._runId}
+                label="Newer run"
+                @click=${this._pickNewerRun}
+                .path=${mdiRayStartArrow}
+              ></ha-icon-button>
+            `
+          : ""}
+      </div>
+      ${this._runs?.length === 0
+        ? html`<div class="container">No runs found</div>`
+        : ""}
+      <div class="content">
+        ${this._events
+          ? html`<assist-render-pipeline-events
+              .hass=${this.hass}
+              .events=${this._events}
+            ></assist-render-pipeline-events>`
+          : ""}
+      </div>
+    </hass-subpage>`;
+  }
+
+  protected willUpdate(changedProperties) {
+    if (changedProperties.has("pipelineId")) {
+      this._fetchRuns();
+    }
+    if (changedProperties.has("_runId")) {
+      this._fetchEvents();
+    }
+  }
+
+  private async _fetchRuns() {
+    if (!this.pipelineId) {
+      this._runs = undefined;
+      return;
+    }
+    try {
+      this._runs = (
+        await listAssistPipelineRuns(this.hass, this.pipelineId)
+      ).pipeline_runs.reverse();
+    } catch (e: any) {
+      showAlertDialog(this, {
+        title: "Failed to fetch pipeline runs",
+        text: e.message,
+      });
+      return;
+    }
+    if (!this._runs.length) {
+      return;
+    }
+    if (
+      !this._runId ||
+      !this._runs.find((run) => run.pipeline_run_id === this._runId)
+    ) {
+      this._runId = this._runs[0].pipeline_run_id;
+      this._fetchEvents();
+    }
+  }
+
+  private async _fetchEvents() {
+    if (!this._runId) {
+      this._events = undefined;
+      return;
+    }
+    try {
+      this._events = (
+        await getAssistPipelineRun(this.hass, this.pipelineId, this._runId)
+      ).events;
+    } catch (e: any) {
+      showAlertDialog(this, {
+        title: "Failed to fetch events",
+        text: e.message,
+      });
+    }
+  }
+
+  private _pickOlderRun() {
+    const curIndex = this._runs!.findIndex(
+      (run) => run.pipeline_run_id === this._runId
+    );
+    this._runId = this._runs![curIndex + 1].pipeline_run_id;
+  }
+
+  private _pickNewerRun() {
+    const curIndex = this._runs!.findIndex(
+      (run) => run.pipeline_run_id === this._runId
+    );
+    this._runId = this._runs![curIndex - 1].pipeline_run_id;
+  }
+
+  private _pickRun(ev) {
+    this._runId = ev.target.value;
+  }
+
+  static styles = [
+    haStyle,
+    css`
+      .toolbar {
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        height: var(--header-height);
+        background-color: var(--primary-background-color);
+        color: var(--app-header-text-color, white);
+        border-bottom: var(--app-header-border-bottom, none);
+        box-sizing: border-box;
+      }
+      .content {
+        padding: 24px 0 32px;
+        max-width: 600px;
+        margin: 0 auto;
+        direction: ltr;
+      }
+      .container {
+        padding: 16px;
+      }
+      assist-render-pipeline-run {
+        padding-top: 16px;
+      }
+    `,
+  ];
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "assist-pipeline-debug": AssistPipelineDebug;
+  }
+}

src/panels/config/voice-assistants/debug/assist-pipeline-run-debug.ts

@@ -1,28 +1,29 @@
 import { css, html, LitElement, TemplateResult } from "lit";
 import { customElement, property, query, state } from "lit/decorators";
-import "../../../../../../components/ha-button";
+import "../../../../components/ha-button";
 import {
   PipelineRun,
   PipelineRunOptions,
   runAssistPipeline,
-} from "../../../../../../data/assist_pipeline";
+} from "../../../../data/assist_pipeline";
-import "../../../../../../layouts/hass-subpage";
+import "../../../../layouts/hass-subpage";
-import "../../../../../../components/ha-formfield";
+import "../../../../components/ha-formfield";
-import "../../../../../../components/ha-checkbox";
+import "../../../../components/ha-checkbox";
-import { haStyle } from "../../../../../../resources/styles";
+import { haStyle } from "../../../../resources/styles";
-import type { HomeAssistant } from "../../../../../../types";
+import type { HomeAssistant } from "../../../../types";
 import {
   showAlertDialog,
   showPromptDialog,
-} from "../../../../../../dialogs/generic/show-dialog-box";
+} from "../../../../dialogs/generic/show-dialog-box";
 import "./assist-render-pipeline-run";
-import type { HaCheckbox } from "../../../../../../components/ha-checkbox";
+import type { HaCheckbox } from "../../../../components/ha-checkbox";
-import type { HaTextField } from "../../../../../../components/ha-textfield";
+import type { HaTextField } from "../../../../components/ha-textfield";
-import "../../../../../../components/ha-textfield";
+import "../../../../components/ha-textfield";
-import { fileDownload } from "../../../../../../util/file_download";
+import { fileDownload } from "../../../../util/file_download";
+import { extractSearchParam } from "../../../../common/url/search-params";
 
-@customElement("assist-pipeline-debug")
+@customElement("assist-pipeline-run-debug")
-export class AssistPipelineDebug extends LitElement {
+export class AssistPipelineRunDebug extends LitElement {
   @property({ attribute: false }) public hass!: HomeAssistant;
 
   @property({ type: Boolean }) public narrow!: boolean;
@@ -39,7 +40,8 @@ export class AssistPipelineDebug extends LitElement {
 
   @state() private _finished = false;
 
-  @state() private _pipelineId?: string;
+  @state() private _pipelineId?: string =
+    extractSearchParam("pipeline") || undefined;
 
   protected render(): TemplateResult {
     return html`
@@ -81,7 +83,7 @@ export class AssistPipelineDebug extends LitElement {
                     Run Audio Pipeline
                   </ha-button>
                 `
-              : this._pipelineRuns[0].init_options.start_stage === "intent"
+              : this._pipelineRuns[0].init_options!.start_stage === "intent"
               ? html`
                   <ha-textfield
                     id="continue-conversation-text"
@@ -364,6 +366,6 @@ export class AssistPipelineDebug extends LitElement {
 
 declare global {
   interface HTMLElementTagNameMap {
-    "assist-pipeline-debug": AssistPipelineDebug;
+    "assist-pipeline-run-debug": AssistPipelineRunDebug;
   }
 }

src/panels/config/voice-assistants/debug/assist-render-pipeline-events.ts

@@ -0,0 +1,57 @@
+import { html, LitElement, TemplateResult } from "lit";
+import { customElement, property } from "lit/decorators";
+import memoizeOne from "memoize-one";
+import {
+  PipelineRun,
+  PipelineRunEvent,
+  processEvent,
+} from "../../../../data/assist_pipeline";
+import type { HomeAssistant } from "../../../../types";
+import "./assist-render-pipeline-run";
+
+@customElement("assist-render-pipeline-events")
+export class AssistPipelineEvents extends LitElement {
+  @property({ attribute: false }) public hass!: HomeAssistant;
+
+  @property() public events!: PipelineRunEvent[];
+
+  private _processEvents = memoizeOne(
+    (events: PipelineRunEvent[]): PipelineRun | undefined => {
+      let run: PipelineRun | undefined;
+      events.forEach((event) => {
+        run = processEvent(run, event);
+      });
+      return run;
+    }
+  );
+
+  protected render(): TemplateResult {
+    const run = this._processEvents(this.events);
+    if (!run) {
+      if (this.events.length) {
+        return html`<ha-alert alert-type="error">Error showing run</ha-alert>
+          <ha-card>
+            <ha-expansion-panel>
+              <span slot="header">Raw</span>
+              <pre>${JSON.stringify(this.events, null, 2)}</pre>
+            </ha-expansion-panel>
+          </ha-card>`;
+      }
+      return html`<ha-alert alert-type="warning"
+        >There where no events in this run.</ha-alert
+      >`;
+    }
+    return html`
+      <assist-render-pipeline-run
+        .hass=${this.hass}
+        .pipelineRun=${run}
+      ></assist-render-pipeline-run>
+    `;
+  }
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "assist-render-pipeline-events": AssistPipelineEvents;
+  }
+}

src/panels/config/voice-assistants/debug/assist-render-pipeline-run.ts

@@ -1,13 +1,13 @@
 import { css, html, LitElement, TemplateResult } from "lit";
 import { customElement, property } from "lit/decorators";
-import "../../../../../../components/ha-card";
+import "../../../../components/ha-card";
-import "../../../../../../components/ha-alert";
+import "../../../../components/ha-alert";
-import "../../../../../../components/ha-button";
+import "../../../../components/ha-button";
-import "../../../../../../components/ha-circular-progress";
+import "../../../../components/ha-circular-progress";
-import "../../../../../../components/ha-expansion-panel";
+import "../../../../components/ha-expansion-panel";
-import type { PipelineRun } from "../../../../../../data/assist_pipeline";
+import type { PipelineRun } from "../../../../data/assist_pipeline";
-import type { HomeAssistant } from "../../../../../../types";
+import type { HomeAssistant } from "../../../../types";
-import { formatNumber } from "../../../../../../common/number/format_number";
+import { formatNumber } from "../../../../common/number/format_number";
 
 const RUN_DATA = {
   pipeline: "Pipeline",
@@ -38,8 +38,10 @@ const STAGES: Record<PipelineRun["stage"], number> = {
 };
 
 const hasStage = (run: PipelineRun, stage: PipelineRun["stage"]) =>
-  STAGES[run.init_options.start_stage] <= STAGES[stage] &&
+  run.init_options
-  STAGES[stage] <= STAGES[run.init_options.end_stage];
+    ? STAGES[run.init_options.start_stage] <= STAGES[stage] &&
+      STAGES[stage] <= STAGES[run.init_options.end_stage]
+    : stage in run;
 
 const maybeRenderError = (
   run: PipelineRun,
@@ -123,21 +125,23 @@ const dataMinusKeysRender = (
 export class AssistPipelineDebug extends LitElement {
   @property({ attribute: false }) public hass!: HomeAssistant;
 
-  @property() private pipelineRun!: PipelineRun;
+  @property() public pipelineRun!: PipelineRun;
 
   protected render(): TemplateResult {
     const lastRunStage: string = this.pipelineRun
-      ? ["tts", "intent", "stt"].find(
+      ? ["tts", "intent", "stt"].find((stage) => stage in this.pipelineRun) ||
-          (stage) => this.pipelineRun![stage] !== undefined
+        "ready"
-        ) || "ready"
       : "ready";
 
     const messages: Array<{ from: string; text: string }> = [];
 
     const userMessage =
-      ("text" in this.pipelineRun.init_options.input
+      (this.pipelineRun.init_options &&
+      "text" in this.pipelineRun.init_options.input
         ? this.pipelineRun.init_options.input.text
-        : undefined) || this.pipelineRun?.stt?.stt_output?.text;
+        : undefined) ||
+      this.pipelineRun?.stt?.stt_output?.text ||
+      this.pipelineRun?.intent?.intent_input;
 
     if (userMessage) {
       messages.push({

src/panels/config/voice-assistants/dialog-voice-assistant-pipeline-detail.ts

@@ -9,12 +9,10 @@ import { SchemaUnion } from "../../../components/ha-form/types";
 import {
   AssistPipeline,
   AssistPipelineMutableParams,
-  getAssistPipelineRun,
-  listAssistPipelineRuns,
 } from "../../../data/assist_pipeline";
-import { showAlertDialog } from "../../../dialogs/generic/show-dialog-box";
 import { haStyleDialog } from "../../../resources/styles";
 import { HomeAssistant } from "../../../types";
+import "./debug/assist-render-pipeline-events";
 import { VoiceAssistantPipelineDetailsDialogParams } from "./show-dialog-voice-assistant-pipeline-detail";
 
 @customElement("dialog-voice-assistant-pipeline-detail")
@@ -94,9 +92,13 @@ export class DialogVoiceAssistantPipelineDetail extends LitElement {
                 @click=${this._setPreferred}
                 >Set as default</ha-button
               >
-              <ha-button slot="secondaryAction" @click=${this._debugPipeline}
+              <a
-                >Debug</ha-button
+                href="/config/voice-assistants/debug/${this._params.pipeline
-              >
+                  .id}"
+                slot="secondaryAction"
+                @click=${this.closeDialog}
+                ><ha-button>Debug</ha-button>
+              </a>
             `
           : nothing}
         <ha-button
@@ -200,25 +202,6 @@ export class DialogVoiceAssistantPipelineDetail extends LitElement {
     this._preferred = true;
   }
 
-  private async _debugPipeline() {
-    const runs = await listAssistPipelineRuns(
-      this.hass,
-      this._params!.pipeline!.id!
-    );
-    if (!runs.pipeline_runs.length) {
-      showAlertDialog(this, { text: "No runs found" });
-      return;
-    }
-    const events = await getAssistPipelineRun(
-      this.hass,
-      this._params!.pipeline!.id!,
-      runs.pipeline_runs[runs.pipeline_runs.length - 1]
-    );
-    showAlertDialog(this, {
-      text: html`<pre>${JSON.stringify(events.events, null, 2)}</pre>`,
-    });
-  }
-
   private async _deletePipeline() {
     this._submitting = true;
     try {

src/panels/config/voice-assistants/ha-config-voice-assistants.ts

@@ -43,11 +43,8 @@ class HaConfigVoiceAssistants extends HassRouterPage {
         load: () => import("./ha-config-voice-assistants-expose"),
       },
       debug: {
-        tag: "assist-pipeline-debug",
+        tag: "assist-debug",
-        load: () =>
+        load: () => import("./debug/assist-debug"),
-          import(
-            "../integrations/integration-panels/voice_assistant/assist/assist-pipeline-debug"
-          ),
       },
     },
   };