Voice assistant (#15841)

* Add basic debug panel for voice assistant pipelines

* Add more info to start event

* Copy on change

* Use latest data model

* Apply suggestions from code review

Co-authored-by: Paul Bottein <paul.bottein@gmail.com>

* Fix CSS

* Also use ha-button

---------

Co-authored-by: Paul Bottein <paul.bottein@gmail.com>

src/data/conversation.ts

@@ -44,7 +44,7 @@ interface IntentResultError extends IntentResultBase {
   };
 }
 
-interface ConversationResult {
+export interface ConversationResult {
   conversation_id: string | null;
   response:
     | IntentResultActionDone

src/data/voice_assistant.ts

@@ -0,0 +1,153 @@
+import { HomeAssistant } from "../types";
+import { ConversationResult } from "./conversation";
+
+interface PipelineEventBase {
+  timestamp: string;
+}
+
+interface PipelineRunStartEvent extends PipelineEventBase {
+  type: "run-start";
+  data: {
+    pipeline: string;
+    language: string;
+  };
+}
+interface PipelineRunFinishEvent extends PipelineEventBase {
+  type: "run-finish";
+  data: Record<string, never>;
+}
+
+interface PipelineErrorEvent extends PipelineEventBase {
+  type: "error";
+  data: {
+    code: string;
+    message: string;
+  };
+}
+
+interface PipelineSTTStartEvent extends PipelineEventBase {
+  type: "stt-start";
+  data: Record<string, never>;
+}
+interface PipelineSTTFinishEvent extends PipelineEventBase {
+  type: "stt-finish";
+  data: Record<string, never>;
+}
+
+interface PipelineIntentStartEvent extends PipelineEventBase {
+  type: "intent-start";
+  data: {
+    engine: string;
+    intent_input: string;
+  };
+}
+interface PipelineIntentFinishEvent extends PipelineEventBase {
+  type: "intent-finish";
+  data: {
+    intent_output: ConversationResult;
+  };
+}
+
+interface PipelineTTSStartEvent extends PipelineEventBase {
+  type: "tts-start";
+  data: Record<string, never>;
+}
+interface PipelineTTSFinishEvent extends PipelineEventBase {
+  type: "tts-finish";
+  data: Record<string, never>;
+}
+
+type PipelineRunEvent =
+  | PipelineRunStartEvent
+  | PipelineRunFinishEvent
+  | PipelineErrorEvent
+  | PipelineSTTStartEvent
+  | PipelineSTTFinishEvent
+  | PipelineIntentStartEvent
+  | PipelineIntentFinishEvent
+  | PipelineTTSStartEvent
+  | PipelineTTSFinishEvent;
+
+interface PipelineRunOptions {
+  pipeline?: string;
+  intent_input?: string;
+  conversation_id?: string | null;
+}
+
+export interface PipelineRun {
+  init_options: PipelineRunOptions;
+  events: PipelineRunEvent[];
+  stage: "ready" | "stt" | "intent" | "tts" | "done" | "error";
+  run: PipelineRunStartEvent["data"];
+  error?: PipelineErrorEvent["data"];
+  stt?: PipelineSTTStartEvent["data"] & Partial<PipelineSTTFinishEvent["data"]>;
+  intent?: PipelineIntentStartEvent["data"] &
+    Partial<PipelineIntentFinishEvent["data"]>;
+  tts?: PipelineTTSStartEvent["data"] & Partial<PipelineTTSFinishEvent["data"]>;
+}
+
+export const runPipelineFromText = (
+  hass: HomeAssistant,
+  callback: (event: PipelineRun) => void,
+  options: PipelineRunOptions = {}
+) => {
+  let run: PipelineRun | undefined;
+
+  const unsubProm = hass.connection.subscribeMessage<PipelineRunEvent>(
+    (updateEvent) => {
+      if (updateEvent.type === "run-start") {
+        run = {
+          init_options: options,
+          stage: "ready",
+          run: updateEvent.data,
+          error: undefined,
+          stt: undefined,
+          intent: undefined,
+          tts: undefined,
+          events: [updateEvent],
+        };
+        callback(run);
+        return;
+      }
+
+      if (!run) {
+        // eslint-disable-next-line no-console
+        console.warn(
+          "Received unexpected event before receiving session",
+          updateEvent
+        );
+        return;
+      }
+
+      run.events.push(updateEvent);
+
+      if (updateEvent.type === "stt-start") {
+        run = { ...run, stage: "stt", stt: updateEvent.data };
+      } else if (updateEvent.type === "stt-finish") {
+        run = { ...run, stt: { ...run.stt!, ...updateEvent.data } };
+      } else if (updateEvent.type === "intent-start") {
+        run = { ...run, stage: "intent", intent: updateEvent.data };
+      } else if (updateEvent.type === "intent-finish") {
+        run = { ...run, intent: { ...run.intent!, ...updateEvent.data } };
+      } else if (updateEvent.type === "tts-start") {
+        run = { ...run, stage: "tts", tts: updateEvent.data };
+      } else if (updateEvent.type === "tts-finish") {
+        run = { ...run, tts: { ...run.tts!, ...updateEvent.data } };
+      } else if (updateEvent.type === "run-finish") {
+        run = { ...run, stage: "done" };
+        unsubProm.then((unsub) => unsub());
+      } else if (updateEvent.type === "error") {
+        run = { ...run, stage: "error", error: updateEvent.data };
+        unsubProm.then((unsub) => unsub());
+      }
+
+      callback(run);
+    },
+    {
+      ...options,
+      type: "voice_assistant/run",
+    }
+  );
+
+  return unsubProm;
+};

src/panels/config/ha-panel-config.ts

@@ -353,6 +353,13 @@ class HaPanelConfig extends HassRouterPage {
         tag: "ha-config-areas",
         load: () => import("./areas/ha-config-areas"),
       },
+      voice_assistant: {
+        tag: "assist-pipeline-debug",
+        load: () =>
+          import(
+            "./integrations/integration-panels/voice_assistant/assist/assist-pipeline-debug"
+          ),
+      },
       automation: {
         tag: "ha-config-automation",
         load: () => import("./automation/ha-config-automation"),

src/panels/config/integrations/integration-panels/voice_assistant/assist/assist-pipeline-debug.ts

@@ -0,0 +1,106 @@
+import { css, html, LitElement, TemplateResult } from "lit";
+import { customElement, property, query, state } from "lit/decorators";
+import "../../../../../../components/ha-card";
+import "../../../../../../components/ha-button";
+import "../../../../../../components/ha-textfield";
+import {
+  PipelineRun,
+  runPipelineFromText,
+} from "../../../../../../data/voice_assistant";
+import "../../../../../../layouts/hass-subpage";
+import { SubscribeMixin } from "../../../../../../mixins/subscribe-mixin";
+import { haStyle } from "../../../../../../resources/styles";
+import { HomeAssistant } from "../../../../../../types";
+
+@customElement("assist-pipeline-debug")
+export class AssistPipelineDebug extends SubscribeMixin(LitElement) {
+  @property({ attribute: false }) public hass!: HomeAssistant;
+
+  @property({ type: Boolean }) public narrow!: boolean;
+
+  @query("#run-input", true)
+  private _newRunInput!: HTMLInputElement;
+
+  @state()
+  private _pipelineRun?: PipelineRun;
+
+  protected render(): TemplateResult {
+    return html`
+      <hass-subpage
+        .narrow=${this.narrow}
+        .hass=${this.hass}
+        header="Assist Pipeline"
+      >
+        <div class="content">
+          <ha-card header="Run pipeline" class="run-pipeline-card">
+            <div class="card-content">
+              <ha-textfield
+                id="run-input"
+                label="Input"
+                value="Are the lights on?"
+              ></ha-textfield>
+            </div>
+            <div class="card-actions">
+              <ha-button
+                @click=${this._runPipeline}
+                .disabled=${this._pipelineRun &&
+                !["error", "done"].includes(this._pipelineRun.stage)}
+              >
+                Run
+              </ha-button>
+            </div>
+          </ha-card>
+          ${this._pipelineRun
+            ? html`
+                <ha-card heading="Pipeline Run">
+                  <div class="card-content">
+                    <pre>${JSON.stringify(this._pipelineRun, null, 2)}</pre>
+                  </div>
+                </ha-card>
+              `
+            : ""}
+        </div>
+      </hass-subpage>
+    `;
+  }
+
+  private _runPipeline(): void {
+    this._pipelineRun = undefined;
+    runPipelineFromText(
+      this.hass,
+      (run) => {
+        this._pipelineRun = run;
+      },
+      {
+        intent_input: this._newRunInput.value,
+      }
+    );
+  }
+
+  static styles = [
+    haStyle,
+    css`
+      .content {
+        padding: 24px 0 32px;
+        max-width: 600px;
+        margin: 0 auto;
+        direction: ltr;
+      }
+      ha-card {
+        margin-bottom: 16px;
+      }
+      .run-pipeline-card ha-textfield {
+        display: block;
+      }
+      pre {
+        margin: 0;
+      }
+    `,
+  ];
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "assist-pipeline-debug": AssistPipelineDebug;
+  }
+}