From 1d20d6979efdf11b2b9edf9364f34834e9b18b96 Mon Sep 17 00:00:00 2001
From: Paul Bottein <paul.bottein@gmail.com>
Date: Fri, 13 Jan 2023 22:20:29 +0100
Subject: [PATCH] Improve voice dialog (#15084)

* Improve voice dialog

* Improve scrolling and dialog size

* Align messages to bottom for better keyboard support

* Add send button

* Simplify label
---
 .../ha-voice-command-dialog.ts                | 207 ++++++++++++------
 src/panels/lovelace/hui-root.ts               |  30 ++-
 src/translations/en.json                      |   8 +-
 3 files changed, 167 insertions(+), 78 deletions(-)

diff --git a/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts b/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts
index fa2ae543cc..2a6dacdd83 100644
--- a/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts
+++ b/src/dialogs/voice-command-dialog/ha-voice-command-dialog.ts
@@ -1,6 +1,6 @@
 /* eslint-disable lit/prefer-static-styles */
 import "@material/mwc-button/mwc-button";
-import { mdiClose, mdiMicrophone } from "@mdi/js";
+import { mdiClose, mdiMicrophone, mdiSend } from "@mdi/js";
 import {
   css,
   CSSResultGroup,
@@ -56,7 +56,11 @@ export class HaVoiceCommandDialog extends LitElement {
 
   @state() private _agentInfo?: AgentInfo;
 
-  @query("ha-dialog", true) private _dialog!: HaDialog;
+  @state() private _showSendButton = false;
+
+  @query("#scroll-container") private _scrollContainer!: HaDialog;
+
+  @query("#message-input") private _messageInput!: HaTextField;
 
   private recognition!: SpeechRecognition;
 
@@ -64,10 +68,8 @@ export class HaVoiceCommandDialog extends LitElement {
 
   public async showDialog(): Promise<void> {
     this._opened = true;
-    if (SpeechRecognition) {
-      this._startListening();
-    }
     this._agentInfo = await getAgentInfo(this.hass);
+    this._scrollMessagesBottom();
   }
 
   public async closeDialog(): Promise<void> {
@@ -83,9 +85,17 @@ export class HaVoiceCommandDialog extends LitElement {
       return html``;
     }
     return html`
-      <ha-dialog open @closed=${this.closeDialog}>
-        <div slot="heading" class="heading">
+      <ha-dialog
+        open
+        @closed=${this.closeDialog}
+        .heading=${this.hass.localize("ui.dialogs.voice_command.title")}
+        flexContent
+      >
+        <div slot="heading">
           <ha-header-bar>
+            <span slot="title">
+              ${this.hass.localize("ui.dialogs.voice_command.title")}
+            </span>
             <ha-icon-button
               slot="navigationIcon"
               dialogAction="cancel"
@@ -94,62 +104,77 @@ export class HaVoiceCommandDialog extends LitElement {
             ></ha-icon-button>
           </ha-header-bar>
         </div>
-        <div>
-          ${this._agentInfo && this._agentInfo.onboarding
-            ? html`
-                <div class="onboarding">
-                  ${this._agentInfo.onboarding.text}
-                  <div class="side-by-side" @click=${this._completeOnboarding}>
-                    <a
-                      class="button"
-                      href=${this._agentInfo.onboarding.url}
-                      target="_blank"
-                      rel="noreferrer"
-                      ><mwc-button unelevated
-                        >${this.hass.localize("ui.common.yes")}!</mwc-button
-                      ></a
-                    >
-                    <mwc-button outlined
-                      >${this.hass.localize("ui.common.no")}</mwc-button
+        <div class="messages">
+          <div class="messages-container" id="scroll-container">
+            ${this._agentInfo && this._agentInfo.onboarding
+              ? html`
+                  <div class="onboarding">
+                    ${this._agentInfo.onboarding.text}
+                    <div
+                      class="side-by-side"
+                      @click=${this._completeOnboarding}
                     >
+                      <a
+                        class="button"
+                        href=${this._agentInfo.onboarding.url}
+                        target="_blank"
+                        rel="noreferrer"
+                        ><mwc-button unelevated
+                          >${this.hass.localize("ui.common.yes")}!</mwc-button
+                        ></a
+                      >
+                      <mwc-button outlined
+                        >${this.hass.localize("ui.common.no")}</mwc-button
+                      >
+                    </div>
                   </div>
+                `
+              : ""}
+            ${this._conversation.map(
+              (message) => html`
+                <div class=${this._computeMessageClasses(message)}>
+                  ${message.text}
                 </div>
               `
-            : ""}
-          ${this._conversation.map(
-            (message) => html`
-              <div class=${this._computeMessageClasses(message)}>
-                ${message.text}
-              </div>
-            `
-          )}
-          ${this.results
-            ? html`
-                <div class="message user">
-                  <span
-                    class=${classMap({
-                      interimTranscript: !this.results.final,
-                    })}
-                    >${this.results.transcript}</span
-                  >${!this.results.final ? "…" : ""}
-                </div>
-              `
-            : ""}
+            )}
+            ${this.results
+              ? html`
+                  <div class="message user">
+                    <span
+                      class=${classMap({
+                        interimTranscript: !this.results.final,
+                      })}
+                      >${this.results.transcript}</span
+                    >${!this.results.final ? "…" : ""}
+                  </div>
+                `
+              : ""}
+          </div>
         </div>
         <div class="input" slot="primaryAction">
           <ha-textfield
+            id="message-input"
             @keyup=${this._handleKeyUp}
-            .label=${this.hass.localize(
-              `ui.dialogs.voice_command.${
-                SpeechRecognition ? "label_voice" : "label"
-              }`
-            )}
+            @input=${this._handleInput}
+            .label=${this.hass.localize(`ui.dialogs.voice_command.input_label`)}
             dialogInitialFocus
             iconTrailing
           >
-            ${SpeechRecognition
-              ? html`
-                  <span slot="trailingIcon">
+            <span slot="trailingIcon">
+              ${this._showSendButton
+                ? html`
+                    <ha-icon-button
+                      class="listening-icon"
+                      .path=${mdiSend}
+                      @click=${this._handleSendMessage}
+                      .label=${this.hass.localize(
+                        "ui.dialogs.voice_command.send_text"
+                      )}
+                    >
+                    </ha-icon-button>
+                  `
+                : SpeechRecognition
+                ? html`
                     ${this.results
                       ? html`
                           <div class="bouncer">
@@ -159,13 +184,17 @@ export class HaVoiceCommandDialog extends LitElement {
                         `
                       : ""}
                     <ha-icon-button
+                      class="listening-icon"
                       .path=${mdiMicrophone}
                       @click=${this._toggleListening}
+                      .label=${this.hass.localize(
+                        "ui.dialogs.voice_command.start_listening"
+                      )}
                     >
                     </ha-icon-button>
-                  </span>
-                `
-              : ""}
+                  `
+                : ""}
+            </span>
           </ha-textfield>
           ${this._agentInfo && this._agentInfo.attribution
             ? html`
@@ -209,6 +238,24 @@ export class HaVoiceCommandDialog extends LitElement {
     if (ev.keyCode === 13 && input.value) {
       this._processText(input.value);
       input.value = "";
+      this._showSendButton = false;
+    }
+  }
+
+  private _handleInput(ev: InputEvent) {
+    const value = (ev.target as HaTextField).value;
+    if (value && !this._showSendButton) {
+      this._showSendButton = true;
+    } else if (!value && this._showSendButton) {
+      this._showSendButton = false;
+    }
+  }
+
+  private _handleSendMessage() {
+    if (this._messageInput.value) {
+      this._processText(this._messageInput.value);
+      this._messageInput.value = "";
+      this._showSendButton = false;
     }
   }
 
@@ -221,6 +268,7 @@ export class HaVoiceCommandDialog extends LitElement {
     this.recognition = new SpeechRecognition();
     this.recognition.interimResults = true;
     this.recognition.lang = this.hass.language;
+    this.recognition.continuous = false;
 
     this.recognition.addEventListener("start", () => {
       this.results = {
@@ -319,7 +367,13 @@ export class HaVoiceCommandDialog extends LitElement {
     if (!this.results) {
       this._startListening();
     } else {
-      this.recognition!.stop();
+      this._stopListening();
+    }
+  }
+
+  private _stopListening() {
+    if (this.recognition) {
+      this.recognition.stop();
     }
   }
 
@@ -340,7 +394,7 @@ export class HaVoiceCommandDialog extends LitElement {
   }
 
   private _scrollMessagesBottom() {
-    this._dialog.scrollToPos(0, 99999);
+    this._scrollContainer.scrollTo(0, 99999);
   }
 
   private _computeMessageClasses(message: Message) {
@@ -351,7 +405,7 @@ export class HaVoiceCommandDialog extends LitElement {
     return [
       haStyleDialog,
       css`
-        ha-icon-button {
+        ha-icon-button.listening-icon {
           color: var(--secondary-text-color);
           margin-right: -24px;
           margin-inline-end: -24px;
@@ -359,7 +413,7 @@ export class HaVoiceCommandDialog extends LitElement {
           direction: var(--direction);
         }
 
-        ha-icon-button[active] {
+        ha-icon-button.listening-icon[active] {
           color: var(--primary-color);
         }
 
@@ -367,21 +421,19 @@ export class HaVoiceCommandDialog extends LitElement {
           --primary-action-button-flex: 1;
           --secondary-action-button-flex: 0;
           --mdc-dialog-max-width: 450px;
+          --mdc-dialog-max-height: 500px;
+          --dialog-content-padding: 0;
         }
         ha-header-bar {
-          display: none;
-        }
-        @media all and (max-width: 450px), all and (max-height: 500px) {
-          ha-header-bar {
-            --mdc-theme-on-primary: var(--primary-text-color);
-            --mdc-theme-primary: var(--mdc-theme-surface);
-            flex-shrink: 0;
-            display: block;
-          }
+          --mdc-theme-on-primary: var(--primary-text-color);
+          --mdc-theme-primary: var(--mdc-theme-surface);
+          display: flex;
+          flex-shrink: 0;
         }
 
         ha-textfield {
           display: block;
+          overflow: hidden;
         }
         a.button {
           text-decoration: none;
@@ -403,6 +455,25 @@ export class HaVoiceCommandDialog extends LitElement {
         .attribution {
           color: var(--secondary-text-color);
         }
+        .messages {
+          display: block;
+          height: 300px;
+          box-sizing: border-box;
+        }
+        @media all and (max-width: 450px), all and (max-height: 500px) {
+          .messages {
+            height: 100%;
+          }
+        }
+        .messages-container {
+          position: absolute;
+          bottom: 0px;
+          right: 0px;
+          left: 0px;
+          padding: 24px;
+          overflow-y: auto;
+          max-height: 100%;
+        }
         .message {
           font-size: 18px;
           clear: both;
diff --git a/src/panels/lovelace/hui-root.ts b/src/panels/lovelace/hui-root.ts
index 78cf52d702..5582c6d67b 100644
--- a/src/panels/lovelace/hui-root.ts
+++ b/src/panels/lovelace/hui-root.ts
@@ -3,13 +3,13 @@ import "@material/mwc-list/mwc-list-item";
 import type { RequestSelectedDetail } from "@material/mwc-list/mwc-list-item";
 import {
   mdiCodeBraces,
+  mdiCommentProcessingOutline,
   mdiDotsVertical,
   mdiFileMultiple,
   mdiFormatListBulletedTriangle,
   mdiHelp,
   mdiHelpCircle,
   mdiMagnify,
-  mdiMicrophone,
   mdiPencil,
   mdiPlus,
   mdiRefresh,
@@ -302,9 +302,9 @@ class HUIRoot extends LitElement {
                     ? html`
                         <ha-icon-button
                           .label=${this.hass!.localize(
-                            "ui.panel.lovelace.menu.start_conversation"
+                            "ui.panel.lovelace.menu.assist"
                           )}
-                          .path=${mdiMicrophone}
+                          .path=${mdiCommentProcessingOutline}
                           @click=${this._showVoiceCommandDialog}
                         ></ha-icon-button>
                       `
@@ -324,7 +324,7 @@ class HUIRoot extends LitElement {
                             ? html`
                                 <mwc-list-item
                                   graphic="icon"
-                                  @request-selected=${this._showQuickBar}
+                                  @request-selected=${this._handleShowQuickBar}
                                 >
                                   ${this.hass!.localize(
                                     "ui.panel.lovelace.menu.search"
@@ -343,15 +343,15 @@ class HUIRoot extends LitElement {
                                 <mwc-list-item
                                   graphic="icon"
                                   @request-selected=${this
-                                    ._showVoiceCommandDialog}
+                                    ._handleShowVoiceCommandDialog}
                                 >
                                   ${this.hass!.localize(
-                                    "ui.panel.lovelace.menu.start_conversation"
+                                    "ui.panel.lovelace.menu.assist"
                                   )}
 
                                   <ha-svg-icon
                                     slot="graphic"
-                                    .path=${mdiMicrophone}
+                                    .path=${mdiCommentProcessingOutline}
                                   ></ha-svg-icon>
                                 </mwc-list-item>
                               `
@@ -711,6 +711,13 @@ class HUIRoot extends LitElement {
     });
   }
 
+  private _handleShowQuickBar(ev: CustomEvent<RequestSelectedDetail>): void {
+    if (!shouldHandleRequestSelectedEvent(ev)) {
+      return;
+    }
+    this._showQuickBar();
+  }
+
   private _showQuickBar(): void {
     showQuickBar(this, {
       commandMode: false,
@@ -762,6 +769,15 @@ class HUIRoot extends LitElement {
     navigate(`${this.route?.prefix}/hass-unused-entities`);
   }
 
+  private _handleShowVoiceCommandDialog(
+    ev: CustomEvent<RequestSelectedDetail>
+  ): void {
+    if (!shouldHandleRequestSelectedEvent(ev)) {
+      return;
+    }
+    this._showVoiceCommandDialog();
+  }
+
   private _showVoiceCommandDialog(): void {
     showVoiceCommandDialog(this);
   }
diff --git a/src/translations/en.json b/src/translations/en.json
index 1b35019433..7de8a95054 100755
--- a/src/translations/en.json
+++ b/src/translations/en.json
@@ -803,13 +803,15 @@
         "nothing_found": "Nothing found!"
       },
       "voice_command": {
+        "title": "Assistant",
         "did_not_hear": "Home Assistant did not hear anything",
         "did_not_understand": "Didn't quite get that",
         "found": "I found the following for you:",
         "error": "Oops, an error has occurred",
         "how_can_i_help": "How can I help?",
-        "label": "Type a question and press 'Enter'",
-        "label_voice": "Type and press 'Enter' or tap the microphone to speak"
+        "input_label": "Enter a request",
+        "send_text": "Send text",
+        "start_listening": "Start listening"
       },
       "generic": {
         "cancel": "Cancel",
@@ -3856,7 +3858,7 @@
           "configure_ui": "Edit Dashboard",
           "help": "Help",
           "search": "Search",
-          "start_conversation": "Start conversation",
+          "assist": "Assist",
           "reload_resources": "Reload resources",
           "exit_edit_mode": "Done",
           "close": "Close"