From f6467a35db40b521a2da3c92b14ad2e4bcaa386e Mon Sep 17 00:00:00 2001 From: Bram Kragten Date: Wed, 26 Mar 2025 14:05:51 +0100 Subject: [PATCH] Update voice wizard (#24750) * Update pipeline step, add option for speech-to-phrase addon * adjust to core changes * Update conversation.ts * Update voice-assistant-setup-step-pipeline.ts * Update voice-assistant-setup-dialog.ts * review --- src/components/ha-language-picker.ts | 85 +-- src/data/conversation.ts | 19 + src/data/wyoming.ts | 22 + .../voice-assistant-setup-dialog.ts | 114 ++- .../voice-assistant-setup-step-local.ts | 210 ++++-- .../voice-assistant-setup-step-pipeline.ts | 661 ++++++++++-------- src/state/quick-bar-mixin.ts | 10 +- src/translations/en.json | 49 +- 8 files changed, 776 insertions(+), 394 deletions(-) create mode 100644 src/data/wyoming.ts diff --git a/src/components/ha-language-picker.ts b/src/components/ha-language-picker.ts index 27776d723d..f771871756 100644 --- a/src/components/ha-language-picker.ts +++ b/src/components/ha-language-picker.ts @@ -13,6 +13,49 @@ import "./ha-list-item"; import "./ha-select"; import type { HaSelect } from "./ha-select"; +export const getLanguageOptions = ( + languages: string[], + nativeName: boolean, + noSort: boolean, + locale?: FrontendLocaleData +) => { + let options: { label: string; value: string }[] = []; + + if (nativeName) { + const translations = translationMetadata.translations; + options = languages.map((lang) => { + let label = translations[lang]?.nativeName; + if (!label) { + try { + // this will not work if Intl.DisplayNames is polyfilled, it will return in the language of the user + label = new Intl.DisplayNames(lang, { + type: "language", + fallback: "code", + }).of(lang)!; + } catch (_err) { + label = lang; + } + } + return { + value: lang, + label, + }; + }); + } else if (locale) { + options = languages.map((lang) => ({ + value: lang, + label: formatLanguageCode(lang, locale), + })); + } + + if (!noSort && locale) { + options.sort((a, b) => + caseInsensitiveStringCompare(a.label, b.label, locale.language) + ); + } + return options; +}; + @customElement("ha-language-picker") export class HaLanguagePicker extends LitElement { @property() public value?: string; @@ -68,6 +111,7 @@ export class HaLanguagePicker extends LitElement { const languageOptions = this._getLanguagesOptions( this.languages ?? this._defaultLanguages, this.nativeName, + this.noSort, this.hass?.locale ); const selectedItemIndex = languageOptions.findIndex( @@ -82,45 +126,7 @@ export class HaLanguagePicker extends LitElement { } } - private _getLanguagesOptions = memoizeOne( - (languages: string[], nativeName: boolean, locale?: FrontendLocaleData) => { - let options: { label: string; value: string }[] = []; - - if (nativeName) { - const translations = translationMetadata.translations; - options = languages.map((lang) => { - let label = translations[lang]?.nativeName; - if (!label) { - try { - // this will not work if Intl.DisplayNames is polyfilled, it will return in the language of the user - label = new Intl.DisplayNames(lang, { - type: "language", - fallback: "code", - }).of(lang)!; - } catch (_err) { - label = lang; - } - } - return { - value: lang, - label, - }; - }); - } else if (locale) { - options = languages.map((lang) => ({ - value: lang, - label: formatLanguageCode(lang, locale), - })); - } - - if (!this.noSort && locale) { - options.sort((a, b) => - caseInsensitiveStringCompare(a.label, b.label, locale.language) - ); - } - return options; - } - ); + private _getLanguagesOptions = memoizeOne(getLanguageOptions); private _computeDefaultLanguageOptions() { this._defaultLanguages = Object.keys(translationMetadata.translations); @@ -130,6 +136,7 @@ export class HaLanguagePicker extends LitElement { const languageOptions = this._getLanguagesOptions( this.languages ?? this._defaultLanguages, this.nativeName, + this.noSort, this.hass?.locale ); diff --git a/src/data/conversation.ts b/src/data/conversation.ts index 3f5282c12e..638ec6824c 100644 --- a/src/data/conversation.ts +++ b/src/data/conversation.ts @@ -124,3 +124,22 @@ export const debugAgent = ( language, device_id, }); + +export interface LanguageScore { + cloud: number; + focused_local: number; + full_local: number; +} + +export type LanguageScores = Record; + +export const getLanguageScores = ( + hass: HomeAssistant, + language?: string, + country?: string +): Promise<{ languages: LanguageScores; preferred_language: string | null }> => + hass.callWS({ + type: "conversation/agent/homeassistant/language_scores", + language, + country, + }); diff --git a/src/data/wyoming.ts b/src/data/wyoming.ts new file mode 100644 index 0000000000..20024d560d --- /dev/null +++ b/src/data/wyoming.ts @@ -0,0 +1,22 @@ +import type { HomeAssistant } from "../types"; + +export interface WyomingInfo { + asr: WyomingAsrInfo[]; + handle: []; + intent: []; + tts: WyomingTtsInfo[]; + wake: []; +} + +interface WyomingBaseInfo { + name: string; + version: string; + attribution: Record; +} + +interface WyomingTtsInfo extends WyomingBaseInfo {} + +interface WyomingAsrInfo extends WyomingBaseInfo {} + +export const fetchWyomingInfo = (hass: HomeAssistant) => + hass.callWS<{ info: Record }>({ type: "wyoming/info" }); diff --git a/src/dialogs/voice-assistant-setup/voice-assistant-setup-dialog.ts b/src/dialogs/voice-assistant-setup/voice-assistant-setup-dialog.ts index b278af890a..5110be9605 100644 --- a/src/dialogs/voice-assistant-setup/voice-assistant-setup-dialog.ts +++ b/src/dialogs/voice-assistant-setup/voice-assistant-setup-dialog.ts @@ -1,14 +1,19 @@ import "@material/mwc-button/mwc-button"; -import { mdiChevronLeft, mdiClose } from "@mdi/js"; +import { mdiChevronLeft, mdiClose, mdiMenuDown } from "@mdi/js"; import type { CSSResultGroup } from "lit"; import { css, html, LitElement, nothing } from "lit"; import { customElement, property, state } from "lit/decorators"; import memoizeOne from "memoize-one"; import { fireEvent } from "../../common/dom/fire_event"; import { computeDomain } from "../../common/entity/compute_domain"; +import { formatLanguageCode } from "../../common/language/format_language"; +import "../../components/chips/ha-assist-chip"; import "../../components/ha-dialog"; +import { getLanguageOptions } from "../../components/ha-language-picker"; +import "../../components/ha-md-button-menu"; import type { AssistSatelliteConfiguration } from "../../data/assist_satellite"; import { fetchAssistSatelliteConfiguration } from "../../data/assist_satellite"; +import { getLanguageScores } from "../../data/conversation"; import { UNAVAILABLE } from "../../data/entity"; import type { EntityRegistryDisplayEntry } from "../../data/entity_registry"; import { haStyleDialog } from "../../resources/styles"; @@ -18,11 +23,11 @@ import "./voice-assistant-setup-step-area"; import "./voice-assistant-setup-step-change-wake-word"; import "./voice-assistant-setup-step-check"; import "./voice-assistant-setup-step-cloud"; +import "./voice-assistant-setup-step-local"; import "./voice-assistant-setup-step-pipeline"; import "./voice-assistant-setup-step-success"; import "./voice-assistant-setup-step-update"; import "./voice-assistant-setup-step-wake-word"; -import "./voice-assistant-setup-step-local"; export const enum STEP { INIT, @@ -49,6 +54,12 @@ export class HaVoiceAssistantSetupDialog extends LitElement { @state() private _error?: string; + @state() private _language?: string; + + @state() private _languages: string[] = []; + + @state() private _localOption?: string; + private _previousSteps: STEP[] = []; private _nextStep?: STEP; @@ -67,6 +78,12 @@ export class HaVoiceAssistantSetupDialog extends LitElement { this.renderRoot.querySelector("ha-dialog")?.close(); } + protected willUpdate(changedProps) { + if (changedProps.has("_step") && this._step === STEP.PIPELINE) { + this._getLanguages(); + } + } + private _dialogClosed() { this._params = undefined; this._assistConfiguration = undefined; @@ -139,9 +156,7 @@ export class HaVoiceAssistantSetupDialog extends LitElement { @click=${this.closeDialog} >` : nothing} - ${this._step === STEP.WAKEWORD || - this._step === STEP.AREA || - this._step === STEP.PIPELINE + ${this._step === STEP.WAKEWORD || this._step === STEP.AREA ? html`` - : nothing} + : this._step === STEP.PIPELINE + ? this._language + ? html` + + + ${getLanguageOptions( + this._languages, + false, + false, + this.hass.locale + ).map( + (lang) => + html` + ${lang.label} + ` + )} + ` + : nothing + : nothing}
` : this._step === STEP.CLOUD ? html`` @@ -233,6 +289,27 @@ export class HaVoiceAssistantSetupDialog extends LitElement { `; } + private async _getLanguages() { + if (this._languages.length) { + return; + } + + const scores = await getLanguageScores(this.hass); + + this._languages = Object.entries(scores.languages) + .filter( + ([_lang, score]) => + score.cloud > 0 || score.full_local > 0 || score.focused_local > 0 + ) + .map(([lang, _score]) => lang); + + this._language = + scores.preferred_language && + this._languages.includes(scores.preferred_language) + ? scores.preferred_language + : undefined; + } + private async _fetchAssistConfiguration() { try { this._assistConfiguration = await fetchAssistSatelliteConfiguration( @@ -248,6 +325,19 @@ export class HaVoiceAssistantSetupDialog extends LitElement { } } + private _handlePickLanguage(ev) { + if (ev.type === "keydown" && ev.key !== "Enter" && ev.key !== " ") return; + + this._language = ev.target.value; + } + + private _languageChanged(ev: CustomEvent) { + if (!ev.detail.value) { + return; + } + this._language = ev.detail.value; + } + private _goToPreviousStep() { if (!this._previousSteps.length) { return; @@ -267,6 +357,9 @@ export class HaVoiceAssistantSetupDialog extends LitElement { } if (ev?.detail?.step) { this._step = ev.detail.step; + if (ev.detail.step === STEP.LOCAL) { + this._localOption = ev.detail.option; + } } else if (this._nextStep) { this._step = this._nextStep; this._nextStep = undefined; @@ -305,6 +398,13 @@ export class HaVoiceAssistantSetupDialog extends LitElement { margin: 24px; display: block; } + ha-md-button-menu { + height: 48px; + display: flex; + align-items: center; + margin-right: 12px; + margin-inline-end: 12px; + } `, ]; } @@ -322,8 +422,10 @@ declare global { updateConfig?: boolean; noPrevious?: boolean; nextStep?: STEP; + option?: string; } | undefined; "prev-step": undefined; + "language-changed": { value: string }; } } diff --git a/src/dialogs/voice-assistant-setup/voice-assistant-setup-step-local.ts b/src/dialogs/voice-assistant-setup/voice-assistant-setup-step-local.ts index 29abe4b8f2..69c1381233 100644 --- a/src/dialogs/voice-assistant-setup/voice-assistant-setup-step-local.ts +++ b/src/dialogs/voice-assistant-setup/voice-assistant-setup-step-local.ts @@ -16,7 +16,10 @@ import { fetchConfigFlowInProgress, handleConfigFlowStep, } from "../../data/config_flow"; -import type { EntityRegistryDisplayEntry } from "../../data/entity_registry"; +import { + type ExtEntityRegistryEntry, + getExtendedEntityRegistryEntries, +} from "../../data/entity_registry"; import { fetchHassioAddonsInfo, installHassioAddon, @@ -24,10 +27,12 @@ import { } from "../../data/hassio/addon"; import { listSTTEngines } from "../../data/stt"; import { listTTSEngines, listTTSVoices } from "../../data/tts"; +import { fetchWyomingInfo } from "../../data/wyoming"; import type { HomeAssistant } from "../../types"; import { documentationUrl } from "../../util/documentation-url"; import { AssistantSetupStyles } from "./styles"; import { STEP } from "./voice-assistant-setup-dialog"; +import { listAgents } from "../../data/conversation"; @customElement("ha-voice-assistant-setup-step-local") export class HaVoiceAssistantSetupStepLocal extends LitElement { @@ -36,6 +41,12 @@ export class HaVoiceAssistantSetupStepLocal extends LitElement { @property({ attribute: false }) public assistConfiguration?: AssistSatelliteConfiguration; + @property({ attribute: false }) public localOption!: + | "focused_local" + | "full_local"; + + @property({ attribute: false }) public language!: string; + @state() private _state: "INSTALLING" | "NOT_SUPPORTED" | "ERROR" | "INTRO" = "INTRO"; @@ -43,9 +54,9 @@ export class HaVoiceAssistantSetupStepLocal extends LitElement { @state() private _error?: string; - @state() private _localTts?: EntityRegistryDisplayEntry[]; + @state() private _localTts?: ExtEntityRegistryEntry[]; - @state() private _localStt?: EntityRegistryDisplayEntry[]; + @state() private _localStt?: ExtEntityRegistryEntry[]; protected override render() { return html`
@@ -159,58 +170,62 @@ export class HaVoiceAssistantSetupStepLocal extends LitElement { } private async _checkLocal() { - this._findLocalEntities(); + await this._findLocalEntities(); if (!this._localTts || !this._localStt) { return; } - if (this._localTts.length && this._localStt.length) { - this._pickOrCreatePipelineExists(); - return; - } - if (!isComponentLoaded(this.hass, "hassio")) { - this._state = "NOT_SUPPORTED"; - return; - } - this._state = "INSTALLING"; try { + if (this._localTts.length && this._localStt.length) { + await this._pickOrCreatePipelineExists(); + return; + } + if (!isComponentLoaded(this.hass, "hassio")) { + this._state = "NOT_SUPPORTED"; + return; + } + this._state = "INSTALLING"; const { addons } = await fetchHassioAddonsInfo(this.hass); - const whisper = addons.find((addon) => addon.slug === "core_whisper"); - const piper = addons.find((addon) => addon.slug === "core_piper"); + const ttsAddon = addons.find( + (addon) => addon.slug === this._ttsAddonName + ); + const sttAddon = addons.find( + (addon) => addon.slug === this._sttAddonName + ); if (!this._localTts.length) { - if (!piper) { + if (!ttsAddon) { this._detailState = this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.local.state.installing_piper" + `ui.panel.config.voice_assistants.satellite_wizard.local.state.installing_${this._ttsProviderName}` ); - await installHassioAddon(this.hass, "core_piper"); + await installHassioAddon(this.hass, this._ttsAddonName); } - if (!piper || piper.state !== "started") { + if (!ttsAddon || ttsAddon.state !== "started") { this._detailState = this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.local.state.starting_piper" + `ui.panel.config.voice_assistants.satellite_wizard.local.state.starting_${this._ttsProviderName}` ); - await startHassioAddon(this.hass, "core_piper"); + await startHassioAddon(this.hass, this._ttsAddonName); } this._detailState = this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.local.state.setup_piper" + `ui.panel.config.voice_assistants.satellite_wizard.local.state.setup_${this._ttsProviderName}` ); - await this._setupConfigEntry("piper"); + await this._setupConfigEntry("tts"); } if (!this._localStt.length) { - if (!whisper) { + if (!sttAddon) { this._detailState = this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.local.state.installing_whisper" + `ui.panel.config.voice_assistants.satellite_wizard.local.state.installing_${this._sttProviderName}` ); - await installHassioAddon(this.hass, "core_whisper"); + await installHassioAddon(this.hass, this._sttAddonName); } - if (!whisper || whisper.state !== "started") { + if (!sttAddon || sttAddon.state !== "started") { this._detailState = this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.local.state.starting_whisper" + `ui.panel.config.voice_assistants.satellite_wizard.local.state.starting_${this._sttProviderName}` ); - await startHassioAddon(this.hass, "core_whisper"); + await startHassioAddon(this.hass, this._sttAddonName); } this._detailState = this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.local.state.setup_whisper" + `ui.panel.config.voice_assistants.satellite_wizard.local.state.setup_${this._sttProviderName}` ); - await this._setupConfigEntry("whisper"); + await this._setupConfigEntry("stt"); } this._detailState = this.hass.localize( "ui.panel.config.voice_assistants.satellite_wizard.local.state.creating_pipeline" @@ -222,20 +237,72 @@ export class HaVoiceAssistantSetupStepLocal extends LitElement { } } - private _findLocalEntities() { + private readonly _ttsProviderName = "piper"; + + private readonly _ttsAddonName = "core_piper"; + + private readonly _ttsHostName = "core-piper"; + + private readonly _ttsPort = "10200"; + + private get _sttProviderName() { + return this.localOption === "focused_local" + ? "speech-to-phrase" + : "faster-whisper"; + } + + private get _sttAddonName() { + return this.localOption === "focused_local" + ? "core_speech-to-phrase" + : "core_whisper"; + } + + private get _sttHostName() { + return this.localOption === "focused_local" + ? "core-speech-to-phrase" + : "core-whisper"; + } + + private readonly _sttPort = "10300"; + + private async _findLocalEntities() { const wyomingEntities = Object.values(this.hass.entities).filter( (entity) => entity.platform === "wyoming" ); - this._localTts = wyomingEntities.filter( - (ent) => computeDomain(ent.entity_id) === "tts" + if (!wyomingEntities.length) { + this._localStt = []; + this._localTts = []; + return; + } + const wyomingInfo = await fetchWyomingInfo(this.hass); + + const entityRegs = Object.values( + await getExtendedEntityRegistryEntries( + this.hass, + wyomingEntities.map((ent) => ent.entity_id) + ) ); - this._localStt = wyomingEntities.filter( - (ent) => computeDomain(ent.entity_id) === "stt" + + this._localTts = entityRegs.filter( + (ent) => + computeDomain(ent.entity_id) === "tts" && + ent.config_entry_id && + wyomingInfo.info[ent.config_entry_id]?.tts.some( + (provider) => provider.name === this._ttsProviderName + ) + ); + this._localStt = entityRegs.filter( + (ent) => + computeDomain(ent.entity_id) === "stt" && + ent.config_entry_id && + wyomingInfo.info[ent.config_entry_id]?.asr.some( + (provider) => provider.name === this._sttProviderName + ) ); } - private async _setupConfigEntry(addon: string) { - const configFlow = await this._findConfigFlowInProgress(addon); + private async _setupConfigEntry(type: "tts" | "stt") { + const configFlow = await this._findConfigFlowInProgress(type); if (configFlow) { const step = await handleConfigFlowStep( @@ -248,30 +315,36 @@ export class HaVoiceAssistantSetupStepLocal extends LitElement { } } - return this._createConfigEntry(addon); + return this._createConfigEntry(type); } - private async _findConfigFlowInProgress(addon: string) { + private async _findConfigFlowInProgress(type: "tts" | "stt") { const configFlows = await fetchConfigFlowInProgress(this.hass.connection); return configFlows.find( (flow) => flow.handler === "wyoming" && flow.context.source === "hassio" && - (flow.context.configuration_url.includes(`core_${addon}`) || - flow.context.title_placeholders.title.toLowerCase().includes(addon)) + (flow.context.configuration_url.includes( + type === "tts" ? this._ttsHostName : this._sttHostName + ) || + flow.context.title_placeholders.title + .toLowerCase() + .includes( + type === "tts" ? this._ttsProviderName : this._sttProviderName + )) ); } - private async _createConfigEntry(addon: string) { + private async _createConfigEntry(type: "tts" | "stt") { const configFlow = await createConfigFlow(this.hass, "wyoming"); const step = await handleConfigFlowStep(this.hass, configFlow.flow_id, { - host: `core-${addon}`, - port: addon === "piper" ? 10200 : 10300, + host: type === "tts" ? this._ttsHostName : this._sttHostName, + port: type === "tts" ? this._ttsPort : this._sttPort, }); if (step.type !== "create_entry") { throw new Error( - `${this.hass.localize("ui.panel.config.voice_assistants.satellite_wizard.local.errors.failed_create_entry", { addon })}${"errors" in step ? `: ${step.errors.base}` : ""}` + `${this.hass.localize("ui.panel.config.voice_assistants.satellite_wizard.local.errors.failed_create_entry", { addon: type === "tts" ? this._ttsProviderName : this._sttProviderName })}${"errors" in step ? `: ${step.errors.base}` : ""}` ); } } @@ -341,27 +414,54 @@ export class HaVoiceAssistantSetupStepLocal extends LitElement { const pipelines = await listAssistPipelines(this.hass); + const agent = ( + await listAgents( + this.hass, + this.language || this.hass.config.language, + this.hass.config.country || undefined + ) + ).agents.find((agnt) => agnt.id === "conversation.home_assistant"); + + if (!agent?.supported_languages.length) { + throw new Error( + "Conversation agent does not support requested language." + ); + } + const ttsEngine = ( await listTTSEngines( this.hass, - this.hass.config.language, + this.language, this.hass.config.country || undefined ) ).providers.find((provider) => provider.engine_id === ttsEntityId); + + if (!ttsEngine?.supported_languages?.length) { + throw new Error("TTS engine does not support requested language."); + } + const ttsVoices = await listTTSVoices( this.hass, ttsEntityId, - ttsEngine?.supported_languages![0] || this.hass.config.language + ttsEngine.supported_languages[0] ); + if (!ttsVoices.voices?.length) { + throw new Error("No voice available for requested language."); + } + const sttEngine = ( await listSTTEngines( this.hass, - this.hass.config.language, + this.language, this.hass.config.country || undefined ) ).providers.find((provider) => provider.engine_id === sttEntityId); + if (!sttEngine?.supported_languages?.length) { + throw new Error("STT engine does not support requested language."); + } + let pipelineName = this.hass.localize( "ui.panel.config.voice_assistants.satellite_wizard.local.local_pipeline" ); @@ -378,21 +478,21 @@ export class HaVoiceAssistantSetupStepLocal extends LitElement { return createAssistPipeline(this.hass, { name: pipelineName, - language: this.hass.config.language.split("-")[0], + language: this.language.split("-")[0], conversation_engine: "conversation.home_assistant", - conversation_language: this.hass.config.language.split("-")[0], + conversation_language: agent.supported_languages[0], stt_engine: sttEntityId, - stt_language: sttEngine!.supported_languages![0], + stt_language: sttEngine.supported_languages[0], tts_engine: ttsEntityId, - tts_language: ttsEngine!.supported_languages![0], - tts_voice: ttsVoices.voices![0].voice_id, + tts_language: ttsEngine.supported_languages[0], + tts_voice: ttsVoices.voices[0].voice_id, wake_word_entity: null, wake_word_id: null, }); } private async _findEntitiesAndCreatePipeline(tryNo = 0) { - this._findLocalEntities(); + await this._findLocalEntities(); if (!this._localTts?.length || !this._localStt?.length) { if (tryNo > 3) { throw new Error( diff --git a/src/dialogs/voice-assistant-setup/voice-assistant-setup-step-pipeline.ts b/src/dialogs/voice-assistant-setup/voice-assistant-setup-step-pipeline.ts index ddab2e8c03..d1fb161d30 100644 --- a/src/dialogs/voice-assistant-setup/voice-assistant-setup-step-pipeline.ts +++ b/src/dialogs/voice-assistant-setup/voice-assistant-setup-step-pipeline.ts @@ -1,22 +1,30 @@ -import { mdiOpenInNew } from "@mdi/js"; import type { PropertyValues } from "lit"; import { css, html, LitElement, nothing } from "lit"; import { customElement, property, state } from "lit/decorators"; +import memoizeOne from "memoize-one"; import { isComponentLoaded } from "../../common/config/is_component_loaded"; import { fireEvent } from "../../common/dom/fire_event"; import { computeDomain } from "../../common/entity/compute_domain"; +import { formatLanguageCode } from "../../common/language/format_language"; +import type { LocalizeFunc } from "../../common/translations/localize"; +import "../../components/ha-select-box"; +import type { SelectBoxOption } from "../../components/ha-select-box"; import { createAssistPipeline, listAssistPipelines, } from "../../data/assist_pipeline"; import type { AssistSatelliteConfiguration } from "../../data/assist_satellite"; import { fetchCloudStatus } from "../../data/cloud"; +import type { LanguageScores } from "../../data/conversation"; +import { getLanguageScores, listAgents } from "../../data/conversation"; import { listSTTEngines } from "../../data/stt"; import { listTTSEngines, listTTSVoices } from "../../data/tts"; import type { HomeAssistant } from "../../types"; -import { documentationUrl } from "../../util/documentation-url"; import { AssistantSetupStyles } from "./styles"; import { STEP } from "./voice-assistant-setup-dialog"; +import { documentationUrl } from "../../util/documentation-url"; + +const OPTIONS = ["cloud", "focused_local", "full_local"] as const; @customElement("ha-voice-assistant-setup-step-pipeline") export class HaVoiceAssistantSetupStepPipeline extends LitElement { @@ -29,166 +37,231 @@ export class HaVoiceAssistantSetupStepPipeline extends LitElement { @property({ attribute: false }) public assistEntityId?: string; + @property({ attribute: false }) public language?: string; + + @property({ attribute: false }) public languages: string[] = []; + @state() private _cloudChecked = false; - @state() private _showFirst = false; + @state() private _value?: (typeof OPTIONS)[number]; - @state() private _showSecond = false; - - @state() private _showThird = false; - - @state() private _showFourth = false; + @state() private _languageScores?: LanguageScores; protected override willUpdate(changedProperties: PropertyValues): void { super.willUpdate(changedProperties); if (!this.hasUpdated) { - this._checkCloud(); + this._fetchData(); + } + + if ( + (changedProperties.has("language") || + changedProperties.has("_languageScores")) && + this.language && + this._languageScores + ) { + const lang = this.language; + if (this._value && this._languageScores[lang][this._value] === 0) { + this._value = undefined; + } + if (!this._value) { + this._value = this._getOptions( + this._languageScores[lang], + this.hass.localize + ).supportedOptions[0]?.value as + | "cloud" + | "focused_local" + | "full_local" + | undefined; + } } } - protected override firstUpdated(changedProperties: PropertyValues) { - super.firstUpdated(changedProperties); - setTimeout(() => { - this._showFirst = true; - }, 200); - setTimeout(() => { - this._showSecond = true; - }, 600); - setTimeout(() => { - this._showThird = true; - }, 2000); - setTimeout(() => { - this._showFourth = true; - }, 3000); - } + private _getOptions = memoizeOne((score, localize: LocalizeFunc) => { + const supportedOptions: SelectBoxOption[] = []; + const unsupportedOptions: SelectBoxOption[] = []; + + OPTIONS.forEach((option) => { + if (score[option] > 0) { + supportedOptions.push({ + label: localize( + `ui.panel.config.voice_assistants.satellite_wizard.pipeline.options.${option}.label` + ), + description: localize( + `ui.panel.config.voice_assistants.satellite_wizard.pipeline.options.${option}.description` + ), + value: option, + }); + } else { + unsupportedOptions.push({ + label: localize( + `ui.panel.config.voice_assistants.satellite_wizard.pipeline.options.${option}.label` + ), + value: option, + }); + } + }); + + return { supportedOptions, unsupportedOptions }; + }); protected override render() { - if (!this._cloudChecked) { + if (!this._cloudChecked || !this._languageScores) { return nothing; } - return html`
-

- ${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.title" - )} -

-

- ${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.secondary" - )} -

-
-
-
- ${!this._showFirst ? "…" : "Turn on the lights in the bedroom"} -
- ${this._showFirst - ? html`
- 0.2 - ${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.seconds" - )} -
` - : nothing} - ${this._showFirst - ? html`
- ${!this._showSecond ? "…" : "Turned on the lights"} -
` - : nothing} - ${this._showSecond - ? html`
- 0.4 - ${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.seconds" - )} -
` - : nothing} -
-

Home Assistant Cloud

-

+ if (!this.language) { + const language = formatLanguageCode( + this.hass.config.language, + this.hass.locale + ); + return html`

+

${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.cloud.description" + "ui.panel.config.voice_assistants.satellite_wizard.pipeline.unsupported_language.header" )} -

- ${this.hass.localize("ui.panel.config.common.learn_more")} + ${this.hass.localize( + "ui.panel.config.voice_assistants.satellite_wizard.pipeline.unsupported_language.secondary", + { language } + )} + + + ${this.hass.localize( + "ui.panel.config.voice_assistants.satellite_wizard.pipeline.unsupported_language.contribute", + { language } + )} -

-
-
-
- ${!this._showThird ? "…" : "Turn on the lights in the bedroom"} -
- ${this._showThird - ? html`
- 2 - ${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.seconds" - )} -
` - : nothing} - ${this._showThird - ? html`
- ${!this._showFourth ? "…" : "Turned on the lights"} -
` - : nothing} - ${this._showFourth - ? html`
- 1 - ${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.seconds" - )} -
` - : nothing} -
-

+

`; + } + + const score = this._languageScores[this.language]; + + const options = this._getOptions( + score || { cloud: 3, focused_local: 0, full_local: 0 }, + this.hass.localize + ); + + const performance = !this._value + ? "" + : this._value === "full_local" + ? "low" + : "high"; + + const commands = !this._value + ? "" + : score?.[this._value] > 2 + ? "high" + : score?.[this._value] > 1 + ? "ready" + : score?.[this._value] > 0 + ? "low" + : ""; + + return html`
+

${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.local.title" + "ui.panel.config.voice_assistants.satellite_wizard.pipeline.title" )} -

-

- ${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.local.description" - )} -

-
- - - - ${this.hass.localize( - "ui.panel.config.common.learn_more" - )} - - +
+ ${this.hass.localize( - "ui.panel.config.voice_assistants.satellite_wizard.pipeline.local.setup" - )}${!performance + ? "" + : this.hass.localize( + `ui.panel.config.voice_assistants.satellite_wizard.pipeline.performance.${performance}` + )}
+
+
+
+
+
+
+ ${this.hass.localize( + "ui.panel.config.voice_assistants.satellite_wizard.pipeline.commands.header" + )}${!commands + ? "" + : this.hass.localize( + `ui.panel.config.voice_assistants.satellite_wizard.pipeline.commands.${commands}` + )} +
+
+
+
+
+
+ + ${options.unsupportedOptions.length + ? html`

+ ${this.hass.localize( + "ui.panel.config.voice_assistants.satellite_wizard.pipeline.unsupported" + )} +

+ ` + : nothing}
-
`; +
+ ${this.hass.localize("ui.common.next")} +
`; } - private async _checkCloud() { - if (!isComponentLoaded(this.hass, "cloud")) { + private async _fetchData() { + const cloud = + (await this._hasCloud()) && (await this._createCloudPipeline()); + if (!cloud) { this._cloudChecked = true; - return; + this._languageScores = (await getLanguageScores(this.hass)).languages; + } + } + + private async _hasCloud(): Promise { + if (!isComponentLoaded(this.hass, "cloud")) { + return false; } const cloudStatus = await fetchCloudStatus(this.hass); if (!cloudStatus.logged_in || !cloudStatus.active_subscription) { - this._cloudChecked = true; - return; + return false; } + return true; + } + + private async _createCloudPipeline(): Promise { let cloudTtsEntityId; let cloudSttEntityId; for (const entity of Object.values(this.hass.entities)) { @@ -206,186 +279,212 @@ export class HaVoiceAssistantSetupStepPipeline extends LitElement { } } } - const pipelines = await listAssistPipelines(this.hass); - const preferredPipeline = pipelines.pipelines.find( - (pipeline) => pipeline.id === pipelines.preferred_pipeline - ); - - if (preferredPipeline) { - if ( - preferredPipeline.conversation_engine === - "conversation.home_assistant" && - preferredPipeline.tts_engine === cloudTtsEntityId && - preferredPipeline.stt_engine === cloudSttEntityId - ) { - await this.hass.callService( - "select", - "select_option", - { option: "preferred" }, - { entity_id: this.assistConfiguration?.pipeline_entity_id } - ); - fireEvent(this, "next-step", { step: STEP.SUCCESS, noPrevious: true }); - return; - } - } - - let cloudPipeline = pipelines.pipelines.find( - (pipeline) => - pipeline.conversation_engine === "conversation.home_assistant" && - pipeline.tts_engine === cloudTtsEntityId && - pipeline.stt_engine === cloudSttEntityId - ); - - if (!cloudPipeline) { - const ttsEngine = ( - await listTTSEngines( - this.hass, - this.hass.config.language, - this.hass.config.country || undefined - ) - ).providers.find((provider) => provider.engine_id === cloudTtsEntityId); - const ttsVoices = await listTTSVoices( - this.hass, - cloudTtsEntityId, - ttsEngine?.supported_languages![0] || this.hass.config.language + try { + const pipelines = await listAssistPipelines(this.hass); + const preferredPipeline = pipelines.pipelines.find( + (pipeline) => pipeline.id === pipelines.preferred_pipeline ); - const sttEngine = ( - await listSTTEngines( - this.hass, - this.hass.config.language, - this.hass.config.country || undefined - ) - ).providers.find((provider) => provider.engine_id === cloudSttEntityId); - - let pipelineName = "Home Assistant Cloud"; - let i = 1; - while ( - pipelines.pipelines.find( - // eslint-disable-next-line no-loop-func - (pipeline) => pipeline.name === pipelineName - ) - ) { - pipelineName = `Home Assistant Cloud ${i}`; - i++; + if (preferredPipeline) { + if ( + preferredPipeline.conversation_engine === + "conversation.home_assistant" && + preferredPipeline.tts_engine === cloudTtsEntityId && + preferredPipeline.stt_engine === cloudSttEntityId + ) { + await this.hass.callService( + "select", + "select_option", + { option: "preferred" }, + { entity_id: this.assistConfiguration?.pipeline_entity_id } + ); + fireEvent(this, "next-step", { + step: STEP.SUCCESS, + noPrevious: true, + }); + return true; + } } - cloudPipeline = await createAssistPipeline(this.hass, { - name: pipelineName, - language: this.hass.config.language.split("-")[0], - conversation_engine: "conversation.home_assistant", - conversation_language: this.hass.config.language.split("-")[0], - stt_engine: cloudSttEntityId, - stt_language: sttEngine!.supported_languages![0], - tts_engine: cloudTtsEntityId, - tts_language: ttsEngine!.supported_languages![0], - tts_voice: ttsVoices.voices![0].voice_id, - wake_word_entity: null, - wake_word_id: null, - }); - } + let cloudPipeline = pipelines.pipelines.find( + (pipeline) => + pipeline.conversation_engine === "conversation.home_assistant" && + pipeline.tts_engine === cloudTtsEntityId && + pipeline.stt_engine === cloudSttEntityId + ); - await this.hass.callService( - "select", - "select_option", - { option: cloudPipeline.name }, - { entity_id: this.assistConfiguration?.pipeline_entity_id } - ); - fireEvent(this, "next-step", { step: STEP.SUCCESS, noPrevious: true }); + if (!cloudPipeline) { + const agent = ( + await listAgents( + this.hass, + this.language || this.hass.config.language, + this.hass.config.country || undefined + ) + ).agents.find((agnt) => agnt.id === "conversation.home_assistant"); + + if (!agent?.supported_languages.length) { + return false; + } + + const ttsEngine = ( + await listTTSEngines( + this.hass, + this.language || this.hass.config.language, + this.hass.config.country || undefined + ) + ).providers.find((provider) => provider.engine_id === cloudTtsEntityId); + + if (!ttsEngine?.supported_languages?.length) { + return false; + } + + const ttsVoices = await listTTSVoices( + this.hass, + cloudTtsEntityId, + ttsEngine.supported_languages[0] + ); + + const sttEngine = ( + await listSTTEngines( + this.hass, + this.language || this.hass.config.language, + this.hass.config.country || undefined + ) + ).providers.find((provider) => provider.engine_id === cloudSttEntityId); + + if (!sttEngine?.supported_languages?.length) { + return false; + } + + let pipelineName = "Home Assistant Cloud"; + let i = 1; + while ( + pipelines.pipelines.find( + // eslint-disable-next-line no-loop-func + (pipeline) => pipeline.name === pipelineName + ) + ) { + pipelineName = `Home Assistant Cloud ${i}`; + i++; + } + + cloudPipeline = await createAssistPipeline(this.hass, { + name: pipelineName, + language: (this.language || this.hass.config.language).split("-")[0], + conversation_engine: "conversation.home_assistant", + conversation_language: agent.supported_languages[0], + stt_engine: cloudSttEntityId, + stt_language: sttEngine.supported_languages[0], + tts_engine: cloudTtsEntityId, + tts_language: ttsEngine.supported_languages[0], + tts_voice: ttsVoices.voices![0].voice_id, + wake_word_entity: null, + wake_word_id: null, + }); + } + + await this.hass.callService( + "select", + "select_option", + { option: cloudPipeline.name }, + { entity_id: this.assistConfiguration?.pipeline_entity_id } + ); + fireEvent(this, "next-step", { step: STEP.SUCCESS, noPrevious: true }); + return true; + } catch (_e) { + return false; + } + } + + private _valueChanged(ev: CustomEvent) { + this._value = ev.detail.value; } private async _setupCloud() { - this._nextStep(STEP.CLOUD); + if (await this._hasCloud()) { + this._createCloudPipeline(); + return; + } + fireEvent(this, "next-step", { step: STEP.CLOUD }); } - private async _setupLocal() { - this._nextStep(STEP.LOCAL); + private _createPipeline() { + if (this._value === "cloud") { + this._setupCloud(); + } else if (this._value === "focused_local") { + this._setupLocalFocused(); + } else { + this._setupLocalFull(); + } } - private _nextStep(step?: STEP) { - fireEvent(this, "next-step", { step }); + private _setupLocalFocused() { + fireEvent(this, "next-step", { step: STEP.LOCAL, option: this._value }); + } + + private _setupLocalFull() { + fireEvent(this, "next-step", { step: STEP.LOCAL, option: this._value }); + } + + private _languageChanged(ev: CustomEvent) { + if (!ev.detail.value) { + return; + } + fireEvent(this, "language-changed", { value: ev.detail.value }); } static styles = [ AssistantSetupStyles, css` - .container { - border-radius: 16px; - border: 1px solid var(--divider-color); - overflow: hidden; - padding-bottom: 16px; + :host { + text-align: left; } - .container:last-child { - margin-top: 16px; - } - .messages-container { - padding: 24px; - box-sizing: border-box; - height: 195px; - background: var(--input-fill-color); + .perf-bar { + width: 100%; + height: 10px; display: flex; - flex-direction: column; - } - .message { - white-space: nowrap; - font-size: 18px; - clear: both; + gap: 4px; margin: 8px 0; - padding: 8px; - border-radius: 15px; - height: 36px; - box-sizing: border-box; - overflow: hidden; - text-overflow: ellipsis; - width: 30px; } - .rpi .message { - transition: width 1s; + .segment { + flex-grow: 1; + background-color: var(--disabled-color); + transition: background-color 0.3s; } - .cloud .message { - transition: width 0.5s; + .segment:first-child { + border-radius: 4px 0 0 4px; } - - .message.user { - margin-left: 24px; - margin-inline-start: 24px; - margin-inline-end: initial; - align-self: self-end; - text-align: right; - border-bottom-right-radius: 0px; - background-color: var(--primary-color); - color: var(--text-primary-color); - direction: var(--direction); + .segment:last-child { + border-radius: 0 4px 4px 0; } - .timing.user { - align-self: self-end; + .perf-bar.high .segment { + background-color: var(--success-color); } - - .message.user.show { - width: 295px; + .perf-bar.ready .segment:nth-child(-n + 2) { + background-color: var(--warning-color); } - - .message.hass { - margin-right: 24px; - margin-inline-end: 24px; - margin-inline-start: initial; - align-self: self-start; - border-bottom-left-radius: 0px; - background-color: var(--secondary-background-color); - color: var(--primary-text-color); - direction: var(--direction); + .perf-bar.low .segment:nth-child(1) { + background-color: var(--error-color); } - .timing.hass { - align-self: self-start; - } - - .message.hass.show { - width: 184px; - } - .row { + .bar-header { display: flex; justify-content: space-between; - margin: 0 16px; + margin: 8px 0; + margin-top: 16px; + } + ha-select-box { + display: block; + } + ha-select-box:first-of-type { + margin-top: 32px; + } + .footer { + margin-top: 16px; + } + ha-language-picker { + display: block; + margin-top: 16px; + margin-bottom: 16px; } `, ]; diff --git a/src/state/quick-bar-mixin.ts b/src/state/quick-bar-mixin.ts index daea444079..116c9d1fbd 100644 --- a/src/state/quick-bar-mixin.ts +++ b/src/state/quick-bar-mixin.ts @@ -187,7 +187,15 @@ export default >(superClass: T) => } private _canOverrideAlphanumericInput(e: KeyboardEvent) { - const el = e.composedPath()[0] as Element; + const composedPath = e.composedPath(); + + if ( + composedPath.some((el) => "tagName" in el && el.tagName === "HA-MENU") + ) { + return false; + } + + const el = composedPath[0] as Element; if (el.tagName === "TEXTAREA") { return false; diff --git a/src/translations/en.json b/src/translations/en.json index 086a3cf2e4..d9bd14dafd 100644 --- a/src/translations/en.json +++ b/src/translations/en.json @@ -3388,16 +3388,38 @@ "no_selection": "Please select an area" }, "pipeline": { - "title": "What hardware do you want to use?", - "secondary": "How quickly your assistant responds depends on the power of the hardware.", - "seconds": "seconds", - "cloud": { - "description": "Ideal if you don't have a powerful system at home." + "title": "How do you want your voice to be processed?", + "performance": { + "header": "Performance on low-powered system", + "low": "Low", + "high": "High" }, - "local": { - "title": "Do-it-yourself", - "description": "Install add-ons or containers to run it on your own system. Powerful hardware is needed for fast responses.", - "setup": "Set up" + "commands": { + "header": "Supported commands", + "low": "Needs work", + "ready": "Ready to be used", + "high": "Fully supported" + }, + "options": { + "cloud": { + "label": "Home Assistant Cloud", + "description": "Offloads speech processing to a fast, private cloud. Offering the highest accuracy and widest language support. Home Assistant Cloud is a subscription service that includes voice processing." + }, + "focused_local": { + "label": "Focused local processing", + "description": "Limited to a set list of common home control phrases, this allows any system to process speech locally and offline." + }, + "full_local": { + "label": "Full local processing", + "description": "Full speech processing is done locally, requiring high processing power for adequate speed and accuracy." + } + }, + "unsupported": "Unsupported", + "unsupported_language": { + "header": "Your language is not supported", + "secondary": "Your language {language} is not supported yet. You can select a different language to continue.", + "language_picker": "Language", + "contribute": "Contribute to the voice initiative for {language}" } }, "cloud": { @@ -3418,9 +3440,12 @@ "installing_piper": "Installing Piper add-on", "starting_piper": "Starting Piper add-on", "setup_piper": "Setting up Piper", - "installing_whisper": "Installing Whisper add-on", - "starting_whisper": "Starting Whisper add-on", - "setup_whisper": "Setting up Whisper", + "installing_faster-whisper": "Installing Whisper add-on", + "starting_faster-whisper": "Starting Whisper add-on", + "setup_faster-whisper": "Setting up Whisper", + "installing_speech-to-phrase": "Installing Speech-to-Phrase add-on", + "starting_speech-to-phrase": "Starting Speech-to-Phrase add-on", + "setup_speech-to-phrase": "Setting up Speech-to-Phrase", "creating_pipeline": "Creating assistant" }, "errors": {