diff --git a/homeassistant/components/conversation/__init__.py b/homeassistant/components/conversation/__init__.py index 9caabc6e8d4..bf18d740821 100644 --- a/homeassistant/components/conversation/__init__.py +++ b/homeassistant/components/conversation/__init__.py @@ -9,7 +9,12 @@ import re from typing import Any, Literal from aiohttp import web -from hassil.recognize import RecognizeResult +from hassil.recognize import ( + MISSING_ENTITY, + RecognizeResult, + UnmatchedRangeEntity, + UnmatchedTextEntity, +) import voluptuous as vol from homeassistant import core @@ -317,37 +322,55 @@ async def websocket_hass_agent_debug( ] # Return results for each sentence in the same order as the input. - connection.send_result( - msg["id"], - { - "results": [ - { - "intent": { - "name": result.intent.name, - }, - "slots": { # direct access to values - entity_key: entity.value - for entity_key, entity in result.entities.items() - }, - "details": { - entity_key: { - "name": entity.name, - "value": entity.value, - "text": entity.text, - } - for entity_key, entity in result.entities.items() - }, - "targets": { - state.entity_id: {"matched": is_matched} - for state, is_matched in _get_debug_targets(hass, result) - }, + result_dicts: list[dict[str, Any] | None] = [] + for result in results: + if result is None: + # Indicate that a recognition failure occurred + result_dicts.append(None) + continue + + successful_match = not result.unmatched_entities + result_dict = { + # Name of the matching intent (or the closest) + "intent": { + "name": result.intent.name, + }, + # Slot values that would be received by the intent + "slots": { # direct access to values + entity_key: entity.value + for entity_key, entity in result.entities.items() + }, + # Extra slot details, such as the originally matched text + "details": { + entity_key: { + "name": entity.name, + "value": entity.value, + "text": entity.text, } - if result is not None - else None - for result in results - ] - }, - ) + for entity_key, entity in result.entities.items() + }, + # Entities/areas/etc. that would be targeted + "targets": {}, + # True if match was successful + "match": successful_match, + # Text of the sentence template that matched (or was closest) + "sentence_template": "", + # When match is incomplete, this will contain the best slot guesses + "unmatched_slots": _get_unmatched_slots(result), + } + + if successful_match: + result_dict["targets"] = { + state.entity_id: {"matched": is_matched} + for state, is_matched in _get_debug_targets(hass, result) + } + + if result.intent_sentence is not None: + result_dict["sentence_template"] = result.intent_sentence.text + + result_dicts.append(result_dict) + + connection.send_result(msg["id"], {"results": result_dicts}) def _get_debug_targets( @@ -393,6 +416,25 @@ def _get_debug_targets( yield state, is_matched +def _get_unmatched_slots( + result: RecognizeResult, +) -> dict[str, str | int]: + """Return a dict of unmatched text/range slot entities.""" + unmatched_slots: dict[str, str | int] = {} + for entity in result.unmatched_entities_list: + if isinstance(entity, UnmatchedTextEntity): + if entity.text == MISSING_ENTITY: + # Don't report since these are just missing context + # slots. + continue + + unmatched_slots[entity.name] = entity.text + elif isinstance(entity, UnmatchedRangeEntity): + unmatched_slots[entity.name] = entity.value + + return unmatched_slots + + class ConversationProcessView(http.HomeAssistantView): """View to process text.""" diff --git a/homeassistant/components/conversation/default_agent.py b/homeassistant/components/conversation/default_agent.py index 3f36e98f85a..3207cde405f 100644 --- a/homeassistant/components/conversation/default_agent.py +++ b/homeassistant/components/conversation/default_agent.py @@ -6,6 +6,7 @@ from collections import defaultdict from collections.abc import Awaitable, Callable, Iterable from dataclasses import dataclass import functools +import itertools import logging from pathlib import Path import re @@ -20,6 +21,7 @@ from hassil.intents import ( WildcardSlotList, ) from hassil.recognize import ( + MISSING_ENTITY, RecognizeResult, UnmatchedEntity, UnmatchedTextEntity, @@ -75,7 +77,7 @@ class LanguageIntents: intents_dict: dict[str, Any] intent_responses: dict[str, Any] error_responses: dict[str, Any] - loaded_components: set[str] + language_variant: str | None @dataclass(slots=True) @@ -181,9 +183,7 @@ class DefaultAgent(AbstractConversationAgent): lang_intents = self._lang_intents.get(language) # Reload intents if missing or new components - if lang_intents is None or ( - lang_intents.loaded_components - self.hass.config.components - ): + if lang_intents is None: # Load intents in executor lang_intents = await self.async_get_or_load_intents(language) @@ -357,6 +357,13 @@ class DefaultAgent(AbstractConversationAgent): intent_context=intent_context, allow_unmatched_entities=True, ): + # Remove missing entities that couldn't be filled from context + for entity_key, entity in list(result.unmatched_entities.items()): + if isinstance(entity, UnmatchedTextEntity) and ( + entity.text == MISSING_ENTITY + ): + result.unmatched_entities.pop(entity_key) + if maybe_result is None: # First result maybe_result = result @@ -364,8 +371,11 @@ class DefaultAgent(AbstractConversationAgent): # Fewer unmatched entities maybe_result = result elif len(result.unmatched_entities) == len(maybe_result.unmatched_entities): - if result.text_chunks_matched > maybe_result.text_chunks_matched: - # More literal text chunks matched + if (result.text_chunks_matched > maybe_result.text_chunks_matched) or ( + (result.text_chunks_matched == maybe_result.text_chunks_matched) + and ("name" in result.unmatched_entities) # prefer entities + ): + # More literal text chunks matched, but prefer entities to areas, etc. maybe_result = result if (maybe_result is not None) and maybe_result.unmatched_entities: @@ -484,84 +494,93 @@ class DefaultAgent(AbstractConversationAgent): if lang_intents is None: intents_dict: dict[str, Any] = {} - loaded_components: set[str] = set() + language_variant: str | None = None else: intents_dict = lang_intents.intents_dict - loaded_components = lang_intents.loaded_components + language_variant = lang_intents.language_variant - # en-US, en_US, en, ... - language_variations = list(_get_language_variations(language)) + domains_langs = get_domains_and_languages() - # Check if any new components have been loaded - intents_changed = False - for component in hass_components: - if component in loaded_components: - continue + if not language_variant: + # Choose a language variant upfront and commit to it for custom + # sentences, etc. + all_language_variants = { + lang.lower(): lang for lang in itertools.chain(*domains_langs.values()) + } - # Don't check component again - loaded_components.add(component) - - # Check for intents for this component with the target language. - # Try en-US, en, etc. - for language_variation in language_variations: - component_intents = get_intents( - component, language_variation, json_load=json_load - ) - if component_intents: - # Merge sentences into existing dictionary - merge_dict(intents_dict, component_intents) - - # Will need to recreate graph - intents_changed = True - _LOGGER.debug( - "Loaded intents component=%s, language=%s (%s)", - component, - language, - language_variation, - ) + # en-US, en_US, en, ... + for maybe_variant in _get_language_variations(language): + matching_variant = all_language_variants.get(maybe_variant.lower()) + if matching_variant: + language_variant = matching_variant break + if not language_variant: + _LOGGER.warning( + "Unable to find supported language variant for %s", language + ) + return None + + # Load intents for all domains supported by this language variant + for domain in domains_langs: + domain_intents = get_intents( + domain, language_variant, json_load=json_load + ) + + if not domain_intents: + continue + + # Merge sentences into existing dictionary + merge_dict(intents_dict, domain_intents) + + # Will need to recreate graph + intents_changed = True + _LOGGER.debug( + "Loaded intents domain=%s, language=%s (%s)", + domain, + language, + language_variant, + ) + # Check for custom sentences in /custom_sentences// if lang_intents is None: # Only load custom sentences once, otherwise they will be re-loaded # when components change. - for language_variation in language_variations: - custom_sentences_dir = Path( - self.hass.config.path("custom_sentences", language_variation) - ) - if custom_sentences_dir.is_dir(): - for custom_sentences_path in custom_sentences_dir.rglob("*.yaml"): - with custom_sentences_path.open( - encoding="utf-8" - ) as custom_sentences_file: - # Merge custom sentences - if isinstance( - custom_sentences_yaml := yaml.safe_load( - custom_sentences_file - ), - dict, - ): - merge_dict(intents_dict, custom_sentences_yaml) - else: - _LOGGER.warning( - "Custom sentences file does not match expected format path=%s", - custom_sentences_file.name, - ) + custom_sentences_dir = Path( + self.hass.config.path("custom_sentences", language_variant) + ) + if custom_sentences_dir.is_dir(): + for custom_sentences_path in custom_sentences_dir.rglob("*.yaml"): + with custom_sentences_path.open( + encoding="utf-8" + ) as custom_sentences_file: + # Merge custom sentences + if isinstance( + custom_sentences_yaml := yaml.safe_load( + custom_sentences_file + ), + dict, + ): + merge_dict(intents_dict, custom_sentences_yaml) + else: + _LOGGER.warning( + "Custom sentences file does not match expected format path=%s", + custom_sentences_file.name, + ) - # Will need to recreate graph - intents_changed = True - _LOGGER.debug( - "Loaded custom sentences language=%s (%s), path=%s", - language, - language_variation, - custom_sentences_path, - ) - - # Stop after first matched language variation - break + # Will need to recreate graph + intents_changed = True + _LOGGER.debug( + "Loaded custom sentences language=%s (%s), path=%s", + language, + language_variant, + custom_sentences_path, + ) # Load sentences from HA config for default language only - if self._config_intents and (language == self.hass.config.language): + if self._config_intents and ( + self.hass.config.language in (language, language_variant) + ): merge_dict( intents_dict, { @@ -598,7 +617,7 @@ class DefaultAgent(AbstractConversationAgent): intents_dict, intent_responses, error_responses, - loaded_components, + language_variant, ) self._lang_intents[language] = lang_intents else: diff --git a/homeassistant/components/conversation/manifest.json b/homeassistant/components/conversation/manifest.json index 5de11d7a41a..2d4a9af346d 100644 --- a/homeassistant/components/conversation/manifest.json +++ b/homeassistant/components/conversation/manifest.json @@ -7,5 +7,5 @@ "integration_type": "system", "iot_class": "local_push", "quality_scale": "internal", - "requirements": ["hassil==1.5.2", "home-assistant-intents==2024.1.2"] + "requirements": ["hassil==1.5.3", "home-assistant-intents==2024.1.2"] } diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 4b84c9a81c5..eb8befd1781 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -26,7 +26,7 @@ ha-av==10.1.1 ha-ffmpeg==3.1.0 habluetooth==2.4.0 hass-nabucasa==0.75.1 -hassil==1.5.2 +hassil==1.5.3 home-assistant-bluetooth==1.12.0 home-assistant-frontend==20240112.0 home-assistant-intents==2024.1.2 diff --git a/requirements_all.txt b/requirements_all.txt index fa636569c70..2ba9def2cf5 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1019,7 +1019,7 @@ hass-nabucasa==0.75.1 hass-splunk==0.1.1 # homeassistant.components.conversation -hassil==1.5.2 +hassil==1.5.3 # homeassistant.components.jewish_calendar hdate==0.10.4 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 76c09f44f76..974bd73385a 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -821,7 +821,7 @@ habluetooth==2.4.0 hass-nabucasa==0.75.1 # homeassistant.components.conversation -hassil==1.5.2 +hassil==1.5.3 # homeassistant.components.jewish_calendar hdate==0.10.4 diff --git a/tests/components/conversation/snapshots/test_init.ambr b/tests/components/conversation/snapshots/test_init.ambr index b68f2fb8701..1d03bf89ad6 100644 --- a/tests/components/conversation/snapshots/test_init.ambr +++ b/tests/components/conversation/snapshots/test_init.ambr @@ -539,7 +539,7 @@ 'speech': dict({ 'plain': dict({ 'extra_data': None, - 'speech': 'No area named kitchen', + 'speech': 'No device or entity named kitchen light', }), }), }), @@ -679,7 +679,7 @@ 'speech': dict({ 'plain': dict({ 'extra_data': None, - 'speech': 'No area named late added', + 'speech': 'No device or entity named late added light', }), }), }), @@ -759,7 +759,7 @@ 'speech': dict({ 'plain': dict({ 'extra_data': None, - 'speech': 'No area named kitchen', + 'speech': 'No device or entity named kitchen light', }), }), }), @@ -779,7 +779,7 @@ 'speech': dict({ 'plain': dict({ 'extra_data': None, - 'speech': 'No area named my cool', + 'speech': 'No device or entity named my cool light', }), }), }), @@ -919,7 +919,7 @@ 'speech': dict({ 'plain': dict({ 'extra_data': None, - 'speech': 'No area named kitchen', + 'speech': 'No device or entity named kitchen light', }), }), }), @@ -969,7 +969,7 @@ 'speech': dict({ 'plain': dict({ 'extra_data': None, - 'speech': 'No area named renamed', + 'speech': 'No device or entity named renamed light', }), }), }), @@ -1403,6 +1403,8 @@ 'intent': dict({ 'name': 'HassTurnOn', }), + 'match': True, + 'sentence_template': ' on ( | [in ])', 'slots': dict({ 'name': 'my cool light', }), @@ -1411,6 +1413,8 @@ 'matched': True, }), }), + 'unmatched_slots': dict({ + }), }), dict({ 'details': dict({ @@ -1423,6 +1427,8 @@ 'intent': dict({ 'name': 'HassTurnOff', }), + 'match': True, + 'sentence_template': '[] ( | [in ]) [to] off', 'slots': dict({ 'name': 'my cool light', }), @@ -1431,6 +1437,8 @@ 'matched': True, }), }), + 'unmatched_slots': dict({ + }), }), dict({ 'details': dict({ @@ -1448,6 +1456,8 @@ 'intent': dict({ 'name': 'HassTurnOn', }), + 'match': True, + 'sentence_template': ' on [all] in ', 'slots': dict({ 'area': 'kitchen', 'domain': 'light', @@ -1457,6 +1467,8 @@ 'matched': True, }), }), + 'unmatched_slots': dict({ + }), }), dict({ 'details': dict({ @@ -1479,6 +1491,8 @@ 'intent': dict({ 'name': 'HassGetState', }), + 'match': True, + 'sentence_template': '[tell me] how many {on_off_domains:domain} (is|are) {on_off_states:state} [in ]', 'slots': dict({ 'area': 'kitchen', 'domain': 'light', @@ -1489,23 +1503,30 @@ 'matched': False, }), }), + 'unmatched_slots': dict({ + }), }), dict({ 'details': dict({ 'domain': dict({ 'name': 'domain', 'text': '', - 'value': 'script', + 'value': 'scene', }), }), 'intent': dict({ 'name': 'HassTurnOn', }), + 'match': False, + 'sentence_template': '[activate|] [scene] [on]', 'slots': dict({ - 'domain': 'script', + 'domain': 'scene', }), 'targets': dict({ }), + 'unmatched_slots': dict({ + 'name': 'this will not match anything', + }), }), ]), }) @@ -1519,3 +1540,74 @@ }), }) # --- +# name: test_ws_hass_agent_debug_null_result + dict({ + 'results': list([ + None, + ]), + }) +# --- +# name: test_ws_hass_agent_debug_out_of_range + dict({ + 'results': list([ + dict({ + 'details': dict({ + 'brightness': dict({ + 'name': 'brightness', + 'text': '100%', + 'value': 100, + }), + 'name': dict({ + 'name': 'name', + 'text': 'test light', + 'value': 'test light', + }), + }), + 'intent': dict({ + 'name': 'HassLightSet', + }), + 'match': True, + 'sentence_template': '[] brightness [to] ', + 'slots': dict({ + 'brightness': 100, + 'name': 'test light', + }), + 'targets': dict({ + 'light.demo_1234': dict({ + 'matched': True, + }), + }), + 'unmatched_slots': dict({ + }), + }), + ]), + }) +# --- +# name: test_ws_hass_agent_debug_out_of_range.1 + dict({ + 'results': list([ + dict({ + 'details': dict({ + 'name': dict({ + 'name': 'name', + 'text': 'test light', + 'value': 'test light', + }), + }), + 'intent': dict({ + 'name': 'HassLightSet', + }), + 'match': False, + 'sentence_template': '[] brightness [to] ', + 'slots': dict({ + 'name': 'test light', + }), + 'targets': dict({ + }), + 'unmatched_slots': dict({ + 'brightness': 1001, + }), + }), + ]), + }) +# --- diff --git a/tests/components/conversation/test_default_agent.py b/tests/components/conversation/test_default_agent.py index 2e815edf1e1..1bd8b5263e5 100644 --- a/tests/components/conversation/test_default_agent.py +++ b/tests/components/conversation/test_default_agent.py @@ -577,3 +577,24 @@ async def test_empty_aliases( names = slot_lists["name"] assert len(names.values) == 1 assert names.values[0].value_out == "kitchen light" + + +async def test_all_domains_loaded( + hass: HomeAssistant, init_components, area_registry: ar.AreaRegistry +) -> None: + """Test that sentences for all domains are always loaded.""" + + # light domain is not loaded + assert "light" not in hass.config.components + + result = await conversation.async_converse( + hass, "set brightness of test light to 100%", None, Context(), None + ) + + # Invalid target vs. no intent recognized + assert result.response.response_type == intent.IntentResponseType.ERROR + assert result.response.error_code == intent.IntentResponseErrorCode.NO_VALID_TARGETS + assert ( + result.response.speech["plain"]["speech"] + == "No device or entity named test light" + ) diff --git a/tests/components/conversation/test_init.py b/tests/components/conversation/test_init.py index b3167d979d5..94ce0932964 100644 --- a/tests/components/conversation/test_init.py +++ b/tests/components/conversation/test_init.py @@ -913,32 +913,6 @@ async def test_language_region(hass: HomeAssistant, init_components) -> None: assert call.data == {"entity_id": ["light.kitchen"]} -async def test_reload_on_new_component(hass: HomeAssistant) -> None: - """Test intents being reloaded when a new component is loaded.""" - language = hass.config.language - assert await async_setup_component(hass, "homeassistant", {}) - assert await async_setup_component(hass, "conversation", {}) - - # Load intents - agent = await conversation._get_agent_manager(hass).async_get_agent() - assert isinstance(agent, conversation.DefaultAgent) - await agent.async_prepare() - - lang_intents = agent._lang_intents.get(language) - assert lang_intents is not None - loaded_components = set(lang_intents.loaded_components) - - # Load another component - assert await async_setup_component(hass, "light", {}) - - # Intents should reload - await agent.async_prepare() - lang_intents = agent._lang_intents.get(language) - assert lang_intents is not None - - assert {"light"} == (lang_intents.loaded_components - loaded_components) - - async def test_non_default_response(hass: HomeAssistant, init_components) -> None: """Test intent response that is not the default.""" hass.states.async_set("cover.front_door", "closed") @@ -1206,7 +1180,7 @@ async def test_ws_hass_agent_debug( "turn my cool light off", "turn on all lights in the kitchen", "how many lights are on in the kitchen?", - "this will not match anything", # null in results + "this will not match anything", # unmatched in results ], } ) @@ -1219,3 +1193,96 @@ async def test_ws_hass_agent_debug( # Light state should not have been changed assert len(on_calls) == 0 assert len(off_calls) == 0 + + +async def test_ws_hass_agent_debug_null_result( + hass: HomeAssistant, + init_components, + hass_ws_client: WebSocketGenerator, + snapshot: SnapshotAssertion, +) -> None: + """Test homeassistant agent debug websocket command with a null result.""" + client = await hass_ws_client(hass) + + async def async_recognize(self, user_input, *args, **kwargs): + if user_input.text == "bad sentence": + return None + + return await self.async_recognize(user_input, *args, **kwargs) + + with patch( + "homeassistant.components.conversation.default_agent.DefaultAgent.async_recognize", + async_recognize, + ): + await client.send_json_auto_id( + { + "type": "conversation/agent/homeassistant/debug", + "sentences": [ + "bad sentence", + ], + } + ) + + msg = await client.receive_json() + + assert msg["success"] + assert msg["result"] == snapshot + assert msg["result"]["results"] == [None] + + +async def test_ws_hass_agent_debug_out_of_range( + hass: HomeAssistant, + init_components, + hass_ws_client: WebSocketGenerator, + snapshot: SnapshotAssertion, + entity_registry: er.EntityRegistry, +) -> None: + """Test homeassistant agent debug websocket command with an out of range entity.""" + test_light = entity_registry.async_get_or_create("light", "demo", "1234") + hass.states.async_set( + test_light.entity_id, "off", attributes={ATTR_FRIENDLY_NAME: "test light"} + ) + + client = await hass_ws_client(hass) + + # Brightness is in range (0-100) + await client.send_json_auto_id( + { + "type": "conversation/agent/homeassistant/debug", + "sentences": [ + "set test light brightness to 100%", + ], + } + ) + + msg = await client.receive_json() + + assert msg["success"] + assert msg["result"] == snapshot + + results = msg["result"]["results"] + assert len(results) == 1 + assert results[0]["match"] + + # Brightness is out of range + await client.send_json_auto_id( + { + "type": "conversation/agent/homeassistant/debug", + "sentences": [ + "set test light brightness to 1001%", + ], + } + ) + + msg = await client.receive_json() + + assert msg["success"] + assert msg["result"] == snapshot + + results = msg["result"]["results"] + assert len(results) == 1 + assert not results[0]["match"] + + # Name matched, but brightness didn't + assert results[0]["slots"] == {"name": "test light"} + assert results[0]["unmatched_slots"] == {"brightness": 1001}