Fix intent loading and incorporate unmatched entities more (#108423)

* Incorporate unmatched entities more

* Don't list targets when match is incomplete

* Add test for out of range

homeassistant/components/conversation/__init__.py

@@ -9,7 +9,12 @@ import re
 from typing import Any, Literal
 
 from aiohttp import web
-from hassil.recognize import RecognizeResult
+from hassil.recognize import (
+    MISSING_ENTITY,
+    RecognizeResult,
+    UnmatchedRangeEntity,
+    UnmatchedTextEntity,
+)
 import voluptuous as vol
 
 from homeassistant import core
@@ -317,37 +322,55 @@ async def websocket_hass_agent_debug(
     ]
 
     # Return results for each sentence in the same order as the input.
-    connection.send_result(
+    result_dicts: list[dict[str, Any] | None] = []
-        msg["id"],
+    for result in results:
-        {
+        if result is None:
-            "results": [
+            # Indicate that a recognition failure occurred
-                {
+            result_dicts.append(None)
-                    "intent": {
+            continue
-                        "name": result.intent.name,
+
-                    },
+        successful_match = not result.unmatched_entities
-                    "slots": {  # direct access to values
+        result_dict = {
-                        entity_key: entity.value
+            # Name of the matching intent (or the closest)
-                        for entity_key, entity in result.entities.items()
+            "intent": {
-                    },
+                "name": result.intent.name,
-                    "details": {
+            },
-                        entity_key: {
+            # Slot values that would be received by the intent
-                            "name": entity.name,
+            "slots": {  # direct access to values
-                            "value": entity.value,
+                entity_key: entity.value
-                            "text": entity.text,
+                for entity_key, entity in result.entities.items()
-                        }
+            },
-                        for entity_key, entity in result.entities.items()
+            # Extra slot details, such as the originally matched text
-                    },
+            "details": {
-                    "targets": {
+                entity_key: {
-                        state.entity_id: {"matched": is_matched}
+                    "name": entity.name,
-                        for state, is_matched in _get_debug_targets(hass, result)
+                    "value": entity.value,
-                    },
+                    "text": entity.text,
                 }
-                if result is not None
+                for entity_key, entity in result.entities.items()
-                else None
+            },
-                for result in results
+            # Entities/areas/etc. that would be targeted
-            ]
+            "targets": {},
-        },
+            # True if match was successful
-    )
+            "match": successful_match,
+            # Text of the sentence template that matched (or was closest)
+            "sentence_template": "",
+            # When match is incomplete, this will contain the best slot guesses
+            "unmatched_slots": _get_unmatched_slots(result),
+        }
+
+        if successful_match:
+            result_dict["targets"] = {
+                state.entity_id: {"matched": is_matched}
+                for state, is_matched in _get_debug_targets(hass, result)
+            }
+
+        if result.intent_sentence is not None:
+            result_dict["sentence_template"] = result.intent_sentence.text
+
+        result_dicts.append(result_dict)
+
+    connection.send_result(msg["id"], {"results": result_dicts})
 
 
 def _get_debug_targets(
@@ -393,6 +416,25 @@ def _get_debug_targets(
         yield state, is_matched
 
 
+def _get_unmatched_slots(
+    result: RecognizeResult,
+) -> dict[str, str | int]:
+    """Return a dict of unmatched text/range slot entities."""
+    unmatched_slots: dict[str, str | int] = {}
+    for entity in result.unmatched_entities_list:
+        if isinstance(entity, UnmatchedTextEntity):
+            if entity.text == MISSING_ENTITY:
+                # Don't report <missing> since these are just missing context
+                # slots.
+                continue
+
+            unmatched_slots[entity.name] = entity.text
+        elif isinstance(entity, UnmatchedRangeEntity):
+            unmatched_slots[entity.name] = entity.value
+
+    return unmatched_slots
+
+
 class ConversationProcessView(http.HomeAssistantView):
     """View to process text."""
 

homeassistant/components/conversation/default_agent.py

@@ -6,6 +6,7 @@ from collections import defaultdict
 from collections.abc import Awaitable, Callable, Iterable
 from dataclasses import dataclass
 import functools
+import itertools
 import logging
 from pathlib import Path
 import re
@@ -20,6 +21,7 @@ from hassil.intents import (
     WildcardSlotList,
 )
 from hassil.recognize import (
+    MISSING_ENTITY,
     RecognizeResult,
     UnmatchedEntity,
     UnmatchedTextEntity,
@@ -75,7 +77,7 @@ class LanguageIntents:
     intents_dict: dict[str, Any]
     intent_responses: dict[str, Any]
     error_responses: dict[str, Any]
-    loaded_components: set[str]
+    language_variant: str | None
 
 
 @dataclass(slots=True)
@@ -181,9 +183,7 @@ class DefaultAgent(AbstractConversationAgent):
         lang_intents = self._lang_intents.get(language)
 
         # Reload intents if missing or new components
-        if lang_intents is None or (
+        if lang_intents is None:
-            lang_intents.loaded_components - self.hass.config.components
-        ):
             # Load intents in executor
             lang_intents = await self.async_get_or_load_intents(language)
 
@@ -357,6 +357,13 @@ class DefaultAgent(AbstractConversationAgent):
             intent_context=intent_context,
             allow_unmatched_entities=True,
         ):
+            # Remove missing entities that couldn't be filled from context
+            for entity_key, entity in list(result.unmatched_entities.items()):
+                if isinstance(entity, UnmatchedTextEntity) and (
+                    entity.text == MISSING_ENTITY
+                ):
+                    result.unmatched_entities.pop(entity_key)
+
             if maybe_result is None:
                 # First result
                 maybe_result = result
@@ -364,8 +371,11 @@ class DefaultAgent(AbstractConversationAgent):
                 # Fewer unmatched entities
                 maybe_result = result
             elif len(result.unmatched_entities) == len(maybe_result.unmatched_entities):
-                if result.text_chunks_matched > maybe_result.text_chunks_matched:
+                if (result.text_chunks_matched > maybe_result.text_chunks_matched) or (
-                    # More literal text chunks matched
+                    (result.text_chunks_matched == maybe_result.text_chunks_matched)
+                    and ("name" in result.unmatched_entities)  # prefer entities
+                ):
+                    # More literal text chunks matched, but prefer entities to areas, etc.
                     maybe_result = result
 
         if (maybe_result is not None) and maybe_result.unmatched_entities:
@@ -484,84 +494,93 @@ class DefaultAgent(AbstractConversationAgent):
 
         if lang_intents is None:
             intents_dict: dict[str, Any] = {}
-            loaded_components: set[str] = set()
+            language_variant: str | None = None
         else:
             intents_dict = lang_intents.intents_dict
-            loaded_components = lang_intents.loaded_components
+            language_variant = lang_intents.language_variant
 
-        # en-US, en_US, en, ...
+        domains_langs = get_domains_and_languages()
-        language_variations = list(_get_language_variations(language))
 
-        # Check if any new components have been loaded
+        if not language_variant:
-        intents_changed = False
+            # Choose a language variant upfront and commit to it for custom
-        for component in hass_components:
+            # sentences, etc.
-            if component in loaded_components:
+            all_language_variants = {
-                continue
+                lang.lower(): lang for lang in itertools.chain(*domains_langs.values())
+            }
 
-            # Don't check component again
+            # en-US, en_US, en, ...
-            loaded_components.add(component)
+            for maybe_variant in _get_language_variations(language):
-
+                matching_variant = all_language_variants.get(maybe_variant.lower())
-            # Check for intents for this component with the target language.
+                if matching_variant:
-            # Try en-US, en, etc.
+                    language_variant = matching_variant
-            for language_variation in language_variations:
-                component_intents = get_intents(
-                    component, language_variation, json_load=json_load
-                )
-                if component_intents:
-                    # Merge sentences into existing dictionary
-                    merge_dict(intents_dict, component_intents)
-
-                    # Will need to recreate graph
-                    intents_changed = True
-                    _LOGGER.debug(
-                        "Loaded intents component=%s, language=%s (%s)",
-                        component,
-                        language,
-                        language_variation,
-                    )
                     break
 
+            if not language_variant:
+                _LOGGER.warning(
+                    "Unable to find supported language variant for %s", language
+                )
+                return None
+
+            # Load intents for all domains supported by this language variant
+            for domain in domains_langs:
+                domain_intents = get_intents(
+                    domain, language_variant, json_load=json_load
+                )
+
+                if not domain_intents:
+                    continue
+
+                # Merge sentences into existing dictionary
+                merge_dict(intents_dict, domain_intents)
+
+                # Will need to recreate graph
+                intents_changed = True
+                _LOGGER.debug(
+                    "Loaded intents domain=%s, language=%s (%s)",
+                    domain,
+                    language,
+                    language_variant,
+                )
+
         # Check for custom sentences in <config>/custom_sentences/<language>/
         if lang_intents is None:
             # Only load custom sentences once, otherwise they will be re-loaded
             # when components change.
-            for language_variation in language_variations:
+            custom_sentences_dir = Path(
-                custom_sentences_dir = Path(
+                self.hass.config.path("custom_sentences", language_variant)
-                    self.hass.config.path("custom_sentences", language_variation)
+            )
-                )
+            if custom_sentences_dir.is_dir():
-                if custom_sentences_dir.is_dir():
+                for custom_sentences_path in custom_sentences_dir.rglob("*.yaml"):
-                    for custom_sentences_path in custom_sentences_dir.rglob("*.yaml"):
+                    with custom_sentences_path.open(
-                        with custom_sentences_path.open(
+                        encoding="utf-8"
-                            encoding="utf-8"
+                    ) as custom_sentences_file:
-                        ) as custom_sentences_file:
+                        # Merge custom sentences
-                            # Merge custom sentences
+                        if isinstance(
-                            if isinstance(
+                            custom_sentences_yaml := yaml.safe_load(
-                                custom_sentences_yaml := yaml.safe_load(
+                                custom_sentences_file
-                                    custom_sentences_file
+                            ),
-                                ),
+                            dict,
-                                dict,
+                        ):
-                            ):
+                            merge_dict(intents_dict, custom_sentences_yaml)
-                                merge_dict(intents_dict, custom_sentences_yaml)
+                        else:
-                            else:
+                            _LOGGER.warning(
-                                _LOGGER.warning(
+                                "Custom sentences file does not match expected format path=%s",
-                                    "Custom sentences file does not match expected format path=%s",
+                                custom_sentences_file.name,
-                                    custom_sentences_file.name,
+                            )
-                                )
 
-                        # Will need to recreate graph
+                    # Will need to recreate graph
-                        intents_changed = True
+                    intents_changed = True
-                        _LOGGER.debug(
+                    _LOGGER.debug(
-                            "Loaded custom sentences language=%s (%s), path=%s",
+                        "Loaded custom sentences language=%s (%s), path=%s",
-                            language,
+                        language,
-                            language_variation,
+                        language_variant,
-                            custom_sentences_path,
+                        custom_sentences_path,
-                        )
+                    )
-
-                    # Stop after first matched language variation
-                    break
 
             # Load sentences from HA config for default language only
-            if self._config_intents and (language == self.hass.config.language):
+            if self._config_intents and (
+                self.hass.config.language in (language, language_variant)
+            ):
                 merge_dict(
                     intents_dict,
                     {
@@ -598,7 +617,7 @@ class DefaultAgent(AbstractConversationAgent):
                 intents_dict,
                 intent_responses,
                 error_responses,
-                loaded_components,
+                language_variant,
             )
             self._lang_intents[language] = lang_intents
         else:

homeassistant/components/conversation/manifest.json

@@ -7,5 +7,5 @@
   "integration_type": "system",
   "iot_class": "local_push",
   "quality_scale": "internal",
-  "requirements": ["hassil==1.5.2", "home-assistant-intents==2024.1.2"]
+  "requirements": ["hassil==1.5.3", "home-assistant-intents==2024.1.2"]
 }

homeassistant/package_constraints.txt

@@ -26,7 +26,7 @@ ha-av==10.1.1
 ha-ffmpeg==3.1.0
 habluetooth==2.4.0
 hass-nabucasa==0.75.1
-hassil==1.5.2
+hassil==1.5.3
 home-assistant-bluetooth==1.12.0
 home-assistant-frontend==20240112.0
 home-assistant-intents==2024.1.2

requirements_all.txt

@@ -1019,7 +1019,7 @@ hass-nabucasa==0.75.1
 hass-splunk==0.1.1
 
 # homeassistant.components.conversation
-hassil==1.5.2
+hassil==1.5.3
 
 # homeassistant.components.jewish_calendar
 hdate==0.10.4

requirements_test_all.txt

@@ -821,7 +821,7 @@ habluetooth==2.4.0
 hass-nabucasa==0.75.1
 
 # homeassistant.components.conversation
-hassil==1.5.2
+hassil==1.5.3
 
 # homeassistant.components.jewish_calendar
 hdate==0.10.4

tests/components/conversation/snapshots/test_init.ambr

@@ -539,7 +539,7 @@
       'speech': dict({
         'plain': dict({
           'extra_data': None,
-          'speech': 'No area named kitchen',
+          'speech': 'No device or entity named kitchen light',
         }),
       }),
     }),
@@ -679,7 +679,7 @@
       'speech': dict({
         'plain': dict({
           'extra_data': None,
-          'speech': 'No area named late added',
+          'speech': 'No device or entity named late added light',
         }),
       }),
     }),
@@ -759,7 +759,7 @@
       'speech': dict({
         'plain': dict({
           'extra_data': None,
-          'speech': 'No area named kitchen',
+          'speech': 'No device or entity named kitchen light',
         }),
       }),
     }),
@@ -779,7 +779,7 @@
       'speech': dict({
         'plain': dict({
           'extra_data': None,
-          'speech': 'No area named my cool',
+          'speech': 'No device or entity named my cool light',
         }),
       }),
     }),
@@ -919,7 +919,7 @@
       'speech': dict({
         'plain': dict({
           'extra_data': None,
-          'speech': 'No area named kitchen',
+          'speech': 'No device or entity named kitchen light',
         }),
       }),
     }),
@@ -969,7 +969,7 @@
       'speech': dict({
         'plain': dict({
           'extra_data': None,
-          'speech': 'No area named renamed',
+          'speech': 'No device or entity named renamed light',
         }),
       }),
     }),
@@ -1403,6 +1403,8 @@
         'intent': dict({
           'name': 'HassTurnOn',
         }),
+        'match': True,
+        'sentence_template': '<turn> on (<area> <name>|<name> [in <area>])',
         'slots': dict({
           'name': 'my cool light',
         }),
@@ -1411,6 +1413,8 @@
             'matched': True,
           }),
         }),
+        'unmatched_slots': dict({
+        }),
       }),
       dict({
         'details': dict({
@@ -1423,6 +1427,8 @@
         'intent': dict({
           'name': 'HassTurnOff',
         }),
+        'match': True,
+        'sentence_template': '[<turn>] (<area> <name>|<name> [in <area>]) [to] off',
         'slots': dict({
           'name': 'my cool light',
         }),
@@ -1431,6 +1437,8 @@
             'matched': True,
           }),
         }),
+        'unmatched_slots': dict({
+        }),
       }),
       dict({
         'details': dict({
@@ -1448,6 +1456,8 @@
         'intent': dict({
           'name': 'HassTurnOn',
         }),
+        'match': True,
+        'sentence_template': '<turn> on [all] <light> in <area>',
         'slots': dict({
           'area': 'kitchen',
           'domain': 'light',
@@ -1457,6 +1467,8 @@
             'matched': True,
           }),
         }),
+        'unmatched_slots': dict({
+        }),
       }),
       dict({
         'details': dict({
@@ -1479,6 +1491,8 @@
         'intent': dict({
           'name': 'HassGetState',
         }),
+        'match': True,
+        'sentence_template': '[tell me] how many {on_off_domains:domain} (is|are) {on_off_states:state} [in <area>]',
         'slots': dict({
           'area': 'kitchen',
           'domain': 'light',
@@ -1489,23 +1503,30 @@
             'matched': False,
           }),
         }),
+        'unmatched_slots': dict({
+        }),
       }),
       dict({
         'details': dict({
           'domain': dict({
             'name': 'domain',
             'text': '',
-            'value': 'script',
+            'value': 'scene',
           }),
         }),
         'intent': dict({
           'name': 'HassTurnOn',
         }),
+        'match': False,
+        'sentence_template': '[activate|<turn>] <name> [scene] [on]',
         'slots': dict({
-          'domain': 'script',
+          'domain': 'scene',
         }),
         'targets': dict({
         }),
+        'unmatched_slots': dict({
+          'name': 'this will not match anything',
+        }),
       }),
     ]),
   })
@@ -1519,3 +1540,74 @@
     }),
   })
 # ---
+# name: test_ws_hass_agent_debug_null_result
+  dict({
+    'results': list([
+      None,
+    ]),
+  })
+# ---
+# name: test_ws_hass_agent_debug_out_of_range
+  dict({
+    'results': list([
+      dict({
+        'details': dict({
+          'brightness': dict({
+            'name': 'brightness',
+            'text': '100%',
+            'value': 100,
+          }),
+          'name': dict({
+            'name': 'name',
+            'text': 'test light',
+            'value': 'test light',
+          }),
+        }),
+        'intent': dict({
+          'name': 'HassLightSet',
+        }),
+        'match': True,
+        'sentence_template': '[<numeric_value_set>] <name> brightness [to] <brightness>',
+        'slots': dict({
+          'brightness': 100,
+          'name': 'test light',
+        }),
+        'targets': dict({
+          'light.demo_1234': dict({
+            'matched': True,
+          }),
+        }),
+        'unmatched_slots': dict({
+        }),
+      }),
+    ]),
+  })
+# ---
+# name: test_ws_hass_agent_debug_out_of_range.1
+  dict({
+    'results': list([
+      dict({
+        'details': dict({
+          'name': dict({
+            'name': 'name',
+            'text': 'test light',
+            'value': 'test light',
+          }),
+        }),
+        'intent': dict({
+          'name': 'HassLightSet',
+        }),
+        'match': False,
+        'sentence_template': '[<numeric_value_set>] <name> brightness [to] <brightness>',
+        'slots': dict({
+          'name': 'test light',
+        }),
+        'targets': dict({
+        }),
+        'unmatched_slots': dict({
+          'brightness': 1001,
+        }),
+      }),
+    ]),
+  })
+# ---

tests/components/conversation/test_default_agent.py

@@ -577,3 +577,24 @@ async def test_empty_aliases(
         names = slot_lists["name"]
         assert len(names.values) == 1
         assert names.values[0].value_out == "kitchen light"
+
+
+async def test_all_domains_loaded(
+    hass: HomeAssistant, init_components, area_registry: ar.AreaRegistry
+) -> None:
+    """Test that sentences for all domains are always loaded."""
+
+    # light domain is not loaded
+    assert "light" not in hass.config.components
+
+    result = await conversation.async_converse(
+        hass, "set brightness of test light to 100%", None, Context(), None
+    )
+
+    # Invalid target vs. no intent recognized
+    assert result.response.response_type == intent.IntentResponseType.ERROR
+    assert result.response.error_code == intent.IntentResponseErrorCode.NO_VALID_TARGETS
+    assert (
+        result.response.speech["plain"]["speech"]
+        == "No device or entity named test light"
+    )

tests/components/conversation/test_init.py

@@ -913,32 +913,6 @@ async def test_language_region(hass: HomeAssistant, init_components) -> None:
     assert call.data == {"entity_id": ["light.kitchen"]}
 
 
-async def test_reload_on_new_component(hass: HomeAssistant) -> None:
-    """Test intents being reloaded when a new component is loaded."""
-    language = hass.config.language
-    assert await async_setup_component(hass, "homeassistant", {})
-    assert await async_setup_component(hass, "conversation", {})
-
-    # Load intents
-    agent = await conversation._get_agent_manager(hass).async_get_agent()
-    assert isinstance(agent, conversation.DefaultAgent)
-    await agent.async_prepare()
-
-    lang_intents = agent._lang_intents.get(language)
-    assert lang_intents is not None
-    loaded_components = set(lang_intents.loaded_components)
-
-    # Load another component
-    assert await async_setup_component(hass, "light", {})
-
-    # Intents should reload
-    await agent.async_prepare()
-    lang_intents = agent._lang_intents.get(language)
-    assert lang_intents is not None
-
-    assert {"light"} == (lang_intents.loaded_components - loaded_components)
-
-
 async def test_non_default_response(hass: HomeAssistant, init_components) -> None:
     """Test intent response that is not the default."""
     hass.states.async_set("cover.front_door", "closed")
@@ -1206,7 +1180,7 @@ async def test_ws_hass_agent_debug(
                 "turn my cool light off",
                 "turn on all lights in the kitchen",
                 "how many lights are on in the kitchen?",
-                "this will not match anything",  # null in results
+                "this will not match anything",  # unmatched in results
             ],
         }
     )
@@ -1219,3 +1193,96 @@ async def test_ws_hass_agent_debug(
     # Light state should not have been changed
     assert len(on_calls) == 0
     assert len(off_calls) == 0
+
+
+async def test_ws_hass_agent_debug_null_result(
+    hass: HomeAssistant,
+    init_components,
+    hass_ws_client: WebSocketGenerator,
+    snapshot: SnapshotAssertion,
+) -> None:
+    """Test homeassistant agent debug websocket command with a null result."""
+    client = await hass_ws_client(hass)
+
+    async def async_recognize(self, user_input, *args, **kwargs):
+        if user_input.text == "bad sentence":
+            return None
+
+        return await self.async_recognize(user_input, *args, **kwargs)
+
+    with patch(
+        "homeassistant.components.conversation.default_agent.DefaultAgent.async_recognize",
+        async_recognize,
+    ):
+        await client.send_json_auto_id(
+            {
+                "type": "conversation/agent/homeassistant/debug",
+                "sentences": [
+                    "bad sentence",
+                ],
+            }
+        )
+
+        msg = await client.receive_json()
+
+    assert msg["success"]
+    assert msg["result"] == snapshot
+    assert msg["result"]["results"] == [None]
+
+
+async def test_ws_hass_agent_debug_out_of_range(
+    hass: HomeAssistant,
+    init_components,
+    hass_ws_client: WebSocketGenerator,
+    snapshot: SnapshotAssertion,
+    entity_registry: er.EntityRegistry,
+) -> None:
+    """Test homeassistant agent debug websocket command with an out of range entity."""
+    test_light = entity_registry.async_get_or_create("light", "demo", "1234")
+    hass.states.async_set(
+        test_light.entity_id, "off", attributes={ATTR_FRIENDLY_NAME: "test light"}
+    )
+
+    client = await hass_ws_client(hass)
+
+    # Brightness is in range (0-100)
+    await client.send_json_auto_id(
+        {
+            "type": "conversation/agent/homeassistant/debug",
+            "sentences": [
+                "set test light brightness to 100%",
+            ],
+        }
+    )
+
+    msg = await client.receive_json()
+
+    assert msg["success"]
+    assert msg["result"] == snapshot
+
+    results = msg["result"]["results"]
+    assert len(results) == 1
+    assert results[0]["match"]
+
+    # Brightness is out of range
+    await client.send_json_auto_id(
+        {
+            "type": "conversation/agent/homeassistant/debug",
+            "sentences": [
+                "set test light brightness to 1001%",
+            ],
+        }
+    )
+
+    msg = await client.receive_json()
+
+    assert msg["success"]
+    assert msg["result"] == snapshot
+
+    results = msg["result"]["results"]
+    assert len(results) == 1
+    assert not results[0]["match"]
+
+    # Name matched, but brightness didn't
+    assert results[0]["slots"] == {"name": "test light"}
+    assert results[0]["unmatched_slots"] == {"brightness": 1001}