Improved Assist debug (#108889)

* Differentiate builtin/custom sentences and triggers in debug

* Refactor so async_process runs triggers

* Report relative path of custom sentences file

* Add sentence trigger test

homeassistant/components/conversation/__init__.py

@@ -31,7 +31,13 @@ from homeassistant.util import language as language_util
 
 from .agent import AbstractConversationAgent, ConversationInput, ConversationResult
 from .const import HOME_ASSISTANT_AGENT
-from .default_agent import DefaultAgent, async_setup as async_setup_default_agent
+from .default_agent import (
+    METADATA_CUSTOM_FILE,
+    METADATA_CUSTOM_SENTENCE,
+    DefaultAgent,
+    SentenceTriggerResult,
+    async_setup as async_setup_default_agent,
+)
 
 __all__ = [
     "DOMAIN",
@@ -324,49 +330,64 @@ async def websocket_hass_agent_debug(
     # Return results for each sentence in the same order as the input.
     result_dicts: list[dict[str, Any] | None] = []
     for result in results:
-        if result is None:
+        result_dict: dict[str, Any] | None = None
-            # Indicate that a recognition failure occurred
+        if isinstance(result, SentenceTriggerResult):
-            result_dicts.append(None)
+            result_dict = {
-            continue
+                # Matched a user-defined sentence trigger.
-
+                # We can't provide the response here without executing the
-        successful_match = not result.unmatched_entities
+                # trigger.
-        result_dict = {
+                "match": True,
-            # Name of the matching intent (or the closest)
+                "source": "trigger",
-            "intent": {
+                "sentence_template": result.sentence_template or "",
-                "name": result.intent.name,
+            }
-            },
+        elif isinstance(result, RecognizeResult):
-            # Slot values that would be received by the intent
+            successful_match = not result.unmatched_entities
-            "slots": {  # direct access to values
+            result_dict = {
-                entity_key: entity.value
+                # Name of the matching intent (or the closest)
-                for entity_key, entity in result.entities.items()
+                "intent": {
-            },
+                    "name": result.intent.name,
-            # Extra slot details, such as the originally matched text
+                },
-            "details": {
+                # Slot values that would be received by the intent
-                entity_key: {
+                "slots": {  # direct access to values
-                    "name": entity.name,
+                    entity_key: entity.value
-                    "value": entity.value,
+                    for entity_key, entity in result.entities.items()
-                    "text": entity.text,
+                },
-                }
+                # Extra slot details, such as the originally matched text
-                for entity_key, entity in result.entities.items()
+                "details": {
-            },
+                    entity_key: {
-            # Entities/areas/etc. that would be targeted
+                        "name": entity.name,
-            "targets": {},
+                        "value": entity.value,
-            # True if match was successful
+                        "text": entity.text,
-            "match": successful_match,
+                    }
-            # Text of the sentence template that matched (or was closest)
+                    for entity_key, entity in result.entities.items()
-            "sentence_template": "",
+                },
-            # When match is incomplete, this will contain the best slot guesses
+                # Entities/areas/etc. that would be targeted
-            "unmatched_slots": _get_unmatched_slots(result),
+                "targets": {},
-        }
+                # True if match was successful
-
+                "match": successful_match,
-        if successful_match:
+                # Text of the sentence template that matched (or was closest)
-            result_dict["targets"] = {
+                "sentence_template": "",
-                state.entity_id: {"matched": is_matched}
+                # When match is incomplete, this will contain the best slot guesses
-                for state, is_matched in _get_debug_targets(hass, result)
+                "unmatched_slots": _get_unmatched_slots(result),
             }
 
-        if result.intent_sentence is not None:
+            if successful_match:
-            result_dict["sentence_template"] = result.intent_sentence.text
+                result_dict["targets"] = {
+                    state.entity_id: {"matched": is_matched}
+                    for state, is_matched in _get_debug_targets(hass, result)
+                }
+
+            if result.intent_sentence is not None:
+                result_dict["sentence_template"] = result.intent_sentence.text
+
+            # Inspect metadata to determine if this matched a custom sentence
+            if result.intent_metadata and result.intent_metadata.get(
+                METADATA_CUSTOM_SENTENCE
+            ):
+                result_dict["source"] = "custom"
+                result_dict["file"] = result.intent_metadata.get(METADATA_CUSTOM_FILE)
+            else:
+                result_dict["source"] = "builtin"
 
         result_dicts.append(result_dict)
 
@@ -402,6 +423,16 @@ def _get_debug_targets(
         # HassGetState only
         state_names = set(cv.ensure_list(entities["state"].value))
 
+    if (
+        (name is None)
+        and (area_name is None)
+        and (not domains)
+        and (not device_classes)
+        and (not state_names)
+    ):
+        # Avoid "matching" all entities when there is no filter
+        return
+
     states = intent.async_match_states(
         hass,
         name=name,

homeassistant/components/conversation/default_agent.py

@@ -62,6 +62,8 @@ _ENTITY_REGISTRY_UPDATE_FIELDS = ["aliases", "name", "original_name"]
 
 REGEX_TYPE = type(re.compile(""))
 TRIGGER_CALLBACK_TYPE = Callable[[str, RecognizeResult], Awaitable[str | None]]
+METADATA_CUSTOM_SENTENCE = "hass_custom_sentence"
+METADATA_CUSTOM_FILE = "hass_custom_file"
 
 
 def json_load(fp: IO[str]) -> JsonObjectType:
@@ -88,6 +90,15 @@ class TriggerData:
     callback: TRIGGER_CALLBACK_TYPE
 
 
+@dataclass(slots=True)
+class SentenceTriggerResult:
+    """Result when matching a sentence trigger in an automation."""
+
+    sentence: str
+    sentence_template: str | None
+    matched_triggers: dict[int, RecognizeResult]
+
+
 def _get_language_variations(language: str) -> Iterable[str]:
     """Generate language codes with and without region."""
     yield language
@@ -177,8 +188,11 @@ class DefaultAgent(AbstractConversationAgent):
 
     async def async_recognize(
         self, user_input: ConversationInput
-    ) -> RecognizeResult | None:
+    ) -> RecognizeResult | SentenceTriggerResult | None:
         """Recognize intent from user input."""
+        if trigger_result := await self._match_triggers(user_input.text):
+            return trigger_result
+
         language = user_input.language or self.hass.config.language
         lang_intents = self._lang_intents.get(language)
 
@@ -208,13 +222,36 @@ class DefaultAgent(AbstractConversationAgent):
 
     async def async_process(self, user_input: ConversationInput) -> ConversationResult:
         """Process a sentence."""
-        if trigger_result := await self._match_triggers(user_input.text):
-            return trigger_result
-
         language = user_input.language or self.hass.config.language
         conversation_id = None  # Not supported
 
         result = await self.async_recognize(user_input)
+
+        # Check if a trigger matched
+        if isinstance(result, SentenceTriggerResult):
+            # Gather callback responses in parallel
+            trigger_responses = await asyncio.gather(
+                *(
+                    self._trigger_sentences[trigger_id].callback(
+                        result.sentence, trigger_result
+                    )
+                    for trigger_id, trigger_result in result.matched_triggers.items()
+                )
+            )
+
+            # Use last non-empty result as response
+            response_text: str | None = None
+            for trigger_response in trigger_responses:
+                response_text = response_text or trigger_response
+
+            # Convert to conversation result
+            response = intent.IntentResponse(language=language)
+            response.response_type = intent.IntentResponseType.ACTION_DONE
+            response.async_set_speech(response_text or "")
+
+            return ConversationResult(response=response)
+
+        # Intent match or failure
         lang_intents = self._lang_intents.get(language)
 
         if result is None:
@@ -561,6 +598,22 @@ class DefaultAgent(AbstractConversationAgent):
                             ),
                             dict,
                         ):
+                            # Add metadata so we can identify custom sentences in the debugger
+                            custom_intents_dict = custom_sentences_yaml.get(
+                                "intents", {}
+                            )
+                            for intent_dict in custom_intents_dict.values():
+                                intent_data_list = intent_dict.get("data", [])
+                                for intent_data in intent_data_list:
+                                    sentence_metadata = intent_data.get("metadata", {})
+                                    sentence_metadata[METADATA_CUSTOM_SENTENCE] = True
+                                    sentence_metadata[METADATA_CUSTOM_FILE] = str(
+                                        custom_sentences_path.relative_to(
+                                            custom_sentences_dir.parent
+                                        )
+                                    )
+                                    intent_data["metadata"] = sentence_metadata
+
                             merge_dict(intents_dict, custom_sentences_yaml)
                         else:
                             _LOGGER.warning(
@@ -807,11 +860,11 @@ class DefaultAgent(AbstractConversationAgent):
         # Force rebuild on next use
         self._trigger_intents = None
 
-    async def _match_triggers(self, sentence: str) -> ConversationResult | None:
+    async def _match_triggers(self, sentence: str) -> SentenceTriggerResult | None:
         """Try to match sentence against registered trigger sentences.
 
-        Calls the registered callbacks if there's a match and returns a positive
+        Calls the registered callbacks if there's a match and returns a sentence
-        conversation result.
+        trigger result.
         """
         if not self._trigger_sentences:
             # No triggers registered
@@ -824,7 +877,11 @@ class DefaultAgent(AbstractConversationAgent):
         assert self._trigger_intents is not None
 
         matched_triggers: dict[int, RecognizeResult] = {}
+        matched_template: str | None = None
         for result in recognize_all(sentence, self._trigger_intents):
+            if result.intent_sentence is not None:
+                matched_template = result.intent_sentence.text
+
             trigger_id = int(result.intent.name)
             if trigger_id in matched_triggers:
                 # Already matched a sentence from this trigger
@@ -843,24 +900,7 @@ class DefaultAgent(AbstractConversationAgent):
             list(matched_triggers),
         )
 
-        # Gather callback responses in parallel
+        return SentenceTriggerResult(sentence, matched_template, matched_triggers)
-        trigger_responses = await asyncio.gather(
-            *(
-                self._trigger_sentences[trigger_id].callback(sentence, result)
-                for trigger_id, result in matched_triggers.items()
-            )
-        )
-
-        # Use last non-empty result as speech response
-        speech: str | None = None
-        for trigger_response in trigger_responses:
-            speech = speech or trigger_response
-
-        response = intent.IntentResponse(language=self.hass.config.language)
-        response.response_type = intent.IntentResponseType.ACTION_DONE
-        response.async_set_speech(speech or "")
-
-        return ConversationResult(response=response)
 
 
 def _make_error_result(

homeassistant/components/conversation/manifest.json

@@ -7,5 +7,5 @@
   "integration_type": "system",
   "iot_class": "local_push",
   "quality_scale": "internal",
-  "requirements": ["hassil==1.5.3", "home-assistant-intents==2024.1.2"]
+  "requirements": ["hassil==1.6.0", "home-assistant-intents==2024.1.2"]
 }

homeassistant/package_constraints.txt

@@ -26,7 +26,7 @@ ha-av==10.1.1
 ha-ffmpeg==3.1.0
 habluetooth==2.4.0
 hass-nabucasa==0.75.1
-hassil==1.5.3
+hassil==1.6.0
 home-assistant-bluetooth==1.12.0
 home-assistant-frontend==20240112.0
 home-assistant-intents==2024.1.2

requirements_all.txt

@@ -1019,7 +1019,7 @@ hass-nabucasa==0.75.1
 hass-splunk==0.1.1
 
 # homeassistant.components.conversation
-hassil==1.5.3
+hassil==1.6.0
 
 # homeassistant.components.jewish_calendar
 hdate==0.10.4

requirements_test_all.txt

@@ -821,7 +821,7 @@ habluetooth==2.4.0
 hass-nabucasa==0.75.1
 
 # homeassistant.components.conversation
-hassil==1.5.3
+hassil==1.6.0
 
 # homeassistant.components.jewish_calendar
 hdate==0.10.4

tests/components/conversation/snapshots/test_init.ambr

@@ -1408,6 +1408,7 @@
         'slots': dict({
           'name': 'my cool light',
         }),
+        'source': 'builtin',
         'targets': dict({
           'light.kitchen': dict({
             'matched': True,
@@ -1432,6 +1433,7 @@
         'slots': dict({
           'name': 'my cool light',
         }),
+        'source': 'builtin',
         'targets': dict({
           'light.kitchen': dict({
             'matched': True,
@@ -1462,6 +1464,7 @@
           'area': 'kitchen',
           'domain': 'light',
         }),
+        'source': 'builtin',
         'targets': dict({
           'light.kitchen': dict({
             'matched': True,
@@ -1498,6 +1501,7 @@
           'domain': 'light',
           'state': 'on',
         }),
+        'source': 'builtin',
         'targets': dict({
           'light.kitchen': dict({
             'matched': False,
@@ -1522,6 +1526,7 @@
         'slots': dict({
           'domain': 'scene',
         }),
+        'source': 'builtin',
         'targets': dict({
         }),
         'unmatched_slots': dict({
@@ -1540,6 +1545,35 @@
     }),
   })
 # ---
+# name: test_ws_hass_agent_debug_custom_sentence
+  dict({
+    'results': list([
+      dict({
+        'details': dict({
+          'beer_style': dict({
+            'name': 'beer_style',
+            'text': 'lager',
+            'value': 'lager',
+          }),
+        }),
+        'file': 'en/beer.yaml',
+        'intent': dict({
+          'name': 'OrderBeer',
+        }),
+        'match': True,
+        'sentence_template': "I'd like to order a {beer_style} [please]",
+        'slots': dict({
+          'beer_style': 'lager',
+        }),
+        'source': 'custom',
+        'targets': dict({
+        }),
+        'unmatched_slots': dict({
+        }),
+      }),
+    ]),
+  })
+# ---
 # name: test_ws_hass_agent_debug_null_result
   dict({
     'results': list([
@@ -1572,6 +1606,7 @@
           'brightness': 100,
           'name': 'test light',
         }),
+        'source': 'builtin',
         'targets': dict({
           'light.demo_1234': dict({
             'matched': True,
@@ -1602,6 +1637,7 @@
         'slots': dict({
           'name': 'test light',
         }),
+        'source': 'builtin',
         'targets': dict({
         }),
         'unmatched_slots': dict({
@@ -1611,3 +1647,14 @@
     ]),
   })
 # ---
+# name: test_ws_hass_agent_debug_sentence_trigger
+  dict({
+    'results': list([
+      dict({
+        'match': True,
+        'sentence_template': 'hello[ world]',
+        'source': 'trigger',
+      }),
+    ]),
+  })
+# ---

tests/components/conversation/test_init.py

@@ -1286,3 +1286,89 @@ async def test_ws_hass_agent_debug_out_of_range(
     # Name matched, but brightness didn't
     assert results[0]["slots"] == {"name": "test light"}
     assert results[0]["unmatched_slots"] == {"brightness": 1001}
+
+
+async def test_ws_hass_agent_debug_custom_sentence(
+    hass: HomeAssistant,
+    init_components,
+    hass_ws_client: WebSocketGenerator,
+    snapshot: SnapshotAssertion,
+    entity_registry: er.EntityRegistry,
+) -> None:
+    """Test homeassistant agent debug websocket command with a custom sentence."""
+    # Expecting testing_config/custom_sentences/en/beer.yaml
+    intent.async_register(hass, OrderBeerIntentHandler())
+
+    client = await hass_ws_client(hass)
+
+    # Brightness is in range (0-100)
+    await client.send_json_auto_id(
+        {
+            "type": "conversation/agent/homeassistant/debug",
+            "sentences": [
+                "I'd like to order a lager, please.",
+            ],
+        }
+    )
+
+    msg = await client.receive_json()
+
+    assert msg["success"]
+    assert msg["result"] == snapshot
+
+    debug_results = msg["result"].get("results", [])
+    assert len(debug_results) == 1
+    assert debug_results[0].get("match")
+    assert debug_results[0].get("source") == "custom"
+    assert debug_results[0].get("file") == "en/beer.yaml"
+
+
+async def test_ws_hass_agent_debug_sentence_trigger(
+    hass: HomeAssistant,
+    init_components,
+    hass_ws_client: WebSocketGenerator,
+    snapshot: SnapshotAssertion,
+) -> None:
+    """Test homeassistant agent debug websocket command with a sentence trigger."""
+    calls = async_mock_service(hass, "test", "automation")
+    assert await async_setup_component(
+        hass,
+        "automation",
+        {
+            "automation": {
+                "trigger": {
+                    "platform": "conversation",
+                    "command": ["hello", "hello[ world]"],
+                },
+                "action": {
+                    "service": "test.automation",
+                    "data_template": {"data": "{{ trigger }}"},
+                },
+            }
+        },
+    )
+
+    client = await hass_ws_client(hass)
+
+    # Use trigger sentence
+    await client.send_json_auto_id(
+        {
+            "type": "conversation/agent/homeassistant/debug",
+            "sentences": ["hello world"],
+        }
+    )
+    await hass.async_block_till_done()
+
+    msg = await client.receive_json()
+
+    assert msg["success"]
+    assert msg["result"] == snapshot
+
+    debug_results = msg["result"].get("results", [])
+    assert len(debug_results) == 1
+    assert debug_results[0].get("match")
+    assert debug_results[0].get("source") == "trigger"
+    assert debug_results[0].get("sentence_template") == "hello[ world]"
+
+    # Trigger should not have been executed
+    assert len(calls) == 0