From 1e69ce9111cd5264c5bbe5d50fe9a6e5e2d15c82 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Mon, 14 Apr 2025 01:05:34 -0700 Subject: [PATCH] Fix quality loss for LLM conversation agent question answering (#142873) * Fix a bug parsing a streaming response with no json * Remove debug lines * Fix quality loss for LLM conversation agent question answering * Update tests --- homeassistant/helpers/llm.py | 32 ++++++++++++++++++++++++++------ tests/helpers/test_llm.py | 28 ++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/homeassistant/helpers/llm.py b/homeassistant/helpers/llm.py index 7f6fe22ec70..df94788a981 100644 --- a/homeassistant/helpers/llm.py +++ b/homeassistant/helpers/llm.py @@ -71,6 +71,19 @@ NO_ENTITIES_PROMPT = ( "to their voice assistant in Home Assistant." ) +DYNAMIC_CONTEXT_PROMPT = """You ARE equipped to answer questions about the current state of +the home using the `GetLiveContext` tool. This is a primary function. Do not state you lack the +functionality if the question requires live data. +If the user asks about device existence/type (e.g., "Do I have lights in the bedroom?"): Answer +from the static context below. +If the user asks about the CURRENT state, value, or mode (e.g., "Is the lock locked?", +"Is the fan on?", "What mode is the thermostat in?", "What is the temperature outside?"): + 1. Recognize this requires live data. + 2. You MUST call `GetLiveContext`. This tool will provide the needed real-time information (like temperature from the local weather, lock status, etc.). + 3. Use the tool's response** to answer the user accurately (e.g., "The temperature outside is [value from tool]."). +For general knowledge questions not about the home: Answer truthfully from internal knowledge. +""" + @callback def async_render_no_api_prompt(hass: HomeAssistant) -> str: @@ -384,6 +397,8 @@ class AssistAPI(API): ): prompt.append("This device is not able to start timers.") + prompt.append(DYNAMIC_CONTEXT_PROMPT) + return prompt @callback @@ -395,7 +410,7 @@ class AssistAPI(API): if exposed_entities and exposed_entities["entities"]: prompt.append( - "An overview of the areas and the devices in this smart home:" + "Static Context: An overview of the areas and the devices in this smart home:" ) prompt.append(yaml_util.dump(list(exposed_entities["entities"].values()))) @@ -457,7 +472,7 @@ class AssistAPI(API): ) if exposed_domains: - tools.append(GetHomeStateTool()) + tools.append(GetLiveContextTool()) return tools @@ -898,7 +913,7 @@ class CalendarGetEventsTool(Tool): return {"success": True, "result": events} -class GetHomeStateTool(Tool): +class GetLiveContextTool(Tool): """Tool for getting the current state of exposed entities. This returns state for all entities that have been exposed to @@ -906,8 +921,13 @@ class GetHomeStateTool(Tool): returns state for entities based on intent parameters. """ - name = "get_home_state" - description = "Get the current state of all devices in the home. " + name = "GetLiveContext" + description = ( + "Use this tool when the user asks a question about the CURRENT state, " + "value, or mode of a specific device, sensor, entity, or area in the " + "smart home, and the answer can be improved with real-time data not " + "available in the static device overview list. " + ) async def async_call( self, @@ -925,7 +945,7 @@ class GetHomeStateTool(Tool): if not exposed_entities["entities"]: return {"success": False, "error": NO_ENTITIES_PROMPT} prompt = [ - "An overview of the areas and the devices in this smart home:", + "Live Context: An overview of the areas and the devices in this smart home:", yaml_util.dump(list(exposed_entities["entities"].values())), ] return { diff --git a/tests/helpers/test_llm.py b/tests/helpers/test_llm.py index 19ada407550..42dd4da6197 100644 --- a/tests/helpers/test_llm.py +++ b/tests/helpers/test_llm.py @@ -181,13 +181,13 @@ async def test_assist_api( assert len(llm.async_get_apis(hass)) == 1 api = await llm.async_get_api(hass, "assist", llm_context) - assert [tool.name for tool in api.tools] == ["get_home_state"] + assert [tool.name for tool in api.tools] == ["GetLiveContext"] # Match all intent_handler.platforms = None api = await llm.async_get_api(hass, "assist", llm_context) - assert [tool.name for tool in api.tools] == ["test_intent", "get_home_state"] + assert [tool.name for tool in api.tools] == ["test_intent", "GetLiveContext"] # Match specific domain intent_handler.platforms = {"light"} @@ -575,7 +575,7 @@ async def test_assist_api_prompt( suggested_area="Test Area 2", ) ) - exposed_entities_prompt = """An overview of the areas and the devices in this smart home: + exposed_entities_prompt = """Live Context: An overview of the areas and the devices in this smart home: - names: Kitchen domain: light state: 'on' @@ -623,7 +623,7 @@ async def test_assist_api_prompt( state: unavailable areas: Test Area 2 """ - stateless_exposed_entities_prompt = """An overview of the areas and the devices in this smart home: + stateless_exposed_entities_prompt = """Static Context: An overview of the areas and the devices in this smart home: - names: Kitchen domain: light - names: Living Room @@ -669,17 +669,30 @@ async def test_assist_api_prompt( "When a user asks to turn on all devices of a specific type, " "ask user to specify an area, unless there is only one device of that type." ) + dynamic_context_prompt = """You ARE equipped to answer questions about the current state of +the home using the `GetLiveContext` tool. This is a primary function. Do not state you lack the +functionality if the question requires live data. +If the user asks about device existence/type (e.g., "Do I have lights in the bedroom?"): Answer +from the static context below. +If the user asks about the CURRENT state, value, or mode (e.g., "Is the lock locked?", +"Is the fan on?", "What mode is the thermostat in?", "What is the temperature outside?"): + 1. Recognize this requires live data. + 2. You MUST call `GetLiveContext`. This tool will provide the needed real-time information (like temperature from the local weather, lock status, etc.). + 3. Use the tool's response** to answer the user accurately (e.g., "The temperature outside is [value from tool]."). +For general knowledge questions not about the home: Answer truthfully from internal knowledge. +""" api = await llm.async_get_api(hass, "assist", llm_context) assert api.api_prompt == ( f"""{first_part_prompt} {area_prompt} {no_timer_prompt} +{dynamic_context_prompt} {stateless_exposed_entities_prompt}""" ) - # Verify that the get_home_state tool returns the same results as the exposed_entities_prompt + # Verify that the GetLiveContext tool returns the same results as the exposed_entities_prompt result = await api.async_call_tool( - llm.ToolInput(tool_name="get_home_state", tool_args={}) + llm.ToolInput(tool_name="GetLiveContext", tool_args={}) ) assert result == { "success": True, @@ -697,6 +710,7 @@ async def test_assist_api_prompt( f"""{first_part_prompt} {area_prompt} {no_timer_prompt} +{dynamic_context_prompt} {stateless_exposed_entities_prompt}""" ) @@ -712,6 +726,7 @@ async def test_assist_api_prompt( f"""{first_part_prompt} {area_prompt} {no_timer_prompt} +{dynamic_context_prompt} {stateless_exposed_entities_prompt}""" ) @@ -723,6 +738,7 @@ async def test_assist_api_prompt( assert api.api_prompt == ( f"""{first_part_prompt} {area_prompt} +{dynamic_context_prompt} {stateless_exposed_entities_prompt}""" )