Add user name and location to the LLM assist prompt (#118071)

Add user name and location to the llm assist prompt

homeassistant/helpers/llm.py

@@ -14,7 +14,7 @@ from homeassistant.core import Context, HomeAssistant, callback
 from homeassistant.exceptions import HomeAssistantError
 from homeassistant.util.json import JsonObjectType
 
-from . import intent
+from . import area_registry, device_registry, floor_registry, intent
 from .singleton import singleton
 
 LLM_API_ASSIST = "assist"
@@ -191,7 +191,25 @@ class AssistAPI(API):
 
     async def async_get_api_prompt(self, tool_input: ToolInput) -> str:
         """Return the prompt for the API."""
-        return "Call the intent tools to control Home Assistant. Just pass the name to the intent."
+        prompt = "Call the intent tools to control Home Assistant. Just pass the name to the intent."
+        if tool_input.device_id:
+            device_reg = device_registry.async_get(self.hass)
+            device = device_reg.async_get(tool_input.device_id)
+            if device:
+                area_reg = area_registry.async_get(self.hass)
+                if device.area_id and (area := area_reg.async_get_area(device.area_id)):
+                    floor_reg = floor_registry.async_get(self.hass)
+                    if area.floor_id and (
+                        floor := floor_reg.async_get_floor(area.floor_id)
+                    ):
+                        prompt += f" You are in {area.name} ({floor.name})."
+                    else:
+                        prompt += f" You are in {area.name}."
+        if tool_input.context and tool_input.context.user_id:
+            user = await self.hass.auth.async_get_user(tool_input.context.user_id)
+            if user:
+                prompt += f" The user name is {user.name}."
+        return prompt
 
     @callback
     def async_get_tools(self) -> list[Tool]:

tests/components/openai_conversation/snapshots/test_conversation.ambr

@@ -1,172 +1,4 @@
 # serializer version: 1
-# name: test_default_prompt[None]
-  list([
-    dict({
-      'content': '''
-        This smart home is controlled by Home Assistant.
-        
-        An overview of the areas and the devices in this smart home:
-        
-        Test Area:
-        - Test Device (Test Model)
-        
-        Test Area 2:
-        - Test Device 2
-        - Test Device 3 (Test Model 3A)
-        - Test Device 4
-        - 1 (3)
-        
-        If the user wants to control a device, tell them to edit the AI configuration and allow access to Home Assistant.
-      ''',
-      'role': 'system',
-    }),
-    dict({
-      'content': 'hello',
-      'role': 'user',
-    }),
-    ChatCompletionMessage(content='Hello, how can I help you?', role='assistant', function_call=None, tool_calls=None),
-  ])
-# ---
-# name: test_default_prompt[config_entry_options0-None]
-  list([
-    dict({
-      'content': '''
-        This smart home is controlled by Home Assistant.
-        
-        An overview of the areas and the devices in this smart home:
-        
-        Test Area:
-        - Test Device (Test Model)
-        
-        Test Area 2:
-        - Test Device 2
-        - Test Device 3 (Test Model 3A)
-        - Test Device 4
-        - 1 (3)
-        
-        Call the intent tools to control the system. Just pass the name to the intent.
-      ''',
-      'role': 'system',
-    }),
-    dict({
-      'content': 'hello',
-      'role': 'user',
-    }),
-    ChatCompletionMessage(content='Hello, how can I help you?', role='assistant', function_call=None, tool_calls=None),
-  ])
-# ---
-# name: test_default_prompt[config_entry_options0-conversation.openai]
-  list([
-    dict({
-      'content': '''
-        This smart home is controlled by Home Assistant.
-        
-        An overview of the areas and the devices in this smart home:
-        
-        Test Area:
-        - Test Device (Test Model)
-        
-        Test Area 2:
-        - Test Device 2
-        - Test Device 3 (Test Model 3A)
-        - Test Device 4
-        - 1 (3)
-        
-        Call the intent tools to control the system. Just pass the name to the intent.
-      ''',
-      'role': 'system',
-    }),
-    dict({
-      'content': 'hello',
-      'role': 'user',
-    }),
-    ChatCompletionMessage(content='Hello, how can I help you?', role='assistant', function_call=None, tool_calls=None),
-  ])
-# ---
-# name: test_default_prompt[config_entry_options1-None]
-  list([
-    dict({
-      'content': '''
-        This smart home is controlled by Home Assistant.
-        
-        An overview of the areas and the devices in this smart home:
-        
-        Test Area:
-        - Test Device (Test Model)
-        
-        Test Area 2:
-        - Test Device 2
-        - Test Device 3 (Test Model 3A)
-        - Test Device 4
-        - 1 (3)
-        
-        Call the intent tools to control the system. Just pass the name to the intent.
-      ''',
-      'role': 'system',
-    }),
-    dict({
-      'content': 'hello',
-      'role': 'user',
-    }),
-    ChatCompletionMessage(content='Hello, how can I help you?', role='assistant', function_call=None, tool_calls=None),
-  ])
-# ---
-# name: test_default_prompt[config_entry_options1-conversation.openai]
-  list([
-    dict({
-      'content': '''
-        This smart home is controlled by Home Assistant.
-        
-        An overview of the areas and the devices in this smart home:
-        
-        Test Area:
-        - Test Device (Test Model)
-        
-        Test Area 2:
-        - Test Device 2
-        - Test Device 3 (Test Model 3A)
-        - Test Device 4
-        - 1 (3)
-        
-        Call the intent tools to control the system. Just pass the name to the intent.
-      ''',
-      'role': 'system',
-    }),
-    dict({
-      'content': 'hello',
-      'role': 'user',
-    }),
-    ChatCompletionMessage(content='Hello, how can I help you?', role='assistant', function_call=None, tool_calls=None),
-  ])
-# ---
-# name: test_default_prompt[conversation.openai]
-  list([
-    dict({
-      'content': '''
-        This smart home is controlled by Home Assistant.
-        
-        An overview of the areas and the devices in this smart home:
-        
-        Test Area:
-        - Test Device (Test Model)
-        
-        Test Area 2:
-        - Test Device 2
-        - Test Device 3 (Test Model 3A)
-        - Test Device 4
-        - 1 (3)
-        
-        If the user wants to control a device, tell them to edit the AI configuration and allow access to Home Assistant.
-      ''',
-      'role': 'system',
-    }),
-    dict({
-      'content': 'hello',
-      'role': 'user',
-    }),
-    ChatCompletionMessage(content='Hello, how can I help you?', role='assistant', function_call=None, tool_calls=None),
-  ])
-# ---
 # name: test_unknown_hass_api
   dict({
     'conversation_id': None,

tests/helpers/test_llm.py

@@ -1,13 +1,22 @@
 """Tests for the llm helpers."""
 
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import pytest
 import voluptuous as vol
 
 from homeassistant.core import Context, HomeAssistant, State
 from homeassistant.exceptions import HomeAssistantError
-from homeassistant.helpers import config_validation as cv, intent, llm
+from homeassistant.helpers import (
+    area_registry as ar,
+    config_validation as cv,
+    device_registry as dr,
+    floor_registry as fr,
+    intent,
+    llm,
+)
+
+from tests.common import MockConfigEntry
 
 
 async def test_get_api_no_existing(hass: HomeAssistant) -> None:
@@ -143,3 +152,65 @@ async def test_assist_api_description(hass: HomeAssistant) -> None:
     tool = tools[0]
     assert tool.name == "test_intent"
     assert tool.description == "my intent handler"
+
+
+async def test_assist_api_prompt(
+    hass: HomeAssistant,
+    device_registry: dr.DeviceRegistry,
+    area_registry: ar.AreaRegistry,
+    floor_registry: fr.FloorRegistry,
+) -> None:
+    """Test prompt for the assist API."""
+    context = Context()
+    tool_input = llm.ToolInput(
+        tool_name=None,
+        tool_args=None,
+        platform="test_platform",
+        context=context,
+        user_prompt="test_text",
+        language="*",
+        assistant="test_assistant",
+        device_id="test_device",
+    )
+    api = llm.async_get_api(hass, "assist")
+    prompt = await api.async_get_api_prompt(tool_input)
+    assert (
+        prompt
+        == "Call the intent tools to control Home Assistant. Just pass the name to the intent."
+    )
+
+    entry = MockConfigEntry(title=None)
+    entry.add_to_hass(hass)
+    tool_input.device_id = device_registry.async_get_or_create(
+        config_entry_id=entry.entry_id,
+        connections={("test", "1234")},
+        name="Test Device",
+        manufacturer="Test Manufacturer",
+        model="Test Model",
+        suggested_area="Test Area",
+    ).id
+    prompt = await api.async_get_api_prompt(tool_input)
+    assert (
+        prompt
+        == "Call the intent tools to control Home Assistant. Just pass the name to the intent. You are in Test Area."
+    )
+
+    floor = floor_registry.async_create("second floor")
+    area = area_registry.async_get_area_by_name("Test Area")
+    area_registry.async_update(area.id, floor_id=floor.floor_id)
+    prompt = await api.async_get_api_prompt(tool_input)
+    assert (
+        prompt
+        == "Call the intent tools to control Home Assistant. Just pass the name to the intent. You are in Test Area (second floor)."
+    )
+
+    context.user_id = "12345"
+    mock_user = Mock()
+    mock_user.id = "12345"
+    mock_user.name = "Test User"
+    with patch("homeassistant.auth.AuthManager.async_get_user", return_value=mock_user):
+        prompt = await api.async_get_api_prompt(tool_input)
+    assert (
+        prompt
+        == "Call the intent tools to control Home Assistant. Just pass the name to the intent. You are in Test Area (second floor). The user name is Test User."
+    )