From f31873a846d5ab78596f32f961bb70549b25498c Mon Sep 17 00:00:00 2001 From: Denis Shulyaka Date: Thu, 16 May 2024 02:16:47 +0300 Subject: [PATCH] Add LLM tools (#115464) * Add llm helper * break out Tool.specification as class members * Format state output * Fix intent tests * Removed auto initialization of intents - let conversation platforms do that * Handle DynamicServiceIntentHandler.extra_slots * Add optional description to IntentTool init * Add device_id and conversation_id parameters * intent tests * Add LLM tools tests * coverage * add agent_id parameter * Apply suggestions from code review * Apply suggestions from code review * Apply suggestions from code review * Apply suggestions from code review * Apply suggestions from code review * Fix tests * Fix intent schema * Allow a Python function to be registered as am LLM tool * Add IntentHandler.effective_slot_schema * Ensure IntentHandler.slot_schema to be vol.Schema * Raise meaningful error on tool not found * Move this change to a separate PR * Update todo integration intent * Remove Tool constructor * Move IntentTool to intent helper * Convert custom serializer into class method * Remove tool_input from FunctionTool auto arguments to avoid recursion * Remove conversion into Open API format * Apply suggestions from code review * Fix tests * Use HassKey for helpers (see #117012) * Add support for functions with typed lists, dicts, and sets as type hints * Remove FunctionTool * Added API to get registered intents * Move IntentTool to the llm library * Return only handlers in intents.async.get * Removed llm tool registration from intent library * Removed tool registration * Add bind_hass back for now * removed area and floor resolving * fix test * Apply suggestions from code review * Improve coverage * Fix intent_type type * Temporary disable HassClimateGetTemperature intent * Remove bind_hass * Fix usage of slot schema * Fix test * Revert some test changes * Don't mutate tool_input --------- Co-authored-by: Paulus Schoutsen Co-authored-by: Paulus Schoutsen --- homeassistant/helpers/intent.py | 6 ++ homeassistant/helpers/llm.py | 122 ++++++++++++++++++++++++++++++++ tests/helpers/test_intent.py | 8 +-- tests/helpers/test_llm.py | 94 ++++++++++++++++++++++++ 4 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 homeassistant/helpers/llm.py create mode 100644 tests/helpers/test_llm.py diff --git a/homeassistant/helpers/intent.py b/homeassistant/helpers/intent.py index 01763fade9d..8b8ea805153 100644 --- a/homeassistant/helpers/intent.py +++ b/homeassistant/helpers/intent.py @@ -87,6 +87,12 @@ def async_remove(hass: HomeAssistant, intent_type: str) -> None: intents.pop(intent_type, None) +@callback +def async_get(hass: HomeAssistant) -> Iterable[IntentHandler]: + """Return registered intents.""" + return hass.data.get(DATA_KEY, {}).values() + + @bind_hass async def async_handle( hass: HomeAssistant, diff --git a/homeassistant/helpers/llm.py b/homeassistant/helpers/llm.py new file mode 100644 index 00000000000..1d91c9e545d --- /dev/null +++ b/homeassistant/helpers/llm.py @@ -0,0 +1,122 @@ +"""Module to coordinate llm tools.""" + +from __future__ import annotations + +from abc import abstractmethod +from collections.abc import Iterable +from dataclasses import dataclass +import logging +from typing import Any + +import voluptuous as vol + +from homeassistant.components.climate.intent import INTENT_GET_TEMPERATURE +from homeassistant.components.weather.intent import INTENT_GET_WEATHER +from homeassistant.core import Context, HomeAssistant, callback +from homeassistant.exceptions import HomeAssistantError +from homeassistant.util.json import JsonObjectType + +from . import intent + +_LOGGER = logging.getLogger(__name__) + +IGNORE_INTENTS = [ + intent.INTENT_NEVERMIND, + intent.INTENT_GET_STATE, + INTENT_GET_WEATHER, + INTENT_GET_TEMPERATURE, +] + + +@dataclass(slots=True) +class ToolInput: + """Tool input to be processed.""" + + tool_name: str + tool_args: dict[str, Any] + platform: str + context: Context | None + user_prompt: str | None + language: str | None + assistant: str | None + + +class Tool: + """LLM Tool base class.""" + + name: str + description: str | None = None + parameters: vol.Schema = vol.Schema({}) + + @abstractmethod + async def async_call( + self, hass: HomeAssistant, tool_input: ToolInput + ) -> JsonObjectType: + """Call the tool.""" + raise NotImplementedError + + def __repr__(self) -> str: + """Represent a string of a Tool.""" + return f"<{self.__class__.__name__} - {self.name}>" + + +@callback +def async_get_tools(hass: HomeAssistant) -> Iterable[Tool]: + """Return a list of LLM tools.""" + for intent_handler in intent.async_get(hass): + if intent_handler.intent_type not in IGNORE_INTENTS: + yield IntentTool(intent_handler) + + +@callback +async def async_call_tool(hass: HomeAssistant, tool_input: ToolInput) -> JsonObjectType: + """Call a LLM tool, validate args and return the response.""" + for tool in async_get_tools(hass): + if tool.name == tool_input.tool_name: + break + else: + raise HomeAssistantError(f'Tool "{tool_input.tool_name}" not found') + + _tool_input = ToolInput( + tool_name=tool.name, + tool_args=tool.parameters(tool_input.tool_args), + platform=tool_input.platform, + context=tool_input.context or Context(), + user_prompt=tool_input.user_prompt, + language=tool_input.language, + assistant=tool_input.assistant, + ) + + return await tool.async_call(hass, _tool_input) + + +class IntentTool(Tool): + """LLM Tool representing an Intent.""" + + def __init__( + self, + intent_handler: intent.IntentHandler, + ) -> None: + """Init the class.""" + self.name = intent_handler.intent_type + self.description = f"Execute Home Assistant {self.name} intent" + if slot_schema := intent_handler.slot_schema: + self.parameters = vol.Schema(slot_schema) + + async def async_call( + self, hass: HomeAssistant, tool_input: ToolInput + ) -> JsonObjectType: + """Handle the intent.""" + slots = {key: {"value": val} for key, val in tool_input.tool_args.items()} + + intent_response = await intent.async_handle( + hass, + tool_input.platform, + self.name, + slots, + tool_input.user_prompt, + tool_input.context, + tool_input.language, + tool_input.assistant, + ) + return intent_response.as_dict() diff --git a/tests/helpers/test_intent.py b/tests/helpers/test_intent.py index 1ac189d8242..f9efd52d727 100644 --- a/tests/helpers/test_intent.py +++ b/tests/helpers/test_intent.py @@ -610,7 +610,7 @@ def test_async_register(hass: HomeAssistant) -> None: intent.async_register(hass, handler) - assert hass.data[intent.DATA_KEY]["test_intent"] == handler + assert list(intent.async_get(hass)) == [handler] def test_async_register_overwrite(hass: HomeAssistant) -> None: @@ -629,7 +629,7 @@ def test_async_register_overwrite(hass: HomeAssistant) -> None: "Intent %s is being overwritten by %s", "test_intent", handler2 ) - assert hass.data[intent.DATA_KEY]["test_intent"] == handler2 + assert list(intent.async_get(hass)) == [handler2] def test_async_remove(hass: HomeAssistant) -> None: @@ -640,7 +640,7 @@ def test_async_remove(hass: HomeAssistant) -> None: intent.async_register(hass, handler) intent.async_remove(hass, "test_intent") - assert "test_intent" not in hass.data[intent.DATA_KEY] + assert not list(intent.async_get(hass)) def test_async_remove_no_existing_entry(hass: HomeAssistant) -> None: @@ -651,7 +651,7 @@ def test_async_remove_no_existing_entry(hass: HomeAssistant) -> None: intent.async_remove(hass, "test_intent2") - assert "test_intent2" not in hass.data[intent.DATA_KEY] + assert list(intent.async_get(hass)) == [handler] def test_async_remove_no_existing(hass: HomeAssistant) -> None: diff --git a/tests/helpers/test_llm.py b/tests/helpers/test_llm.py new file mode 100644 index 00000000000..3cb2078967d --- /dev/null +++ b/tests/helpers/test_llm.py @@ -0,0 +1,94 @@ +"""Tests for the llm helpers.""" + +from unittest.mock import patch + +import pytest +import voluptuous as vol + +from homeassistant.core import Context, HomeAssistant, State +from homeassistant.exceptions import HomeAssistantError +from homeassistant.helpers import config_validation as cv, intent, llm + + +async def test_call_tool_no_existing(hass: HomeAssistant) -> None: + """Test calling an llm tool where no config exists.""" + with pytest.raises(HomeAssistantError): + await llm.async_call_tool( + hass, + llm.ToolInput( + "test_tool", + {}, + "test_platform", + None, + None, + None, + None, + ), + ) + + +async def test_intent_tool(hass: HomeAssistant) -> None: + """Test IntentTool class.""" + schema = { + vol.Optional("area"): cv.string, + vol.Optional("floor"): cv.string, + } + + class MyIntentHandler(intent.IntentHandler): + intent_type = "test_intent" + slot_schema = schema + + intent_handler = MyIntentHandler() + + intent.async_register(hass, intent_handler) + + assert len(list(llm.async_get_tools(hass))) == 1 + tool = list(llm.async_get_tools(hass))[0] + assert tool.name == "test_intent" + assert tool.description == "Execute Home Assistant test_intent intent" + assert tool.parameters == vol.Schema(intent_handler.slot_schema) + assert str(tool) == "" + + test_context = Context() + intent_response = intent.IntentResponse("*") + intent_response.matched_states = [State("light.matched", "on")] + intent_response.unmatched_states = [State("light.unmatched", "on")] + tool_input = llm.ToolInput( + tool_name="test_intent", + tool_args={"area": "kitchen", "floor": "ground_floor"}, + platform="test_platform", + context=test_context, + user_prompt="test_text", + language="*", + assistant="test_assistant", + ) + + with patch( + "homeassistant.helpers.intent.async_handle", return_value=intent_response + ) as mock_intent_handle: + response = await llm.async_call_tool(hass, tool_input) + + mock_intent_handle.assert_awaited_once_with( + hass, + "test_platform", + "test_intent", + { + "area": {"value": "kitchen"}, + "floor": {"value": "ground_floor"}, + }, + "test_text", + test_context, + "*", + "test_assistant", + ) + assert response == { + "card": {}, + "data": { + "failed": [], + "success": [], + "targets": [], + }, + "language": "*", + "response_type": "action_done", + "speech": {}, + }