diff --git a/homeassistant/components/google_generative_ai_conversation/config_flow.py b/homeassistant/components/google_generative_ai_conversation/config_flow.py index 00a016143f4..b413f9c9a62 100644 --- a/homeassistant/components/google_generative_ai_conversation/config_flow.py +++ b/homeassistant/components/google_generative_ai_conversation/config_flow.py @@ -44,6 +44,7 @@ from .const import ( CONF_TEMPERATURE, CONF_TOP_K, CONF_TOP_P, + CONF_USE_GOOGLE_SEARCH_TOOL, DOMAIN, RECOMMENDED_CHAT_MODEL, RECOMMENDED_HARM_BLOCK_THRESHOLD, @@ -51,6 +52,7 @@ from .const import ( RECOMMENDED_TEMPERATURE, RECOMMENDED_TOP_K, RECOMMENDED_TOP_P, + RECOMMENDED_USE_GOOGLE_SEARCH_TOOL, TIMEOUT_MILLIS, ) @@ -341,6 +343,13 @@ async def google_generative_ai_config_option_schema( }, default=RECOMMENDED_HARM_BLOCK_THRESHOLD, ): harm_block_thresholds_selector, + vol.Optional( + CONF_USE_GOOGLE_SEARCH_TOOL, + description={ + "suggested_value": options.get(CONF_USE_GOOGLE_SEARCH_TOOL), + }, + default=RECOMMENDED_USE_GOOGLE_SEARCH_TOOL, + ): bool, } ) return schema diff --git a/homeassistant/components/google_generative_ai_conversation/const.py b/homeassistant/components/google_generative_ai_conversation/const.py index 35834f6e7f9..108ffe1891d 100644 --- a/homeassistant/components/google_generative_ai_conversation/const.py +++ b/homeassistant/components/google_generative_ai_conversation/const.py @@ -22,5 +22,7 @@ CONF_HATE_BLOCK_THRESHOLD = "hate_block_threshold" CONF_SEXUAL_BLOCK_THRESHOLD = "sexual_block_threshold" CONF_DANGEROUS_BLOCK_THRESHOLD = "dangerous_block_threshold" RECOMMENDED_HARM_BLOCK_THRESHOLD = "BLOCK_MEDIUM_AND_ABOVE" +CONF_USE_GOOGLE_SEARCH_TOOL = "enable_google_search_tool" +RECOMMENDED_USE_GOOGLE_SEARCH_TOOL = False TIMEOUT_MILLIS = 10000 diff --git a/homeassistant/components/google_generative_ai_conversation/conversation.py b/homeassistant/components/google_generative_ai_conversation/conversation.py index e35346cc745..36e402a62fe 100644 --- a/homeassistant/components/google_generative_ai_conversation/conversation.py +++ b/homeassistant/components/google_generative_ai_conversation/conversation.py @@ -12,6 +12,7 @@ from google.genai.types import ( Content, FunctionDeclaration, GenerateContentConfig, + GoogleSearch, HarmCategory, Part, SafetySetting, @@ -39,6 +40,7 @@ from .const import ( CONF_TEMPERATURE, CONF_TOP_K, CONF_TOP_P, + CONF_USE_GOOGLE_SEARCH_TOOL, DOMAIN, LOGGER, RECOMMENDED_CHAT_MODEL, @@ -296,6 +298,13 @@ class GoogleGenerativeAIConversationEntity( for tool in chat_log.llm_api.tools ] + # Using search grounding allows the model to retrieve information from the web, + # however, it may interfere with how the model decides to use some tools, or entities + # for example weather entity may be disregarded if the model chooses to Google it. + if options.get(CONF_USE_GOOGLE_SEARCH_TOOL) is True: + tools = tools or [] + tools.append(Tool(google_search=GoogleSearch())) + model_name = self.entry.options.get(CONF_CHAT_MODEL, RECOMMENDED_CHAT_MODEL) # Gemini 1.0 doesn't support system_instruction while 1.5 does. # Assume future versions will support it (if not, the request fails with a diff --git a/homeassistant/components/google_generative_ai_conversation/strings.json b/homeassistant/components/google_generative_ai_conversation/strings.json index 7bf1831a34b..b814f89469a 100644 --- a/homeassistant/components/google_generative_ai_conversation/strings.json +++ b/homeassistant/components/google_generative_ai_conversation/strings.json @@ -36,7 +36,8 @@ "harassment_block_threshold": "Negative or harmful comments targeting identity and/or protected attributes", "hate_block_threshold": "Content that is rude, disrespectful, or profane", "sexual_block_threshold": "Contains references to sexual acts or other lewd content", - "dangerous_block_threshold": "Promotes, facilitates, or encourages harmful acts" + "dangerous_block_threshold": "Promotes, facilitates, or encourages harmful acts", + "enable_google_search_tool": "Enable Google Search tool" }, "data_description": { "prompt": "Instruct how the LLM should respond. This can be a template." diff --git a/tests/components/google_generative_ai_conversation/conftest.py b/tests/components/google_generative_ai_conversation/conftest.py index 2bc81b10ce4..6ec147da2ab 100644 --- a/tests/components/google_generative_ai_conversation/conftest.py +++ b/tests/components/google_generative_ai_conversation/conftest.py @@ -4,6 +4,9 @@ from unittest.mock import Mock, patch import pytest +from homeassistant.components.google_generative_ai_conversation.conversation import ( + CONF_USE_GOOGLE_SEARCH_TOOL, +) from homeassistant.config_entries import ConfigEntry from homeassistant.const import CONF_LLM_HASS_API from homeassistant.core import HomeAssistant @@ -41,6 +44,23 @@ async def mock_config_entry_with_assist( return mock_config_entry +@pytest.fixture +async def mock_config_entry_with_google_search( + hass: HomeAssistant, mock_config_entry: MockConfigEntry +) -> MockConfigEntry: + """Mock a config entry with assist.""" + with patch("google.genai.models.AsyncModels.get"): + hass.config_entries.async_update_entry( + mock_config_entry, + options={ + CONF_LLM_HASS_API: llm.LLM_API_ASSIST, + CONF_USE_GOOGLE_SEARCH_TOOL: True, + }, + ) + await hass.async_block_till_done() + return mock_config_entry + + @pytest.fixture async def mock_init_component( hass: HomeAssistant, mock_config_entry: ConfigEntry diff --git a/tests/components/google_generative_ai_conversation/snapshots/test_conversation.ambr b/tests/components/google_generative_ai_conversation/snapshots/test_conversation.ambr index c840f7da324..2a20ce37a57 100644 --- a/tests/components/google_generative_ai_conversation/snapshots/test_conversation.ambr +++ b/tests/components/google_generative_ai_conversation/snapshots/test_conversation.ambr @@ -61,3 +61,34 @@ ), ]) # --- +# name: test_use_google_search + list([ + tuple( + '', + tuple( + ), + dict({ + 'config': GenerateContentConfig(http_options=None, system_instruction="Current time is 05:00:00. Today's date is 2024-05-24.\nYou are a voice assistant for Home Assistant.\nAnswer questions about the world truthfully.\nAnswer in plain text. Keep it simple and to the point.\nOnly if the user wants to control a device, tell them to expose entities to their voice assistant in Home Assistant.", temperature=1.0, top_p=0.95, top_k=64.0, candidate_count=None, max_output_tokens=150, stop_sequences=None, response_logprobs=None, logprobs=None, presence_penalty=None, frequency_penalty=None, seed=None, response_mime_type=None, response_schema=None, routing_config=None, safety_settings=[SafetySetting(method=None, category=, threshold=), SafetySetting(method=None, category=, threshold=), SafetySetting(method=None, category=, threshold=), SafetySetting(method=None, category=, threshold=)], tools=[Tool(function_declarations=[FunctionDeclaration(response=None, description='Test function', name='test_tool', parameters=Schema(min_items=None, example=None, property_ordering=None, pattern=None, minimum=None, default=None, any_of=None, max_length=None, title=None, min_length=None, min_properties=None, max_items=None, maximum=None, nullable=None, max_properties=None, type=, description=None, enum=None, format=None, items=None, properties={'param1': Schema(min_items=None, example=None, property_ordering=None, pattern=None, minimum=None, default=None, any_of=None, max_length=None, title=None, min_length=None, min_properties=None, max_items=None, maximum=None, nullable=None, max_properties=None, type=, description='Test parameters', enum=None, format=None, items=Schema(min_items=None, example=None, property_ordering=None, pattern=None, minimum=None, default=None, any_of=None, max_length=None, title=None, min_length=None, min_properties=None, max_items=None, maximum=None, nullable=None, max_properties=None, type=, description=None, enum=None, format=None, items=None, properties=None, required=None), properties=None, required=None), 'param2': Schema(min_items=None, example=None, property_ordering=None, pattern=None, minimum=None, default=None, any_of=None, max_length=None, title=None, min_length=None, min_properties=None, max_items=None, maximum=None, nullable=None, max_properties=None, type=None, description=None, enum=None, format=None, items=None, properties=None, required=None), 'param3': Schema(min_items=None, example=None, property_ordering=None, pattern=None, minimum=None, default=None, any_of=None, max_length=None, title=None, min_length=None, min_properties=None, max_items=None, maximum=None, nullable=None, max_properties=None, type=, description=None, enum=None, format=None, items=None, properties={'json': Schema(min_items=None, example=None, property_ordering=None, pattern=None, minimum=None, default=None, any_of=None, max_length=None, title=None, min_length=None, min_properties=None, max_items=None, maximum=None, nullable=None, max_properties=None, type=, description=None, enum=None, format=None, items=None, properties=None, required=None)}, required=[])}, required=[]))], retrieval=None, google_search=None, google_search_retrieval=None, code_execution=None), Tool(function_declarations=None, retrieval=None, google_search=GoogleSearch(), google_search_retrieval=None, code_execution=None)], tool_config=None, labels=None, cached_content=None, response_modalities=None, media_resolution=None, speech_config=None, audio_timestamp=None, automatic_function_calling=AutomaticFunctionCallingConfig(disable=True, maximum_remote_calls=None, ignore_call_history=None), thinking_config=None), + 'history': list([ + ]), + 'model': 'models/gemini-2.0-flash', + }), + ), + tuple( + '().send_message', + tuple( + ), + dict({ + 'message': 'Please call the test function', + }), + ), + tuple( + '().send_message', + tuple( + ), + dict({ + 'message': Content(parts=[Part(video_metadata=None, thought=None, code_execution_result=None, executable_code=None, file_data=None, function_call=None, function_response=FunctionResponse(id=None, name='test_tool', response={'result': 'Test response'}), inline_data=None, text=None)], role=None), + }), + ), + ]) +# --- diff --git a/tests/components/google_generative_ai_conversation/test_config_flow.py b/tests/components/google_generative_ai_conversation/test_config_flow.py index 30c9d6c46e6..f7635c0b45e 100644 --- a/tests/components/google_generative_ai_conversation/test_config_flow.py +++ b/tests/components/google_generative_ai_conversation/test_config_flow.py @@ -21,12 +21,14 @@ from homeassistant.components.google_generative_ai_conversation.const import ( CONF_TEMPERATURE, CONF_TOP_K, CONF_TOP_P, + CONF_USE_GOOGLE_SEARCH_TOOL, DOMAIN, RECOMMENDED_CHAT_MODEL, RECOMMENDED_HARM_BLOCK_THRESHOLD, RECOMMENDED_MAX_TOKENS, RECOMMENDED_TOP_K, RECOMMENDED_TOP_P, + RECOMMENDED_USE_GOOGLE_SEARCH_TOOL, ) from homeassistant.const import CONF_LLM_HASS_API from homeassistant.core import HomeAssistant @@ -143,6 +145,7 @@ async def test_form(hass: HomeAssistant) -> None: CONF_HATE_BLOCK_THRESHOLD: RECOMMENDED_HARM_BLOCK_THRESHOLD, CONF_SEXUAL_BLOCK_THRESHOLD: RECOMMENDED_HARM_BLOCK_THRESHOLD, CONF_DANGEROUS_BLOCK_THRESHOLD: RECOMMENDED_HARM_BLOCK_THRESHOLD, + CONF_USE_GOOGLE_SEARCH_TOOL: RECOMMENDED_USE_GOOGLE_SEARCH_TOOL, }, ), ( diff --git a/tests/components/google_generative_ai_conversation/test_conversation.py b/tests/components/google_generative_ai_conversation/test_conversation.py index 22bc079a21f..82ad4affaf3 100644 --- a/tests/components/google_generative_ai_conversation/test_conversation.py +++ b/tests/components/google_generative_ai_conversation/test_conversation.py @@ -176,6 +176,72 @@ async def test_function_call( } +@patch( + "homeassistant.components.google_generative_ai_conversation.conversation.llm.AssistAPI._async_get_tools" +) +@pytest.mark.usefixtures("mock_init_component") +@pytest.mark.usefixtures("mock_ulid_tools") +async def test_use_google_search( + mock_get_tools, + hass: HomeAssistant, + mock_config_entry_with_google_search: MockConfigEntry, + snapshot: SnapshotAssertion, +) -> None: + """Test function calling.""" + agent_id = "conversation.google_generative_ai_conversation" + context = Context() + + mock_tool = AsyncMock() + mock_tool.name = "test_tool" + mock_tool.description = "Test function" + mock_tool.parameters = vol.Schema( + { + vol.Optional("param1", description="Test parameters"): [ + vol.All(str, vol.Lower) + ], + vol.Optional("param2"): vol.Any(float, int), + vol.Optional("param3"): dict, + } + ) + + mock_get_tools.return_value = [mock_tool] + + with patch("google.genai.chats.AsyncChats.create") as mock_create: + mock_chat = AsyncMock() + mock_create.return_value.send_message = mock_chat + chat_response = Mock(prompt_feedback=None) + mock_chat.return_value = chat_response + mock_part = Mock() + mock_part.text = "" + mock_part.function_call = FunctionCall( + name="test_tool", + args={ + "param1": ["test_value", "param1\\'s value"], + "param2": 2.7, + }, + ) + + def tool_call( + hass: HomeAssistant, tool_input: llm.ToolInput, tool_context: llm.LLMContext + ) -> dict[str, Any]: + mock_part.function_call = None + mock_part.text = "Hi there!" + return {"result": "Test response"} + + mock_tool.async_call.side_effect = tool_call + chat_response.candidates = [Mock(content=Mock(parts=[mock_part]))] + await conversation.async_converse( + hass, + "Please call the test function", + None, + context, + agent_id=agent_id, + device_id="test_device", + ) + + assert [tuple(mock_call) for mock_call in mock_create.mock_calls] == snapshot + + @patch( "homeassistant.components.google_generative_ai_conversation.conversation.llm.AssistAPI._async_get_tools" )