From 7d641e4d3e5c4166b0ef897f6b12720b7e1dd018 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Wed, 25 Jan 2023 11:30:13 -0500 Subject: [PATCH] Add OpenAI integration (#86621) * Add OpenAI integration * Remove empty manifest fields * More prompt tweaks * Update manifest * Update homeassistant/components/openai_conversation/config_flow.py Co-authored-by: Franck Nijhof * Address comments * Add full integration tests * Cripple the integration * Test single instance Co-authored-by: Franck Nijhof --- CODEOWNERS | 2 + .../openai_conversation/__init__.py | 141 ++++++++++++++++++ .../openai_conversation/config_flow.py | 70 +++++++++ .../components/openai_conversation/const.py | 23 +++ .../openai_conversation/manifest.json | 11 ++ .../openai_conversation/strings.json | 19 +++ .../openai_conversation/translations/en.json | 19 +++ homeassistant/generated/config_flows.py | 1 + homeassistant/generated/integrations.json | 6 + requirements_all.txt | 3 + requirements_test_all.txt | 3 + .../openai_conversation/__init__.py | 1 + .../openai_conversation/conftest.py | 31 ++++ .../openai_conversation/test_config_flow.py | 79 ++++++++++ .../openai_conversation/test_init.py | 63 ++++++++ 15 files changed, 472 insertions(+) create mode 100644 homeassistant/components/openai_conversation/__init__.py create mode 100644 homeassistant/components/openai_conversation/config_flow.py create mode 100644 homeassistant/components/openai_conversation/const.py create mode 100644 homeassistant/components/openai_conversation/manifest.json create mode 100644 homeassistant/components/openai_conversation/strings.json create mode 100644 homeassistant/components/openai_conversation/translations/en.json create mode 100644 tests/components/openai_conversation/__init__.py create mode 100644 tests/components/openai_conversation/conftest.py create mode 100644 tests/components/openai_conversation/test_config_flow.py create mode 100644 tests/components/openai_conversation/test_init.py diff --git a/CODEOWNERS b/CODEOWNERS index a4a5171e3bc..07637da634a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -838,6 +838,8 @@ build.json @home-assistant/supervisor /tests/components/onvif/ @hunterjm /homeassistant/components/open_meteo/ @frenck /tests/components/open_meteo/ @frenck +/homeassistant/components/openai_conversation/ @balloob +/tests/components/openai_conversation/ @balloob /homeassistant/components/openerz/ @misialq /tests/components/openerz/ @misialq /homeassistant/components/openexchangerates/ @MartinHjelmare diff --git a/homeassistant/components/openai_conversation/__init__.py b/homeassistant/components/openai_conversation/__init__.py new file mode 100644 index 00000000000..78cdc927c10 --- /dev/null +++ b/homeassistant/components/openai_conversation/__init__.py @@ -0,0 +1,141 @@ +"""The OpenAI Conversation integration.""" +from __future__ import annotations + +from functools import partial +import logging +from typing import cast + +import openai +from openai import error + +from homeassistant.components import conversation +from homeassistant.config_entries import ConfigEntry +from homeassistant.const import CONF_API_KEY +from homeassistant.core import HomeAssistant +from homeassistant.exceptions import ConfigEntryNotReady, TemplateError +from homeassistant.helpers import area_registry, device_registry, intent, template +from homeassistant.util import ulid + +from .const import DEFAULT_MODEL, DEFAULT_PROMPT + +_LOGGER = logging.getLogger(__name__) + + +async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: + """Set up OpenAI Conversation from a config entry.""" + openai.api_key = entry.data[CONF_API_KEY] + + try: + await hass.async_add_executor_job( + partial(openai.Engine.list, request_timeout=10) + ) + except error.AuthenticationError as err: + _LOGGER.error("Invalid API key: %s", err) + return False + except error.OpenAIError as err: + raise ConfigEntryNotReady(err) from err + + conversation.async_set_agent(hass, entry, OpenAIAgent(hass, entry)) + return True + + +async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: + """Unload OpenAI.""" + openai.api_key = None + conversation.async_unset_agent(hass, entry) + return True + + +class OpenAIAgent(conversation.AbstractConversationAgent): + """OpenAI conversation agent.""" + + def __init__(self, hass: HomeAssistant, entry: ConfigEntry) -> None: + """Initialize the agent.""" + self.hass = hass + self.entry = entry + self.history: dict[str, str] = {} + + @property + def attribution(self): + """Return the attribution.""" + return {"name": "Powered by OpenAI", "url": "https://www.openai.com"} + + async def async_process( + self, user_input: conversation.ConversationInput + ) -> conversation.ConversationResult: + """Process a sentence.""" + model = DEFAULT_MODEL + + if user_input.conversation_id in self.history: + conversation_id = user_input.conversation_id + prompt = self.history[conversation_id] + else: + conversation_id = ulid.ulid() + try: + prompt = self._async_generate_prompt() + except TemplateError as err: + intent_response = intent.IntentResponse(language=user_input.language) + intent_response.async_set_error( + intent.IntentResponseErrorCode.UNKNOWN, + f"Sorry, I had a problem with my template: {err}", + ) + return conversation.ConversationResult( + response=intent_response, conversation_id=conversation_id + ) + + user_name = "User" + if ( + user_input.context.user_id + and ( + user := await self.hass.auth.async_get_user(user_input.context.user_id) + ) + and user.name + ): + user_name = user.name + + prompt += f"\n{user_name}: {user_input.text}\nSmart home: " + + _LOGGER.debug("Prompt for %s: %s", model, prompt) + + result = await self.hass.async_add_executor_job( + partial( + openai.Completion.create, + engine=model, + prompt=prompt, + max_tokens=150, + user=conversation_id, + ) + ) + _LOGGER.debug("Response %s", result) + response = result["choices"][0]["text"].strip() + self.history[conversation_id] = prompt + response + + stripped_response = response + if response.startswith("Smart home:"): + stripped_response = response[11:].strip() + + intent_response = intent.IntentResponse(language=user_input.language) + intent_response.async_set_speech(stripped_response) + return conversation.ConversationResult( + response=intent_response, conversation_id=conversation_id + ) + + def _async_generate_prompt(self) -> str: + """Generate a prompt for the user.""" + dev_reg = device_registry.async_get(self.hass) + return template.Template(DEFAULT_PROMPT, self.hass).async_render( + { + "ha_name": self.hass.config.location_name, + "areas": [ + area + for area in area_registry.async_get(self.hass).areas.values() + # Filter out areas without devices + if any( + not dev.disabled_by + for dev in device_registry.async_entries_for_area( + dev_reg, cast(str, area.id) + ) + ) + ], + } + ) diff --git a/homeassistant/components/openai_conversation/config_flow.py b/homeassistant/components/openai_conversation/config_flow.py new file mode 100644 index 00000000000..88253d63a44 --- /dev/null +++ b/homeassistant/components/openai_conversation/config_flow.py @@ -0,0 +1,70 @@ +"""Config flow for OpenAI Conversation integration.""" +from __future__ import annotations + +from functools import partial +import logging +from typing import Any + +import openai +from openai import error +import voluptuous as vol + +from homeassistant import config_entries +from homeassistant.const import CONF_API_KEY +from homeassistant.core import HomeAssistant +from homeassistant.data_entry_flow import FlowResult + +from .const import DOMAIN + +_LOGGER = logging.getLogger(__name__) + +STEP_USER_DATA_SCHEMA = vol.Schema( + { + vol.Required(CONF_API_KEY): str, + } +) + + +async def validate_input(hass: HomeAssistant, data: dict[str, Any]) -> None: + """Validate the user input allows us to connect. + + Data has the keys from STEP_USER_DATA_SCHEMA with values provided by the user. + """ + openai.api_key = data[CONF_API_KEY] + await hass.async_add_executor_job(partial(openai.Engine.list, request_timeout=10)) + + +class ConfigFlow(config_entries.ConfigFlow, domain=DOMAIN): + """Handle a config flow for OpenAI Conversation.""" + + VERSION = 1 + + async def async_step_user( + self, user_input: dict[str, Any] | None = None + ) -> FlowResult: + """Handle the initial step.""" + if self._async_current_entries(): + return self.async_abort(reason="single_instance_allowed") + + if user_input is None: + return self.async_show_form( + step_id="user", data_schema=STEP_USER_DATA_SCHEMA + ) + + errors = {} + + try: + await validate_input(self.hass, user_input) + except error.APIConnectionError: + errors["base"] = "cannot_connect" + except error.AuthenticationError: + errors["base"] = "invalid_auth" + except Exception: # pylint: disable=broad-except + _LOGGER.exception("Unexpected exception") + errors["base"] = "unknown" + else: + return self.async_create_entry(title="OpenAI Conversation", data=user_input) + + return self.async_show_form( + step_id="user", data_schema=STEP_USER_DATA_SCHEMA, errors=errors + ) diff --git a/homeassistant/components/openai_conversation/const.py b/homeassistant/components/openai_conversation/const.py new file mode 100644 index 00000000000..035a02a5b2e --- /dev/null +++ b/homeassistant/components/openai_conversation/const.py @@ -0,0 +1,23 @@ +"""Constants for the OpenAI Conversation integration.""" + +DOMAIN = "openai_conversation" +CONF_PROMPT = "prompt" +DEFAULT_MODEL = "text-davinci-003" +DEFAULT_PROMPT = """ +You are a conversational AI for a smart home named {{ ha_name }}. +If a user wants to control a device, reject the request and suggest using the Home Assistant UI. + +An overview of the areas and the devices in this smart home: +{% for area in areas %} +{{ area.name }}: +{% for device in area_devices(area.name) -%} +{%- if not device_attr(device, "disabled_by") %} +- {{ device_attr(device, "name") }} ({{ device_attr(device, "model") }} by {{ device_attr(device, "manufacturer") }}) +{%- endif %} +{%- endfor %} +{% endfor %} + +Now finish this conversation: + +Smart home: How can I assist? +""" diff --git a/homeassistant/components/openai_conversation/manifest.json b/homeassistant/components/openai_conversation/manifest.json new file mode 100644 index 00000000000..233f7b5a5b2 --- /dev/null +++ b/homeassistant/components/openai_conversation/manifest.json @@ -0,0 +1,11 @@ +{ + "domain": "openai_conversation", + "name": "OpenAI Conversation", + "config_flow": true, + "documentation": "https://www.home-assistant.io/integrations/openai_conversation", + "requirements": ["openai==0.26.2"], + "dependencies": ["conversation"], + "codeowners": ["@balloob"], + "iot_class": "cloud_polling", + "integration_type": "service" +} diff --git a/homeassistant/components/openai_conversation/strings.json b/homeassistant/components/openai_conversation/strings.json new file mode 100644 index 00000000000..9ebf1c64a21 --- /dev/null +++ b/homeassistant/components/openai_conversation/strings.json @@ -0,0 +1,19 @@ +{ + "config": { + "step": { + "user": { + "data": { + "api_key": "[%key:common::config_flow::data::api_key%]" + } + } + }, + "error": { + "cannot_connect": "[%key:common::config_flow::error::cannot_connect%]", + "invalid_auth": "[%key:common::config_flow::error::invalid_auth%]", + "unknown": "[%key:common::config_flow::error::unknown%]" + }, + "abort": { + "single_instance_allowed": "[%key:common::config_flow::abort::single_instance_allowed%]" + } + } +} diff --git a/homeassistant/components/openai_conversation/translations/en.json b/homeassistant/components/openai_conversation/translations/en.json new file mode 100644 index 00000000000..7665a5535ab --- /dev/null +++ b/homeassistant/components/openai_conversation/translations/en.json @@ -0,0 +1,19 @@ +{ + "config": { + "abort": { + "single_instance_allowed": "Already configured. Only a single configuration possible." + }, + "error": { + "cannot_connect": "Failed to connect", + "invalid_auth": "Invalid authentication", + "unknown": "Unexpected error" + }, + "step": { + "user": { + "data": { + "api_key": "API Key" + } + } + } + } +} \ No newline at end of file diff --git a/homeassistant/generated/config_flows.py b/homeassistant/generated/config_flows.py index c0c737f4b46..1d891403f1b 100644 --- a/homeassistant/generated/config_flows.py +++ b/homeassistant/generated/config_flows.py @@ -297,6 +297,7 @@ FLOWS = { "onewire", "onvif", "open_meteo", + "openai_conversation", "openexchangerates", "opengarage", "opentherm_gw", diff --git a/homeassistant/generated/integrations.json b/homeassistant/generated/integrations.json index 4c3ed17e726..3ce71e2b37d 100644 --- a/homeassistant/generated/integrations.json +++ b/homeassistant/generated/integrations.json @@ -3828,6 +3828,12 @@ "config_flow": true, "iot_class": "cloud_polling" }, + "openai_conversation": { + "name": "OpenAI Conversation", + "integration_type": "service", + "config_flow": true, + "iot_class": "cloud_polling" + }, "openalpr_cloud": { "name": "OpenALPR Cloud", "integration_type": "hub", diff --git a/requirements_all.txt b/requirements_all.txt index 35d527fc839..ff0a1bcbf75 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1268,6 +1268,9 @@ open-garage==0.2.0 # homeassistant.components.open_meteo open-meteo==0.2.1 +# homeassistant.components.openai_conversation +openai==0.26.2 + # homeassistant.components.opencv # opencv-python-headless==4.6.0.66 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index d9b54c6dd73..b7f37da52ec 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -937,6 +937,9 @@ open-garage==0.2.0 # homeassistant.components.open_meteo open-meteo==0.2.1 +# homeassistant.components.openai_conversation +openai==0.26.2 + # homeassistant.components.openerz openerz-api==0.2.0 diff --git a/tests/components/openai_conversation/__init__.py b/tests/components/openai_conversation/__init__.py new file mode 100644 index 00000000000..dda2fe16a63 --- /dev/null +++ b/tests/components/openai_conversation/__init__.py @@ -0,0 +1 @@ +"""Tests for the OpenAI Conversation integration.""" diff --git a/tests/components/openai_conversation/conftest.py b/tests/components/openai_conversation/conftest.py new file mode 100644 index 00000000000..9f00290600e --- /dev/null +++ b/tests/components/openai_conversation/conftest.py @@ -0,0 +1,31 @@ +"""Tests helpers.""" +from unittest.mock import patch + +import pytest + +from homeassistant.setup import async_setup_component + +from tests.common import MockConfigEntry + + +@pytest.fixture +def mock_config_entry(hass): + """Mock a config entry.""" + entry = MockConfigEntry( + domain="openai_conversation", + data={ + "api_key": "bla", + }, + ) + entry.add_to_hass(hass) + return entry + + +@pytest.fixture +async def mock_init_component(hass, mock_config_entry): + """Initialize integration.""" + with patch( + "openai.Engine.list", + ): + assert await async_setup_component(hass, "openai_conversation", {}) + await hass.async_block_till_done() diff --git a/tests/components/openai_conversation/test_config_flow.py b/tests/components/openai_conversation/test_config_flow.py new file mode 100644 index 00000000000..1510b986b59 --- /dev/null +++ b/tests/components/openai_conversation/test_config_flow.py @@ -0,0 +1,79 @@ +"""Test the OpenAI Conversation config flow.""" +from unittest.mock import patch + +from openai.error import APIConnectionError, AuthenticationError, InvalidRequestError +import pytest + +from homeassistant import config_entries +from homeassistant.components.openai_conversation.const import DOMAIN +from homeassistant.core import HomeAssistant +from homeassistant.data_entry_flow import FlowResultType + + +async def test_single_instance_allowed( + hass: HomeAssistant, mock_config_entry: config_entries.ConfigEntry +) -> None: + """Test that config flow only allows a single instance.""" + result = await hass.config_entries.flow.async_init( + DOMAIN, context={"source": config_entries.SOURCE_USER} + ) + assert result["type"] == FlowResultType.ABORT + assert result["reason"] == "single_instance_allowed" + + +async def test_form(hass: HomeAssistant) -> None: + """Test we get the form.""" + result = await hass.config_entries.flow.async_init( + DOMAIN, context={"source": config_entries.SOURCE_USER} + ) + assert result["type"] == FlowResultType.FORM + assert result["errors"] is None + + with patch( + "homeassistant.components.openai_conversation.config_flow.openai.Engine.list", + ), patch( + "homeassistant.components.openai_conversation.async_setup_entry", + return_value=True, + ) as mock_setup_entry: + result2 = await hass.config_entries.flow.async_configure( + result["flow_id"], + { + "api_key": "bla", + }, + ) + await hass.async_block_till_done() + + assert result2["type"] == FlowResultType.CREATE_ENTRY + assert result2["data"] == { + "api_key": "bla", + } + assert len(mock_setup_entry.mock_calls) == 1 + + +@pytest.mark.parametrize( + "side_effect, error", + [ + (APIConnectionError(""), "cannot_connect"), + (AuthenticationError, "invalid_auth"), + (InvalidRequestError, "unknown"), + ], +) +async def test_form_invalid_auth(hass: HomeAssistant, side_effect, error) -> None: + """Test we handle invalid auth.""" + result = await hass.config_entries.flow.async_init( + DOMAIN, context={"source": config_entries.SOURCE_USER} + ) + + with patch( + "homeassistant.components.openai_conversation.config_flow.openai.Engine.list", + side_effect=side_effect, + ): + result2 = await hass.config_entries.flow.async_configure( + result["flow_id"], + { + "api_key": "bla", + }, + ) + + assert result2["type"] == FlowResultType.FORM + assert result2["errors"] == {"base": error} diff --git a/tests/components/openai_conversation/test_init.py b/tests/components/openai_conversation/test_init.py new file mode 100644 index 00000000000..6597d81bffb --- /dev/null +++ b/tests/components/openai_conversation/test_init.py @@ -0,0 +1,63 @@ +"""Tests for the OpenAI integration.""" +from unittest.mock import patch + +from homeassistant.components import conversation +from homeassistant.core import Context +from homeassistant.helpers import device_registry + + +async def test_default_prompt(hass, mock_init_component): + """Test that the default prompt works.""" + device_reg = device_registry.async_get(hass) + + device_reg.async_get_or_create( + config_entry_id="1234", + connections={("test", "1234")}, + name="Test Device", + manufacturer="Test Manufacturer", + model="Test Model", + suggested_area="Test Area", + ) + device_reg.async_get_or_create( + config_entry_id="1234", + connections={("test", "5678")}, + name="Test Device 2", + manufacturer="Test Manufacturer 2", + model="Test Model 2", + suggested_area="Test Area 2", + ) + device_reg.async_get_or_create( + config_entry_id="1234", + connections={("test", "9876")}, + name="Test Device 3", + manufacturer="Test Manufacturer 3", + model="Test Model 3", + suggested_area="Test Area 2", + ) + + with patch("openai.Completion.create") as mock_create: + await conversation.async_converse(hass, "hello", None, Context()) + + assert ( + mock_create.mock_calls[0][2]["prompt"] + == """You are a conversational AI for a smart home named test home. +If a user wants to control a device, reject the request and suggest using the Home Assistant UI. + +An overview of the areas and the devices in this smart home: + +Test Area: + +- Test Device (Test Model by Test Manufacturer) + +Test Area 2: + +- Test Device 2 (Test Model 2 by Test Manufacturer 2) +- Test Device 3 (Test Model 3 by Test Manufacturer 3) + + +Now finish this conversation: + +Smart home: How can I assist? +User: hello +Smart home: """ + )