Add OpenAI integration (#86621)

* Add OpenAI integration

* Remove empty manifest fields

* More prompt tweaks

* Update manifest

* Update homeassistant/components/openai_conversation/config_flow.py

Co-authored-by: Franck Nijhof <git@frenck.dev>

* Address comments

* Add full integration tests

* Cripple the integration

* Test single instance

Co-authored-by: Franck Nijhof <git@frenck.dev>

CODEOWNERS

@@ -838,6 +838,8 @@ build.json @home-assistant/supervisor
 /tests/components/onvif/ @hunterjm
 /homeassistant/components/open_meteo/ @frenck
 /tests/components/open_meteo/ @frenck
+/homeassistant/components/openai_conversation/ @balloob
+/tests/components/openai_conversation/ @balloob
 /homeassistant/components/openerz/ @misialq
 /tests/components/openerz/ @misialq
 /homeassistant/components/openexchangerates/ @MartinHjelmare

homeassistant/components/openai_conversation/__init__.py

@@ -0,0 +1,141 @@
+"""The OpenAI Conversation integration."""
+from __future__ import annotations
+
+from functools import partial
+import logging
+from typing import cast
+
+import openai
+from openai import error
+
+from homeassistant.components import conversation
+from homeassistant.config_entries import ConfigEntry
+from homeassistant.const import CONF_API_KEY
+from homeassistant.core import HomeAssistant
+from homeassistant.exceptions import ConfigEntryNotReady, TemplateError
+from homeassistant.helpers import area_registry, device_registry, intent, template
+from homeassistant.util import ulid
+
+from .const import DEFAULT_MODEL, DEFAULT_PROMPT
+
+_LOGGER = logging.getLogger(__name__)
+
+
+async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
+    """Set up OpenAI Conversation from a config entry."""
+    openai.api_key = entry.data[CONF_API_KEY]
+
+    try:
+        await hass.async_add_executor_job(
+            partial(openai.Engine.list, request_timeout=10)
+        )
+    except error.AuthenticationError as err:
+        _LOGGER.error("Invalid API key: %s", err)
+        return False
+    except error.OpenAIError as err:
+        raise ConfigEntryNotReady(err) from err
+
+    conversation.async_set_agent(hass, entry, OpenAIAgent(hass, entry))
+    return True
+
+
+async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
+    """Unload OpenAI."""
+    openai.api_key = None
+    conversation.async_unset_agent(hass, entry)
+    return True
+
+
+class OpenAIAgent(conversation.AbstractConversationAgent):
+    """OpenAI conversation agent."""
+
+    def __init__(self, hass: HomeAssistant, entry: ConfigEntry) -> None:
+        """Initialize the agent."""
+        self.hass = hass
+        self.entry = entry
+        self.history: dict[str, str] = {}
+
+    @property
+    def attribution(self):
+        """Return the attribution."""
+        return {"name": "Powered by OpenAI", "url": "https://www.openai.com"}
+
+    async def async_process(
+        self, user_input: conversation.ConversationInput
+    ) -> conversation.ConversationResult:
+        """Process a sentence."""
+        model = DEFAULT_MODEL
+
+        if user_input.conversation_id in self.history:
+            conversation_id = user_input.conversation_id
+            prompt = self.history[conversation_id]
+        else:
+            conversation_id = ulid.ulid()
+            try:
+                prompt = self._async_generate_prompt()
+            except TemplateError as err:
+                intent_response = intent.IntentResponse(language=user_input.language)
+                intent_response.async_set_error(
+                    intent.IntentResponseErrorCode.UNKNOWN,
+                    f"Sorry, I had a problem with my template: {err}",
+                )
+                return conversation.ConversationResult(
+                    response=intent_response, conversation_id=conversation_id
+                )
+
+        user_name = "User"
+        if (
+            user_input.context.user_id
+            and (
+                user := await self.hass.auth.async_get_user(user_input.context.user_id)
+            )
+            and user.name
+        ):
+            user_name = user.name
+
+        prompt += f"\n{user_name}: {user_input.text}\nSmart home: "
+
+        _LOGGER.debug("Prompt for %s: %s", model, prompt)
+
+        result = await self.hass.async_add_executor_job(
+            partial(
+                openai.Completion.create,
+                engine=model,
+                prompt=prompt,
+                max_tokens=150,
+                user=conversation_id,
+            )
+        )
+        _LOGGER.debug("Response %s", result)
+        response = result["choices"][0]["text"].strip()
+        self.history[conversation_id] = prompt + response
+
+        stripped_response = response
+        if response.startswith("Smart home:"):
+            stripped_response = response[11:].strip()
+
+        intent_response = intent.IntentResponse(language=user_input.language)
+        intent_response.async_set_speech(stripped_response)
+        return conversation.ConversationResult(
+            response=intent_response, conversation_id=conversation_id
+        )
+
+    def _async_generate_prompt(self) -> str:
+        """Generate a prompt for the user."""
+        dev_reg = device_registry.async_get(self.hass)
+        return template.Template(DEFAULT_PROMPT, self.hass).async_render(
+            {
+                "ha_name": self.hass.config.location_name,
+                "areas": [
+                    area
+                    for area in area_registry.async_get(self.hass).areas.values()
+                    # Filter out areas without devices
+                    if any(
+                        not dev.disabled_by
+                        for dev in device_registry.async_entries_for_area(
+                            dev_reg, cast(str, area.id)
+                        )
+                    )
+                ],
+            }
+        )

homeassistant/components/openai_conversation/config_flow.py

@@ -0,0 +1,70 @@
+"""Config flow for OpenAI Conversation integration."""
+from __future__ import annotations
+
+from functools import partial
+import logging
+from typing import Any
+
+import openai
+from openai import error
+import voluptuous as vol
+
+from homeassistant import config_entries
+from homeassistant.const import CONF_API_KEY
+from homeassistant.core import HomeAssistant
+from homeassistant.data_entry_flow import FlowResult
+
+from .const import DOMAIN
+
+_LOGGER = logging.getLogger(__name__)
+
+STEP_USER_DATA_SCHEMA = vol.Schema(
+    {
+        vol.Required(CONF_API_KEY): str,
+    }
+)
+
+
+async def validate_input(hass: HomeAssistant, data: dict[str, Any]) -> None:
+    """Validate the user input allows us to connect.
+
+    Data has the keys from STEP_USER_DATA_SCHEMA with values provided by the user.
+    """
+    openai.api_key = data[CONF_API_KEY]
+    await hass.async_add_executor_job(partial(openai.Engine.list, request_timeout=10))
+
+
+class ConfigFlow(config_entries.ConfigFlow, domain=DOMAIN):
+    """Handle a config flow for OpenAI Conversation."""
+
+    VERSION = 1
+
+    async def async_step_user(
+        self, user_input: dict[str, Any] | None = None
+    ) -> FlowResult:
+        """Handle the initial step."""
+        if self._async_current_entries():
+            return self.async_abort(reason="single_instance_allowed")
+
+        if user_input is None:
+            return self.async_show_form(
+                step_id="user", data_schema=STEP_USER_DATA_SCHEMA
+            )
+
+        errors = {}
+
+        try:
+            await validate_input(self.hass, user_input)
+        except error.APIConnectionError:
+            errors["base"] = "cannot_connect"
+        except error.AuthenticationError:
+            errors["base"] = "invalid_auth"
+        except Exception:  # pylint: disable=broad-except
+            _LOGGER.exception("Unexpected exception")
+            errors["base"] = "unknown"
+        else:
+            return self.async_create_entry(title="OpenAI Conversation", data=user_input)
+
+        return self.async_show_form(
+            step_id="user", data_schema=STEP_USER_DATA_SCHEMA, errors=errors
+        )

homeassistant/components/openai_conversation/const.py

@@ -0,0 +1,23 @@
+"""Constants for the OpenAI Conversation integration."""
+
+DOMAIN = "openai_conversation"
+CONF_PROMPT = "prompt"
+DEFAULT_MODEL = "text-davinci-003"
+DEFAULT_PROMPT = """
+You are a conversational AI for a smart home named {{ ha_name }}.
+If a user wants to control a device, reject the request and suggest using the Home Assistant UI.
+
+An overview of the areas and the devices in this smart home:
+{% for area in areas %}
+{{ area.name }}:
+{% for device in area_devices(area.name) -%}
+{%- if not device_attr(device, "disabled_by") %}
+- {{ device_attr(device, "name") }} ({{ device_attr(device, "model") }} by {{ device_attr(device, "manufacturer") }})
+{%- endif %}
+{%- endfor %}
+{% endfor %}
+
+Now finish this conversation:
+
+Smart home: How can I assist?
+"""

homeassistant/components/openai_conversation/manifest.json

@@ -0,0 +1,11 @@
+{
+  "domain": "openai_conversation",
+  "name": "OpenAI Conversation",
+  "config_flow": true,
+  "documentation": "https://www.home-assistant.io/integrations/openai_conversation",
+  "requirements": ["openai==0.26.2"],
+  "dependencies": ["conversation"],
+  "codeowners": ["@balloob"],
+  "iot_class": "cloud_polling",
+  "integration_type": "service"
+}

homeassistant/components/openai_conversation/strings.json

@@ -0,0 +1,19 @@
+{
+  "config": {
+    "step": {
+      "user": {
+        "data": {
+          "api_key": "[%key:common::config_flow::data::api_key%]"
+        }
+      }
+    },
+    "error": {
+      "cannot_connect": "[%key:common::config_flow::error::cannot_connect%]",
+      "invalid_auth": "[%key:common::config_flow::error::invalid_auth%]",
+      "unknown": "[%key:common::config_flow::error::unknown%]"
+    },
+    "abort": {
+      "single_instance_allowed": "[%key:common::config_flow::abort::single_instance_allowed%]"
+    }
+  }
+}

homeassistant/components/openai_conversation/translations/en.json

@@ -0,0 +1,19 @@
+{
+    "config": {
+        "abort": {
+            "single_instance_allowed": "Already configured. Only a single configuration possible."
+        },
+        "error": {
+            "cannot_connect": "Failed to connect",
+            "invalid_auth": "Invalid authentication",
+            "unknown": "Unexpected error"
+        },
+        "step": {
+            "user": {
+                "data": {
+                    "api_key": "API Key"
+                }
+            }
+        }
+    }
+}

homeassistant/generated/config_flows.py

@@ -297,6 +297,7 @@ FLOWS = {
         "onewire",
         "onvif",
         "open_meteo",
+        "openai_conversation",
         "openexchangerates",
         "opengarage",
         "opentherm_gw",

homeassistant/generated/integrations.json

@@ -3828,6 +3828,12 @@
       "config_flow": true,
       "iot_class": "cloud_polling"
     },
+    "openai_conversation": {
+      "name": "OpenAI Conversation",
+      "integration_type": "service",
+      "config_flow": true,
+      "iot_class": "cloud_polling"
+    },
     "openalpr_cloud": {
       "name": "OpenALPR Cloud",
       "integration_type": "hub",

requirements_all.txt

@@ -1268,6 +1268,9 @@ open-garage==0.2.0
 # homeassistant.components.open_meteo
 open-meteo==0.2.1
 
+# homeassistant.components.openai_conversation
+openai==0.26.2
+
 # homeassistant.components.opencv
 # opencv-python-headless==4.6.0.66
 

requirements_test_all.txt

@@ -937,6 +937,9 @@ open-garage==0.2.0
 # homeassistant.components.open_meteo
 open-meteo==0.2.1
 
+# homeassistant.components.openai_conversation
+openai==0.26.2
+
 # homeassistant.components.openerz
 openerz-api==0.2.0
 

tests/components/openai_conversation/__init__.py

@@ -0,0 +1 @@
+"""Tests for the OpenAI Conversation integration."""

tests/components/openai_conversation/conftest.py

@@ -0,0 +1,31 @@
+"""Tests helpers."""
+from unittest.mock import patch
+
+import pytest
+
+from homeassistant.setup import async_setup_component
+
+from tests.common import MockConfigEntry
+
+
+@pytest.fixture
+def mock_config_entry(hass):
+    """Mock a config entry."""
+    entry = MockConfigEntry(
+        domain="openai_conversation",
+        data={
+            "api_key": "bla",
+        },
+    )
+    entry.add_to_hass(hass)
+    return entry
+
+
+@pytest.fixture
+async def mock_init_component(hass, mock_config_entry):
+    """Initialize integration."""
+    with patch(
+        "openai.Engine.list",
+    ):
+        assert await async_setup_component(hass, "openai_conversation", {})
+        await hass.async_block_till_done()

tests/components/openai_conversation/test_config_flow.py

@@ -0,0 +1,79 @@
+"""Test the OpenAI Conversation config flow."""
+from unittest.mock import patch
+
+from openai.error import APIConnectionError, AuthenticationError, InvalidRequestError
+import pytest
+
+from homeassistant import config_entries
+from homeassistant.components.openai_conversation.const import DOMAIN
+from homeassistant.core import HomeAssistant
+from homeassistant.data_entry_flow import FlowResultType
+
+
+async def test_single_instance_allowed(
+    hass: HomeAssistant, mock_config_entry: config_entries.ConfigEntry
+) -> None:
+    """Test that config flow only allows a single instance."""
+    result = await hass.config_entries.flow.async_init(
+        DOMAIN, context={"source": config_entries.SOURCE_USER}
+    )
+    assert result["type"] == FlowResultType.ABORT
+    assert result["reason"] == "single_instance_allowed"
+
+
+async def test_form(hass: HomeAssistant) -> None:
+    """Test we get the form."""
+    result = await hass.config_entries.flow.async_init(
+        DOMAIN, context={"source": config_entries.SOURCE_USER}
+    )
+    assert result["type"] == FlowResultType.FORM
+    assert result["errors"] is None
+
+    with patch(
+        "homeassistant.components.openai_conversation.config_flow.openai.Engine.list",
+    ), patch(
+        "homeassistant.components.openai_conversation.async_setup_entry",
+        return_value=True,
+    ) as mock_setup_entry:
+        result2 = await hass.config_entries.flow.async_configure(
+            result["flow_id"],
+            {
+                "api_key": "bla",
+            },
+        )
+        await hass.async_block_till_done()
+
+    assert result2["type"] == FlowResultType.CREATE_ENTRY
+    assert result2["data"] == {
+        "api_key": "bla",
+    }
+    assert len(mock_setup_entry.mock_calls) == 1
+
+
+@pytest.mark.parametrize(
+    "side_effect, error",
+    [
+        (APIConnectionError(""), "cannot_connect"),
+        (AuthenticationError, "invalid_auth"),
+        (InvalidRequestError, "unknown"),
+    ],
+)
+async def test_form_invalid_auth(hass: HomeAssistant, side_effect, error) -> None:
+    """Test we handle invalid auth."""
+    result = await hass.config_entries.flow.async_init(
+        DOMAIN, context={"source": config_entries.SOURCE_USER}
+    )
+
+    with patch(
+        "homeassistant.components.openai_conversation.config_flow.openai.Engine.list",
+        side_effect=side_effect,
+    ):
+        result2 = await hass.config_entries.flow.async_configure(
+            result["flow_id"],
+            {
+                "api_key": "bla",
+            },
+        )
+
+    assert result2["type"] == FlowResultType.FORM
+    assert result2["errors"] == {"base": error}

tests/components/openai_conversation/test_init.py

@@ -0,0 +1,63 @@
+"""Tests for the OpenAI integration."""
+from unittest.mock import patch
+
+from homeassistant.components import conversation
+from homeassistant.core import Context
+from homeassistant.helpers import device_registry
+
+
+async def test_default_prompt(hass, mock_init_component):
+    """Test that the default prompt works."""
+    device_reg = device_registry.async_get(hass)
+
+    device_reg.async_get_or_create(
+        config_entry_id="1234",
+        connections={("test", "1234")},
+        name="Test Device",
+        manufacturer="Test Manufacturer",
+        model="Test Model",
+        suggested_area="Test Area",
+    )
+    device_reg.async_get_or_create(
+        config_entry_id="1234",
+        connections={("test", "5678")},
+        name="Test Device 2",
+        manufacturer="Test Manufacturer 2",
+        model="Test Model 2",
+        suggested_area="Test Area 2",
+    )
+    device_reg.async_get_or_create(
+        config_entry_id="1234",
+        connections={("test", "9876")},
+        name="Test Device 3",
+        manufacturer="Test Manufacturer 3",
+        model="Test Model 3",
+        suggested_area="Test Area 2",
+    )
+
+    with patch("openai.Completion.create") as mock_create:
+        await conversation.async_converse(hass, "hello", None, Context())
+
+    assert (
+        mock_create.mock_calls[0][2]["prompt"]
+        == """You are a conversational AI for a smart home named test home.
+If a user wants to control a device, reject the request and suggest using the Home Assistant UI.
+
+An overview of the areas and the devices in this smart home:
+
+Test Area:
+
+- Test Device (Test Model by Test Manufacturer)
+
+Test Area 2:
+
+- Test Device 2 (Test Model 2 by Test Manufacturer 2)
+- Test Device 3 (Test Model 3 by Test Manufacturer 3)
+
+
+Now finish this conversation:
+
+Smart home: How can I assist?
+User: hello
+Smart home: """
+    )