Fix pipeline conversation language (#132896)

homeassistant/components/assist_pipeline/pipeline.py

@@ -29,6 +29,7 @@ from homeassistant.components import (
 from homeassistant.components.tts import (
     generate_media_source_id as tts_generate_media_source_id,
 )
+from homeassistant.const import MATCH_ALL
 from homeassistant.core import Context, HomeAssistant, callback
 from homeassistant.exceptions import HomeAssistantError
 from homeassistant.helpers import intent
@@ -1009,12 +1010,19 @@ class PipelineRun:
         if self.intent_agent is None:
             raise RuntimeError("Recognize intent was not prepared")
 
+        if self.pipeline.conversation_language == MATCH_ALL:
+            # LLMs support all languages ('*') so use pipeline language for
+            # intent fallback.
+            input_language = self.pipeline.language
+        else:
+            input_language = self.pipeline.conversation_language
+
         self.process_event(
             PipelineEvent(
                 PipelineEventType.INTENT_START,
                 {
                     "engine": self.intent_agent,
-                    "language": self.pipeline.conversation_language,
+                    "language": input_language,
                     "intent_input": intent_input,
                     "conversation_id": conversation_id,
                     "device_id": device_id,
@@ -1029,7 +1037,7 @@ class PipelineRun:
                 context=self.context,
                 conversation_id=conversation_id,
                 device_id=device_id,
-                language=self.pipeline.language,
+                language=input_language,
                 agent_id=self.intent_agent,
             )
             processed_locally = self.intent_agent == conversation.HOME_ASSISTANT_AGENT

tests/components/assist_pipeline/snapshots/test_init.ambr

@@ -142,7 +142,7 @@
             'data': dict({
               'code': 'no_intent_match',
             }),
-            'language': 'en',
+            'language': 'en-US',
             'response_type': 'error',
             'speech': dict({
               'plain': dict({
@@ -233,7 +233,7 @@
             'data': dict({
               'code': 'no_intent_match',
             }),
-            'language': 'en',
+            'language': 'en-US',
             'response_type': 'error',
             'speech': dict({
               'plain': dict({
@@ -387,6 +387,57 @@
     }),
   ])
 # ---
+# name: test_pipeline_language_used_instead_of_conversation_language
+  list([
+    dict({
+      'data': dict({
+        'language': 'en',
+        'pipeline': <ANY>,
+      }),
+      'type': <PipelineEventType.RUN_START: 'run-start'>,
+    }),
+    dict({
+      'data': dict({
+        'conversation_id': None,
+        'device_id': None,
+        'engine': 'conversation.home_assistant',
+        'intent_input': 'test input',
+        'language': 'en',
+        'prefer_local_intents': False,
+      }),
+      'type': <PipelineEventType.INTENT_START: 'intent-start'>,
+    }),
+    dict({
+      'data': dict({
+        'intent_output': dict({
+          'conversation_id': None,
+          'response': dict({
+            'card': dict({
+            }),
+            'data': dict({
+              'failed': list([
+              ]),
+              'success': list([
+              ]),
+              'targets': list([
+              ]),
+            }),
+            'language': 'en',
+            'response_type': 'action_done',
+            'speech': dict({
+            }),
+          }),
+        }),
+        'processed_locally': True,
+      }),
+      'type': <PipelineEventType.INTENT_END: 'intent-end'>,
+    }),
+    dict({
+      'data': None,
+      'type': <PipelineEventType.RUN_END: 'run-end'>,
+    }),
+  ])
+# ---
 # name: test_wake_word_detection_aborted
   list([
     dict({

tests/components/assist_pipeline/test_init.py

@@ -23,6 +23,7 @@ from homeassistant.components.assist_pipeline.const import (
     CONF_DEBUG_RECORDING_DIR,
     DOMAIN,
 )
+from homeassistant.const import MATCH_ALL
 from homeassistant.core import Context, HomeAssistant
 from homeassistant.helpers import intent
 from homeassistant.setup import async_setup_component
@@ -1098,3 +1099,77 @@ async def test_prefer_local_intents(
             ]
             == "Order confirmed"
         )
+
+
+async def test_pipeline_language_used_instead_of_conversation_language(
+    hass: HomeAssistant,
+    hass_ws_client: WebSocketGenerator,
+    init_components,
+    snapshot: SnapshotAssertion,
+) -> None:
+    """Test that the pipeline language is used when the conversation language is '*' (all languages)."""
+    client = await hass_ws_client(hass)
+
+    events: list[assist_pipeline.PipelineEvent] = []
+
+    await client.send_json_auto_id(
+        {
+            "type": "assist_pipeline/pipeline/create",
+            "conversation_engine": "homeassistant",
+            "conversation_language": MATCH_ALL,
+            "language": "en",
+            "name": "test_name",
+            "stt_engine": "test",
+            "stt_language": "en-US",
+            "tts_engine": "test",
+            "tts_language": "en-US",
+            "tts_voice": "Arnold Schwarzenegger",
+            "wake_word_entity": None,
+            "wake_word_id": None,
+        }
+    )
+    msg = await client.receive_json()
+    assert msg["success"]
+    pipeline_id = msg["result"]["id"]
+    pipeline = assist_pipeline.async_get_pipeline(hass, pipeline_id)
+
+    pipeline_input = assist_pipeline.pipeline.PipelineInput(
+        intent_input="test input",
+        run=assist_pipeline.pipeline.PipelineRun(
+            hass,
+            context=Context(),
+            pipeline=pipeline,
+            start_stage=assist_pipeline.PipelineStage.INTENT,
+            end_stage=assist_pipeline.PipelineStage.INTENT,
+            event_callback=events.append,
+        ),
+    )
+    await pipeline_input.validate()
+
+    with patch(
+        "homeassistant.components.assist_pipeline.pipeline.conversation.async_converse",
+        return_value=conversation.ConversationResult(
+            intent.IntentResponse(pipeline.language)
+        ),
+    ) as mock_async_converse:
+        await pipeline_input.execute()
+
+        # Check intent start event
+        assert process_events(events) == snapshot
+        intent_start: assist_pipeline.PipelineEvent | None = None
+        for event in events:
+            if event.type == assist_pipeline.PipelineEventType.INTENT_START:
+                intent_start = event
+                break
+
+        assert intent_start is not None
+
+        # Pipeline language (en) should be used instead of '*'
+        assert intent_start.data.get("language") == pipeline.language
+
+        # Check input to async_converse
+        mock_async_converse.assert_called_once()
+        assert (
+            mock_async_converse.call_args_list[0].kwargs.get("language")
+            == pipeline.language
+        )

tests/components/conversation/test_default_agent.py

@@ -30,6 +30,7 @@ from homeassistant.const import (
     ATTR_DEVICE_CLASS,
     ATTR_FRIENDLY_NAME,
     STATE_CLOSED,
+    STATE_OFF,
     STATE_ON,
     STATE_UNKNOWN,
     EntityCategory,
@@ -3049,3 +3050,49 @@ async def test_entities_names_are_not_templates(hass: HomeAssistant) -> None:
 
     assert result is not None
     assert result.response.response_type == intent.IntentResponseType.ERROR
+
+
+@pytest.mark.parametrize(
+    ("language", "light_name", "on_sentence", "off_sentence"),
+    [
+        ("en", "test light", "turn on test light", "turn off test light"),
+        ("zh-cn", "卧室灯", "打开卧室灯", "关闭卧室灯"),
+        ("zh-hk", "睡房燈", "打開睡房燈", "關閉睡房燈"),
+        ("zh-tw", "臥室檯燈", "打開臥室檯燈", "關臥室檯燈"),
+    ],
+)
+@pytest.mark.usefixtures("init_components")
+async def test_turn_on_off(
+    hass: HomeAssistant,
+    language: str,
+    light_name: str,
+    on_sentence: str,
+    off_sentence: str,
+) -> None:
+    """Test turn on/off in multiple languages."""
+    entity_id = "light.light1234"
+    hass.states.async_set(
+        entity_id, STATE_OFF, attributes={ATTR_FRIENDLY_NAME: light_name}
+    )
+
+    on_calls = async_mock_service(hass, LIGHT_DOMAIN, "turn_on")
+    await conversation.async_converse(
+        hass,
+        on_sentence,
+        None,
+        Context(),
+        language=language,
+    )
+    assert len(on_calls) == 1
+    assert on_calls[0].data.get("entity_id") == [entity_id]
+
+    off_calls = async_mock_service(hass, LIGHT_DOMAIN, "turn_off")
+    await conversation.async_converse(
+        hass,
+        off_sentence,
+        None,
+        Context(),
+        language=language,
+    )
+    assert len(off_calls) == 1
+    assert off_calls[0].data.get("entity_id") == [entity_id]