diff --git a/homeassistant/components/esphome/manager.py b/homeassistant/components/esphome/manager.py
index 56c2998a3cc..e119d152d09 100644
--- a/homeassistant/components/esphome/manager.py
+++ b/homeassistant/components/esphome/manager.py
@@ -568,7 +568,7 @@ class ESPHomeManager:
 
     async def on_connect_error(self, err: Exception) -> None:
         """Start reauth flow if appropriate connect error type."""
-        if isinstance(
+        if not isinstance(
             err,
             (
                 EncryptionPlaintextAPIError,
@@ -577,7 +577,36 @@ class ESPHomeManager:
                 InvalidAuthAPIError,
             ),
         ):
-            self.entry.async_start_reauth(self.hass)
+            return
+        if isinstance(err, InvalidEncryptionKeyAPIError):
+            if (
+                (received_name := err.received_name)
+                and (received_mac := err.received_mac)
+                and (unique_id := self.entry.unique_id)
+                and ":" in unique_id
+            ):
+                formatted_received_mac = format_mac(received_mac)
+                formatted_expected_mac = format_mac(unique_id)
+                if formatted_received_mac != formatted_expected_mac:
+                    _LOGGER.error(
+                        "Unexpected device found at %s; "
+                        "expected `%s` with mac address `%s`, "
+                        "found `%s` with mac address `%s`",
+                        self.host,
+                        self.entry.data.get(CONF_DEVICE_NAME),
+                        formatted_expected_mac,
+                        received_name,
+                        formatted_received_mac,
+                    )
+                    # If the device comes back online, discovery
+                    # will update the config entry with the new IP address
+                    # and reload which will try again to connect to the device.
+                    # In the mean time we stop the reconnect logic
+                    # so we don't keep trying to connect to the wrong device.
+                    if self.reconnect_logic:
+                        await self.reconnect_logic.stop()
+                    return
+        self.entry.async_start_reauth(self.hass)
 
     @callback
     def _async_handle_logging_changed(self, _event: Event) -> None:
diff --git a/tests/components/esphome/test_manager.py b/tests/components/esphome/test_manager.py
index f4cae1e1499..02a32190437 100644
--- a/tests/components/esphome/test_manager.py
+++ b/tests/components/esphome/test_manager.py
@@ -9,6 +9,7 @@ from aioesphomeapi import (
     APIClient,
     APIConnectionError,
     DeviceInfo,
+    EncryptionPlaintextAPIError,
     EntityInfo,
     EntityState,
     HomeassistantServiceCall,
@@ -1316,6 +1317,7 @@ async def test_disconnects_at_close_event(
 @pytest.mark.parametrize(
     "error",
     [
+        EncryptionPlaintextAPIError,
         RequiresEncryptionAPIError,
         InvalidEncryptionKeyAPIError,
         InvalidAuthAPIError,
@@ -1349,6 +1351,42 @@ async def test_start_reauth(
     assert flow["context"]["source"] == "reauth"
 
 
+async def test_no_reauth_wrong_mac(
+    hass: HomeAssistant,
+    mock_client: APIClient,
+    mock_esphome_device: Callable[
+        [APIClient, list[EntityInfo], list[UserService], list[EntityState]],
+        Awaitable[MockESPHomeDevice],
+    ],
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test exceptions on connect error trigger reauth."""
+    device = await mock_esphome_device(
+        mock_client=mock_client,
+        entity_info=[],
+        user_service=[],
+        device_info={"compilation_time": "comp_time"},
+        states=[],
+    )
+    await hass.async_block_till_done()
+
+    await device.mock_connect_error(
+        InvalidEncryptionKeyAPIError(
+            "fail", received_mac="aabbccddeeff", received_name="test"
+        )
+    )
+    await hass.async_block_till_done()
+
+    # Reauth should not be triggered
+    flows = hass.config_entries.flow.async_progress(DOMAIN)
+    assert len(flows) == 0
+    assert (
+        "Unexpected device found at test.local; expected `test` "
+        "with mac address `11:22:33:44:55:aa`, found `test` "
+        "with mac address `aa:bb:cc:dd:ee:ff`" in caplog.text
+    )
+
+
 async def test_entry_missing_unique_id(
     hass: HomeAssistant,
     mock_client: APIClient,