diff --git a/homeassistant/components/matter/__init__.py b/homeassistant/components/matter/__init__.py
index ddd6db3e50e..8aa79aae86b 100644
--- a/homeassistant/components/matter/__init__.py
+++ b/homeassistant/components/matter/__init__.py
@@ -9,6 +9,7 @@ from matter_server.client import MatterClient
 from matter_server.client.exceptions import (
     CannotConnect,
     InvalidServerVersion,
+    NotConnected,
     ServerVersionTooNew,
     ServerVersionTooOld,
 )
@@ -132,6 +133,14 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
         listen_task.cancel()
         raise ConfigEntryNotReady("Matter client not ready") from err
 
+    # Set default fabric
+    try:
+        await matter_client.set_default_fabric_label(
+            hass.config.location_name or "Home"
+        )
+    except (NotConnected, MatterError) as err:
+        raise ConfigEntryNotReady("Failed to set default fabric label") from err
+
     if DOMAIN not in hass.data:
         hass.data[DOMAIN] = {}
 
diff --git a/homeassistant/components/matter/manifest.json b/homeassistant/components/matter/manifest.json
index 24229fad5d9..295b0a23735 100644
--- a/homeassistant/components/matter/manifest.json
+++ b/homeassistant/components/matter/manifest.json
@@ -6,6 +6,6 @@
   "dependencies": ["websocket_api"],
   "documentation": "https://www.home-assistant.io/integrations/matter",
   "iot_class": "local_push",
-  "requirements": ["python-matter-server==6.5.2"],
+  "requirements": ["python-matter-server==6.6.0"],
   "zeroconf": ["_matter._tcp.local.", "_matterc._udp.local."]
 }
diff --git a/requirements_all.txt b/requirements_all.txt
index 5e0d59561f5..f836f0800b3 100644
--- a/requirements_all.txt
+++ b/requirements_all.txt
@@ -2352,7 +2352,7 @@ python-linkplay==0.0.12
 # python-lirc==1.2.3
 
 # homeassistant.components.matter
-python-matter-server==6.5.2
+python-matter-server==6.6.0
 
 # homeassistant.components.xiaomi_miio
 python-miio==0.5.12
diff --git a/requirements_test_all.txt b/requirements_test_all.txt
index aaee40ec5c5..b48235c7b52 100644
--- a/requirements_test_all.txt
+++ b/requirements_test_all.txt
@@ -1870,7 +1870,7 @@ python-kasa[speedups]==0.7.4
 python-linkplay==0.0.12
 
 # homeassistant.components.matter
-python-matter-server==6.5.2
+python-matter-server==6.6.0
 
 # homeassistant.components.xiaomi_miio
 python-miio==0.5.12
diff --git a/tests/components/matter/test_init.py b/tests/components/matter/test_init.py
index 5492ff29535..4e9f922bfa2 100644
--- a/tests/components/matter/test_init.py
+++ b/tests/components/matter/test_init.py
@@ -9,6 +9,7 @@ from unittest.mock import AsyncMock, MagicMock, call, patch
 from aiohasupervisor import SupervisorError
 from matter_server.client.exceptions import (
     CannotConnect,
+    NotConnected,
     ServerVersionTooNew,
     ServerVersionTooOld,
 )
@@ -64,6 +65,7 @@ async def test_entry_setup_unload(
     await hass.async_block_till_done()
 
     assert matter_client.connect.call_count == 1
+    assert matter_client.set_default_fabric_label.call_count == 1
     assert entry.state is ConfigEntryState.LOADED
     entity_state = hass.states.get("light.mock_onoff_light_light")
     assert entity_state
@@ -108,6 +110,26 @@ async def test_connect_failed(
     assert entry.state is ConfigEntryState.SETUP_RETRY
 
 
+@pytest.mark.parametrize("expected_lingering_tasks", [True])
+async def test_set_default_fabric_label_failed(
+    hass: HomeAssistant,
+    matter_client: MagicMock,
+) -> None:
+    """Test failure during client connection."""
+    entry = MockConfigEntry(domain=DOMAIN, data={"url": "ws://localhost:5580/ws"})
+    entry.add_to_hass(hass)
+
+    matter_client.set_default_fabric_label.side_effect = NotConnected()
+
+    await hass.config_entries.async_setup(entry.entry_id)
+    await hass.async_block_till_done()
+
+    assert matter_client.connect.call_count == 1
+    assert matter_client.set_default_fabric_label.call_count == 1
+
+    assert entry.state is ConfigEntryState.SETUP_RETRY
+
+
 async def test_connect_timeout(
     hass: HomeAssistant,
     matter_client: MagicMock,