diff --git a/homeassistant/components/homekit/__init__.py b/homeassistant/components/homekit/__init__.py
index 7c787c7e7be..8b606036a48 100644
--- a/homeassistant/components/homekit/__init__.py
+++ b/homeassistant/components/homekit/__init__.py
@@ -117,6 +117,8 @@ STATUS_RUNNING = 1
 STATUS_STOPPED = 2
 STATUS_WAIT = 3
 
+PORT_CLEANUP_CHECK_INTERVAL_SECS = 1
+
 
 def _has_all_unique_names_and_ports(bridges):
     """Validate that each homekit bridge configured has a unique name."""
@@ -306,12 +308,16 @@ async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry):
     if homekit.status == STATUS_RUNNING:
         await homekit.async_stop()
 
+    logged_shutdown_wait = False
     for _ in range(0, SHUTDOWN_TIMEOUT):
-        if not await hass.async_add_executor_job(
-            port_is_available, entry.data[CONF_PORT]
-        ):
+        if await hass.async_add_executor_job(port_is_available, entry.data[CONF_PORT]):
+            break
+
+        if not logged_shutdown_wait:
             _LOGGER.info("Waiting for the HomeKit server to shutdown")
-            await asyncio.sleep(1)
+            logged_shutdown_wait = True
+
+        await asyncio.sleep(PORT_CLEANUP_CHECK_INTERVAL_SECS)
 
     hass.data[DOMAIN].pop(entry.entry_id)
 
diff --git a/tests/components/homekit/test_homekit.py b/tests/components/homekit/test_homekit.py
index 6d5f3faae95..5895aae351e 100644
--- a/tests/components/homekit/test_homekit.py
+++ b/tests/components/homekit/test_homekit.py
@@ -9,7 +9,7 @@ from pyhap.const import CATEGORY_CAMERA, CATEGORY_TELEVISION
 import pytest
 
 from homeassistant import config as hass_config
-from homeassistant.components import zeroconf
+from homeassistant.components import homekit as homekit_base, zeroconf
 from homeassistant.components.binary_sensor import (
     DEVICE_CLASS_BATTERY_CHARGING,
     DEVICE_CLASS_MOTION,
@@ -1167,3 +1167,36 @@ async def test_homekit_start_in_accessory_mode(
     )
     assert hk_driver_start.called
     assert homekit.status == STATUS_RUNNING
+
+
+async def test_wait_for_port_to_free(hass, hk_driver, mock_zeroconf, caplog):
+    """Test we wait for the port to free before declaring unload success."""
+    await async_setup_component(hass, "persistent_notification", {})
+    entry = MockConfigEntry(
+        domain=DOMAIN,
+        data={CONF_NAME: BRIDGE_NAME, CONF_PORT: DEFAULT_PORT},
+        options={},
+    )
+    entry.add_to_hass(hass)
+
+    with patch("pyhap.accessory_driver.AccessoryDriver.async_start"), patch(
+        f"{PATH_HOMEKIT}.HomeKit.async_stop"
+    ), patch(f"{PATH_HOMEKIT}.port_is_available", return_value=True) as port_mock:
+        assert await hass.config_entries.async_setup(entry.entry_id)
+        await hass.async_block_till_done()
+        assert await hass.config_entries.async_unload(entry.entry_id)
+        await hass.async_block_till_done()
+        assert "Waiting for the HomeKit server to shutdown" not in caplog.text
+        assert port_mock.called
+
+    with patch("pyhap.accessory_driver.AccessoryDriver.async_start"), patch(
+        f"{PATH_HOMEKIT}.HomeKit.async_stop"
+    ), patch.object(homekit_base, "PORT_CLEANUP_CHECK_INTERVAL_SECS", 0), patch(
+        f"{PATH_HOMEKIT}.port_is_available", return_value=False
+    ) as port_mock:
+        assert await hass.config_entries.async_setup(entry.entry_id)
+        await hass.async_block_till_done()
+        assert await hass.config_entries.async_unload(entry.entry_id)
+        await hass.async_block_till_done()
+        assert "Waiting for the HomeKit server to shutdown" in caplog.text
+        assert port_mock.called