diff --git a/homeassistant/components/arcam_fmj/__init__.py b/homeassistant/components/arcam_fmj/__init__.py
index aa11e66d49c..008266e5a45 100644
--- a/homeassistant/components/arcam_fmj/__init__.py
+++ b/homeassistant/components/arcam_fmj/__init__.py
@@ -27,6 +27,7 @@ from .const import (
     DOMAIN,
     DOMAIN_DATA_CONFIG,
     DOMAIN_DATA_ENTRIES,
+    DOMAIN_DATA_TASKS,
     SIGNAL_CLIENT_DATA,
     SIGNAL_CLIENT_STARTED,
     SIGNAL_CLIENT_STOPPED,
@@ -73,6 +74,15 @@ DEVICE_SCHEMA = vol.Schema(
     )
 )
 
+
+async def _await_cancel(task):
+    task.cancel()
+    try:
+        await task
+    except asyncio.CancelledError:
+        pass
+
+
 CONFIG_SCHEMA = vol.Schema(
     {DOMAIN: vol.All(cv.ensure_list, [DEVICE_SCHEMA])}, extra=vol.ALLOW_EXTRA
 )
@@ -81,6 +91,7 @@ CONFIG_SCHEMA = vol.Schema(
 async def async_setup(hass: HomeAssistantType, config: ConfigType):
     """Set up the component."""
     hass.data[DOMAIN_DATA_ENTRIES] = {}
+    hass.data[DOMAIN_DATA_TASKS] = {}
     hass.data[DOMAIN_DATA_CONFIG] = {}
 
     for device in config[DOMAIN]:
@@ -94,6 +105,13 @@ async def async_setup(hass: HomeAssistantType, config: ConfigType):
             )
         )
 
+    async def _stop(_):
+        asyncio.gather(
+            *[_await_cancel(task) for task in hass.data[DOMAIN_DATA_TASKS].values()]
+        )
+
+    hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, _stop)
+
     return True
 
 
@@ -107,13 +125,15 @@ async def async_setup_entry(hass: HomeAssistantType, entry: config_entries.Confi
             {CONF_HOST: entry.data[CONF_HOST], CONF_PORT: entry.data[CONF_PORT]}
         ),
     )
+    tasks = hass.data.setdefault(DOMAIN_DATA_TASKS, {})
 
     hass.data[DOMAIN_DATA_ENTRIES][entry.entry_id] = {
         "client": client,
         "config": config,
     }
 
-    asyncio.ensure_future(_run_client(hass, client, config[CONF_SCAN_INTERVAL]))
+    task = asyncio.create_task(_run_client(hass, client, DEFAULT_SCAN_INTERVAL))
+    tasks[entry.entry_id] = task
 
     hass.async_create_task(
         hass.config_entries.async_forward_entry_setup(entry, "media_player")
@@ -122,22 +142,23 @@ async def async_setup_entry(hass: HomeAssistantType, entry: config_entries.Confi
     return True
 
 
+async def async_unload_entry(hass, entry):
+    """Cleanup before removing config entry."""
+    await hass.config_entries.async_forward_entry_unload(entry, "media_player")
+
+    task = hass.data[DOMAIN_DATA_TASKS].pop(entry.entry_id)
+    await _await_cancel(task)
+
+    hass.data[DOMAIN_DATA_ENTRIES].pop(entry.entry_id)
+
+    return True
+
+
 async def _run_client(hass, client, interval):
-    task = asyncio.Task.current_task()
-    run = True
-
-    async def _stop(_):
-        nonlocal run
-        run = False
-        task.cancel()
-        await task
-
-    hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, _stop)
-
     def _listen(_):
         hass.helpers.dispatcher.async_dispatcher_send(SIGNAL_CLIENT_DATA, client.host)
 
-    while run:
+    while True:
         try:
             with async_timeout.timeout(interval):
                 await client.start()
@@ -163,7 +184,7 @@ async def _run_client(hass, client, interval):
         except asyncio.TimeoutError:
             continue
         except asyncio.CancelledError:
-            return
+            raise
         except Exception:  # pylint: disable=broad-except
             _LOGGER.exception("Unexpected exception, aborting arcam client")
             return
diff --git a/homeassistant/components/arcam_fmj/const.py b/homeassistant/components/arcam_fmj/const.py
index 5270eb706dc..180abf2c960 100644
--- a/homeassistant/components/arcam_fmj/const.py
+++ b/homeassistant/components/arcam_fmj/const.py
@@ -12,4 +12,5 @@ DEFAULT_NAME = "Arcam FMJ"
 DEFAULT_SCAN_INTERVAL = 5
 
 DOMAIN_DATA_ENTRIES = f"{DOMAIN}.entries"
+DOMAIN_DATA_TASKS = f"{DOMAIN}.tasks"
 DOMAIN_DATA_CONFIG = f"{DOMAIN}.config"
diff --git a/homeassistant/components/arcam_fmj/manifest.json b/homeassistant/components/arcam_fmj/manifest.json
index c304d7bf351..ff89641667a 100644
--- a/homeassistant/components/arcam_fmj/manifest.json
+++ b/homeassistant/components/arcam_fmj/manifest.json
@@ -3,6 +3,6 @@
   "name": "Arcam FMJ Receivers",
   "config_flow": false,
   "documentation": "https://www.home-assistant.io/integrations/arcam_fmj",
-  "requirements": ["arcam-fmj==0.4.4"],
+  "requirements": ["arcam-fmj==0.4.6"],
   "codeowners": ["@elupus"]
 }
diff --git a/requirements_all.txt b/requirements_all.txt
index 753d2cc29aa..47176f2b679 100644
--- a/requirements_all.txt
+++ b/requirements_all.txt
@@ -266,7 +266,7 @@ aprslib==0.6.46
 aqualogic==1.0
 
 # homeassistant.components.arcam_fmj
-arcam-fmj==0.4.4
+arcam-fmj==0.4.6
 
 # homeassistant.components.arris_tg2492lg
 arris-tg2492lg==1.0.0
diff --git a/requirements_test_all.txt b/requirements_test_all.txt
index 1b5f57dbed6..9fbb1f0d8cd 100644
--- a/requirements_test_all.txt
+++ b/requirements_test_all.txt
@@ -131,7 +131,7 @@ apprise==0.8.5
 aprslib==0.6.46
 
 # homeassistant.components.arcam_fmj
-arcam-fmj==0.4.4
+arcam-fmj==0.4.6
 
 # homeassistant.components.dlna_dmr
 # homeassistant.components.upnp