diff --git a/homeassistant/components/stream/const.py b/homeassistant/components/stream/const.py
index 9406c24eaf6..181808e549e 100644
--- a/homeassistant/components/stream/const.py
+++ b/homeassistant/components/stream/const.py
@@ -23,3 +23,6 @@ MAX_TIMESTAMP_GAP = 10000  # seconds - anything from 10 to 50000 is probably rea
 
 MAX_MISSING_DTS = 6  # Number of packets missing DTS to allow
 STREAM_TIMEOUT = 30  # Timeout for reading stream
+
+STREAM_RESTART_INCREMENT = 10  # Increase wait_timeout by this amount each retry
+STREAM_RESTART_RESET_TIME = 300  # Reset wait_timeout after this many seconds
diff --git a/homeassistant/components/stream/worker.py b/homeassistant/components/stream/worker.py
index 40f9ef873ea..aa6a5d350a9 100644
--- a/homeassistant/components/stream/worker.py
+++ b/homeassistant/components/stream/worker.py
@@ -11,6 +11,8 @@ from .const import (
     MAX_TIMESTAMP_GAP,
     MIN_SEGMENT_DURATION,
     PACKETS_TO_WAIT_FOR_AUDIO,
+    STREAM_RESTART_INCREMENT,
+    STREAM_RESTART_RESET_TIME,
     STREAM_TIMEOUT,
 )
 from .core import Segment, StreamBuffer
@@ -56,8 +58,13 @@ def stream_worker(hass, stream, quit_event):
             _LOGGER.exception("Stream connection failed: %s", stream.source)
         if not stream.keepalive or quit_event.is_set():
             break
-        # To avoid excessive restarts, don't restart faster than once every 40 seconds.
-        wait_timeout = max(40 - (time.time() - start_time), 0)
+        # To avoid excessive restarts, wait before restarting
+        # As the required recovery time may be different for different setups, start
+        # with trying a short wait_timeout and increase it on each reconnection attempt.
+        # Reset the wait_timeout after the worker has been up for several minutes
+        if time.time() - start_time > STREAM_RESTART_RESET_TIME:
+            wait_timeout = 0
+        wait_timeout += STREAM_RESTART_INCREMENT
         _LOGGER.debug(
             "Restarting stream worker in %d seconds: %s",
             wait_timeout,
@@ -68,7 +75,13 @@ def stream_worker(hass, stream, quit_event):
 def _stream_worker_internal(hass, stream, quit_event):
     """Handle consuming streams."""
 
-    container = av.open(stream.source, options=stream.options, timeout=STREAM_TIMEOUT)
+    try:
+        container = av.open(
+            stream.source, options=stream.options, timeout=STREAM_TIMEOUT
+        )
+    except av.AVError:
+        _LOGGER.error("Error opening stream %s", stream.source)
+        return
     try:
         video_stream = container.streams.video[0]
     except (KeyError, IndexError):
diff --git a/tests/components/stream/test_hls.py b/tests/components/stream/test_hls.py
index 863513c8157..16d2d724f22 100644
--- a/tests/components/stream/test_hls.py
+++ b/tests/components/stream/test_hls.py
@@ -147,7 +147,9 @@ async def test_stream_keepalive(hass):
 
     with patch("av.open") as av_open, patch(
         "homeassistant.components.stream.worker.time"
-    ) as mock_time:
+    ) as mock_time, patch(
+        "homeassistant.components.stream.worker.STREAM_RESTART_INCREMENT", 0
+    ):
         av_open.side_effect = av.error.InvalidDataError(-2, "error")
         mock_time.time.side_effect = time_side_effect
         # Request stream