diff --git a/homeassistant/components/stream/__init__.py b/homeassistant/components/stream/__init__.py
index 365ea946f51..66929fff79c 100644
--- a/homeassistant/components/stream/__init__.py
+++ b/homeassistant/components/stream/__init__.py
@@ -344,7 +344,9 @@ class Stream:
             stream_state.discontinuity()
             if not _should_retry() or self._thread_quit.is_set():
                 if self._fast_restart_once:
-                    # The stream source is updated, restart without any delay.
+                    # The stream source is updated, restart without any delay and reset the retry
+                    # backoff for the new url.
+                    wait_timeout = 0
                     self._fast_restart_once = False
                     self._thread_quit.clear()
                     continue