Improve microphone connection handling in WebRTC player

Co-Authored-By: Claude Fable 5.2 <[email protected]>

src/common/webrtc/get-ice-path.ts

@@ -0,0 +1,54 @@
+import type { IcePath } from "./limit-ice-servers";
+
+// The fields of the transport, candidate pair and local candidate stats
+interface IceStats {
+  id: string;
+  type: string;
+  selectedCandidatePairId?: string;
+  selected?: boolean;
+  localCandidateId?: string;
+  candidateType?: string;
+  relayProtocol?: string;
+  url?: string;
+}
+
+/**
+ * Reads the network path a connected peer connection uses. Returns
+ * `undefined` if the browser doesn't report it.
+ */
+export const getIcePath = async (
+  peerConnection: RTCPeerConnection
+): Promise<IcePath | undefined> => {
+  const stats: Record<string, IceStats> = {};
+  try {
+    (await peerConnection.getStats()).forEach((report: IceStats) => {
+      stats[report.id] = report;
+    });
+  } catch (_err: unknown) {
+    return undefined;
+  }
+  const reports = Object.values(stats);
+  const pairId = reports.find(
+    (report) => report.type === "transport" && report.selectedCandidatePairId
+  )?.selectedCandidatePairId;
+  const pair = pairId
+    ? stats[pairId]
+    : // Firefox has no transport stats
+      reports.find(
+        (report) => report.type === "candidate-pair" && report.selected
+      );
+  const candidate = pair?.localCandidateId
+    ? stats[pair.localCandidateId]
+    : undefined;
+  if (!candidate?.candidateType) {
+    return undefined;
+  }
+  if (candidate.url?.startsWith("turn")) {
+    return { relayUrl: candidate.url };
+  }
+  if (candidate.candidateType === "relay" || candidate.relayProtocol) {
+    // Not all browsers report which TURN URL a relayed candidate came from
+    return undefined;
+  }
+  return {};
+};

src/common/webrtc/limit-ice-servers.ts

@@ -0,0 +1,103 @@
+import { ensureArray } from "../array/ensure-array";
+
+const isTurn = (url: string) => url.startsWith("turn");
+
+const isTcpTurn = (url: string) =>
+  url.startsWith("turns:") || url.includes("transport=tcp");
+
+const isTls443 = (url: string) =>
+  url.startsWith("turns:") && /:443(\?|$)/.test(url);
+
+// "turn:host:3478?transport=udp" -> "host"
+const getHost = (url: string) =>
+  url
+    .replace(/^[a-z]+:/i, "")
+    .replace(/\?.*$/, "")
+    .replace(/:\d+$/, "")
+    .toLowerCase();
+
+/**
+ * Reduces the URLs of every ICE server to one STUN URL, one UDP TURN URL and
+ * one TCP TURN URL (preferring TLS on port 443, which passes most firewalls)
+ * per host.
+ *
+ * The browser gathers a candidate for every URL on every network interface and
+ * checks each of them against every remote candidate. Redundant URLs of the
+ * same host do not add a new network path, but multiply the number of checks
+ * and delay the connection by many seconds.
+ */
+export const limitIceServers = (
+  iceServers: RTCIceServer[] | undefined
+): RTCIceServer[] | undefined =>
+  iceServers?.map((server) => {
+    const urls = ensureArray(server.urls);
+    if (urls.length < 2) {
+      return server;
+    }
+    const hosts = [...new Set(urls.map(getHost))];
+    return {
+      ...server,
+      urls: hosts.flatMap((host) => {
+        const hostUrls = urls.filter((url) => getHost(url) === host);
+        const turnUrls = hostUrls.filter(isTurn);
+        const tcpTurnUrls = turnUrls.filter(isTcpTurn);
+        return [
+          hostUrls.find((url) => !isTurn(url)),
+          turnUrls.find((url) => !isTcpTurn(url)),
+          tcpTurnUrls.find(isTls443) ??
+            tcpTurnUrls.find((url) => url.startsWith("turns:")) ??
+            tcpTurnUrls[0],
+        ].filter((url) => url !== undefined);
+      }),
+    };
+  });
+
+// "turns:host" -> "turns:host:5349?transport=tcp", browsers report the URL of
+// a candidate with the port and transport that were left out
+const normalizeUrl = (url: string) => {
+  const scheme = url.slice(0, url.indexOf(":")).toLowerCase();
+  const port =
+    /:(\d+)(\?|$)/.exec(url)?.[1] ?? (scheme.endsWith("s") ? "5349" : "3478");
+  const transport =
+    /[?&]transport=(\w+)/i.exec(url)?.[1].toLowerCase() ??
+    (scheme === "turns" ? "tcp" : "udp");
+  return `${scheme}:${getHost(url)}:${port}?transport=${transport}`;
+};
+
+/** The network path a connection uses, without `relayUrl` it is direct. */
+export interface IcePath {
+  relayUrl?: string;
+}
+
+/**
+ * Reduces the ICE servers to what a new connection needs to take the same
+ * path as a connection that is already running: only the TURN URL it relays
+ * over, or no TCP TURN URLs when it is direct. The UDP TURN URLs stay, as
+ * browsers also use them to find their public address.
+ *
+ * Returns `undefined` if the relay is not part of the ICE servers.
+ */
+export const limitIceServersToPath = (
+  iceServers: RTCIceServer[] | undefined,
+  path: IcePath
+): RTCIceServer[] | undefined => {
+  const { relayUrl } = path;
+  if (relayUrl) {
+    const normalizedUrl = normalizeUrl(relayUrl);
+    for (const server of iceServers ?? []) {
+      const url = ensureArray(server.urls).find(
+        (serverUrl) => normalizeUrl(serverUrl) === normalizedUrl
+      );
+      if (url) {
+        return [{ ...server, urls: [url] }];
+      }
+    }
+    return undefined;
+  }
+  return limitIceServers(iceServers)
+    ?.map((server) => ({
+      ...server,
+      urls: ensureArray(server.urls).filter((url) => !isTcpTurn(url)),
+    }))
+    .filter((server) => server.urls.length);
+};

src/components/ha-camera-stream.ts

@@ -6,7 +6,7 @@ import { repeat } from "lit/directives/repeat";
 import { styleMap } from "lit/directives/style-map";
 import memoizeOne from "memoize-one";
 import { consume } from "../common/decorators/consume";
-import { fireEvent } from "../common/dom/fire_event";
+import { fireEvent, type HASSDomEvent } from "../common/dom/fire_event";
 import { supportsFeature } from "../common/entity/supports-feature";
 import {
   CameraEntityFeature,
@@ -17,8 +17,16 @@ import {
   STREAM_TYPE_WEB_RTC,
   type CameraCapabilities,
   type CameraEntity,
-  type StreamType,
 } from "../data/camera";
+import {
+  hasWebRtcWorked,
+  MJPEG_STREAM,
+  selectCameraStreams,
+  setWebRtcWorked,
+  type CameraStream,
+  type CameraStreamTracks,
+  type CameraStreamType,
+} from "../data/camera-streams";
 import {
   apiContext,
   configContext,
@@ -29,14 +37,10 @@ import "./ha-hls-player";
 import "./ha-web-rtc-player";
 import type { HaWebRtcPlayer } from "./ha-web-rtc-player";
 
-const MJPEG_STREAM = "mjpeg";
+export type { CameraStreamType } from "../data/camera-streams";
 
-export type CameraStreamType = StreamType | typeof MJPEG_STREAM;
-
-interface Stream {
-  type: CameraStreamType;
-  visible: boolean;
-}
+// How long WebRTC gets on its own before HLS is started as well
+const WEB_RTC_FIRST_TIMEOUT = 12000;
 
 @customElement("ha-camera-stream")
 export class HaCameraStream extends LitElement {
@@ -78,13 +82,25 @@ export class HaCameraStream extends LitElement {
 
   @state() private _capabilities?: CameraCapabilities;
 
-  @state() private _hlsStreams?: { hasAudio: boolean; hasVideo: boolean };
+  @state() private _hlsStreams?: CameraStreamTracks;
 
-  @state() private _webRtcStreams?: { hasAudio: boolean; hasVideo: boolean };
+  @state() private _webRtcStreams?: CameraStreamTracks;
+
+  // WebRTC worked before for this camera, so HLS is not started with it
+  @state() private _webRtcFirst = false;
 
   @query("ha-web-rtc-player") private _webRtcPlayer?: HaWebRtcPlayer;
 
-  private _visibleStreamType?: Stream["type"];
+  private _visibleStreamType?: CameraStreamType;
+
+  private _webRtcFirstTimeout?: number;
+
+  private _capabilitiesRequest = 0;
+
+  private _streamsStartedAt = 0;
+
+  // If WebRTC loaded fast enough to be worth waiting for without HLS
+  private _webRtcLoadedInTime?: boolean;
 
   private _thumbnailApi = memoizeOne(
     (
@@ -132,11 +148,16 @@ export class HaCameraStream extends LitElement {
   public connectedCallback() {
     super.connectedCallback();
     this._connected = true;
+    if (this.hasUpdated && this._capabilities && this.stateObj) {
+      // The players start again when reattached, so their results do as well
+      this._startStreams(this.stateObj.entity_id, this._capabilities);
+    }
   }
 
   public disconnectedCallback() {
     super.disconnectedCallback();
     this._connected = false;
+    this._stopWebRtcFirst();
   }
 
   protected updated(changedProps: PropertyValues<this>): void {
@@ -146,13 +167,34 @@ export class HaCameraStream extends LitElement {
         capabilities: this._capabilities,
       });
     }
-    const visibleStreamType = this._currentStreams().find(
-      (stream) => stream.visible
-    )?.type;
+    const streams = this._currentStreams();
+    const visibleStreamType = streams.find((stream) => stream.visible)?.type;
     if (visibleStreamType !== this._visibleStreamType) {
       this._visibleStreamType = visibleStreamType;
       fireEvent(this, "stream-type-changed", { type: visibleStreamType });
     }
+    this._rememberWebRtc(streams);
+  }
+
+  /**
+   * Remembers per camera if WebRTC is the stream that ends up playing in this
+   * browser, to start it without HLS the next time.
+   */
+  private _rememberWebRtc(streams: CameraStream[]) {
+    if (
+      !this.stateObj ||
+      !this._webRtcStreams ||
+      (this._capabilities?.frontend_stream_types.length ?? 0) < 2 ||
+      streams.length !== 1
+    ) {
+      // No choice between streams, or the choice is not final yet
+      return;
+    }
+    setWebRtcWorked(
+      this.stateObj.entity_id,
+      streams[0].type === STREAM_TYPE_WEB_RTC &&
+        this._webRtcLoadedInTime === true
+    );
   }
 
   protected render() {
@@ -167,7 +209,7 @@ export class HaCameraStream extends LitElement {
     )}`;
   }
 
-  private _renderStream(stream: Stream) {
+  private _renderStream(stream: CameraStream) {
     if (!this.stateObj) {
       return nothing;
     }
@@ -230,14 +272,14 @@ export class HaCameraStream extends LitElement {
       this._capabilities?.frontend_stream_types,
       this._hlsStreams,
       this._webRtcStreams,
-      this.muted
+      this._webRtcFirst
     );
   }
 
   private async _getCapabilities() {
     this._capabilities = undefined;
-    this._hlsStreams = undefined;
-    this._webRtcStreams = undefined;
+    this._resetStreams();
+    const request = ++this._capabilitiesRequest;
     if (!supportsFeature(this.stateObj!, CameraEntityFeature.STREAM)) {
       this._capabilities = {
         frontend_stream_types: [],
@@ -245,10 +287,62 @@ export class HaCameraStream extends LitElement {
       };
       return;
     }
-    this._capabilities = await fetchCameraCapabilities(
-      this._api,
-      this.stateObj!.entity_id
-    );
+    const entityId = this.stateObj!.entity_id;
+    const capabilities = await fetchCameraCapabilities(this._api, entityId);
+    if (request !== this._capabilitiesRequest) {
+      return;
+    }
+    if (this.isConnected) {
+      // Otherwise started when connected
+      this._startStreams(entityId, capabilities);
+    }
+    this._capabilities = capabilities;
+  }
+
+  private _resetStreams() {
+    this._hlsStreams = undefined;
+    this._webRtcStreams = undefined;
+    this._webRtcLoadedInTime = undefined;
+    this._stopWebRtcFirst();
+  }
+
+  private _startStreams(entityId: string, capabilities: CameraCapabilities) {
+    this._resetStreams();
+    this._streamsStartedAt = performance.now();
+    if (
+      capabilities.frontend_stream_types.length > 1 &&
+      hasWebRtcWorked(entityId)
+    ) {
+      this._webRtcFirst = true;
+      this._webRtcFirstTimeout = window.setTimeout(
+        this._webRtcFirstTimedOut,
+        WEB_RTC_FIRST_TIMEOUT
+      );
+    }
+  }
+
+  private _webRtcFirstTimedOut = () => {
+    if (document.hidden) {
+      // A player started in the background is not stopped again, so wait
+      this._webRtcFirstTimeout = window.setTimeout(
+        this._webRtcFirstTimedOut,
+        WEB_RTC_FIRST_TIMEOUT
+      );
+      return;
+    }
+    this._webRtcFirstTimeout = undefined;
+    // Start HLS as well, WebRTC keeps connecting and takes over when it loads
+    this._webRtcFirst = false;
+    if (this.stateObj) {
+      // Don't wait for it the next time, also when it never reports a result
+      setWebRtcWorked(this.stateObj.entity_id, false);
+    }
+  };
+
+  private _stopWebRtcFirst() {
+    clearTimeout(this._webRtcFirstTimeout);
+    this._webRtcFirstTimeout = undefined;
+    this._webRtcFirst = false;
   }
 
   private async _getPosterUrl(): Promise<void> {
@@ -269,89 +363,46 @@ export class HaCameraStream extends LitElement {
     fireEvent(this, "load");
   }
 
-  private _handleHlsStreams(ev: CustomEvent) {
+  private _handleHlsStreams(ev: HASSDomEvent<HASSDomEvents["streams"]>) {
     this._hlsStreams = ev.detail;
   }
 
-  private _handleWebRtcStreams(ev: CustomEvent) {
+  private _handleWebRtcStreams(ev: HASSDomEvent<HASSDomEvents["streams"]>) {
+    clearTimeout(this._webRtcFirstTimeout);
+    this._webRtcFirstTimeout = undefined;
+    if (
+      !ev.detail.hasVideo &&
+      !ev.detail.hasAudio &&
+      this._capabilities?.frontend_stream_types.length === 1 &&
+      typeof RTCPeerConnection !== "undefined"
+    ) {
+      // Without another stream type, keep the player to show why it failed.
+      // Only a browser without WebRTC falls back to MJPEG.
+      return;
+    }
+    if (ev.detail.hasVideo && this._webRtcLoadedInTime === undefined) {
+      this._webRtcLoadedInTime =
+        performance.now() - this._streamsStartedAt <= WEB_RTC_FIRST_TIMEOUT;
+    }
     this._webRtcStreams = ev.detail;
   }
 
   private _streams = memoizeOne(
     (
-      supportedTypes?: StreamType[],
-      hlsStreams?: { hasAudio: boolean; hasVideo: boolean },
-      webRtcStreams?: { hasAudio: boolean; hasVideo: boolean },
-      muted?: boolean
-    ): Stream[] => {
+      supportedTypes?: CameraCapabilities["frontend_stream_types"],
+      hlsStreams?: CameraStreamTracks,
+      webRtcStreams?: CameraStreamTracks,
+      webRtcFirst?: boolean
+    ): CameraStream[] => {
       if (__DEMO__) {
         return [{ type: MJPEG_STREAM, visible: true }];
       }
-      if (!supportedTypes) {
-        return [];
-      }
-      if (supportedTypes.length === 0) {
-        // doesn't support any stream type, fallback to mjpeg
-        return [{ type: MJPEG_STREAM, visible: true }];
-      }
-      if (supportedTypes.length === 1) {
-        // only 1 stream type, no need to choose
-        if (
-          (supportedTypes[0] === STREAM_TYPE_HLS &&
-            hlsStreams?.hasVideo === false) ||
-          (supportedTypes[0] === STREAM_TYPE_WEB_RTC &&
-            webRtcStreams?.hasVideo === false)
-        ) {
-          // stream failed to load, fallback to mjpeg
-          return [{ type: MJPEG_STREAM, visible: true }];
-        }
-        return [{ type: supportedTypes[0], visible: true }];
-      }
-      if (hlsStreams && webRtcStreams) {
-        // fully loaded
-        if (
-          hlsStreams.hasVideo &&
-          hlsStreams.hasAudio &&
-          !webRtcStreams.hasAudio &&
-          !muted
-        ) {
-          // webRTC stream is missing audio and audio is not muted, use HLS
-          return [{ type: STREAM_TYPE_HLS, visible: true }];
-        }
-        if (webRtcStreams.hasVideo) {
-          return [{ type: STREAM_TYPE_WEB_RTC, visible: true }];
-        }
-        // both streams failed to load, fallback to mjpeg
-        return [{ type: MJPEG_STREAM, visible: true }];
-      }
-
-      if (hlsStreams?.hasVideo !== webRtcStreams?.hasVideo) {
-        // one of the two streams is loaded, or errored
-        // choose the one that has video or is still loading
-        if (hlsStreams?.hasVideo) {
-          return [
-            { type: STREAM_TYPE_HLS, visible: true },
-            { type: STREAM_TYPE_WEB_RTC, visible: false },
-          ];
-        }
-        if (hlsStreams?.hasVideo === false) {
-          return [{ type: STREAM_TYPE_WEB_RTC, visible: true }];
-        }
-        if (webRtcStreams?.hasVideo) {
-          return [
-            { type: STREAM_TYPE_WEB_RTC, visible: true },
-            { type: STREAM_TYPE_HLS, visible: false },
-          ];
-        }
-        if (webRtcStreams?.hasVideo === false) {
-          return [{ type: STREAM_TYPE_HLS, visible: true }];
-        }
-      }
-
-      return [
-        { type: STREAM_TYPE_HLS, visible: true },
-        { type: STREAM_TYPE_WEB_RTC, visible: false },
-      ];
+      return selectCameraStreams({
+        supportedTypes,
+        hlsStreams,
+        webRtcStreams,
+        webRtcFirst,
+      });
     }
   );
 

src/components/ha-web-rtc-player.ts

@@ -9,6 +9,12 @@ import { consume } from "../common/decorators/consume";
 import { consumeLocalize } from "../common/decorators/consume-context-entry";
 import { fireEvent } from "../common/dom/fire_event";
 import type { LocalizeFunc } from "../common/translations/localize";
+import { getIcePath } from "../common/webrtc/get-ice-path";
+import type { IcePath } from "../common/webrtc/limit-ice-servers";
+import {
+  limitIceServers,
+  limitIceServersToPath,
+} from "../common/webrtc/limit-ice-servers";
 import {
   addWebRtcCandidate,
   fetchWebRtcClientConfiguration,
@@ -24,8 +30,21 @@ import { showToast } from "../util/toast";
 
 const HIDDEN_CLEANUP_DELAY = 60000;
 
+// How long the backend gets to answer the offer, it has to connect to the camera
+const ANSWER_TIMEOUT = 20000;
+
+// How long finding a network path gets after the answer. Browsers can take
+// minutes to give up on their own, or never do.
+const ICE_TIMEOUT = 30000;
+
+// How often a stream that lost or never got its connection is started again
+const MAX_RETRIES = 2;
+
+// Gives the backend time to tear down the failed session before the next one
+const RETRY_DELAY = 1000;
+
 interface WebRtcPlayerError {
-  type: "not_supported" | "start_failed" | "connect_failed";
+  type: "not_supported" | "start_failed" | "connect_failed" | "no_connection";
   message?: string;
 }
 
@@ -99,7 +118,7 @@ export class HaWebRtcPlayer extends LitElement {
     if (this._microphoneDenied) {
       return "denied";
     }
-    if (this._microphoneConnecting) {
+    if (this._microphoneConnecting || this._retryMicrophoneTrack) {
       return "connecting";
     }
     return this._localReturnAudioTrack ? "on" : "off";
@@ -110,11 +129,17 @@ export class HaWebRtcPlayer extends LitElement {
   }
 
   public async toggleMicrophone() {
-    if (this._localReturnAudioTrack || this._microphoneConnecting) {
+    if (
+      this._localReturnAudioTrack ||
+      this._microphoneConnecting ||
+      this._retryMicrophoneTrack
+    ) {
       // Stop (not disable) so the browser releases the microphone, also cancels connecting
       this._microphoneRequest++;
       this._localReturnAudioTrack?.stop();
       this._localReturnAudioTrack = undefined;
+      this._retryMicrophoneTrack?.stop();
+      this._retryMicrophoneTrack = undefined;
       this._microphoneConnecting = false;
       this._fireMicrophoneChanged();
       await this._microphoneSender?.replaceTrack(null).catch(() => undefined);
@@ -167,8 +192,16 @@ export class HaWebRtcPlayer extends LitElement {
     // Renegotiating the existing connection is not supported by all providers
     // (go2rtc answers with a new DTLS fingerprint, which Firefox rejects), so
     // start a new connection and session with the microphone track instead.
+    // The new connection only looks for the path the running stream uses
+    const icePath = this._peerConnection
+      ? await getIcePath(this._peerConnection)
+      : undefined;
+    if (request !== this._microphoneRequest || !this.isConnected) {
+      track.stop();
+      return;
+    }
     this._captureReconnectFrame();
-    await this._startWebRtc(track);
+    await this._startWebRtc(track, icePath);
   }
 
   private _captureReconnectFrame() {
@@ -197,6 +230,17 @@ export class HaWebRtcPlayer extends LitElement {
 
   private _cleanUpCount = 0;
 
+  private _connectTimeout?: number;
+
+  private _retryTimeout?: number;
+
+  private _retryMicrophoneTrack?: MediaStreamTrack;
+
+  private _retries = 0;
+
+  // The stream only loaded with all ICE servers, so keep using them
+  private _allIceServers = false;
+
   private _handleVisibilityChange = () => {
     if (document.pictureInPictureElement) {
       // video is playing in picture-in-picture mode, don't do anything
@@ -212,6 +256,7 @@ export class HaWebRtcPlayer extends LitElement {
       clearTimeout(this._hiddenCleanupTimeout);
       this._hiddenCleanupTimeout = undefined;
     } else {
+      this._retries = 0;
       this._startWebRtc();
     }
   };
@@ -253,6 +298,7 @@ export class HaWebRtcPlayer extends LitElement {
   public override connectedCallback() {
     super.connectedCallback();
     if (this.hasUpdated && this.entityid) {
+      this._retries = 0;
       this._startWebRtc();
     }
     document.addEventListener("visibilitychange", this._handleVisibilityChange);
@@ -274,11 +320,14 @@ export class HaWebRtcPlayer extends LitElement {
     if (!changedProperties.has("entityid")) {
       return;
     }
+    this._retries = 0;
+    this._allIceServers = false;
     this._startWebRtc();
   }
 
   private async _startWebRtc(
-    localReturnAudioTrack?: MediaStreamTrack
+    localReturnAudioTrack?: MediaStreamTrack,
+    icePath?: IcePath
   ): Promise<void> {
     this._cleanUp();
 
@@ -333,9 +382,21 @@ export class HaWebRtcPlayer extends LitElement {
 
     this._logEvent("end clientConfig", this._clientConfig);
 
-    this._peerConnection = new RTCPeerConnection(
-      this._clientConfig.configuration
-    );
+    const { iceServers } = this._clientConfig.configuration;
+    const allIceServers = this._allIceServers || this._retries >= MAX_RETRIES;
+
+    this._peerConnection = new RTCPeerConnection({
+      ...this._clientConfig.configuration,
+      // The last attempt uses all ICE servers, in case a removed URL is the
+      // only one that works in this network
+      // A known relay is taken from all of them. A direct path is not, as
+      // the URL it needs can be one that is removed.
+      iceServers:
+        (icePath &&
+          (icePath.relayUrl || !allIceServers) &&
+          limitIceServersToPath(iceServers, icePath)) ||
+        (allIceServers ? iceServers : limitIceServers(iceServers)),
+    });
 
     if (this._clientConfig.dataChannel) {
       // Some cameras (such as nest) require a data channel to establish a stream
@@ -415,7 +476,7 @@ export class HaWebRtcPlayer extends LitElement {
     while (this._candidatesList.length) {
       const candidate = this._candidatesList.pop();
       if (candidate) {
-        candidates += `a=${candidate}\r\n`;
+        candidates += `a=${candidate.candidate}\r\n`;
       }
     }
 
@@ -423,6 +484,8 @@ export class HaWebRtcPlayer extends LitElement {
 
     this._logEvent("start webRtcOffer", offer_sdp);
 
+    this._startConnectTimeout(ANSWER_TIMEOUT);
+
     this._unsub = webRtcOffer(
       this._connection,
       this.entityid,
@@ -443,11 +506,58 @@ export class HaWebRtcPlayer extends LitElement {
       "ice connection state change",
       this._peerConnection?.iceConnectionState
     );
-    if (this._peerConnection?.iceConnectionState === "failed") {
-      this._peerConnection.restartIce();
+    const iceState = this._peerConnection?.iceConnectionState;
+    if (iceState === "connected" || iceState === "completed") {
+      clearTimeout(this._connectTimeout);
+      this._connectTimeout = undefined;
+    } else if (iceState === "failed") {
+      // An ICE restart needs a new offer, which the backend answers from a new
+      // peer that can't continue this connection, so start a new one instead.
+      this._retryOrFail({ type: "no_connection" });
     }
   };
 
+  private _startConnectTimeout(timeout: number) {
+    clearTimeout(this._connectTimeout);
+    this._connectTimeout = window.setTimeout(() => {
+      this._connectTimeout = undefined;
+      this._logEvent("connect timeout");
+      this._retryOrFail({ type: "no_connection" });
+    }, timeout);
+  }
+
+  private _retryOrFail(error: WebRtcPlayerError) {
+    if (
+      this._retries >= MAX_RETRIES ||
+      // A microphone connect the user started is handled by starting again
+      // without the microphone. A retry keeps the microphone of the stream.
+      (this._localReturnAudioTrack &&
+        this._microphoneConnecting &&
+        this._retries === 0)
+    ) {
+      this._failStart(error);
+      return;
+    }
+    this._retries++;
+    this._logEvent("retry", this._retries, error);
+    // Keep showing the last frame instead of the poster while starting again
+    this._captureReconnectFrame();
+    // Taken out before the clean up, so a running microphone is not stopped
+    const microphoneTrack = this._localReturnAudioTrack;
+    this._localReturnAudioTrack = undefined;
+    this._cleanUp();
+    if (microphoneTrack) {
+      this._retryMicrophoneTrack = microphoneTrack;
+      this._fireMicrophoneChanged();
+    }
+    this._retryTimeout = window.setTimeout(() => {
+      this._retryTimeout = undefined;
+      const track = this._retryMicrophoneTrack;
+      this._retryMicrophoneTrack = undefined;
+      this._startWebRtc(track);
+    }, RETRY_DELAY);
+  }
+
   private async _handleOfferEvent(
     peerConnection: RTCPeerConnection,
     event: WebRtcOfferEvent
@@ -470,6 +580,10 @@ export class HaWebRtcPlayer extends LitElement {
       this._candidatesList = [];
     }
     if (event.type === "answer") {
+      const iceState = peerConnection.iceConnectionState;
+      if (iceState !== "connected" && iceState !== "completed") {
+        this._startConnectTimeout(ICE_TIMEOUT);
+      }
       this._logEvent("answer", event.answer);
 
       this._handleAnswer(peerConnection, event);
@@ -495,7 +609,8 @@ export class HaWebRtcPlayer extends LitElement {
       }
     }
     if (event.type === "error") {
-      this._failStart({ type: "start_failed", message: event.message });
+      // Often temporary, like the backend still closing the previous session
+      this._retryOrFail({ type: "start_failed", message: event.message });
     }
   }
 
@@ -587,8 +702,13 @@ export class HaWebRtcPlayer extends LitElement {
   }
 
   private _failStart(error: WebRtcPlayerError) {
-    // The camera can reject the microphone, keep the video running without it
-    if (this._localReturnAudioTrack && this._microphoneConnecting) {
+    // The camera can reject the microphone, keep the video running without it.
+    // After retries it is the connection that failed, not the microphone.
+    if (
+      this._localReturnAudioTrack &&
+      this._microphoneConnecting &&
+      this._retries === 0
+    ) {
       showToast(this, {
         message: this._localize(
           "ui.components.web-rtc-player.microphone_connect_failed",
@@ -599,12 +719,19 @@ export class HaWebRtcPlayer extends LitElement {
       return;
     }
     this._error = error;
+    this._reconnectPoster = undefined;
     this._cleanUp();
+    // Tell the camera stream this player has no video, so it can fall back
+    fireEvent(this, "streams", { hasAudio: false, hasVideo: false });
   }
 
   private _cleanUp() {
     this._cleanUpCount++;
     this._microphoneRequest++;
+    clearTimeout(this._connectTimeout);
+    this._connectTimeout = undefined;
+    clearTimeout(this._retryTimeout);
+    this._retryTimeout = undefined;
     if (this._remoteStream) {
       this._remoteStream.getTracks().forEach((track) => {
         track.stop();
@@ -612,9 +739,15 @@ export class HaWebRtcPlayer extends LitElement {
 
       this._remoteStream = undefined;
     }
-    if (this._localReturnAudioTrack || this._microphoneConnecting) {
+    if (
+      this._localReturnAudioTrack ||
+      this._microphoneConnecting ||
+      this._retryMicrophoneTrack
+    ) {
       this._localReturnAudioTrack?.stop();
       this._localReturnAudioTrack = undefined;
+      this._retryMicrophoneTrack?.stop();
+      this._retryMicrophoneTrack = undefined;
       this._microphoneConnecting = false;
       this._fireMicrophoneChanged();
     }
@@ -666,6 +799,10 @@ export class HaWebRtcPlayer extends LitElement {
 
     this._logEvent("loadedData", data);
     this._stopTimer();
+    if (this._retries >= MAX_RETRIES) {
+      this._allIceServers = true;
+    }
+    this._retries = 0;
 
     // Without a track the microphone is still waiting for permission
     if (this._microphoneConnecting && this._localReturnAudioTrack) {

src/data/camera-streams.ts

@@ -0,0 +1,115 @@
+import { addRecent } from "../common/array/add-recent";
+import { getStorage } from "../common/decorators/storage";
+
+export const STREAM_TYPE_HLS = "hls";
+export const STREAM_TYPE_WEB_RTC = "web_rtc";
+
+export type StreamType = typeof STREAM_TYPE_HLS | typeof STREAM_TYPE_WEB_RTC;
+
+export const MJPEG_STREAM = "mjpeg";
+
+export type CameraStreamType = StreamType | typeof MJPEG_STREAM;
+
+export interface CameraStream {
+  type: CameraStreamType;
+  visible: boolean;
+}
+
+/** What a player reported, `hasVideo: false` means it failed to load. */
+export interface CameraStreamTracks {
+  hasAudio: boolean;
+  hasVideo: boolean;
+}
+
+export interface CameraStreamSelectionOptions {
+  supportedTypes?: StreamType[];
+  hlsStreams?: CameraStreamTracks;
+  webRtcStreams?: CameraStreamTracks;
+  /** Only start WebRTC and keep HLS off until WebRTC fails or takes too long. */
+  webRtcFirst?: boolean;
+}
+
+/**
+ * Decides which players are rendered and which one is shown. Hidden players
+ * are still loading in the background.
+ */
+export const selectCameraStreams = ({
+  supportedTypes,
+  hlsStreams,
+  webRtcStreams,
+  webRtcFirst,
+}: CameraStreamSelectionOptions): CameraStream[] => {
+  if (!supportedTypes) {
+    return [];
+  }
+  if (supportedTypes.length === 0) {
+    // doesn't support any stream type, fallback to mjpeg
+    return [{ type: MJPEG_STREAM, visible: true }];
+  }
+  if (supportedTypes.length === 1) {
+    // only 1 stream type, no need to choose
+    if (
+      (supportedTypes[0] === STREAM_TYPE_HLS &&
+        hlsStreams?.hasVideo === false) ||
+      (supportedTypes[0] === STREAM_TYPE_WEB_RTC &&
+        webRtcStreams?.hasVideo === false)
+    ) {
+      // stream failed to load, fallback to mjpeg
+      return [{ type: MJPEG_STREAM, visible: true }];
+    }
+    return [{ type: supportedTypes[0], visible: true }];
+  }
+
+  const hlsFailed = hlsStreams?.hasVideo === false;
+
+  if (webRtcStreams?.hasVideo) {
+    // WebRTC is preferred whenever it plays, it has the lowest latency
+    return [{ type: STREAM_TYPE_WEB_RTC, visible: true }];
+  }
+
+  if (webRtcStreams?.hasVideo === false) {
+    if (hlsFailed) {
+      // both streams failed to load, fallback to mjpeg
+      return [{ type: MJPEG_STREAM, visible: true }];
+    }
+    return [{ type: STREAM_TYPE_HLS, visible: true }];
+  }
+
+  // WebRTC is still loading
+  if (hlsFailed || (webRtcFirst && !hlsStreams)) {
+    return [{ type: STREAM_TYPE_WEB_RTC, visible: true }];
+  }
+  return [
+    { type: STREAM_TYPE_HLS, visible: true },
+    { type: STREAM_TYPE_WEB_RTC, visible: false },
+  ];
+};
+
+const WEB_RTC_ENTITIES_KEY = "cameraWebRtcEntities";
+
+const MAX_WEB_RTC_ENTITIES = 100;
+
+const getWebRtcEntities = (): string[] => {
+  const entities = getStorage().getValue<string[]>(WEB_RTC_ENTITIES_KEY);
+  return Array.isArray(entities) ? entities : [];
+};
+
+/**
+ * If WebRTC was the stream that ended up playing for this camera in this
+ * browser. Stored per browser, as it depends on the network of the client.
+ */
+export const hasWebRtcWorked = (entityId: string): boolean =>
+  getWebRtcEntities().includes(entityId);
+
+export const setWebRtcWorked = (entityId: string, worked: boolean): void => {
+  const entities = getWebRtcEntities();
+  if (entities.includes(entityId) === worked) {
+    return;
+  }
+  getStorage().setValue(
+    WEB_RTC_ENTITIES_KEY,
+    worked
+      ? addRecent(entities, entityId, MAX_WEB_RTC_ENTITIES)
+      : entities.filter((entity) => entity !== entityId)
+  );
+};

src/data/camera.ts

@@ -7,13 +7,15 @@ import { timeCacheEntityPromiseFunc } from "../common/util/time-cache-entity-pro
 import type { HomeAssistant, HomeAssistantApi } from "../types";
 import { fileDownload } from "../util/file_download";
 import { getSignedPath } from "./auth";
+import type { StreamType } from "./camera-streams";
 
 export const CAMERA_ORIENTATIONS = [1, 2, 3, 4, 6, 8];
 
-export const STREAM_TYPE_HLS = "hls";
-export const STREAM_TYPE_WEB_RTC = "web_rtc";
-
-export type StreamType = typeof STREAM_TYPE_HLS | typeof STREAM_TYPE_WEB_RTC;
+export {
+  STREAM_TYPE_HLS,
+  STREAM_TYPE_WEB_RTC,
+  type StreamType,
+} from "./camera-streams";
 
 export { CameraEntityFeature } from "./feature/camera_entity_feature";
 

src/translations/en.json

@@ -1333,6 +1333,7 @@
         "not_supported": "WebRTC is not supported in this browser.",
         "start_failed": "Failed to start the WebRTC stream: {message}",
         "connect_failed": "Failed to connect the WebRTC stream: {message}",
+        "no_connection": "Could not connect the WebRTC stream.",
         "microphone_failed": "Could not access the microphone: {message}",
         "microphone_connect_failed": "Could not connect the microphone to the camera: {message}"
       },

test/common/webrtc/limit-ice-servers.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it } from "vitest";
+import {
+  limitIceServers,
+  limitIceServersToPath,
+} from "../../../src/common/webrtc/limit-ice-servers";
+
+// Redundant URLs cost seconds of connection time, while dropping the wrong
+// one (the UDP or the firewall friendly TLS relay) breaks remote access.
+describe("limitIceServers", () => {
+  it("works", () => {
+    expect(limitIceServers(undefined)).toBeUndefined();
+    expect(
+      limitIceServers([
+        { urls: "stun:stun.example.com:3478" },
+        { urls: ["stun:a.example.com:3478", "stun:a.example.com:80"] },
+        {
+          urls: [
+            "turn:t.example.com:3478?transport=udp",
+            "turn:t.example.com:3478?transport=tcp",
+            "turns:t.example.com:5349?transport=tcp",
+            "turn:t.example.com:443?transport=udp",
+            "turn:t.example.com:80?transport=tcp",
+            "turns:t.example.com:443?transport=tcp",
+          ],
+          username: "user",
+          credential: "secret",
+        },
+        {
+          urls: [
+            "turn:u.example.com:80?transport=tcp",
+            "turn:u.example.com:443?transport=tcp",
+          ],
+        },
+        // Different hosts are different network paths, all are kept
+        {
+          urls: [
+            "stun:v.example.com",
+            "turn:v.example.com:3478",
+            "turn:v.example.com:3479",
+            "turn:w.example.com:3478",
+          ],
+        },
+      ])
+    ).toEqual([
+      { urls: "stun:stun.example.com:3478" },
+      { urls: ["stun:a.example.com:3478"] },
+      {
+        urls: [
+          "turn:t.example.com:3478?transport=udp",
+          "turns:t.example.com:443?transport=tcp",
+        ],
+        username: "user",
+        credential: "secret",
+      },
+      { urls: ["turn:u.example.com:80?transport=tcp"] },
+      {
+        urls: [
+          "stun:v.example.com",
+          "turn:v.example.com:3478",
+          "turn:w.example.com:3478",
+        ],
+      },
+    ]);
+
+    const iceServers = [
+      { urls: ["stun:a.example.com:3478", "stun:a.example.com:80"] },
+      {
+        urls: ["turn:t.example.com", "turns:t.example.com:443"],
+        username: "user",
+      },
+    ];
+    // A direct path keeps the UDP relay, which is also used to find the
+    // public address. A relayed one only needs its own relay.
+    expect(limitIceServersToPath(iceServers, {})).toEqual([
+      { urls: ["stun:a.example.com:3478"] },
+      { urls: ["turn:t.example.com"], username: "user" },
+    ]);
+    // Browsers report the URL with the port and transport that were left out
+    expect(
+      limitIceServersToPath(iceServers, {
+        relayUrl: "turns:t.example.com:443?transport=tcp",
+      })
+    ).toEqual([{ urls: ["turns:t.example.com:443"], username: "user" }]);
+    expect(
+      limitIceServersToPath(iceServers, {
+        relayUrl: "turn:t.example.com:3478?transport=udp",
+      })
+    ).toEqual([{ urls: ["turn:t.example.com"], username: "user" }]);
+    expect(
+      limitIceServersToPath(iceServers, { relayUrl: "turn:other.example.com" })
+    ).toBeUndefined();
+  });
+});

test/data/camera-streams.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import {
+  selectCameraStreams,
+  type CameraStreamSelectionOptions,
+} from "../../src/data/camera-streams";
+
+const ok = { hasVideo: true, hasAudio: true };
+const silent = { hasVideo: true, hasAudio: false };
+const failed = { hasVideo: false, hasAudio: false };
+
+const select = (
+  options: Omit<CameraStreamSelectionOptions, "supportedTypes">
+) =>
+  selectCameraStreams({ supportedTypes: ["web_rtc", "hls"], ...options })
+    .map((stream) => (stream.visible ? stream.type : `(${stream.type})`))
+    .join(" ");
+
+// The player has no other fallback: a wrong choice here shows a broken or
+// silent stream while a working one is available.
+describe("selectCameraStreams", () => {
+  it("works", () => {
+    expect(selectCameraStreams({})).toEqual([]);
+    expect(selectCameraStreams({ supportedTypes: [] })).toEqual([
+      { type: "mjpeg", visible: true },
+    ]);
+    expect(
+      selectCameraStreams({ supportedTypes: ["hls"], hlsStreams: failed })
+    ).toEqual([{ type: "mjpeg", visible: true }]);
+
+    // Loading: HLS is shown until WebRTC is ready, unless WebRTC goes first
+    expect(select({})).toBe("hls (web_rtc)");
+    expect(select({ hlsStreams: ok })).toBe("hls (web_rtc)");
+    expect(select({ webRtcFirst: true })).toBe("web_rtc");
+    expect(select({ webRtcFirst: true, hlsStreams: ok })).toBe("hls (web_rtc)");
+    expect(select({ hlsStreams: failed })).toBe("web_rtc");
+
+    // WebRTC loaded: always preferred, also when only HLS has audio
+    expect(select({ webRtcStreams: ok })).toBe("web_rtc");
+    expect(select({ webRtcStreams: silent, hlsStreams: ok })).toBe("web_rtc");
+    expect(select({ webRtcStreams: ok, hlsStreams: failed })).toBe("web_rtc");
+
+    // WebRTC failed
+    expect(select({ webRtcStreams: failed })).toBe("hls");
+    expect(select({ webRtcStreams: failed, hlsStreams: ok })).toBe("hls");
+    expect(select({ webRtcStreams: failed, hlsStreams: failed })).toBe("mjpeg");
+  });
+});