Make web rtc player ice resolving async (#22312)

* Make web rtc player ice resolving async

* rename getCandidatesUpfront

* dont send empty candidates, catch answer when signalingState is stable

* Update src/components/ha-web-rtc-player.ts

Co-authored-by: Paulus Schoutsen <balloob@gmail.com>

* review

* Update ha-web-rtc-player.ts

---------

Co-authored-by: Paulus Schoutsen <balloob@gmail.com>

src/components/ha-web-rtc-player.ts

@@ -1,4 +1,5 @@
 /* eslint-disable no-console */
+import { UnsubscribeFunc } from "home-assistant-js-websocket";
 import {
   css,
   CSSResultGroup,
@@ -11,9 +12,12 @@ import { customElement, property, query, state } from "lit/decorators";
 import { ifDefined } from "lit/directives/if-defined";
 import { fireEvent } from "../common/dom/fire_event";
 import {
+  addWebRtcCandidate,
   fetchWebRtcClientConfiguration,
-  handleWebRtcOffer,
   WebRtcAnswer,
+  WebRTCClientConfiguration,
+  webRtcOffer,
+  WebRtcOfferEvent,
 } from "../data/camera";
 import type { HomeAssistant } from "../types";
 import "./ha-alert";
@@ -27,7 +31,7 @@ import "./ha-alert";
 class HaWebRtcPlayer extends LitElement {
   @property({ attribute: false }) public hass!: HomeAssistant;
 
-  @property() public entityid!: string;
+  @property() public entityid?: string;
 
   @property({ type: Boolean, attribute: "controls" })
   public controls = false;
@@ -45,12 +49,20 @@ class HaWebRtcPlayer extends LitElement {
 
   @state() private _error?: string;
 
-  @query("#remote-stream", true) private _videoEl!: HTMLVideoElement;
+  @query("#remote-stream") private _videoEl!: HTMLVideoElement;
+
+  private _clientConfig?: WebRTCClientConfiguration;
 
   private _peerConnection?: RTCPeerConnection;
 
   private _remoteStream?: MediaStream;
 
+  private _unsub?: Promise<UnsubscribeFunc>;
+
+  private _sessionId?: string;
+
+  private _candidatesList: string[] = [];
+
   protected override render(): TemplateResult {
     if (this._error) {
       return html`<ha-alert alert-type="error">${this._error}</ha-alert>`;
@@ -70,7 +82,7 @@ class HaWebRtcPlayer extends LitElement {
 
   public override connectedCallback() {
     super.connectedCallback();
-    if (this.hasUpdated) {
+    if (this.hasUpdated && this.entityid) {
       this._startWebRtc();
     }
   }
@@ -80,7 +92,8 @@ class HaWebRtcPlayer extends LitElement {
     this._cleanUp();
   }
 
-  protected override updated(changedProperties: PropertyValues<this>) {
+  protected override willUpdate(changedProperties: PropertyValues<this>) {
+    super.willUpdate(changedProperties);
     if (!changedProperties.has("entityid")) {
       return;
     }
@@ -88,28 +101,68 @@ class HaWebRtcPlayer extends LitElement {
   }
 
   private async _startWebRtc(): Promise<void> {
+    this._cleanUp();
+
+    if (!this.hass || !this.entityid) {
+      return;
+    }
+
     console.time("WebRTC");
 
     this._error = undefined;
 
     console.timeLog("WebRTC", "start clientConfig");
 
-    const clientConfig = await fetchWebRtcClientConfiguration(
+    this._clientConfig = await fetchWebRtcClientConfiguration(
       this.hass,
       this.entityid
     );
 
-    console.timeLog("WebRTC", "end clientConfig", clientConfig);
+    console.timeLog("WebRTC", "end clientConfig", this._clientConfig);
 
-    const peerConnection = new RTCPeerConnection(clientConfig.configuration);
+    this._peerConnection = new RTCPeerConnection(
+      this._clientConfig.configuration
+    );
 
-    if (clientConfig.dataChannel) {
+    if (this._clientConfig.dataChannel) {
       // Some cameras (such as nest) require a data channel to establish a stream
       // however, not used by any integrations.
-      peerConnection.createDataChannel(clientConfig.dataChannel);
+      this._peerConnection.createDataChannel(this._clientConfig.dataChannel);
+    }
+
+    this._peerConnection.onnegotiationneeded = this._startNegotiation;
+
+    this._peerConnection.onicecandidate = this._handleIceCandidate;
+    this._peerConnection.oniceconnectionstatechange =
+      this._iceConnectionStateChanged;
+
+    // just for debugging
+    this._peerConnection.onsignalingstatechange = (ev) => {
+      switch ((ev.target as RTCPeerConnection).signalingState) {
+        case "stable":
+          console.timeLog("WebRTC", "ICE negotiation complete");
+          break;
+        default:
+          console.timeLog(
+            "WebRTC",
+            "Signaling state changed",
+            (ev.target as RTCPeerConnection).signalingState
+          );
+      }
+    };
+
+    // Setup callbacks to render remote stream once media tracks are discovered.
+    this._remoteStream = new MediaStream();
+    this._peerConnection.ontrack = this._addTrack;
+
+    this._peerConnection.addTransceiver("audio", { direction: "recvonly" });
+    this._peerConnection.addTransceiver("video", { direction: "recvonly" });
+  }
+
+  private _startNegotiation = async () => {
+    if (!this._peerConnection) {
+      return;
     }
-    peerConnection.addTransceiver("audio", { direction: "recvonly" });
-    peerConnection.addTransceiver("video", { direction: "recvonly" });
 
     const offerOptions: RTCOfferOptions = {
       offerToReceiveAudio: true,
@@ -119,98 +172,218 @@ class HaWebRtcPlayer extends LitElement {
     console.timeLog("WebRTC", "start createOffer", offerOptions);
 
     const offer: RTCSessionDescriptionInit =
-      await peerConnection.createOffer(offerOptions);
+      await this._peerConnection.createOffer(offerOptions);
+
+    if (!this._peerConnection) {
+      return;
+    }
 
     console.timeLog("WebRTC", "end createOffer", offer);
 
     console.timeLog("WebRTC", "start setLocalDescription");
 
-    await peerConnection.setLocalDescription(offer);
+    await this._peerConnection.setLocalDescription(offer);
 
-    console.timeLog("WebRTC", "end setLocalDescription");
+    if (!this._peerConnection || !this.entityid) {
-
-    console.timeLog("WebRTC", "start iceResolver");
-
-    let candidates = ""; // Build an Offer SDP string with ice candidates
-    const iceResolver = new Promise<void>((resolve) => {
-      peerConnection.addEventListener("icecandidate", (event) => {
-        if (!event.candidate?.candidate) {
-          resolve(); // Gathering complete
-          return;
-        }
-        console.timeLog("WebRTC", "iceResolver candidate", event.candidate);
-        candidates += `a=${event.candidate.candidate}\r\n`;
-      });
-    });
-    await iceResolver;
-
-    console.timeLog("WebRTC", "end iceResolver", candidates);
-
-    const offer_sdp = offer.sdp! + candidates;
-
-    let webRtcAnswer: WebRtcAnswer;
-    try {
-      console.timeLog("WebRTC", "start WebRTCOffer", offer_sdp);
-      webRtcAnswer = await handleWebRtcOffer(
-        this.hass,
-        this.entityid,
-        offer_sdp
-      );
-      console.timeLog("WebRTC", "end webRtcOffer", webRtcAnswer);
-    } catch (err: any) {
-      this._error = "Failed to start WebRTC stream: " + err.message;
-      peerConnection.close();
       return;
     }
 
-    // Setup callbacks to render remote stream once media tracks are discovered.
+    console.timeLog("WebRTC", "end setLocalDescription");
-    const remoteStream = new MediaStream();
+
-    peerConnection.addEventListener("track", (event) => {
+    let candidates = "";
-      console.timeLog("WebRTC", "track", event);
+
-      remoteStream.addTrack(event.track);
+    if (this._clientConfig?.getCandidatesUpfront) {
-      this._videoEl.srcObject = remoteStream;
+      await new Promise<void>((resolve) => {
-    });
+        this._peerConnection!.onicegatheringstatechange = (ev: Event) => {
-    this._remoteStream = remoteStream;
+          const iceGatheringState = (ev.target as RTCPeerConnection)
+            .iceGatheringState;
+          if (iceGatheringState === "complete") {
+            this._peerConnection!.onicegatheringstatechange = null;
+            resolve();
+          }
+
+          console.timeLog(
+            "WebRTC",
+            "Ice gathering state changed",
+            iceGatheringState
+          );
+        };
+      });
+
+      if (!this._peerConnection || !this.entityid) {
+        return;
+      }
+    }
+
+    while (this._candidatesList.length) {
+      const candidate = this._candidatesList.pop();
+      if (candidate) {
+        candidates += `a=${candidate}\r\n`;
+      }
+    }
+
+    const offer_sdp = offer.sdp! + candidates;
+
+    console.timeLog("WebRTC", "start webRtcOffer", offer_sdp);
+
+    try {
+      this._unsub = webRtcOffer(this.hass, this.entityid, offer_sdp, (event) =>
+        this._handleOfferEvent(event)
+      );
+    } catch (err: any) {
+      this._error = "Failed to start WebRTC stream: " + err.message;
+      this._cleanUp();
+    }
+  };
+
+  private _iceConnectionStateChanged = () => {
+    console.timeLog(
+      "WebRTC",
+      "ice connection state change",
+      this._peerConnection?.iceConnectionState
+    );
+    if (this._peerConnection?.iceConnectionState === "failed") {
+      this._peerConnection.restartIce();
+    }
+  };
+
+  private async _handleOfferEvent(event: WebRtcOfferEvent) {
+    if (!this.entityid) {
+      return;
+    }
+    if (event.type === "session") {
+      this._sessionId = event.session_id;
+      this._candidatesList.forEach((candidate) =>
+        addWebRtcCandidate(
+          this.hass,
+          this.entityid!,
+          event.session_id,
+          candidate
+        )
+      );
+      this._candidatesList = [];
+    }
+    if (event.type === "answer") {
+      console.timeLog("WebRTC", "answer", event.answer);
+
+      this._handleAnswer(event);
+    }
+    if (event.type === "candidate") {
+      console.timeLog("WebRTC", "remote ice candidate", event.candidate);
+
+      try {
+        await this._peerConnection?.addIceCandidate(
+          new RTCIceCandidate({ candidate: event.candidate, sdpMid: "0" })
+        );
+      } catch (err: any) {
+        console.error(err);
+      }
+    }
+    if (event.type === "error") {
+      this._error = "Failed to start WebRTC stream: " + event.message;
+      this._cleanUp();
+    }
+  }
+
+  private _handleIceCandidate = (event: RTCPeerConnectionIceEvent) => {
+    if (!this.entityid || !event.candidate?.candidate) {
+      return;
+    }
+
+    console.timeLog(
+      "WebRTC",
+      "local ice candidate",
+      event.candidate?.candidate
+    );
+
+    if (this._sessionId) {
+      addWebRtcCandidate(
+        this.hass,
+        this.entityid,
+        this._sessionId,
+        event.candidate?.candidate
+      );
+    } else {
+      this._candidatesList.push(event.candidate?.candidate);
+    }
+  };
+
+  private _addTrack = async (event: RTCTrackEvent) => {
+    if (!this._remoteStream) {
+      return;
+    }
+    this._remoteStream.addTrack(event.track);
+    if (!this.hasUpdated) {
+      await this.updateComplete;
+    }
+    this._videoEl.srcObject = this._remoteStream;
+  };
+
+  private async _handleAnswer(event: WebRtcAnswer) {
+    if (
+      !this._peerConnection?.signalingState ||
+      ["stable", "closed"].includes(this._peerConnection.signalingState)
+    ) {
+      return;
+    }
 
     // Initiate the stream with the remote device
     const remoteDesc = new RTCSessionDescription({
       type: "answer",
-      sdp: webRtcAnswer.answer,
+      sdp: event.answer,
     });
     try {
       console.timeLog("WebRTC", "start setRemoteDescription", remoteDesc);
-      await peerConnection.setRemoteDescription(remoteDesc);
+      await this._peerConnection.setRemoteDescription(remoteDesc);
-      console.timeLog("WebRTC", "end setRemoteDescription");
     } catch (err: any) {
       this._error = "Failed to connect WebRTC stream: " + err.message;
-      peerConnection.close();
+      this._cleanUp();
-      return;
     }
-    this._peerConnection = peerConnection;
+    console.timeLog("WebRTC", "end setRemoteDescription");
   }
 
   private _cleanUp() {
+    console.timeLog("WebRTC", "stopped");
+    console.timeEnd("WebRTC");
+
     if (this._remoteStream) {
       this._remoteStream.getTracks().forEach((track) => {
         track.stop();
       });
+
       this._remoteStream = undefined;
     }
-    if (this._videoEl) {
+    const videoEl = this._videoEl;
-      this._videoEl.removeAttribute("src");
+    if (videoEl) {
-      this._videoEl.load();
+      videoEl.removeAttribute("src");
+      videoEl.load();
     }
     if (this._peerConnection) {
       this._peerConnection.close();
+
+      this._peerConnection.onnegotiationneeded = null;
+      this._peerConnection.onicecandidate = null;
+      this._peerConnection.oniceconnectionstatechange = null;
+      this._peerConnection.onicegatheringstatechange = null;
+      this._peerConnection.ontrack = null;
+
+      // just for debugging
+      this._peerConnection.onsignalingstatechange = null;
+
       this._peerConnection = undefined;
     }
+    this._unsub?.then((unsub) => unsub());
+    this._unsub = undefined;
+    this._sessionId = undefined;
+    this._candidatesList = [];
   }
 
   private _loadedData() {
-    console.timeLog("WebRTC", "loadedData");
-    console.timeEnd("WebRTC");
     // @ts-ignore
     fireEvent(this, "load");
+
+    console.timeLog("WebRTC", "loadedData");
+    console.timeEnd("WebRTC");
   }
 
   static get styles(): CSSResultGroup {

src/data/camera.ts

@@ -39,10 +39,37 @@ export interface Stream {
   url: string;
 }
 
+export type WebRtcOfferEvent =
+  | WebRtcId
+  | WebRtcAnswer
+  | WebRtcCandidate
+  | WebRtcError;
+
+export interface WebRtcId {
+  type: "session";
+  session_id: string;
+}
+
 export interface WebRtcAnswer {
+  type: "answer";
   answer: string;
 }
 
+export interface WebRtcCandidate {
+  type: "candidate";
+  candidate: string;
+}
+
+export interface WebRtcError {
+  type: "error";
+  code: string;
+  message: string;
+}
+
+export interface WebRtcOfferResponse {
+  id: string;
+}
+
 export const cameraUrlWithWidthHeight = (
   base_url: string,
   width: number,
@@ -94,15 +121,29 @@ export const fetchStreamUrl = async (
   return stream;
 };
 
-export const handleWebRtcOffer = (
+export const webRtcOffer = (
   hass: HomeAssistant,
-  entityId: string,
+  entity_id: string,
-  offer: string
+  offer: string,
+  callback: (event: WebRtcOfferEvent) => void
 ) =>
-  hass.callWS<WebRtcAnswer>({
+  hass.connection.subscribeMessage<WebRtcOfferEvent>(callback, {
-    type: "camera/web_rtc_offer",
+    type: "camera/webrtc/offer",
-    entity_id: entityId,
+    entity_id,
-    offer: offer,
+    offer,
+  });
+
+export const addWebRtcCandidate = (
+  hass: HomeAssistant,
+  entity_id: string,
+  session_id: string,
+  candidate: string
+) =>
+  hass.callWS({
+    type: "camera/webrtc/candidate",
+    entity_id,
+    session_id,
+    candidate,
   });
 
 export const fetchCameraPrefs = (hass: HomeAssistant, entityId: string) =>
@@ -137,6 +178,7 @@ export const getEntityIdFromCameraMediaSource = (mediaContentId: string) =>
 export interface WebRTCClientConfiguration {
   configuration: RTCConfiguration;
   dataChannel?: string;
+  getCandidatesUpfront: boolean;
 }
 
 export const fetchWebRtcClientConfiguration = async (