Add WebRTC stream player (#10193)

Co-authored-by: Bram Kragten <mail@bramkragten.nl>

src/components/ha-camera-stream.ts

@@ -15,9 +15,12 @@ import {
   CAMERA_SUPPORT_STREAM,
   computeMJPEGStreamUrl,
   fetchStreamUrl,
+  STREAM_TYPE_HLS,
+  STREAM_TYPE_WEB_RTC,
 } from "../data/camera";
 import { HomeAssistant } from "../types";
 import "./ha-hls-player";
+import "./ha-web-rtc-player";
 
 @customElement("ha-camera-stream")
 class HaCameraStream extends LitElement {
@@ -34,8 +37,8 @@ class HaCameraStream extends LitElement {
   @property({ type: Boolean, attribute: "allow-exoplayer" })
   public allowExoPlayer = false;
 
-  // We keep track if we should force MJPEG with a string
+  // We keep track if we should force MJPEG if there was a failure
-  // that way it automatically resets if we change entity.
+  // to get the HLS stream url. This is reset if we change entities.
   @state() private _forceMJPEG?: string;
 
   @state() private _url?: string;
@@ -48,7 +51,8 @@ class HaCameraStream extends LitElement {
       !this._shouldRenderMJPEG &&
       this.stateObj &&
       (changedProps.get("stateObj") as CameraEntity | undefined)?.entity_id !==
-        this.stateObj.entity_id
+        this.stateObj.entity_id &&
+      this.stateObj!.attributes.stream_type === STREAM_TYPE_HLS
     ) {
       this._forceMJPEG = undefined;
       this._url = undefined;
@@ -70,43 +74,64 @@ class HaCameraStream extends LitElement {
     if (!this.stateObj) {
       return html``;
     }
-
+    if (__DEMO__ || this._shouldRenderMJPEG) {
-    return html`
+      return html` <img
-      ${__DEMO__ || this._shouldRenderMJPEG
+        .src=${__DEMO__
-        ? html`
+          ? this.stateObj.attributes.entity_picture!
-            <img
+          : this._connected
-              .src=${__DEMO__
+          ? computeMJPEGStreamUrl(this.stateObj)
-                ? this.stateObj!.attributes.entity_picture!
+          : ""}
-                : this._connected
+        .alt=${`Preview of the ${computeStateName(this.stateObj)} camera.`}
-                ? computeMJPEGStreamUrl(this.stateObj)
+      />`;
-                : ""}
+    }
-              .alt=${`Preview of the ${computeStateName(
+    if (this.stateObj.attributes.stream_type === STREAM_TYPE_HLS && true) {
-                this.stateObj
+      return this._url
-              )} camera.`}
+        ? html` <ha-hls-player
-            />
+            autoplay
-          `
+            playsinline
-        : this._url
+            .allowExoPlayer=${this.allowExoPlayer}
-        ? html`
+            .muted=${this.muted}
-            <ha-hls-player
+            .controls=${this.controls}
-              autoplay
+            .hass=${this.hass}
-              playsinline
+            .url=${this._url}
-              .allowExoPlayer=${this.allowExoPlayer}
+          ></ha-hls-player>`
-              .muted=${this.muted}
+        : html``;
-              .controls=${this.controls}
+    }
-              .hass=${this.hass}
+    if (this.stateObj.attributes.stream_type === STREAM_TYPE_WEB_RTC) {
-              .url=${this._url}
+      return html` <ha-web-rtc-player
-            ></ha-hls-player>
+        autoplay
-          `
+        playsinline
-        : ""}
+        .muted=${this.muted}
-    `;
+        .controls=${this.controls}
+        .hass=${this.hass}
+        .entityid=${this.stateObj.entity_id}
+      ></ha-web-rtc-player>`;
+    }
+    return html``;
   }
 
   private get _shouldRenderMJPEG() {
-    return (
+    if (this._forceMJPEG === this.stateObj!.entity_id) {
-      this._forceMJPEG === this.stateObj!.entity_id ||
+      // Fallback when unable to fetch stream url
+      return true;
+    }
+    if (
       !isComponentLoaded(this.hass!, "stream") ||
       !supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM)
-    );
+    ) {
+      // Steaming is not supported by the camera so fallback to MJPEG stream
+      return true;
+    }
+    if (
+      this.stateObj!.attributes.stream_type === STREAM_TYPE_WEB_RTC &&
+      typeof RTCPeerConnection === "undefined"
+    ) {
+      // Stream requires WebRTC but browser does not support, so fallback to
+      // MJPEG stream.
+      return true;
+    }
+    // Render stream
+    return false;
   }
 
   private async _getStreamUrl(): Promise<void> {

src/components/ha-web-rtc-player.ts

@@ -0,0 +1,135 @@
+import {
+  css,
+  CSSResultGroup,
+  html,
+  LitElement,
+  PropertyValues,
+  TemplateResult,
+} from "lit";
+import { customElement, property, state, query } from "lit/decorators";
+import { handleWebRtcOffer, WebRtcAnswer } from "../data/camera";
+import type { HomeAssistant } from "../types";
+import "./ha-alert";
+
+/**
+ * A WebRTC stream is established by first sending an offer through a signal
+ * path via an integration. An answer is returned, then the rest of the stream
+ * is handled entirely client side.
+ */
+@customElement("ha-web-rtc-player")
+class HaWebRtcPlayer extends LitElement {
+  @property({ attribute: false }) public hass!: HomeAssistant;
+
+  @property() public entityid!: string;
+
+  @property({ type: Boolean, attribute: "controls" })
+  public controls = false;
+
+  @property({ type: Boolean, attribute: "muted" })
+  public muted = false;
+
+  @property({ type: Boolean, attribute: "autoplay" })
+  public autoPlay = false;
+
+  @property({ type: Boolean, attribute: "playsinline" })
+  public playsInline = false;
+
+  @state() private _error?: string;
+
+  // don't cache this, as we remove it on disconnects
+  @query("#remote-stream") private _videoEl!: HTMLVideoElement;
+
+  protected render(): TemplateResult {
+    if (this._error) {
+      return html`<ha-alert alert-type="error">${this._error}</ha-alert>`;
+    }
+    return html`
+      <video
+        id="remote-stream"
+        ?autoplay=${this.autoPlay}
+        .muted=${this.muted}
+        ?playsinline=${this.playsInline}
+        ?controls=${this.controls}
+      ></video>
+    `;
+  }
+
+  public disconnectedCallback() {
+    super.disconnectedCallback();
+    this._cleanUp();
+  }
+
+  protected updated(changedProperties: PropertyValues<this>) {
+    if (!changedProperties.has("entityid")) {
+      return;
+    }
+    if (!this._videoEl) {
+      return;
+    }
+    this._startWebRtc();
+  }
+
+  private async _startWebRtc(): Promise<void> {
+    this._error = undefined;
+    const peerConnection = new RTCPeerConnection();
+    // Some cameras (such as nest) require a data channel to establish a stream
+    // however, not used by any integrations.
+    peerConnection.createDataChannel("dataSendChannel");
+    const offerOptions: RTCOfferOptions = {
+      offerToReceiveAudio: true,
+      offerToReceiveVideo: true,
+    };
+    const offer: RTCSessionDescriptionInit = await peerConnection.createOffer(
+      offerOptions
+    );
+    await peerConnection.setLocalDescription(offer);
+
+    let webRtcAnswer: WebRtcAnswer;
+    try {
+      webRtcAnswer = await handleWebRtcOffer(
+        this.hass,
+        this.entityid,
+        offer.sdp!
+      );
+    } catch (err: any) {
+      this._error = "Failed to start WebRTC stream: " + err.message;
+      return;
+    }
+
+    // Setup callbacks to render remote stream once media tracks are discovered.
+    const remoteStream = new MediaStream();
+    peerConnection.addEventListener("track", (event) => {
+      remoteStream.addTrack(event.track);
+      this._videoEl.srcObject = remoteStream;
+    });
+
+    // Initiate the stream with the remote device
+    const remoteDesc = new RTCSessionDescription({
+      type: "answer",
+      sdp: webRtcAnswer.answer,
+    });
+    await peerConnection.setRemoteDescription(remoteDesc);
+  }
+
+  private _cleanUp() {
+    if (this._videoEl) {
+      const videoEl = this._videoEl;
+      videoEl.removeAttribute("src");
+      videoEl.load();
+    }
+  }
+
+  static get styles(): CSSResultGroup {
+    return css`
+      video {
+        display: block;
+      }
+    `;
+  }
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "ha-web-rtc-player": HaWebRtcPlayer;
+  }
+}

src/data/camera.ts

@@ -9,11 +9,15 @@ import { getSignedPath } from "./auth";
 export const CAMERA_SUPPORT_ON_OFF = 1;
 export const CAMERA_SUPPORT_STREAM = 2;
 
+export const STREAM_TYPE_HLS = "hls";
+export const STREAM_TYPE_WEB_RTC = "web_rtc";
+
 interface CameraEntityAttributes extends HassEntityAttributeBase {
   model_name: string;
   access_token: string;
   brand: string;
   motion_detection: boolean;
+  stream_type: string;
 }
 
 export interface CameraEntity extends HassEntityBase {
@@ -33,6 +37,10 @@ export interface Stream {
   url: string;
 }
 
+export interface WebRtcAnswer {
+  answer: string;
+}
+
 export const computeMJPEGStreamUrl = (entity: CameraEntity) =>
   `/api/camera_proxy_stream/${entity.entity_id}?token=${entity.attributes.access_token}`;
 
@@ -78,6 +86,17 @@ export const fetchStreamUrl = async (
   return stream;
 };
 
+export const handleWebRtcOffer = (
+  hass: HomeAssistant,
+  entityId: string,
+  offer: string
+) =>
+  hass.callWS<WebRtcAnswer>({
+    type: "camera/web_rtc_offer",
+    entity_id: entityId,
+    offer: offer,
+  });
+
 export const fetchCameraPrefs = (hass: HomeAssistant, entityId: string) =>
   hass.callWS<CameraPreferences>({
     type: "camera/get_prefs",

src/dialogs/more-info/controls/more-info-camera.ts

@@ -17,6 +17,7 @@ import {
   CameraPreferences,
   CAMERA_SUPPORT_STREAM,
   fetchCameraPrefs,
+  STREAM_TYPE_HLS,
   updateCameraPrefs,
 } from "../../../data/camera";
 import type { HomeAssistant } from "../../../types";
@@ -82,7 +83,10 @@ class MoreInfoCamera extends LitElement {
     if (
       curEntityId &&
       isComponentLoaded(this.hass!, "stream") &&
-      supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM)
+      supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM) &&
+      // The stream component for HLS streams supports a server-side pre-load
+      // option that client initiated WebRTC streams do not
+      this.stateObj!.attributes.stream_type === STREAM_TYPE_HLS
     ) {
       // Fetch in background while we set up the video.
       this._fetchCameraPrefs();