Try both HLS and webRTC and pick best stream (#22585)

src/components/ha-camera-stream.ts

@@ -3,21 +3,22 @@ import {
   CSSResultGroup,
   html,
   LitElement,
-  PropertyValues,
   nothing,
+  PropertyValues,
 } from "lit";
 import { customElement, property, state } from "lit/decorators";
-import { isComponentLoaded } from "../common/config/is_component_loaded";
 import { computeStateName } from "../common/entity/compute_state_name";
 import { supportsFeature } from "../common/entity/supports-feature";
 import {
-  CameraEntity,
   CAMERA_SUPPORT_STREAM,
+  CameraCapabilities,
+  CameraEntity,
   computeMJPEGStreamUrl,
-  fetchStreamUrl,
+  fetchCameraCapabilities,
   fetchThumbnailUrlWithCache,
   STREAM_TYPE_HLS,
   STREAM_TYPE_WEB_RTC,
+  StreamType,
 } from "../data/camera";
 import { HomeAssistant } from "../types";
 import "./ha-hls-player";
@@ -41,14 +42,16 @@ export class HaCameraStream extends LitElement {
   // Video background image before its loaded
   @state() private _posterUrl?: string;
 
-  // We keep track if we should force MJPEG if there was a failure
-  // to get the HLS stream url. This is reset if we change entities.
-  @state() private _forceMJPEG?: string;
-
-  @state() private _url?: string;
-
   @state() private _connected = false;
 
+  @state() private _capabilities?: CameraCapabilities;
+
+  @state() private _streamType?: StreamType;
+
+  @state() private _hlsStreams?: { hasAudio: boolean; hasVideo: boolean };
+
+  @state() private _webRtcStreams?: { hasAudio: boolean; hasVideo: boolean };
+
   public willUpdate(changedProps: PropertyValues): void {
     if (
       changedProps.has("stateObj") &&
@@ -57,12 +60,8 @@ export class HaCameraStream extends LitElement {
       (changedProps.get("stateObj") as CameraEntity | undefined)?.entity_id !==
         this.stateObj.entity_id
     ) {
+      this._getCapabilities();
       this._getPosterUrl();
-      if (this.stateObj!.attributes.frontend_stream_type === STREAM_TYPE_HLS) {
-        this._forceMJPEG = undefined;
-        this._url = undefined;
-        this._getStreamUrl();
-      }
     }
   }
 
@@ -87,54 +86,79 @@ export class HaCameraStream extends LitElement {
           : this._connected
             ? computeMJPEGStreamUrl(this.stateObj)
             : ""}
-        .alt=${`Preview of the ${computeStateName(this.stateObj)} camera.`}
+        alt=${`Preview of the ${computeStateName(this.stateObj)} camera.`}
       />`;
     }
-    if (this.stateObj.attributes.frontend_stream_type === STREAM_TYPE_HLS) {
+    return html`${this._streamType === STREAM_TYPE_HLS ||
-      return this._url
+    (!this._streamType &&
-        ? html`<ha-hls-player
+      this._capabilities?.frontend_stream_types.includes(STREAM_TYPE_HLS))
-            autoplay
+      ? html`<ha-hls-player
-            playsinline
+          autoplay
-            .allowExoPlayer=${this.allowExoPlayer}
+          playsinline
-            .muted=${this.muted}
+          .allowExoPlayer=${this.allowExoPlayer}
-            .controls=${this.controls}
+          .muted=${this.muted}
-            .hass=${this.hass}
+          .controls=${this.controls}
-            .url=${this._url}
+          .hass=${this.hass}
-            .posterUrl=${this._posterUrl}
+          .entityid=${this.stateObj.entity_id}
-          ></ha-hls-player>`
+          .posterUrl=${this._posterUrl}
-        : nothing;
+          @streams=${this._handleHlsStreams}
+          class=${!this._streamType && this._webRtcStreams ? "hidden" : ""}
+        ></ha-hls-player>`
+      : nothing}
+    ${this._streamType === STREAM_TYPE_WEB_RTC ||
+    (!this._streamType &&
+      this._capabilities?.frontend_stream_types.includes(STREAM_TYPE_WEB_RTC))
+      ? html`<ha-web-rtc-player
+          autoplay
+          playsinline
+          .muted=${this.muted}
+          .controls=${this.controls}
+          .hass=${this.hass}
+          .entityid=${this.stateObj.entity_id}
+          .posterUrl=${this._posterUrl}
+          @streams=${this._handleWebRtcStreams}
+          class=${this._streamType !== STREAM_TYPE_WEB_RTC &&
+          !this._webRtcStreams
+            ? "hidden"
+            : ""}
+        ></ha-web-rtc-player>`
+      : nothing}`;
+  }
+
+  private async _getCapabilities() {
+    this._capabilities = undefined;
+    this._hlsStreams = undefined;
+    this._webRtcStreams = undefined;
+    if (!supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM)) {
+      return;
     }
-    if (this.stateObj.attributes.frontend_stream_type === STREAM_TYPE_WEB_RTC) {
+    this._capabilities = await fetchCameraCapabilities(
-      return html`<ha-web-rtc-player
+      this.hass!,
-        autoplay
+      this.stateObj!.entity_id
-        playsinline
+    );
-        .muted=${this.muted}
+    if (this._capabilities.frontend_stream_types.length === 1) {
-        .controls=${this.controls}
+      this._streamType = this._capabilities.frontend_stream_types[0];
-        .hass=${this.hass}
-        .entityid=${this.stateObj.entity_id}
-        .posterUrl=${this._posterUrl}
-      ></ha-web-rtc-player>`;
     }
-    return nothing;
   }
 
   private get _shouldRenderMJPEG() {
-    if (this._forceMJPEG === this.stateObj!.entity_id) {
-      // Fallback when unable to fetch stream url
-      return true;
-    }
     if (!supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM)) {
       // Steaming is not supported by the camera so fallback to MJPEG stream
       return true;
     }
     if (
-      this.stateObj!.attributes.frontend_stream_type === STREAM_TYPE_WEB_RTC
+      this._capabilities &&
+      (!this._capabilities.frontend_stream_types.includes(STREAM_TYPE_HLS) ||
+        this._hlsStreams?.hasVideo === false) &&
+      (!this._capabilities.frontend_stream_types.includes(
+        STREAM_TYPE_WEB_RTC
+      ) ||
+        this._webRtcStreams?.hasVideo === false)
     ) {
-      // Browser support required for WebRTC
+      // No video in HLS stream and no video in WebRTC stream
-      return typeof RTCPeerConnection === "undefined";
+      return true;
     }
-    // Server side stream component required for HLS
+    return false;
-    return !isComponentLoaded(this.hass!, "stream");
   }
 
   private async _getPosterUrl(): Promise<void> {
@@ -151,20 +175,28 @@ export class HaCameraStream extends LitElement {
     }
   }
 
-  private async _getStreamUrl(): Promise<void> {
+  private _handleHlsStreams(ev: CustomEvent) {
-    try {
+    this._hlsStreams = ev.detail;
-      const { url } = await fetchStreamUrl(
+    this._pickStreamType();
-        this.hass!,
+  }
-        this.stateObj!.entity_id
-      );
 
-      this._url = url;
+  private _handleWebRtcStreams(ev: CustomEvent) {
-    } catch (err: any) {
+    this._webRtcStreams = ev.detail;
-      // Fails if we were unable to get a stream
+    this._pickStreamType();
-      // eslint-disable-next-line
+  }
-      console.error(err);
 
-      this._forceMJPEG = this.stateObj!.entity_id;
+  private _pickStreamType() {
+    if (!this._hlsStreams || !this._webRtcStreams) {
+      return;
+    }
+    if (
+      this._hlsStreams.hasVideo &&
+      this._hlsStreams.hasAudio &&
+      !this._webRtcStreams.hasAudio
+    ) {
+      this._streamType = STREAM_TYPE_HLS;
+    } else if (this._webRtcStreams.hasVideo) {
+      this._streamType = STREAM_TYPE_WEB_RTC;
     }
   }
 
@@ -178,6 +210,10 @@ export class HaCameraStream extends LitElement {
       img {
         width: 100%;
       }
+
+      .hidden {
+        display: none;
+      }
     `;
   }
 }
@@ -186,4 +222,12 @@ declare global {
   interface HTMLElementTagNameMap {
     "ha-camera-stream": HaCameraStream;
   }
+  interface HASSDomEvents {
+    load: undefined;
+    streams: {
+      hasAudio: boolean;
+      hasVideo: boolean;
+      codecs?: string[];
+    };
+  }
 }

src/components/ha-hls-player.ts

@@ -12,6 +12,8 @@ import { fireEvent } from "../common/dom/fire_event";
 import { nextRender } from "../common/util/render-status";
 import type { HomeAssistant } from "../types";
 import "./ha-alert";
+import { fetchStreamUrl } from "../data/camera";
+import { isComponentLoaded } from "../common/config/is_component_loaded";
 
 type HlsLite = Omit<
   HlsType,
@@ -22,9 +24,9 @@ type HlsLite = Omit<
 class HaHLSPlayer extends LitElement {
   @property({ attribute: false }) public hass!: HomeAssistant;
 
-  @property() public url!: string;
+  @property() public entityid?: string;
 
-  @property() public posterUrl!: string;
+  @property({ attribute: "poster-url" }) public posterUrl?: string;
 
   @property({ type: Boolean, attribute: "controls" })
   public controls = false;
@@ -48,6 +50,8 @@ class HaHLSPlayer extends LitElement {
 
   @state() private _errorIsFatal = false;
 
+  @state() private _url!: string;
+
   private _hlsPolyfillInstance?: HlsLite;
 
   private _exoPlayer = false;
@@ -95,19 +99,44 @@ class HaHLSPlayer extends LitElement {
   protected updated(changedProps: PropertyValues) {
     super.updated(changedProps);
 
-    const urlChanged = changedProps.has("url");
+    const entityChanged = changedProps.has("entityid");
 
-    if (!urlChanged) {
+    if (!entityChanged) {
+      return;
+    }
+    this._getStreamUrl();
+  }
+
+  private async _getStreamUrl(): Promise<void> {
+    this._cleanUp();
+    this._resetError();
+
+    if (!isComponentLoaded(this.hass!, "stream")) {
+      this._setFatalError("Streaming component is not loaded.");
       return;
     }
 
-    this._cleanUp();
+    if (!this.entityid) {
-    this._resetError();
+      return;
-    this._startHls();
+    }
+    try {
+      const { url } = await fetchStreamUrl(this.hass!, this.entityid);
+
+      this._url = url;
+      this._cleanUp();
+      this._resetError();
+      this._startHls();
+    } catch (err: any) {
+      // Fails if we were unable to get a stream
+      // eslint-disable-next-line
+      console.error(err);
+
+      fireEvent(this, "streams", { hasAudio: false, hasVideo: false });
+    }
   }
 
   private async _startHls(): Promise<void> {
-    const masterPlaylistPromise = fetch(this.url);
+    const masterPlaylistPromise = fetch(this._url);
 
     const Hls: typeof HlsType = (await import("hls.js/dist/hls.light.mjs"))
       .default;
@@ -138,10 +167,10 @@ class HaHLSPlayer extends LitElement {
       return;
     }
 
-    // Parse playlist assuming it is a master playlist. Match group 1 is whether hevc, match group 2 is regular playlist url
+    // Parse playlist assuming it is a master playlist. Match group 1 and 2 are codec, match group 3 is regular playlist url
     // See https://tools.ietf.org/html/rfc8216 for HLS spec details
     const playlistRegexp =
-      /#EXT-X-STREAM-INF:.*?(?:CODECS=".*?(hev1|hvc1)?\..*?".*?)?(?:\n|\r\n)(.+)/g;
+      /#EXT-X-STREAM-INF:.*?(?:CODECS=".*?([^.]*)?\..*?,([^.]*)?\..*?".*?)?(?:\n|\r\n)(.+)/g;
     const match = playlistRegexp.exec(masterPlaylist);
     const matchTwice = playlistRegexp.exec(masterPlaylist);
 
@@ -150,13 +179,20 @@ class HaHLSPlayer extends LitElement {
     let playlist_url: string;
     if (match !== null && matchTwice === null) {
       // Only send the regular playlist url if we match exactly once
-      playlist_url = new URL(match[2], this.url).href;
+      playlist_url = new URL(match[3], this._url).href;
     } else {
-      playlist_url = this.url;
+      playlist_url = this._url;
     }
 
+    const codecs = match ? `${match[1]},${match[2]}` : undefined;
+
+    this._reportStreams(codecs);
+
     // If codec is HEVC and ExoPlayer is supported, use ExoPlayer.
-    if (useExoPlayer && match !== null && match[1] !== undefined) {
+    if (
+      useExoPlayer &&
+      (codecs?.includes("hevc") || codecs?.includes("hev1"))
+    ) {
       this._renderHLSExoPlayer(playlist_url);
     } else if (Hls.isSupported()) {
       this._renderHLSPolyfill(this._videoEl, Hls, playlist_url);
@@ -313,15 +349,26 @@ class HaHLSPlayer extends LitElement {
   private _setFatalError(errorMessage: string) {
     this._error = errorMessage;
     this._errorIsFatal = true;
+    fireEvent(this, "streams", { hasAudio: false, hasVideo: false });
   }
 
   private _setRetryableError(errorMessage: string) {
     this._error = errorMessage;
     this._errorIsFatal = false;
+    fireEvent(this, "streams", { hasAudio: false, hasVideo: false });
+  }
+
+  private _reportStreams(codecs?: string) {
+    const codec = codecs?.split(",");
+    fireEvent(this, "streams", {
+      hasAudio: codec?.includes("mp4a") ?? false,
+      hasVideo: codec?.includes("mp4a")
+        ? codec?.length > 1
+        : Boolean(codec?.length),
+    });
   }
 
   private _loadedData() {
-    // @ts-ignore
     fireEvent(this, "load");
   }
 

src/components/ha-web-rtc-player.ts

@@ -103,6 +103,13 @@ class HaWebRtcPlayer extends LitElement {
   private async _startWebRtc(): Promise<void> {
     this._cleanUp();
 
+    // Browser support required for WebRTC
+    if (typeof RTCPeerConnection === "undefined") {
+      this._error = "WebRTC is not supported in this browser";
+      fireEvent(this, "streams", { hasAudio: false, hasVideo: false });
+      return;
+    }
+
     if (!this.hass || !this.entityid) {
       return;
     }
@@ -379,11 +386,17 @@ class HaWebRtcPlayer extends LitElement {
   }
 
   private _loadedData() {
-    // @ts-ignore
-    fireEvent(this, "load");
-
     console.timeLog("WebRTC", "loadedData");
     console.timeEnd("WebRTC");
+
+    const video = this._videoEl;
+    const stream = video.srcObject as MediaStream;
+
+    fireEvent(this, "load");
+    fireEvent(this, "streams", {
+      hasAudio: Boolean(stream?.getAudioTracks().length),
+      hasVideo: Boolean(stream?.getVideoTracks().length),
+    });
   }
 
   static get styles(): CSSResultGroup {

src/data/camera.ts

@@ -13,6 +13,8 @@ export const CAMERA_SUPPORT_STREAM = 2;
 export const STREAM_TYPE_HLS = "hls";
 export const STREAM_TYPE_WEB_RTC = "web_rtc";
 
+export type StreamType = typeof STREAM_TYPE_HLS | typeof STREAM_TYPE_WEB_RTC;
+
 interface CameraEntityAttributes extends HassEntityAttributeBase {
   model_name: string;
   access_token: string;
@@ -175,6 +177,16 @@ export const isCameraMediaSource = (mediaContentId: string) =>
 export const getEntityIdFromCameraMediaSource = (mediaContentId: string) =>
   mediaContentId.substring(CAMERA_MEDIA_SOURCE_PREFIX.length);
 
+export interface CameraCapabilities {
+  frontend_stream_types: StreamType[];
+}
+
+export const fetchCameraCapabilities = async (
+  hass: HomeAssistant,
+  entity_id: string
+) =>
+  hass.callWS<CameraCapabilities>({ type: "camera/capabilities", entity_id });
+
 export interface WebRTCClientConfiguration {
   configuration: RTCConfiguration;
   dataChannel?: string;

src/panels/config/entities/editor-tabs/settings/entity-settings-helper-tab.ts

@@ -100,8 +100,8 @@ export class EntitySettingsHelperTab extends LitElement {
           .entry=${this.entry}
           .disabled=${this._submitting}
           @change=${this._entityRegistryChanged}
-          hideName
+          hide-name
-          hideIcon
+          hide-icon
         ></entity-registry-settings-editor>
       </div>
       <div class="buttons">

src/panels/config/entities/entity-registry-settings-editor.ts

@@ -44,6 +44,7 @@ import {
   CAMERA_ORIENTATIONS,
   CAMERA_SUPPORT_STREAM,
   CameraPreferences,
+  fetchCameraCapabilities,
   fetchCameraPrefs,
   STREAM_TYPE_HLS,
   updateCameraPrefs,
@@ -145,9 +146,9 @@ export class EntityRegistrySettingsEditor extends LitElement {
 
   @property({ type: Object }) public entry!: ExtEntityRegistryEntry;
 
-  @property({ type: Boolean }) public hideName = false;
+  @property({ type: Boolean, attribute: "hide-name" }) public hideName = false;
 
-  @property({ type: Boolean }) public hideIcon = false;
+  @property({ type: Boolean, attribute: "hide-icon" }) public hideIcon = false;
 
   @property({ type: Boolean }) public disabled = false;
 
@@ -236,13 +237,7 @@ export class EntityRegistrySettingsEditor extends LitElement {
     if (domain === "camera" && isComponentLoaded(this.hass, "stream")) {
       const stateObj: HassEntity | undefined =
         this.hass.states[this.entry.entity_id];
-      if (
+      if (stateObj && supportsFeature(stateObj, CAMERA_SUPPORT_STREAM)) {
-        stateObj &&
-        supportsFeature(stateObj, CAMERA_SUPPORT_STREAM) &&
-        // The stream component for HLS streams supports a server-side pre-load
-        // option that client initiated WebRTC streams do not
-        stateObj.attributes.frontend_stream_type === STREAM_TYPE_HLS
-      ) {
         this._fetchCameraPrefs();
       }
     }
@@ -1396,6 +1391,19 @@ export class EntityRegistrySettingsEditor extends LitElement {
   }
 
   private async _fetchCameraPrefs() {
+    const capabilities = await fetchCameraCapabilities(
+      this.hass,
+      this.entry.entity_id
+    );
+
+    // The stream component for HLS streams supports a server-side pre-load
+    // option that client initiated WebRTC streams do not
+
+    if (!capabilities.frontend_stream_types.includes(STREAM_TYPE_HLS)) {
+      this._cameraPrefs = undefined;
+      return;
+    }
+
     this._cameraPrefs = await fetchCameraPrefs(this.hass, this.entry.entity_id);
   }