From 67852125e5f2aa74b8b92d4279138807f390374b Mon Sep 17 00:00:00 2001 From: Bram Kragten Date: Tue, 29 Oct 2024 21:38:22 +0100 Subject: [PATCH] Try both HLS and webRTC and pick best stream (#22585) --- src/components/ha-camera-stream.ts | 166 +++++++++++------- src/components/ha-hls-player.ts | 75 ++++++-- src/components/ha-web-rtc-player.ts | 19 +- src/data/camera.ts | 12 ++ .../settings/entity-settings-helper-tab.ts | 4 +- .../entity-registry-settings-editor.ts | 26 ++- 6 files changed, 213 insertions(+), 89 deletions(-) diff --git a/src/components/ha-camera-stream.ts b/src/components/ha-camera-stream.ts index d17ca1f2c4..3d67eb9e89 100644 --- a/src/components/ha-camera-stream.ts +++ b/src/components/ha-camera-stream.ts @@ -3,21 +3,22 @@ import { CSSResultGroup, html, LitElement, - PropertyValues, nothing, + PropertyValues, } from "lit"; import { customElement, property, state } from "lit/decorators"; -import { isComponentLoaded } from "../common/config/is_component_loaded"; import { computeStateName } from "../common/entity/compute_state_name"; import { supportsFeature } from "../common/entity/supports-feature"; import { - CameraEntity, CAMERA_SUPPORT_STREAM, + CameraCapabilities, + CameraEntity, computeMJPEGStreamUrl, - fetchStreamUrl, + fetchCameraCapabilities, fetchThumbnailUrlWithCache, STREAM_TYPE_HLS, STREAM_TYPE_WEB_RTC, + StreamType, } from "../data/camera"; import { HomeAssistant } from "../types"; import "./ha-hls-player"; @@ -41,14 +42,16 @@ export class HaCameraStream extends LitElement { // Video background image before its loaded @state() private _posterUrl?: string; - // We keep track if we should force MJPEG if there was a failure - // to get the HLS stream url. This is reset if we change entities. - @state() private _forceMJPEG?: string; - - @state() private _url?: string; - @state() private _connected = false; + @state() private _capabilities?: CameraCapabilities; + + @state() private _streamType?: StreamType; + + @state() private _hlsStreams?: { hasAudio: boolean; hasVideo: boolean }; + + @state() private _webRtcStreams?: { hasAudio: boolean; hasVideo: boolean }; + public willUpdate(changedProps: PropertyValues): void { if ( changedProps.has("stateObj") && @@ -57,12 +60,8 @@ export class HaCameraStream extends LitElement { (changedProps.get("stateObj") as CameraEntity | undefined)?.entity_id !== this.stateObj.entity_id ) { + this._getCapabilities(); this._getPosterUrl(); - if (this.stateObj!.attributes.frontend_stream_type === STREAM_TYPE_HLS) { - this._forceMJPEG = undefined; - this._url = undefined; - this._getStreamUrl(); - } } } @@ -87,54 +86,79 @@ export class HaCameraStream extends LitElement { : this._connected ? computeMJPEGStreamUrl(this.stateObj) : ""} - .alt=${`Preview of the ${computeStateName(this.stateObj)} camera.`} + alt=${`Preview of the ${computeStateName(this.stateObj)} camera.`} />`; } - if (this.stateObj.attributes.frontend_stream_type === STREAM_TYPE_HLS) { - return this._url - ? html`` - : nothing; + return html`${this._streamType === STREAM_TYPE_HLS || + (!this._streamType && + this._capabilities?.frontend_stream_types.includes(STREAM_TYPE_HLS)) + ? html`` + : nothing} + ${this._streamType === STREAM_TYPE_WEB_RTC || + (!this._streamType && + this._capabilities?.frontend_stream_types.includes(STREAM_TYPE_WEB_RTC)) + ? html`` + : nothing}`; + } + + private async _getCapabilities() { + this._capabilities = undefined; + this._hlsStreams = undefined; + this._webRtcStreams = undefined; + if (!supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM)) { + return; } - if (this.stateObj.attributes.frontend_stream_type === STREAM_TYPE_WEB_RTC) { - return html``; + this._capabilities = await fetchCameraCapabilities( + this.hass!, + this.stateObj!.entity_id + ); + if (this._capabilities.frontend_stream_types.length === 1) { + this._streamType = this._capabilities.frontend_stream_types[0]; } - return nothing; } private get _shouldRenderMJPEG() { - if (this._forceMJPEG === this.stateObj!.entity_id) { - // Fallback when unable to fetch stream url - return true; - } if (!supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM)) { // Steaming is not supported by the camera so fallback to MJPEG stream return true; } if ( - this.stateObj!.attributes.frontend_stream_type === STREAM_TYPE_WEB_RTC + this._capabilities && + (!this._capabilities.frontend_stream_types.includes(STREAM_TYPE_HLS) || + this._hlsStreams?.hasVideo === false) && + (!this._capabilities.frontend_stream_types.includes( + STREAM_TYPE_WEB_RTC + ) || + this._webRtcStreams?.hasVideo === false) ) { - // Browser support required for WebRTC - return typeof RTCPeerConnection === "undefined"; + // No video in HLS stream and no video in WebRTC stream + return true; } - // Server side stream component required for HLS - return !isComponentLoaded(this.hass!, "stream"); + return false; } private async _getPosterUrl(): Promise { @@ -151,20 +175,28 @@ export class HaCameraStream extends LitElement { } } - private async _getStreamUrl(): Promise { - try { - const { url } = await fetchStreamUrl( - this.hass!, - this.stateObj!.entity_id - ); + private _handleHlsStreams(ev: CustomEvent) { + this._hlsStreams = ev.detail; + this._pickStreamType(); + } - this._url = url; - } catch (err: any) { - // Fails if we were unable to get a stream - // eslint-disable-next-line - console.error(err); + private _handleWebRtcStreams(ev: CustomEvent) { + this._webRtcStreams = ev.detail; + this._pickStreamType(); + } - this._forceMJPEG = this.stateObj!.entity_id; + private _pickStreamType() { + if (!this._hlsStreams || !this._webRtcStreams) { + return; + } + if ( + this._hlsStreams.hasVideo && + this._hlsStreams.hasAudio && + !this._webRtcStreams.hasAudio + ) { + this._streamType = STREAM_TYPE_HLS; + } else if (this._webRtcStreams.hasVideo) { + this._streamType = STREAM_TYPE_WEB_RTC; } } @@ -178,6 +210,10 @@ export class HaCameraStream extends LitElement { img { width: 100%; } + + .hidden { + display: none; + } `; } } @@ -186,4 +222,12 @@ declare global { interface HTMLElementTagNameMap { "ha-camera-stream": HaCameraStream; } + interface HASSDomEvents { + load: undefined; + streams: { + hasAudio: boolean; + hasVideo: boolean; + codecs?: string[]; + }; + } } diff --git a/src/components/ha-hls-player.ts b/src/components/ha-hls-player.ts index cf3ed5a526..056e038892 100644 --- a/src/components/ha-hls-player.ts +++ b/src/components/ha-hls-player.ts @@ -12,6 +12,8 @@ import { fireEvent } from "../common/dom/fire_event"; import { nextRender } from "../common/util/render-status"; import type { HomeAssistant } from "../types"; import "./ha-alert"; +import { fetchStreamUrl } from "../data/camera"; +import { isComponentLoaded } from "../common/config/is_component_loaded"; type HlsLite = Omit< HlsType, @@ -22,9 +24,9 @@ type HlsLite = Omit< class HaHLSPlayer extends LitElement { @property({ attribute: false }) public hass!: HomeAssistant; - @property() public url!: string; + @property() public entityid?: string; - @property() public posterUrl!: string; + @property({ attribute: "poster-url" }) public posterUrl?: string; @property({ type: Boolean, attribute: "controls" }) public controls = false; @@ -48,6 +50,8 @@ class HaHLSPlayer extends LitElement { @state() private _errorIsFatal = false; + @state() private _url!: string; + private _hlsPolyfillInstance?: HlsLite; private _exoPlayer = false; @@ -95,19 +99,44 @@ class HaHLSPlayer extends LitElement { protected updated(changedProps: PropertyValues) { super.updated(changedProps); - const urlChanged = changedProps.has("url"); + const entityChanged = changedProps.has("entityid"); - if (!urlChanged) { + if (!entityChanged) { + return; + } + this._getStreamUrl(); + } + + private async _getStreamUrl(): Promise { + this._cleanUp(); + this._resetError(); + + if (!isComponentLoaded(this.hass!, "stream")) { + this._setFatalError("Streaming component is not loaded."); return; } - this._cleanUp(); - this._resetError(); - this._startHls(); + if (!this.entityid) { + return; + } + try { + const { url } = await fetchStreamUrl(this.hass!, this.entityid); + + this._url = url; + this._cleanUp(); + this._resetError(); + this._startHls(); + } catch (err: any) { + // Fails if we were unable to get a stream + // eslint-disable-next-line + console.error(err); + + fireEvent(this, "streams", { hasAudio: false, hasVideo: false }); + } } private async _startHls(): Promise { - const masterPlaylistPromise = fetch(this.url); + const masterPlaylistPromise = fetch(this._url); const Hls: typeof HlsType = (await import("hls.js/dist/hls.light.mjs")) .default; @@ -138,10 +167,10 @@ class HaHLSPlayer extends LitElement { return; } - // Parse playlist assuming it is a master playlist. Match group 1 is whether hevc, match group 2 is regular playlist url + // Parse playlist assuming it is a master playlist. Match group 1 and 2 are codec, match group 3 is regular playlist url // See https://tools.ietf.org/html/rfc8216 for HLS spec details const playlistRegexp = - /#EXT-X-STREAM-INF:.*?(?:CODECS=".*?(hev1|hvc1)?\..*?".*?)?(?:\n|\r\n)(.+)/g; + /#EXT-X-STREAM-INF:.*?(?:CODECS=".*?([^.]*)?\..*?,([^.]*)?\..*?".*?)?(?:\n|\r\n)(.+)/g; const match = playlistRegexp.exec(masterPlaylist); const matchTwice = playlistRegexp.exec(masterPlaylist); @@ -150,13 +179,20 @@ class HaHLSPlayer extends LitElement { let playlist_url: string; if (match !== null && matchTwice === null) { // Only send the regular playlist url if we match exactly once - playlist_url = new URL(match[2], this.url).href; + playlist_url = new URL(match[3], this._url).href; } else { - playlist_url = this.url; + playlist_url = this._url; } + const codecs = match ? `${match[1]},${match[2]}` : undefined; + + this._reportStreams(codecs); + // If codec is HEVC and ExoPlayer is supported, use ExoPlayer. - if (useExoPlayer && match !== null && match[1] !== undefined) { + if ( + useExoPlayer && + (codecs?.includes("hevc") || codecs?.includes("hev1")) + ) { this._renderHLSExoPlayer(playlist_url); } else if (Hls.isSupported()) { this._renderHLSPolyfill(this._videoEl, Hls, playlist_url); @@ -313,15 +349,26 @@ class HaHLSPlayer extends LitElement { private _setFatalError(errorMessage: string) { this._error = errorMessage; this._errorIsFatal = true; + fireEvent(this, "streams", { hasAudio: false, hasVideo: false }); } private _setRetryableError(errorMessage: string) { this._error = errorMessage; this._errorIsFatal = false; + fireEvent(this, "streams", { hasAudio: false, hasVideo: false }); + } + + private _reportStreams(codecs?: string) { + const codec = codecs?.split(","); + fireEvent(this, "streams", { + hasAudio: codec?.includes("mp4a") ?? false, + hasVideo: codec?.includes("mp4a") + ? codec?.length > 1 + : Boolean(codec?.length), + }); } private _loadedData() { - // @ts-ignore fireEvent(this, "load"); } diff --git a/src/components/ha-web-rtc-player.ts b/src/components/ha-web-rtc-player.ts index a2ae49e472..562bb382ff 100644 --- a/src/components/ha-web-rtc-player.ts +++ b/src/components/ha-web-rtc-player.ts @@ -103,6 +103,13 @@ class HaWebRtcPlayer extends LitElement { private async _startWebRtc(): Promise { this._cleanUp(); + // Browser support required for WebRTC + if (typeof RTCPeerConnection === "undefined") { + this._error = "WebRTC is not supported in this browser"; + fireEvent(this, "streams", { hasAudio: false, hasVideo: false }); + return; + } + if (!this.hass || !this.entityid) { return; } @@ -379,11 +386,17 @@ class HaWebRtcPlayer extends LitElement { } private _loadedData() { - // @ts-ignore - fireEvent(this, "load"); - console.timeLog("WebRTC", "loadedData"); console.timeEnd("WebRTC"); + + const video = this._videoEl; + const stream = video.srcObject as MediaStream; + + fireEvent(this, "load"); + fireEvent(this, "streams", { + hasAudio: Boolean(stream?.getAudioTracks().length), + hasVideo: Boolean(stream?.getVideoTracks().length), + }); } static get styles(): CSSResultGroup { diff --git a/src/data/camera.ts b/src/data/camera.ts index 84df934a74..553853c328 100644 --- a/src/data/camera.ts +++ b/src/data/camera.ts @@ -13,6 +13,8 @@ export const CAMERA_SUPPORT_STREAM = 2; export const STREAM_TYPE_HLS = "hls"; export const STREAM_TYPE_WEB_RTC = "web_rtc"; +export type StreamType = typeof STREAM_TYPE_HLS | typeof STREAM_TYPE_WEB_RTC; + interface CameraEntityAttributes extends HassEntityAttributeBase { model_name: string; access_token: string; @@ -175,6 +177,16 @@ export const isCameraMediaSource = (mediaContentId: string) => export const getEntityIdFromCameraMediaSource = (mediaContentId: string) => mediaContentId.substring(CAMERA_MEDIA_SOURCE_PREFIX.length); +export interface CameraCapabilities { + frontend_stream_types: StreamType[]; +} + +export const fetchCameraCapabilities = async ( + hass: HomeAssistant, + entity_id: string +) => + hass.callWS({ type: "camera/capabilities", entity_id }); + export interface WebRTCClientConfiguration { configuration: RTCConfiguration; dataChannel?: string; diff --git a/src/panels/config/entities/editor-tabs/settings/entity-settings-helper-tab.ts b/src/panels/config/entities/editor-tabs/settings/entity-settings-helper-tab.ts index 59b425b47d..87b027166a 100644 --- a/src/panels/config/entities/editor-tabs/settings/entity-settings-helper-tab.ts +++ b/src/panels/config/entities/editor-tabs/settings/entity-settings-helper-tab.ts @@ -100,8 +100,8 @@ export class EntitySettingsHelperTab extends LitElement { .entry=${this.entry} .disabled=${this._submitting} @change=${this._entityRegistryChanged} - hideName - hideIcon + hide-name + hide-icon >
diff --git a/src/panels/config/entities/entity-registry-settings-editor.ts b/src/panels/config/entities/entity-registry-settings-editor.ts index da34ca591b..158d281f77 100644 --- a/src/panels/config/entities/entity-registry-settings-editor.ts +++ b/src/panels/config/entities/entity-registry-settings-editor.ts @@ -44,6 +44,7 @@ import { CAMERA_ORIENTATIONS, CAMERA_SUPPORT_STREAM, CameraPreferences, + fetchCameraCapabilities, fetchCameraPrefs, STREAM_TYPE_HLS, updateCameraPrefs, @@ -145,9 +146,9 @@ export class EntityRegistrySettingsEditor extends LitElement { @property({ type: Object }) public entry!: ExtEntityRegistryEntry; - @property({ type: Boolean }) public hideName = false; + @property({ type: Boolean, attribute: "hide-name" }) public hideName = false; - @property({ type: Boolean }) public hideIcon = false; + @property({ type: Boolean, attribute: "hide-icon" }) public hideIcon = false; @property({ type: Boolean }) public disabled = false; @@ -236,13 +237,7 @@ export class EntityRegistrySettingsEditor extends LitElement { if (domain === "camera" && isComponentLoaded(this.hass, "stream")) { const stateObj: HassEntity | undefined = this.hass.states[this.entry.entity_id]; - if ( - stateObj && - supportsFeature(stateObj, CAMERA_SUPPORT_STREAM) && - // The stream component for HLS streams supports a server-side pre-load - // option that client initiated WebRTC streams do not - stateObj.attributes.frontend_stream_type === STREAM_TYPE_HLS - ) { + if (stateObj && supportsFeature(stateObj, CAMERA_SUPPORT_STREAM)) { this._fetchCameraPrefs(); } } @@ -1396,6 +1391,19 @@ export class EntityRegistrySettingsEditor extends LitElement { } private async _fetchCameraPrefs() { + const capabilities = await fetchCameraCapabilities( + this.hass, + this.entry.entity_id + ); + + // The stream component for HLS streams supports a server-side pre-load + // option that client initiated WebRTC streams do not + + if (!capabilities.frontend_stream_types.includes(STREAM_TYPE_HLS)) { + this._cameraPrefs = undefined; + return; + } + this._cameraPrefs = await fetchCameraPrefs(this.hass, this.entry.entity_id); }