From 4ad005f0bfe7ced82be683929b2cecdab66c2495 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Wed, 13 Oct 2021 01:14:33 -0700 Subject: [PATCH] Add WebRTC stream player (#10193) Co-authored-by: Bram Kragten --- src/components/ha-camera-stream.ts | 95 +++++++----- src/components/ha-web-rtc-player.ts | 135 ++++++++++++++++++ src/data/camera.ts | 19 +++ .../more-info/controls/more-info-camera.ts | 6 +- 4 files changed, 219 insertions(+), 36 deletions(-) create mode 100644 src/components/ha-web-rtc-player.ts diff --git a/src/components/ha-camera-stream.ts b/src/components/ha-camera-stream.ts index 7444e98c27..c19ee6d71f 100644 --- a/src/components/ha-camera-stream.ts +++ b/src/components/ha-camera-stream.ts @@ -15,9 +15,12 @@ import { CAMERA_SUPPORT_STREAM, computeMJPEGStreamUrl, fetchStreamUrl, + STREAM_TYPE_HLS, + STREAM_TYPE_WEB_RTC, } from "../data/camera"; import { HomeAssistant } from "../types"; import "./ha-hls-player"; +import "./ha-web-rtc-player"; @customElement("ha-camera-stream") class HaCameraStream extends LitElement { @@ -34,8 +37,8 @@ class HaCameraStream extends LitElement { @property({ type: Boolean, attribute: "allow-exoplayer" }) public allowExoPlayer = false; - // We keep track if we should force MJPEG with a string - // that way it automatically resets if we change entity. + // We keep track if we should force MJPEG if there was a failure + // to get the HLS stream url. This is reset if we change entities. @state() private _forceMJPEG?: string; @state() private _url?: string; @@ -48,7 +51,8 @@ class HaCameraStream extends LitElement { !this._shouldRenderMJPEG && this.stateObj && (changedProps.get("stateObj") as CameraEntity | undefined)?.entity_id !== - this.stateObj.entity_id + this.stateObj.entity_id && + this.stateObj!.attributes.stream_type === STREAM_TYPE_HLS ) { this._forceMJPEG = undefined; this._url = undefined; @@ -70,43 +74,64 @@ class HaCameraStream extends LitElement { if (!this.stateObj) { return html``; } - - return html` - ${__DEMO__ || this._shouldRenderMJPEG - ? html` - - ` - : this._url - ? html` - - ` - : ""} - `; + if (__DEMO__ || this._shouldRenderMJPEG) { + return html` `; + } + if (this.stateObj.attributes.stream_type === STREAM_TYPE_HLS && true) { + return this._url + ? html` ` + : html``; + } + if (this.stateObj.attributes.stream_type === STREAM_TYPE_WEB_RTC) { + return html` `; + } + return html``; } private get _shouldRenderMJPEG() { - return ( - this._forceMJPEG === this.stateObj!.entity_id || + if (this._forceMJPEG === this.stateObj!.entity_id) { + // Fallback when unable to fetch stream url + return true; + } + if ( !isComponentLoaded(this.hass!, "stream") || !supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM) - ); + ) { + // Steaming is not supported by the camera so fallback to MJPEG stream + return true; + } + if ( + this.stateObj!.attributes.stream_type === STREAM_TYPE_WEB_RTC && + typeof RTCPeerConnection === "undefined" + ) { + // Stream requires WebRTC but browser does not support, so fallback to + // MJPEG stream. + return true; + } + // Render stream + return false; } private async _getStreamUrl(): Promise { diff --git a/src/components/ha-web-rtc-player.ts b/src/components/ha-web-rtc-player.ts new file mode 100644 index 0000000000..99faf1953f --- /dev/null +++ b/src/components/ha-web-rtc-player.ts @@ -0,0 +1,135 @@ +import { + css, + CSSResultGroup, + html, + LitElement, + PropertyValues, + TemplateResult, +} from "lit"; +import { customElement, property, state, query } from "lit/decorators"; +import { handleWebRtcOffer, WebRtcAnswer } from "../data/camera"; +import type { HomeAssistant } from "../types"; +import "./ha-alert"; + +/** + * A WebRTC stream is established by first sending an offer through a signal + * path via an integration. An answer is returned, then the rest of the stream + * is handled entirely client side. + */ +@customElement("ha-web-rtc-player") +class HaWebRtcPlayer extends LitElement { + @property({ attribute: false }) public hass!: HomeAssistant; + + @property() public entityid!: string; + + @property({ type: Boolean, attribute: "controls" }) + public controls = false; + + @property({ type: Boolean, attribute: "muted" }) + public muted = false; + + @property({ type: Boolean, attribute: "autoplay" }) + public autoPlay = false; + + @property({ type: Boolean, attribute: "playsinline" }) + public playsInline = false; + + @state() private _error?: string; + + // don't cache this, as we remove it on disconnects + @query("#remote-stream") private _videoEl!: HTMLVideoElement; + + protected render(): TemplateResult { + if (this._error) { + return html`${this._error}`; + } + return html` + + `; + } + + public disconnectedCallback() { + super.disconnectedCallback(); + this._cleanUp(); + } + + protected updated(changedProperties: PropertyValues) { + if (!changedProperties.has("entityid")) { + return; + } + if (!this._videoEl) { + return; + } + this._startWebRtc(); + } + + private async _startWebRtc(): Promise { + this._error = undefined; + const peerConnection = new RTCPeerConnection(); + // Some cameras (such as nest) require a data channel to establish a stream + // however, not used by any integrations. + peerConnection.createDataChannel("dataSendChannel"); + const offerOptions: RTCOfferOptions = { + offerToReceiveAudio: true, + offerToReceiveVideo: true, + }; + const offer: RTCSessionDescriptionInit = await peerConnection.createOffer( + offerOptions + ); + await peerConnection.setLocalDescription(offer); + + let webRtcAnswer: WebRtcAnswer; + try { + webRtcAnswer = await handleWebRtcOffer( + this.hass, + this.entityid, + offer.sdp! + ); + } catch (err: any) { + this._error = "Failed to start WebRTC stream: " + err.message; + return; + } + + // Setup callbacks to render remote stream once media tracks are discovered. + const remoteStream = new MediaStream(); + peerConnection.addEventListener("track", (event) => { + remoteStream.addTrack(event.track); + this._videoEl.srcObject = remoteStream; + }); + + // Initiate the stream with the remote device + const remoteDesc = new RTCSessionDescription({ + type: "answer", + sdp: webRtcAnswer.answer, + }); + await peerConnection.setRemoteDescription(remoteDesc); + } + + private _cleanUp() { + if (this._videoEl) { + const videoEl = this._videoEl; + videoEl.removeAttribute("src"); + videoEl.load(); + } + } + + static get styles(): CSSResultGroup { + return css` + video { + display: block; + } + `; + } +} + +declare global { + interface HTMLElementTagNameMap { + "ha-web-rtc-player": HaWebRtcPlayer; + } +} diff --git a/src/data/camera.ts b/src/data/camera.ts index 945fe474f9..e2a3f04d68 100644 --- a/src/data/camera.ts +++ b/src/data/camera.ts @@ -9,11 +9,15 @@ import { getSignedPath } from "./auth"; export const CAMERA_SUPPORT_ON_OFF = 1; export const CAMERA_SUPPORT_STREAM = 2; +export const STREAM_TYPE_HLS = "hls"; +export const STREAM_TYPE_WEB_RTC = "web_rtc"; + interface CameraEntityAttributes extends HassEntityAttributeBase { model_name: string; access_token: string; brand: string; motion_detection: boolean; + stream_type: string; } export interface CameraEntity extends HassEntityBase { @@ -33,6 +37,10 @@ export interface Stream { url: string; } +export interface WebRtcAnswer { + answer: string; +} + export const computeMJPEGStreamUrl = (entity: CameraEntity) => `/api/camera_proxy_stream/${entity.entity_id}?token=${entity.attributes.access_token}`; @@ -78,6 +86,17 @@ export const fetchStreamUrl = async ( return stream; }; +export const handleWebRtcOffer = ( + hass: HomeAssistant, + entityId: string, + offer: string +) => + hass.callWS({ + type: "camera/web_rtc_offer", + entity_id: entityId, + offer: offer, + }); + export const fetchCameraPrefs = (hass: HomeAssistant, entityId: string) => hass.callWS({ type: "camera/get_prefs", diff --git a/src/dialogs/more-info/controls/more-info-camera.ts b/src/dialogs/more-info/controls/more-info-camera.ts index ce3b02140f..58b7096c3c 100644 --- a/src/dialogs/more-info/controls/more-info-camera.ts +++ b/src/dialogs/more-info/controls/more-info-camera.ts @@ -17,6 +17,7 @@ import { CameraPreferences, CAMERA_SUPPORT_STREAM, fetchCameraPrefs, + STREAM_TYPE_HLS, updateCameraPrefs, } from "../../../data/camera"; import type { HomeAssistant } from "../../../types"; @@ -82,7 +83,10 @@ class MoreInfoCamera extends LitElement { if ( curEntityId && isComponentLoaded(this.hass!, "stream") && - supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM) + supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM) && + // The stream component for HLS streams supports a server-side pre-load + // option that client initiated WebRTC streams do not + this.stateObj!.attributes.stream_type === STREAM_TYPE_HLS ) { // Fetch in background while we set up the video. this._fetchCameraPrefs();