diff --git a/homeassistant/components/nest/__init__.py b/homeassistant/components/nest/__init__.py index 4347c86596b..ec7a732e0b0 100644 --- a/homeassistant/components/nest/__init__.py +++ b/homeassistant/components/nest/__init__.py @@ -1,12 +1,15 @@ """Support for Nest devices.""" from __future__ import annotations -from abc import ABC +from abc import ABC, abstractmethod +import asyncio from collections.abc import Awaitable, Callable from http import HTTPStatus import logging from aiohttp import web +from google_nest_sdm.camera_traits import CameraClipPreviewTrait +from google_nest_sdm.device import Device from google_nest_sdm.event import EventMessage from google_nest_sdm.event_media import Media from google_nest_sdm.exceptions import ( @@ -57,7 +60,11 @@ from .const import ( ) from .events import EVENT_NAME_MAP, NEST_EVENT from .legacy import async_setup_legacy, async_setup_legacy_entry -from .media_source import async_get_media_event_store, get_media_source_devices +from .media_source import ( + async_get_media_event_store, + async_get_transcoder, + get_media_source_devices, +) _LOGGER = logging.getLogger(__name__) @@ -234,6 +241,7 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: subscriber.cache_policy.fetch = True # Use disk backed event media store subscriber.cache_policy.store = await async_get_media_event_store(hass, subscriber) + subscriber.cache_policy.transcoder = await async_get_transcoder(hass) async def async_config_reload() -> None: await hass.config_entries.async_reload(entry.entry_id) @@ -333,9 +341,7 @@ class NestEventViewBase(HomeAssistantView, ABC): f"No Nest Device found for '{device_id}'", HTTPStatus.NOT_FOUND ) try: - media = await nest_device.event_media_manager.get_media_from_token( - event_token - ) + media = await self.load_media(nest_device, event_token) except DecodeException as err: raise HomeAssistantError( "Even token was invalid: %s" % event_token @@ -348,9 +354,14 @@ class NestEventViewBase(HomeAssistantView, ABC): ) return await self.handle_media(media) - async def handle_media(self, media: Media) -> web.StreamResponse: + @abstractmethod + async def load_media(self, nest_device: Device, event_token: str) -> Media | None: """Load the specified media.""" + @abstractmethod + async def handle_media(self, media: Media) -> web.StreamResponse: + """Process the specified media.""" + def _json_error(self, message: str, status: HTTPStatus) -> web.StreamResponse: """Return a json error message with additional logging.""" _LOGGER.debug(message) @@ -367,8 +378,12 @@ class NestEventMediaView(NestEventViewBase): url = "/api/nest/event_media/{device_id}/{event_token}" name = "api:nest:event_media" + async def load_media(self, nest_device: Device, event_token: str) -> Media | None: + """Load the specified media.""" + return await nest_device.event_media_manager.get_media_from_token(event_token) + async def handle_media(self, media: Media) -> web.StreamResponse: - """Start a GET request.""" + """Process the specified media.""" return web.Response(body=media.contents, content_type=media.content_type) @@ -377,15 +392,38 @@ class NestEventMediaThumbnailView(NestEventViewBase): This is primarily used to render media for events for MediaSource. The media type depends on the specific device e.g. an image, or a movie clip preview. + + mp4 clips are transcoded and thumbnailed by the SDM transcoder. jpgs are thumbnailed + from the original in this view. """ url = "/api/nest/event_media/{device_id}/{event_token}/thumbnail" name = "api:nest:event_media" + def __init__(self, hass: HomeAssistant) -> None: + """Initialize NestEventMediaThumbnailView.""" + super().__init__(hass) + self._lock = asyncio.Lock() + self.hass = hass + + async def load_media(self, nest_device: Device, event_token: str) -> Media | None: + """Load the specified media.""" + if CameraClipPreviewTrait.NAME in nest_device.traits: + async with self._lock: # Only one transcode subprocess at a time + return ( + await nest_device.event_media_manager.get_clip_thumbnail_from_token( + event_token + ) + ) + return await nest_device.event_media_manager.get_media_from_token(event_token) + async def handle_media(self, media: Media) -> web.StreamResponse: """Start a GET request.""" - image = Image(media.event_image_type.content_type, media.contents) - contents = img_util.scale_jpeg_camera_image( - image, THUMBNAIL_SIZE_PX, THUMBNAIL_SIZE_PX - ) - return web.Response(body=contents, content_type=media.content_type) + contents = media.contents + content_type = media.content_type + if content_type == "image/jpeg": + image = Image(media.event_image_type.content_type, contents) + contents = img_util.scale_jpeg_camera_image( + image, THUMBNAIL_SIZE_PX, THUMBNAIL_SIZE_PX + ) + return web.Response(body=contents, content_type=content_type) diff --git a/homeassistant/components/nest/manifest.json b/homeassistant/components/nest/manifest.json index 6ea607b8d74..22098c4e770 100644 --- a/homeassistant/components/nest/manifest.json +++ b/homeassistant/components/nest/manifest.json @@ -4,7 +4,7 @@ "config_flow": true, "dependencies": ["ffmpeg", "http", "media_source"], "documentation": "https://www.home-assistant.io/integrations/nest", - "requirements": ["python-nest==4.1.0", "google-nest-sdm==1.4.0"], + "requirements": ["python-nest==4.1.0", "google-nest-sdm==1.5.0"], "codeowners": ["@allenporter"], "quality_scale": "platinum", "dhcp": [ diff --git a/homeassistant/components/nest/media_source.py b/homeassistant/components/nest/media_source.py index e287d41438e..225afd67595 100644 --- a/homeassistant/components/nest/media_source.py +++ b/homeassistant/components/nest/media_source.py @@ -32,7 +32,9 @@ from google_nest_sdm.event_media import ( ImageSession, ) from google_nest_sdm.google_nest_subscriber import GoogleNestSubscriber +from google_nest_sdm.transcoder import Transcoder +from homeassistant.components.ffmpeg import get_ffmpeg_manager from homeassistant.components.media_player.const import ( MEDIA_CLASS_DIRECTORY, MEDIA_CLASS_IMAGE, @@ -91,6 +93,13 @@ async def async_get_media_event_store( return NestEventMediaStore(hass, subscriber, store, media_path) +async def async_get_transcoder(hass: HomeAssistant) -> Transcoder: + """Get a nest clip transcoder.""" + media_path = hass.config.path(MEDIA_PATH) + ffmpeg_manager = get_ffmpeg_manager(hass) + return Transcoder(ffmpeg_manager.binary, media_path) + + class NestEventMediaStore(EventMediaStore): """Storage hook to locally persist nest media for events. @@ -176,6 +185,15 @@ class NestEventMediaStore(EventMediaStore): event_type_str = EVENT_NAME_MAP.get(event.event_type, "event") return f"{device_id_str}/{time_str}-{event_type_str}.mp4" + def get_clip_preview_thumbnail_media_key( + self, device_id: str, event: ImageEventBase + ) -> str: + """Return the filename for clip preview thumbnail media for an event session.""" + device_id_str = self._map_device_id(device_id) + time_str = str(int(event.timestamp.timestamp())) + event_type_str = EVENT_NAME_MAP.get(event.event_type, "event") + return f"{device_id_str}/{time_str}-{event_type_str}_thumb.gif" + def get_media_filename(self, media_key: str) -> str: """Return the filename in storage for a media key.""" return f"{self._media_path}/{media_key}" @@ -381,9 +399,11 @@ class NestMediaSource(MediaSource): browse_device.children = [] for clip in clips.values(): event_id = MediaId(media_id.device_id, clip.event_token) - browse_device.children.append( - _browse_clip_preview(event_id, device, clip) - ) + browse_event = _browse_clip_preview(event_id, device, clip) + browse_device.children.append(browse_event) + # Use thumbnail for first event in the list as the device thumbnail + if browse_device.thumbnail is None: + browse_device.thumbnail = browse_event.thumbnail return browse_device # Browse a specific event @@ -485,7 +505,9 @@ def _browse_clip_preview( ), can_play=True, can_expand=False, - thumbnail=None, + thumbnail=EVENT_THUMBNAIL_URL_FORMAT.format( + device_id=event_id.device_id, event_token=event_id.event_token + ), children=[], ) diff --git a/requirements_all.txt b/requirements_all.txt index 2998f33bb77..328979e27c3 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -764,7 +764,7 @@ google-cloud-pubsub==2.9.0 google-cloud-texttospeech==0.4.0 # homeassistant.components.nest -google-nest-sdm==1.4.0 +google-nest-sdm==1.5.0 # homeassistant.components.google_travel_time googlemaps==2.5.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index a106d6f0575..ad084189d9f 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -489,7 +489,7 @@ google-api-python-client==1.6.4 google-cloud-pubsub==2.9.0 # homeassistant.components.nest -google-nest-sdm==1.4.0 +google-nest-sdm==1.5.0 # homeassistant.components.google_travel_time googlemaps==2.5.1 diff --git a/tests/components/nest/test_media_source.py b/tests/components/nest/test_media_source.py index c593415e66b..8f968638d1d 100644 --- a/tests/components/nest/test_media_source.py +++ b/tests/components/nest/test_media_source.py @@ -7,11 +7,14 @@ as media in the media source. from collections.abc import Generator import datetime from http import HTTPStatus +import io from unittest.mock import patch import aiohttp +import av from google_nest_sdm.device import Device from google_nest_sdm.event import EventMessage +import numpy as np import pytest from homeassistant.components import media_source @@ -75,6 +78,51 @@ IMAGE_AUTHORIZATION_HEADERS = {"Authorization": "Basic g.0.eventToken"} NEST_EVENT = "nest_event" +def frame_image_data(frame_i, total_frames): + """Generate image content for a frame of a video.""" + img = np.empty((480, 320, 3)) + img[:, :, 0] = 0.5 + 0.5 * np.sin(2 * np.pi * (0 / 3 + frame_i / total_frames)) + img[:, :, 1] = 0.5 + 0.5 * np.sin(2 * np.pi * (1 / 3 + frame_i / total_frames)) + img[:, :, 2] = 0.5 + 0.5 * np.sin(2 * np.pi * (2 / 3 + frame_i / total_frames)) + + img = np.round(255 * img).astype(np.uint8) + img = np.clip(img, 0, 255) + return img + + +@pytest.fixture +def mp4() -> io.BytesIO: + """Generate test mp4 clip.""" + + total_frames = 10 + fps = 10 + output = io.BytesIO() + output.name = "test.mp4" + container = av.open(output, mode="w", format="mp4") + + stream = container.add_stream("libx264", rate=fps) + stream.width = 480 + stream.height = 320 + stream.pix_fmt = "yuv420p" + # stream.options.update({"g": str(fps), "keyint_min": str(fps)}) + + for frame_i in range(total_frames): + img = frame_image_data(frame_i, total_frames) + frame = av.VideoFrame.from_ndarray(img, format="rgb24") + for packet in stream.encode(frame): + container.mux(packet) + + # Flush stream + for packet in stream.encode(): + container.mux(packet) + + # Close the file + container.close() + output.seek(0) + + return output + + async def async_setup_devices(hass, auth, device_type, traits={}, events=[]): """Set up the platform and prerequisites.""" devices = { @@ -685,7 +733,7 @@ async def test_resolve_invalid_event_id(hass, auth): assert media.mime_type == "image/jpeg" -async def test_camera_event_clip_preview(hass, auth, hass_client): +async def test_camera_event_clip_preview(hass, auth, hass_client, mp4): """Test an event for a battery camera video clip.""" subscriber = await async_setup_devices( hass, auth, CAMERA_DEVICE_TYPE, BATTERY_CAMERA_TRAITS @@ -695,7 +743,7 @@ async def test_camera_event_clip_preview(hass, auth, hass_client): received_events = async_capture_events(hass, NEST_EVENT) auth.responses = [ - aiohttp.web.Response(body=IMAGE_BYTES_FROM_EVENT), + aiohttp.web.Response(body=mp4.getvalue()), ] event_timestamp = dt_util.now() await subscriber.async_receive_event( @@ -730,8 +778,10 @@ async def test_camera_event_clip_preview(hass, auth, hass_client): assert browse.identifier == device.id assert browse.title == "Front: Recent Events" assert browse.can_expand - # No thumbnail support for mp4 clips yet - assert browse.thumbnail is None + assert ( + browse.thumbnail + == f"/api/nest/event_media/{device.id}/{event_identifier}/thumbnail" + ) # The device expands recent events assert len(browse.children) == 1 assert browse.children[0].domain == DOMAIN @@ -742,7 +792,10 @@ async def test_camera_event_clip_preview(hass, auth, hass_client): assert len(browse.children[0].children) == 0 assert browse.children[0].can_play # No thumbnail support for mp4 clips yet - assert browse.children[0].thumbnail is None + assert ( + browse.children[0].thumbnail + == f"/api/nest/event_media/{device.id}/{event_identifier}/thumbnail" + ) # Verify received event and media ids match assert browse.children[0].identifier == f"{device.id}/{event_identifier}" @@ -769,7 +822,14 @@ async def test_camera_event_clip_preview(hass, auth, hass_client): response = await client.get(media.url) assert response.status == HTTPStatus.OK, "Response not matched: %s" % response contents = await response.read() - assert contents == IMAGE_BYTES_FROM_EVENT + assert contents == mp4.getvalue() + + # Verify thumbnail for mp4 clip + response = await client.get( + f"/api/nest/event_media/{device.id}/{event_identifier}/thumbnail" + ) + assert response.status == HTTPStatus.OK, "Response not matched: %s" % response + await response.read() # Animated gif format not tested async def test_event_media_render_invalid_device_id(hass, auth, hass_client): @@ -1327,6 +1387,7 @@ async def test_camera_image_resize(hass, auth, hass_client): hass, f"{const.URI_SCHEME}{DOMAIN}/{device.id}/{event_identifier}" ) assert browse.domain == DOMAIN + assert browse.identifier == f"{device.id}/{event_identifier}" assert "Person" in browse.title assert not browse.can_expand assert not browse.children