Bump google-nest-sdm to 1.5.0 and add nest mp4 clip transcoding to animated gif (#64155)

homeassistant/components/nest/__init__.py

@@ -1,12 +1,15 @@
 """Support for Nest devices."""
 from __future__ import annotations
 
-from abc import ABC
+from abc import ABC, abstractmethod
+import asyncio
 from collections.abc import Awaitable, Callable
 from http import HTTPStatus
 import logging
 
 from aiohttp import web
+from google_nest_sdm.camera_traits import CameraClipPreviewTrait
+from google_nest_sdm.device import Device
 from google_nest_sdm.event import EventMessage
 from google_nest_sdm.event_media import Media
 from google_nest_sdm.exceptions import (
@@ -57,7 +60,11 @@ from .const import (
 )
 from .events import EVENT_NAME_MAP, NEST_EVENT
 from .legacy import async_setup_legacy, async_setup_legacy_entry
-from .media_source import async_get_media_event_store, get_media_source_devices
+from .media_source import (
+    async_get_media_event_store,
+    async_get_transcoder,
+    get_media_source_devices,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -234,6 +241,7 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
     subscriber.cache_policy.fetch = True
     # Use disk backed event media store
     subscriber.cache_policy.store = await async_get_media_event_store(hass, subscriber)
+    subscriber.cache_policy.transcoder = await async_get_transcoder(hass)
 
     async def async_config_reload() -> None:
         await hass.config_entries.async_reload(entry.entry_id)
@@ -333,9 +341,7 @@ class NestEventViewBase(HomeAssistantView, ABC):
                 f"No Nest Device found for '{device_id}'", HTTPStatus.NOT_FOUND
             )
         try:
-            media = await nest_device.event_media_manager.get_media_from_token(
-                event_token
-            )
+            media = await self.load_media(nest_device, event_token)
         except DecodeException as err:
             raise HomeAssistantError(
                 "Even token was invalid: %s" % event_token
@@ -348,9 +354,14 @@ class NestEventViewBase(HomeAssistantView, ABC):
             )
         return await self.handle_media(media)
 
-    async def handle_media(self, media: Media) -> web.StreamResponse:
+    @abstractmethod
+    async def load_media(self, nest_device: Device, event_token: str) -> Media | None:
         """Load the specified media."""
 
+    @abstractmethod
+    async def handle_media(self, media: Media) -> web.StreamResponse:
+        """Process the specified media."""
+
     def _json_error(self, message: str, status: HTTPStatus) -> web.StreamResponse:
         """Return a json error message with additional logging."""
         _LOGGER.debug(message)
@@ -367,8 +378,12 @@ class NestEventMediaView(NestEventViewBase):
     url = "/api/nest/event_media/{device_id}/{event_token}"
     name = "api:nest:event_media"
 
+    async def load_media(self, nest_device: Device, event_token: str) -> Media | None:
+        """Load the specified media."""
+        return await nest_device.event_media_manager.get_media_from_token(event_token)
+
     async def handle_media(self, media: Media) -> web.StreamResponse:
-        """Start a GET request."""
+        """Process the specified media."""
         return web.Response(body=media.contents, content_type=media.content_type)
 
 
@@ -377,15 +392,38 @@ class NestEventMediaThumbnailView(NestEventViewBase):
 
     This is primarily used to render media for events for MediaSource. The media type
     depends on the specific device e.g. an image, or a movie clip preview.
+
+    mp4 clips are transcoded and thumbnailed by the SDM transcoder. jpgs are thumbnailed
+    from the original in this view.
     """
 
     url = "/api/nest/event_media/{device_id}/{event_token}/thumbnail"
     name = "api:nest:event_media"
 
+    def __init__(self, hass: HomeAssistant) -> None:
+        """Initialize NestEventMediaThumbnailView."""
+        super().__init__(hass)
+        self._lock = asyncio.Lock()
+        self.hass = hass
+
+    async def load_media(self, nest_device: Device, event_token: str) -> Media | None:
+        """Load the specified media."""
+        if CameraClipPreviewTrait.NAME in nest_device.traits:
+            async with self._lock:  # Only one transcode subprocess at a time
+                return (
+                    await nest_device.event_media_manager.get_clip_thumbnail_from_token(
+                        event_token
+                    )
+                )
+        return await nest_device.event_media_manager.get_media_from_token(event_token)
+
     async def handle_media(self, media: Media) -> web.StreamResponse:
         """Start a GET request."""
-        image = Image(media.event_image_type.content_type, media.contents)
-        contents = img_util.scale_jpeg_camera_image(
-            image, THUMBNAIL_SIZE_PX, THUMBNAIL_SIZE_PX
-        )
-        return web.Response(body=contents, content_type=media.content_type)
+        contents = media.contents
+        content_type = media.content_type
+        if content_type == "image/jpeg":
+            image = Image(media.event_image_type.content_type, contents)
+            contents = img_util.scale_jpeg_camera_image(
+                image, THUMBNAIL_SIZE_PX, THUMBNAIL_SIZE_PX
+            )
+        return web.Response(body=contents, content_type=content_type)

homeassistant/components/nest/manifest.json

@@ -4,7 +4,7 @@
   "config_flow": true,
   "dependencies": ["ffmpeg", "http", "media_source"],
   "documentation": "https://www.home-assistant.io/integrations/nest",
-  "requirements": ["python-nest==4.1.0", "google-nest-sdm==1.4.0"],
+  "requirements": ["python-nest==4.1.0", "google-nest-sdm==1.5.0"],
   "codeowners": ["@allenporter"],
   "quality_scale": "platinum",
   "dhcp": [

homeassistant/components/nest/media_source.py

@@ -32,7 +32,9 @@ from google_nest_sdm.event_media import (
     ImageSession,
 )
 from google_nest_sdm.google_nest_subscriber import GoogleNestSubscriber
+from google_nest_sdm.transcoder import Transcoder
 
+from homeassistant.components.ffmpeg import get_ffmpeg_manager
 from homeassistant.components.media_player.const import (
     MEDIA_CLASS_DIRECTORY,
     MEDIA_CLASS_IMAGE,
@@ -91,6 +93,13 @@ async def async_get_media_event_store(
     return NestEventMediaStore(hass, subscriber, store, media_path)
 
 
+async def async_get_transcoder(hass: HomeAssistant) -> Transcoder:
+    """Get a nest clip transcoder."""
+    media_path = hass.config.path(MEDIA_PATH)
+    ffmpeg_manager = get_ffmpeg_manager(hass)
+    return Transcoder(ffmpeg_manager.binary, media_path)
+
+
 class NestEventMediaStore(EventMediaStore):
     """Storage hook to locally persist nest media for events.
 
@@ -176,6 +185,15 @@ class NestEventMediaStore(EventMediaStore):
         event_type_str = EVENT_NAME_MAP.get(event.event_type, "event")
         return f"{device_id_str}/{time_str}-{event_type_str}.mp4"
 
+    def get_clip_preview_thumbnail_media_key(
+        self, device_id: str, event: ImageEventBase
+    ) -> str:
+        """Return the filename for clip preview thumbnail media for an event session."""
+        device_id_str = self._map_device_id(device_id)
+        time_str = str(int(event.timestamp.timestamp()))
+        event_type_str = EVENT_NAME_MAP.get(event.event_type, "event")
+        return f"{device_id_str}/{time_str}-{event_type_str}_thumb.gif"
+
     def get_media_filename(self, media_key: str) -> str:
         """Return the filename in storage for a media key."""
         return f"{self._media_path}/{media_key}"
@@ -381,9 +399,11 @@ class NestMediaSource(MediaSource):
                 browse_device.children = []
                 for clip in clips.values():
                     event_id = MediaId(media_id.device_id, clip.event_token)
-                    browse_device.children.append(
-                        _browse_clip_preview(event_id, device, clip)
-                    )
+                    browse_event = _browse_clip_preview(event_id, device, clip)
+                    browse_device.children.append(browse_event)
+                    # Use thumbnail for first event in the list as the device thumbnail
+                    if browse_device.thumbnail is None:
+                        browse_device.thumbnail = browse_event.thumbnail
                 return browse_device
 
             # Browse a specific event
@@ -485,7 +505,9 @@ def _browse_clip_preview(
         ),
         can_play=True,
         can_expand=False,
-        thumbnail=None,
+        thumbnail=EVENT_THUMBNAIL_URL_FORMAT.format(
+            device_id=event_id.device_id, event_token=event_id.event_token
+        ),
         children=[],
     )
 

requirements_all.txt

@@ -764,7 +764,7 @@ google-cloud-pubsub==2.9.0
 google-cloud-texttospeech==0.4.0
 
 # homeassistant.components.nest
-google-nest-sdm==1.4.0
+google-nest-sdm==1.5.0
 
 # homeassistant.components.google_travel_time
 googlemaps==2.5.1

requirements_test_all.txt

@@ -489,7 +489,7 @@ google-api-python-client==1.6.4
 google-cloud-pubsub==2.9.0
 
 # homeassistant.components.nest
-google-nest-sdm==1.4.0
+google-nest-sdm==1.5.0
 
 # homeassistant.components.google_travel_time
 googlemaps==2.5.1

tests/components/nest/test_media_source.py

@@ -7,11 +7,14 @@ as media in the media source.
 from collections.abc import Generator
 import datetime
 from http import HTTPStatus
+import io
 from unittest.mock import patch
 
 import aiohttp
+import av
 from google_nest_sdm.device import Device
 from google_nest_sdm.event import EventMessage
+import numpy as np
 import pytest
 
 from homeassistant.components import media_source
@@ -75,6 +78,51 @@ IMAGE_AUTHORIZATION_HEADERS = {"Authorization": "Basic g.0.eventToken"}
 NEST_EVENT = "nest_event"
 
 
+def frame_image_data(frame_i, total_frames):
+    """Generate image content for a frame of a video."""
+    img = np.empty((480, 320, 3))
+    img[:, :, 0] = 0.5 + 0.5 * np.sin(2 * np.pi * (0 / 3 + frame_i / total_frames))
+    img[:, :, 1] = 0.5 + 0.5 * np.sin(2 * np.pi * (1 / 3 + frame_i / total_frames))
+    img[:, :, 2] = 0.5 + 0.5 * np.sin(2 * np.pi * (2 / 3 + frame_i / total_frames))
+
+    img = np.round(255 * img).astype(np.uint8)
+    img = np.clip(img, 0, 255)
+    return img
+
+
+@pytest.fixture
+def mp4() -> io.BytesIO:
+    """Generate test mp4 clip."""
+
+    total_frames = 10
+    fps = 10
+    output = io.BytesIO()
+    output.name = "test.mp4"
+    container = av.open(output, mode="w", format="mp4")
+
+    stream = container.add_stream("libx264", rate=fps)
+    stream.width = 480
+    stream.height = 320
+    stream.pix_fmt = "yuv420p"
+    #    stream.options.update({"g": str(fps), "keyint_min": str(fps)})
+
+    for frame_i in range(total_frames):
+        img = frame_image_data(frame_i, total_frames)
+        frame = av.VideoFrame.from_ndarray(img, format="rgb24")
+        for packet in stream.encode(frame):
+            container.mux(packet)
+
+    # Flush stream
+    for packet in stream.encode():
+        container.mux(packet)
+
+    # Close the file
+    container.close()
+    output.seek(0)
+
+    return output
+
+
 async def async_setup_devices(hass, auth, device_type, traits={}, events=[]):
     """Set up the platform and prerequisites."""
     devices = {
@@ -685,7 +733,7 @@ async def test_resolve_invalid_event_id(hass, auth):
     assert media.mime_type == "image/jpeg"
 
 
-async def test_camera_event_clip_preview(hass, auth, hass_client):
+async def test_camera_event_clip_preview(hass, auth, hass_client, mp4):
     """Test an event for a battery camera video clip."""
     subscriber = await async_setup_devices(
         hass, auth, CAMERA_DEVICE_TYPE, BATTERY_CAMERA_TRAITS
@@ -695,7 +743,7 @@ async def test_camera_event_clip_preview(hass, auth, hass_client):
     received_events = async_capture_events(hass, NEST_EVENT)
 
     auth.responses = [
-        aiohttp.web.Response(body=IMAGE_BYTES_FROM_EVENT),
+        aiohttp.web.Response(body=mp4.getvalue()),
     ]
     event_timestamp = dt_util.now()
     await subscriber.async_receive_event(
@@ -730,8 +778,10 @@ async def test_camera_event_clip_preview(hass, auth, hass_client):
     assert browse.identifier == device.id
     assert browse.title == "Front: Recent Events"
     assert browse.can_expand
-    # No thumbnail support for mp4 clips yet
-    assert browse.thumbnail is None
+    assert (
+        browse.thumbnail
+        == f"/api/nest/event_media/{device.id}/{event_identifier}/thumbnail"
+    )
     # The device expands recent events
     assert len(browse.children) == 1
     assert browse.children[0].domain == DOMAIN
@@ -742,7 +792,10 @@ async def test_camera_event_clip_preview(hass, auth, hass_client):
     assert len(browse.children[0].children) == 0
     assert browse.children[0].can_play
     # No thumbnail support for mp4 clips yet
-    assert browse.children[0].thumbnail is None
+    assert (
+        browse.children[0].thumbnail
+        == f"/api/nest/event_media/{device.id}/{event_identifier}/thumbnail"
+    )
 
     # Verify received event and media ids match
     assert browse.children[0].identifier == f"{device.id}/{event_identifier}"
@@ -769,7 +822,14 @@ async def test_camera_event_clip_preview(hass, auth, hass_client):
     response = await client.get(media.url)
     assert response.status == HTTPStatus.OK, "Response not matched: %s" % response
     contents = await response.read()
-    assert contents == IMAGE_BYTES_FROM_EVENT
+    assert contents == mp4.getvalue()
+
+    # Verify thumbnail for mp4 clip
+    response = await client.get(
+        f"/api/nest/event_media/{device.id}/{event_identifier}/thumbnail"
+    )
+    assert response.status == HTTPStatus.OK, "Response not matched: %s" % response
+    await response.read()  # Animated gif format not tested
 
 
 async def test_event_media_render_invalid_device_id(hass, auth, hass_client):
@@ -1327,6 +1387,7 @@ async def test_camera_image_resize(hass, auth, hass_client):
         hass, f"{const.URI_SCHEME}{DOMAIN}/{device.id}/{event_identifier}"
     )
     assert browse.domain == DOMAIN
+    assert browse.identifier == f"{device.id}/{event_identifier}"
     assert "Person" in browse.title
     assert not browse.can_expand
     assert not browse.children