Add thumbnails to nest media player (#62532)

homeassistant/components/nest/__init__.py

@@ -1,12 +1,14 @@
 """Support for Nest devices."""
 from __future__ import annotations
 
+from abc import ABC
 from collections.abc import Awaitable, Callable
 from http import HTTPStatus
 import logging
 
 from aiohttp import web
 from google_nest_sdm.event import EventMessage
+from google_nest_sdm.event_media import Media
 from google_nest_sdm.exceptions import (
     ApiException,
     AuthException,
@@ -17,6 +19,7 @@ from google_nest_sdm.exceptions import (
 import voluptuous as vol
 
 from homeassistant.auth.permissions.const import POLICY_READ
+from homeassistant.components.camera import Image, img_util
 from homeassistant.components.http.const import KEY_HASS_USER
 from homeassistant.components.http.view import HomeAssistantView
 from homeassistant.config_entries import ConfigEntry
@@ -96,6 +99,8 @@ INSTALLED_AUTH_DOMAIN = f"{DOMAIN}.installed"
 # for event history not not filling the disk.
 EVENT_MEDIA_CACHE_SIZE = 1024  # number of events
 
+THUMBNAIL_SIZE_PX = 175
+
 
 class WebAuth(config_entry_oauth2_flow.LocalOAuth2Implementation):
     """OAuth implementation using OAuth for web applications."""
@@ -173,6 +178,7 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
     )
 
     hass.http.register_view(NestEventMediaView(hass))
+    hass.http.register_view(NestEventMediaThumbnailView(hass))
 
     return True
 
@@ -304,18 +310,11 @@ async def async_remove_entry(hass: HomeAssistant, entry: ConfigEntry) -> None:
         subscriber.stop_async()
 
 
-class NestEventMediaView(HomeAssistantView):
+class NestEventViewBase(HomeAssistantView, ABC):
-    """Returns media for related to events for a specific device.
+    """Base class for media event APIs."""
-
-    This is primarily used to render media for events for MediaSource. The media type
-    depends on the specific device e.g. an image, or a movie clip preview.
-    """
-
-    url = "/api/nest/event_media/{device_id}/{event_token}"
-    name = "api:nest:event_media"
 
     def __init__(self, hass: HomeAssistant) -> None:
-        """Initialize NestEventMediaView."""
+        """Initialize NestEventViewBase."""
         self.hass = hass
 
     async def get(
@@ -347,9 +346,46 @@ class NestEventMediaView(HomeAssistantView):
             return self._json_error(
                 f"No event found for event_id '{event_token}'", HTTPStatus.NOT_FOUND
             )
-        return web.Response(body=media.contents, content_type=media.content_type)
+        return await self.handle_media(media)
+
+    async def handle_media(self, media: Media) -> web.StreamResponse:
+        """Load the specified media."""
 
     def _json_error(self, message: str, status: HTTPStatus) -> web.StreamResponse:
         """Return a json error message with additional logging."""
         _LOGGER.debug(message)
         return self.json_message(message, status)
+
+
+class NestEventMediaView(NestEventViewBase):
+    """Returns media for related to events for a specific device.
+
+    This is primarily used to render media for events for MediaSource. The media type
+    depends on the specific device e.g. an image, or a movie clip preview.
+    """
+
+    url = "/api/nest/event_media/{device_id}/{event_token}"
+    name = "api:nest:event_media"
+
+    async def handle_media(self, media: Media) -> web.StreamResponse:
+        """Start a GET request."""
+        return web.Response(body=media.contents, content_type=media.content_type)
+
+
+class NestEventMediaThumbnailView(NestEventViewBase):
+    """Returns media for related to events for a specific device.
+
+    This is primarily used to render media for events for MediaSource. The media type
+    depends on the specific device e.g. an image, or a movie clip preview.
+    """
+
+    url = "/api/nest/event_media/{device_id}/{event_token}/thumbnail"
+    name = "api:nest:event_media"
+
+    async def handle_media(self, media: Media) -> web.StreamResponse:
+        """Start a GET request."""
+        image = Image(media.event_image_type.content_type, media.contents)
+        contents = img_util.scale_jpeg_camera_image(
+            image, THUMBNAIL_SIZE_PX, THUMBNAIL_SIZE_PX
+        )
+        return web.Response(body=contents, content_type=media.content_type)

homeassistant/components/nest/media_source.py

@@ -64,6 +64,7 @@ MEDIA_SOURCE_TITLE = "Nest"
 DEVICE_TITLE_FORMAT = "{device_name}: Recent Events"
 CLIP_TITLE_FORMAT = "{event_name} @ {event_time}"
 EVENT_MEDIA_API_URL_FORMAT = "/api/nest/event_media/{device_id}/{event_token}"
+EVENT_THUMBNAIL_URL_FORMAT = "/api/nest/event_media/{device_id}/{event_token}/thumbnail"
 
 STORAGE_KEY = "nest.event_media"
 STORAGE_VERSION = 1
@@ -400,9 +401,11 @@ class NestMediaSource(MediaSource):
             browse_device.children = []
             for image in images.values():
                 event_id = MediaId(media_id.device_id, image.event_token)
-                browse_device.children.append(
+                browse_event = _browse_image_event(event_id, device, image)
-                    _browse_image_event(event_id, device, image)
+                browse_device.children.append(browse_event)
-                )
+                # Use thumbnail for first event in the list as the device thumbnail
+                if browse_device.thumbnail is None:
+                    browse_device.thumbnail = browse_event.thumbnail
             return browse_device
 
         # Browse a specific event
@@ -502,6 +505,8 @@ def _browse_image_event(
         ),
         can_play=False,
         can_expand=False,
-        thumbnail=None,
+        thumbnail=EVENT_THUMBNAIL_URL_FORMAT.format(
+            device_id=event_id.device_id, event_token=event_id.event_token
+        ),
         children=[],
     )

tests/components/nest/test_media_source.py

@@ -38,8 +38,8 @@ DEVICE_ID = "example/api/device/id"
 DEVICE_NAME = "Front"
 PLATFORM = "camera"
 NEST_EVENT = "nest_event"
-EVENT_ID = "1aXEvi9ajKVTdDsXdJda8fzfCa..."
+EVENT_ID = "1aXEvi9ajKVTdDsXdJda8fzfCa"
-EVENT_SESSION_ID = "CjY5Y3VKaTZwR3o4Y19YbTVfMF..."
+EVENT_SESSION_ID = "CjY5Y3VKaTZwR3o4Y19YbTVfMF"
 CAMERA_DEVICE_TYPE = "sdm.devices.types.CAMERA"
 CAMERA_TRAITS = {
     "sdm.devices.traits.Info": {
@@ -730,6 +730,8 @@ async def test_camera_event_clip_preview(hass, auth, hass_client):
     assert browse.identifier == device.id
     assert browse.title == "Front: Recent Events"
     assert browse.can_expand
+    # No thumbnail support for mp4 clips yet
+    assert browse.thumbnail is None
     # The device expands recent events
     assert len(browse.children) == 1
     assert browse.children[0].domain == DOMAIN
@@ -739,6 +741,8 @@ async def test_camera_event_clip_preview(hass, auth, hass_client):
     assert not browse.children[0].can_expand
     assert len(browse.children[0].children) == 0
     assert browse.children[0].can_play
+    # No thumbnail support for mp4 clips yet
+    assert browse.children[0].thumbnail is None
 
     # Verify received event and media ids match
     assert browse.children[0].identifier == f"{device.id}/{event_identifier}"
@@ -1270,3 +1274,80 @@ async def test_camera_event_media_eviction(hass, auth, hass_client):
         contents = await response.read()
         assert contents == f"image-bytes-{i}".encode()
         await hass.async_block_till_done()
+
+
+async def test_camera_image_resize(hass, auth, hass_client):
+    """Test scaling a thumbnail for an event image."""
+    event_timestamp = dt_util.now()
+    subscriber = await async_setup_devices(
+        hass,
+        auth,
+        CAMERA_DEVICE_TYPE,
+        CAMERA_TRAITS,
+        events=[
+            create_event(
+                EVENT_SESSION_ID,
+                EVENT_ID,
+                PERSON_EVENT,
+                timestamp=event_timestamp,
+            ),
+        ],
+    )
+
+    device_registry = dr.async_get(hass)
+    device = device_registry.async_get_device({(DOMAIN, DEVICE_ID)})
+    assert device
+    assert device.name == DEVICE_NAME
+
+    # Capture any events published
+    received_events = async_capture_events(hass, NEST_EVENT)
+
+    auth.responses = [
+        aiohttp.web.json_response(GENERATE_IMAGE_URL_RESPONSE),
+        aiohttp.web.Response(body=IMAGE_BYTES_FROM_EVENT),
+    ]
+    event_timestamp = dt_util.now()
+    await subscriber.async_receive_event(
+        create_event(
+            EVENT_SESSION_ID,
+            EVENT_ID,
+            PERSON_EVENT,
+            timestamp=event_timestamp,
+        )
+    )
+    await hass.async_block_till_done()
+
+    assert len(received_events) == 1
+    received_event = received_events[0]
+    assert received_event.data["device_id"] == device.id
+    assert received_event.data["type"] == "camera_person"
+    event_identifier = received_event.data["nest_event_id"]
+
+    browse = await media_source.async_browse_media(
+        hass, f"{const.URI_SCHEME}{DOMAIN}/{device.id}/{event_identifier}"
+    )
+    assert browse.domain == DOMAIN
+    assert "Person" in browse.title
+    assert not browse.can_expand
+    assert not browse.children
+    assert (
+        browse.thumbnail
+        == f"/api/nest/event_media/{device.id}/{event_identifier}/thumbnail"
+    )
+
+    client = await hass_client()
+    response = await client.get(browse.thumbnail)
+    assert response.status == HTTPStatus.OK, "Response not matched: %s" % response
+    contents = await response.read()
+    assert contents == IMAGE_BYTES_FROM_EVENT
+
+    # The event thumbnail is used for the device thumbnail
+    browse = await media_source.async_browse_media(
+        hass, f"{const.URI_SCHEME}{DOMAIN}/{device.id}"
+    )
+    assert browse.domain == DOMAIN
+    assert browse.identifier == device.id
+    assert (
+        browse.thumbnail
+        == f"/api/nest/event_media/{device.id}/{event_identifier}/thumbnail"
+    )