Update Google Calendar to synchronize calendar events efficiently (#80925)

* Sync google calendar and serve from local storage

Update to use new gcal_sync APIs
Update google calendar filter logic
Remove storage on config entry removal
Make timeline queries timezone aware
Do not block startup while syncing

* Minor readability tweaks

* Remove unnecessary args to async_add_entities

* Change how task is created on startup

* Update homeassistant/components/google/calendar.py

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

* Revert min time between updates

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

homeassistant/components/google/__init__.py

@@ -36,6 +36,7 @@ from homeassistant.helpers.entity import generate_entity_id
 from .api import ApiAuthImpl, get_feature_access
 from .const import (
     DATA_SERVICE,
+    DATA_STORE,
     DOMAIN,
     EVENT_DESCRIPTION,
     EVENT_END_DATE,
@@ -49,6 +50,7 @@ from .const import (
     EVENT_TYPES_CONF,
     FeatureAccess,
 )
+from .store import LocalCalendarStore
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -171,6 +173,9 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
         ApiAuthImpl(async_get_clientsession(hass), session)
     )
     hass.data[DOMAIN][entry.entry_id][DATA_SERVICE] = calendar_service
+    hass.data[DOMAIN][entry.entry_id][DATA_STORE] = LocalCalendarStore(
+        hass, entry.entry_id
+    )
 
     if entry.unique_id is None:
         try:
@@ -213,6 +218,12 @@ async def async_reload_entry(hass: HomeAssistant, entry: ConfigEntry) -> None:
         await hass.config_entries.async_reload(entry.entry_id)
 
 
+async def async_remove_entry(hass: HomeAssistant, entry: ConfigEntry) -> None:
+    """Handle removal of a local storage."""
+    store = LocalCalendarStore(hass, entry.entry_id)
+    await store.async_remove()
+
+
 async def async_setup_add_event_service(
     hass: HomeAssistant,
     calendar_service: GoogleCalendarService,

homeassistant/components/google/calendar.py

@@ -2,13 +2,17 @@
 
 from __future__ import annotations
 
+import asyncio
 from datetime import datetime, timedelta
 import logging
 from typing import Any
 
-from gcal_sync.api import GoogleCalendarService, ListEventsRequest
+from gcal_sync.api import SyncEventsRequest
 from gcal_sync.exceptions import ApiException
 from gcal_sync.model import DateOrDatetime, Event
+from gcal_sync.store import ScopedCalendarStore
+from gcal_sync.sync import CalendarEventSyncManager
+from gcal_sync.timeline import Timeline
 import voluptuous as vol
 
 from homeassistant.components.calendar import (
@@ -34,12 +38,12 @@ from homeassistant.helpers.update_coordinator import (
     DataUpdateCoordinator,
     UpdateFailed,
 )
+from homeassistant.util import dt as dt_util
 
 from . import (
     CONF_IGNORE_AVAILABILITY,
     CONF_SEARCH,
     CONF_TRACK,
-    DATA_SERVICE,
     DEFAULT_CONF_OFFSET,
     DOMAIN,
     YAML_DEVICES,
@@ -49,6 +53,8 @@ from . import (
 )
 from .api import get_feature_access
 from .const import (
+    DATA_SERVICE,
+    DATA_STORE,
     EVENT_DESCRIPTION,
     EVENT_END_DATE,
     EVENT_END_DATETIME,
@@ -66,6 +72,10 @@ _LOGGER = logging.getLogger(__name__)
 
 MIN_TIME_BETWEEN_UPDATES = timedelta(minutes=15)
 
+# Avoid syncing super old data on initial syncs. Note that old but active
+# recurring events are still included.
+SYNC_EVENT_MIN_TIME = timedelta(days=-90)
+
 # Events have a transparency that determine whether or not they block time on calendar.
 # When an event is opaque, it means "Show me as busy" which is the default.  Events that
 # are not opaque are ignored by default.
@@ -115,6 +125,7 @@ async def async_setup_entry(
 ) -> None:
     """Set up the google calendar platform."""
     calendar_service = hass.data[DOMAIN][config_entry.entry_id][DATA_SERVICE]
+    store = hass.data[DOMAIN][config_entry.entry_id][DATA_STORE]
     try:
         result = await calendar_service.async_list_calendars()
     except ApiException as err:
@@ -185,12 +196,20 @@ async def async_setup_entry(
                     entity_registry.async_remove(
                         entity_entry.entity_id,
                     )
+            request_template = SyncEventsRequest(
+                calendar_id=calendar_id,
+                search=data.get(CONF_SEARCH),
+                start_time=dt_util.now() + SYNC_EVENT_MIN_TIME,
+            )
+            sync = CalendarEventSyncManager(
+                calendar_service,
+                store=ScopedCalendarStore(store, unique_id or entity_name),
+                request_template=request_template,
+            )
             coordinator = CalendarUpdateCoordinator(
                 hass,
-                calendar_service,
+                sync,
                 data[CONF_NAME],
-                calendar_id,
-                data.get(CONF_SEARCH),
             )
             entities.append(
                 GoogleCalendarEntity(
@@ -203,7 +222,7 @@ async def async_setup_entry(
                 )
             )
 
-    async_add_entities(entities, True)
+    async_add_entities(entities)
 
     if calendars and new_calendars:
 
@@ -223,16 +242,14 @@ async def async_setup_entry(
         )
 
 
-class CalendarUpdateCoordinator(DataUpdateCoordinator):
+class CalendarUpdateCoordinator(DataUpdateCoordinator[Timeline]):
     """Coordinator for calendar RPC calls."""
 
     def __init__(
         self,
         hass: HomeAssistant,
-        calendar_service: GoogleCalendarService,
+        sync: CalendarEventSyncManager,
         name: str,
-        calendar_id: str,
-        search: str | None,
     ) -> None:
         """Create the Calendar event device."""
         super().__init__(
@@ -241,38 +258,18 @@ class CalendarUpdateCoordinator(DataUpdateCoordinator):
             name=name,
             update_interval=MIN_TIME_BETWEEN_UPDATES,
         )
-        self.calendar_service = calendar_service
+        self.sync = sync
-        self.calendar_id = calendar_id
-        self._search = search
 
-    async def async_get_events(
+    async def _async_update_data(self) -> Timeline:
-        self, start_date: datetime, end_date: datetime
-    ) -> list[Event]:
-        """Get all events in a specific time frame."""
-        request = ListEventsRequest(
-            calendar_id=self.calendar_id,
-            start_time=start_date,
-            end_time=end_date,
-            search=self._search,
-        )
-        result_items = []
-        try:
-            result = await self.calendar_service.async_list_events(request)
-            async for result_page in result:
-                result_items.extend(result_page.items)
-        except ApiException as err:
-            self.async_set_update_error(err)
-            raise HomeAssistantError(str(err)) from err
-        return result_items
-
-    async def _async_update_data(self) -> list[Event]:
         """Fetch data from API endpoint."""
-        request = ListEventsRequest(calendar_id=self.calendar_id, search=self._search)
         try:
-            result = await self.calendar_service.async_list_events(request)
+            await self.sync.run()
         except ApiException as err:
             raise UpdateFailed(f"Error communicating with API: {err}") from err
-        return result.items
+
+        return await self.sync.store_service.async_get_timeline(
+            dt_util.DEFAULT_TIME_ZONE
+        )
 
 
 class GoogleCalendarEntity(CoordinatorEntity, CalendarEntity):
@@ -341,16 +338,28 @@ class GoogleCalendarEntity(CoordinatorEntity, CalendarEntity):
     async def async_added_to_hass(self) -> None:
         """When entity is added to hass."""
         await super().async_added_to_hass()
+
         # We do not ask for an update with async_add_entities()
-        # because it will update disabled entities
+        # because it will update disabled entities. This is started as a
+        # task to let if sync in the background without blocking startup
+        async def refresh() -> None:
             await self.coordinator.async_request_refresh()
             self._apply_coordinator_update()
 
+        asyncio.create_task(refresh())
+
     async def async_get_events(
         self, hass: HomeAssistant, start_date: datetime, end_date: datetime
     ) -> list[CalendarEvent]:
         """Get all events in a specific time frame."""
-        result_items = await self.coordinator.async_get_events(start_date, end_date)
+        if not (timeline := self.coordinator.data):
+            raise HomeAssistantError(
+                "Unable to get events: Sync from server has not completed"
+            )
+        result_items = timeline.overlapping(
+            dt_util.as_local(start_date),
+            dt_util.as_local(end_date),
+        )
         return [
             _get_calendar_event(event)
             for event in filter(self._event_filter, result_items)
@@ -358,13 +367,21 @@ class GoogleCalendarEntity(CoordinatorEntity, CalendarEntity):
 
     def _apply_coordinator_update(self) -> None:
         """Copy state from the coordinator to this entity."""
-        events = self.coordinator.data
+        if (timeline := self.coordinator.data) and (
-        api_event = next(filter(self._event_filter, iter(events)), None)
+            api_event := next(
-        self._event = _get_calendar_event(api_event) if api_event else None
+                filter(
-        if self._event:
+                    self._event_filter,
+                    timeline.active_after(dt_util.now()),
+                ),
+                None,
+            )
+        ):
+            self._event = _get_calendar_event(api_event)
             (self._event.summary, self._offset_value) = extract_offset(
                 self._event.summary, self._offset
             )
+        else:
+            self._event = None
 
     @callback
     def _handle_coordinator_update(self) -> None:
@@ -432,7 +449,7 @@ async def async_create_event(entity: GoogleCalendarEntity, call: ServiceCall) ->
         raise ValueError("Missing required fields to set start or end date/datetime")
 
     try:
-        await entity.coordinator.calendar_service.async_create_event(
+        await entity.coordinator.sync.api.async_create_event(
             entity.calendar_id,
             Event(
                 summary=call.data[EVENT_SUMMARY],
@@ -443,3 +460,4 @@ async def async_create_event(entity: GoogleCalendarEntity, call: ServiceCall) ->
         )
     except ApiException as err:
         raise HomeAssistantError(str(err)) from err
+    entity.async_write_ha_state()

homeassistant/components/google/const.py

@@ -10,6 +10,7 @@ CONF_CALENDAR_ACCESS = "calendar_access"
 DATA_CALENDARS = "calendars"
 DATA_SERVICE = "service"
 DATA_CONFIG = "config"
+DATA_STORE = "store"
 
 
 class FeatureAccess(Enum):

homeassistant/components/google/store.py

@@ -0,0 +1,53 @@
+"""Google Calendar local storage."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from gcal_sync.store import CalendarStore
+
+from homeassistant.core import HomeAssistant
+from homeassistant.helpers.storage import Store
+
+from .const import DOMAIN
+
+_LOGGER = logging.getLogger(__name__)
+
+STORAGE_KEY_FORMAT = "{domain}.{entry_id}"
+STORAGE_VERSION = 1
+# Buffer writes every few minutes (plus guaranteed to be written at shutdown)
+STORAGE_SAVE_DELAY_SECONDS = 120
+
+
+class LocalCalendarStore(CalendarStore):
+    """Storage for local persistence of calendar and event data."""
+
+    def __init__(self, hass: HomeAssistant, entry_id: str) -> None:
+        """Initialize LocalCalendarStore."""
+        self._store = Store[dict[str, Any]](
+            hass,
+            STORAGE_VERSION,
+            STORAGE_KEY_FORMAT.format(domain=DOMAIN, entry_id=entry_id),
+            private=True,
+        )
+        self._data: dict[str, Any] | None = None
+
+    async def async_load(self) -> dict[str, Any] | None:
+        """Load data."""
+        if self._data is None:
+            self._data = await self._store.async_load() or {}
+        return self._data
+
+    async def async_save(self, data: dict[str, Any]) -> None:
+        """Save data."""
+        self._data = data
+
+        def provide_data() -> dict:
+            return data
+
+        self._store.async_delay_save(provide_data, STORAGE_SAVE_DELAY_SECONDS)
+
+    async def async_remove(self) -> None:
+        """Remove data."""
+        await self._store.async_remove()

tests/components/google/conftest.py

@@ -206,9 +206,13 @@ def mock_events_list(
     ) -> None:
         if calendar_id is None:
             calendar_id = CALENDAR_ID
+        resp = {
+            **response,
+            "nextSyncToken": "sync-token",
+        }
         aioclient_mock.get(
             f"{API_BASE_URL}/calendars/{calendar_id}/events",
-            json=response,
+            json=resp,
             exc=exc,
         )
         return
@@ -236,9 +240,13 @@ def mock_calendars_list(
     """Fixture to construct a fake calendar list API response."""
 
     def _result(response: dict[str, Any], exc: ClientError | None = None) -> None:
+        resp = {
+            **response,
+            "nextSyncToken": "sync-token",
+        }
         aioclient_mock.get(
             f"{API_BASE_URL}/users/me/calendarList",
-            json=response,
+            json=resp,
             exc=exc,
         )
         return

tests/components/google/test_calendar.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import copy
 import datetime
 from http import HTTPStatus
 from typing import Any
@@ -10,7 +9,6 @@ from unittest.mock import patch
 import urllib
 
 from aiohttp.client_exceptions import ClientError
-from gcal_sync.auth import API_BASE_URL
 import pytest
 
 from homeassistant.components.google.const import DOMAIN
@@ -28,7 +26,6 @@ from .conftest import (
 )
 
 from tests.common import async_fire_time_changed
-from tests.test_util.aiohttp import AiohttpClientMockResponse
 
 TEST_ENTITY = TEST_API_ENTITY
 TEST_ENTITY_NAME = TEST_API_ENTITY_NAME
@@ -76,6 +73,11 @@ def mock_test_setup(
     return
 
 
+def get_events_url(entity: str, start: str, end: str) -> str:
+    """Create a url to get events during the specified time range."""
+    return f"/api/calendars/{entity}?start={urllib.parse.quote(start)}&end={urllib.parse.quote(end)}"
+
+
 def upcoming() -> dict[str, Any]:
     """Create a test event with an arbitrary start/end time fetched from the api url."""
     now = dt_util.now()
@@ -90,7 +92,7 @@ def upcoming_event_url(entity: str = TEST_ENTITY) -> str:
     now = dt_util.now()
     start = (now - datetime.timedelta(minutes=60)).isoformat()
     end = (now + datetime.timedelta(minutes=60)).isoformat()
-    return f"/api/calendars/{entity}?start={urllib.parse.quote(start)}&end={urllib.parse.quote(end)}"
+    return get_events_url(entity, start, end)
 
 
 async def test_all_day_event(hass, mock_events_list_items, component_setup):
@@ -406,14 +408,12 @@ async def test_http_event_api_failure(
     aioclient_mock,
 ):
     """Test the Rest API response during a calendar failure."""
-    mock_events_list({})
+    mock_events_list({}, exc=ClientError())
+
     assert await component_setup()
 
     client = await hass_client()
 
-    aioclient_mock.clear_requests()
-    mock_events_list({}, exc=ClientError())
-
     response = await client.get(upcoming_event_url())
     assert response.status == HTTPStatus.INTERNAL_SERVER_ERROR
 
@@ -472,66 +472,6 @@ async def test_http_api_all_day_event(
     }
 
 
-@pytest.mark.freeze_time("2022-03-27 12:05:00+00:00")
-async def test_http_api_event_paging(
-    hass, hass_client, aioclient_mock, component_setup
-):
-    """Test paging through results from the server."""
-    hass.config.set_time_zone("Asia/Baghdad")
-
-    responses = [
-        {
-            "nextPageToken": "page-token",
-            "items": [
-                {
-                    **TEST_EVENT,
-                    "summary": "event 1",
-                    **upcoming(),
-                }
-            ],
-        },
-        {
-            "items": [
-                {
-                    **TEST_EVENT,
-                    "summary": "event 2",
-                    **upcoming(),
-                }
-            ],
-        },
-    ]
-
-    def next_response(response_list):
-        results = copy.copy(response_list)
-
-        async def get(method, url, data):
-            return AiohttpClientMockResponse(method, url, json=results.pop(0))
-
-        return get
-
-    # Setup response for initial entity load
-    aioclient_mock.get(
-        f"{API_BASE_URL}/calendars/{CALENDAR_ID}/events",
-        side_effect=next_response(responses),
-    )
-    assert await component_setup()
-
-    # Setup response for API request
-    aioclient_mock.clear_requests()
-    aioclient_mock.get(
-        f"{API_BASE_URL}/calendars/{CALENDAR_ID}/events",
-        side_effect=next_response(responses),
-    )
-
-    client = await hass_client()
-    response = await client.get(upcoming_event_url())
-    assert response.status == HTTPStatus.OK
-    events = await response.json()
-    assert len(events) == 2
-    assert events[0]["summary"] == "event 1"
-    assert events[1]["summary"] == "event 2"
-
-
 @pytest.mark.parametrize(
     "calendars_config_ignore_availability,transparency,expect_visible_event",
     [
@@ -781,3 +721,58 @@ async def test_invalid_unique_id_cleanup(
         entity_registry, config_entry.entry_id
     )
     assert not registry_entries
+
+
+@pytest.mark.parametrize(
+    "time_zone,event_order",
+    [
+        ("America/Los_Angeles", ["One", "Two", "All Day Event"]),
+        ("America/Regina", ["One", "Two", "All Day Event"]),
+        ("UTC", ["One", "All Day Event", "Two"]),
+        ("Asia/Tokyo", ["All Day Event", "One", "Two"]),
+    ],
+)
+async def test_all_day_iter_order(
+    hass,
+    hass_client,
+    mock_events_list_items,
+    component_setup,
+    time_zone,
+    event_order,
+):
+    """Test the sort order of an all day events depending on the time zone."""
+    hass.config.set_time_zone(time_zone)
+    mock_events_list_items(
+        [
+            {
+                **TEST_EVENT,
+                "id": "event-id-3",
+                "summary": "All Day Event",
+                "start": {"date": "2022-10-08"},
+                "end": {"date": "2022-10-09"},
+            },
+            {
+                **TEST_EVENT,
+                "id": "event-id-1",
+                "summary": "One",
+                "start": {"dateTime": "2022-10-07T23:00:00+00:00"},
+                "end": {"dateTime": "2022-10-07T23:30:00+00:00"},
+            },
+            {
+                **TEST_EVENT,
+                "id": "event-id-2",
+                "summary": "Two",
+                "start": {"dateTime": "2022-10-08T01:00:00+00:00"},
+                "end": {"dateTime": "2022-10-08T02:00:00+00:00"},
+            },
+        ]
+    )
+    assert await component_setup()
+
+    client = await hass_client()
+    response = await client.get(
+        get_events_url(TEST_ENTITY, "2022-10-06T00:00:00Z", "2022-10-09T00:00:00Z")
+    )
+    assert response.status == HTTPStatus.OK
+    events = await response.json()
+    assert [event["summary"] for event in events] == event_order

tests/components/google/test_init.py

@@ -818,3 +818,24 @@ async def test_assign_unique_id_failure(
 
     assert config_entry.state is config_entry_status
     assert config_entry.unique_id is None
+
+
+async def test_remove_entry(
+    hass: HomeAssistant,
+    mock_calendars_list: ApiResult,
+    component_setup: ComponentSetup,
+    test_api_calendar: dict[str, Any],
+    mock_events_list: ApiResult,
+) -> None:
+    """Test load and remove of a ConfigEntry."""
+    mock_calendars_list({"items": [test_api_calendar]})
+    mock_events_list({})
+    assert await component_setup()
+
+    entries = hass.config_entries.async_entries(DOMAIN)
+    assert len(entries) == 1
+    entry = entries[0]
+    assert entry.state is ConfigEntryState.LOADED
+
+    assert await hass.config_entries.async_remove(entry.entry_id)
+    assert entry.state == ConfigEntryState.NOT_LOADED