From 0e2bea038d1f6ef9b5f4ff507198d61342eaa208 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Wed, 26 Oct 2022 07:57:49 -0700 Subject: [PATCH] Update Google Calendar to synchronize calendar events efficiently (#80925) * Sync google calendar and serve from local storage Update to use new gcal_sync APIs Update google calendar filter logic Remove storage on config entry removal Make timeline queries timezone aware Do not block startup while syncing * Minor readability tweaks * Remove unnecessary args to async_add_entities * Change how task is created on startup * Update homeassistant/components/google/calendar.py Co-authored-by: Martin Hjelmare * Revert min time between updates Co-authored-by: Martin Hjelmare --- homeassistant/components/google/__init__.py | 11 ++ homeassistant/components/google/calendar.py | 110 +++++++++------- homeassistant/components/google/const.py | 1 + homeassistant/components/google/store.py | 53 ++++++++ tests/components/google/conftest.py | 12 +- tests/components/google/test_calendar.py | 131 ++++++++++---------- tests/components/google/test_init.py | 21 ++++ 7 files changed, 223 insertions(+), 116 deletions(-) create mode 100644 homeassistant/components/google/store.py diff --git a/homeassistant/components/google/__init__.py b/homeassistant/components/google/__init__.py index 33a1216085d..c4e51739efd 100644 --- a/homeassistant/components/google/__init__.py +++ b/homeassistant/components/google/__init__.py @@ -36,6 +36,7 @@ from homeassistant.helpers.entity import generate_entity_id from .api import ApiAuthImpl, get_feature_access from .const import ( DATA_SERVICE, + DATA_STORE, DOMAIN, EVENT_DESCRIPTION, EVENT_END_DATE, @@ -49,6 +50,7 @@ from .const import ( EVENT_TYPES_CONF, FeatureAccess, ) +from .store import LocalCalendarStore _LOGGER = logging.getLogger(__name__) @@ -171,6 +173,9 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: ApiAuthImpl(async_get_clientsession(hass), session) ) hass.data[DOMAIN][entry.entry_id][DATA_SERVICE] = calendar_service + hass.data[DOMAIN][entry.entry_id][DATA_STORE] = LocalCalendarStore( + hass, entry.entry_id + ) if entry.unique_id is None: try: @@ -213,6 +218,12 @@ async def async_reload_entry(hass: HomeAssistant, entry: ConfigEntry) -> None: await hass.config_entries.async_reload(entry.entry_id) +async def async_remove_entry(hass: HomeAssistant, entry: ConfigEntry) -> None: + """Handle removal of a local storage.""" + store = LocalCalendarStore(hass, entry.entry_id) + await store.async_remove() + + async def async_setup_add_event_service( hass: HomeAssistant, calendar_service: GoogleCalendarService, diff --git a/homeassistant/components/google/calendar.py b/homeassistant/components/google/calendar.py index 995fb1ec98f..ca1228759cd 100644 --- a/homeassistant/components/google/calendar.py +++ b/homeassistant/components/google/calendar.py @@ -2,13 +2,17 @@ from __future__ import annotations +import asyncio from datetime import datetime, timedelta import logging from typing import Any -from gcal_sync.api import GoogleCalendarService, ListEventsRequest +from gcal_sync.api import SyncEventsRequest from gcal_sync.exceptions import ApiException from gcal_sync.model import DateOrDatetime, Event +from gcal_sync.store import ScopedCalendarStore +from gcal_sync.sync import CalendarEventSyncManager +from gcal_sync.timeline import Timeline import voluptuous as vol from homeassistant.components.calendar import ( @@ -34,12 +38,12 @@ from homeassistant.helpers.update_coordinator import ( DataUpdateCoordinator, UpdateFailed, ) +from homeassistant.util import dt as dt_util from . import ( CONF_IGNORE_AVAILABILITY, CONF_SEARCH, CONF_TRACK, - DATA_SERVICE, DEFAULT_CONF_OFFSET, DOMAIN, YAML_DEVICES, @@ -49,6 +53,8 @@ from . import ( ) from .api import get_feature_access from .const import ( + DATA_SERVICE, + DATA_STORE, EVENT_DESCRIPTION, EVENT_END_DATE, EVENT_END_DATETIME, @@ -66,6 +72,10 @@ _LOGGER = logging.getLogger(__name__) MIN_TIME_BETWEEN_UPDATES = timedelta(minutes=15) +# Avoid syncing super old data on initial syncs. Note that old but active +# recurring events are still included. +SYNC_EVENT_MIN_TIME = timedelta(days=-90) + # Events have a transparency that determine whether or not they block time on calendar. # When an event is opaque, it means "Show me as busy" which is the default. Events that # are not opaque are ignored by default. @@ -115,6 +125,7 @@ async def async_setup_entry( ) -> None: """Set up the google calendar platform.""" calendar_service = hass.data[DOMAIN][config_entry.entry_id][DATA_SERVICE] + store = hass.data[DOMAIN][config_entry.entry_id][DATA_STORE] try: result = await calendar_service.async_list_calendars() except ApiException as err: @@ -185,12 +196,20 @@ async def async_setup_entry( entity_registry.async_remove( entity_entry.entity_id, ) + request_template = SyncEventsRequest( + calendar_id=calendar_id, + search=data.get(CONF_SEARCH), + start_time=dt_util.now() + SYNC_EVENT_MIN_TIME, + ) + sync = CalendarEventSyncManager( + calendar_service, + store=ScopedCalendarStore(store, unique_id or entity_name), + request_template=request_template, + ) coordinator = CalendarUpdateCoordinator( hass, - calendar_service, + sync, data[CONF_NAME], - calendar_id, - data.get(CONF_SEARCH), ) entities.append( GoogleCalendarEntity( @@ -203,7 +222,7 @@ async def async_setup_entry( ) ) - async_add_entities(entities, True) + async_add_entities(entities) if calendars and new_calendars: @@ -223,16 +242,14 @@ async def async_setup_entry( ) -class CalendarUpdateCoordinator(DataUpdateCoordinator): +class CalendarUpdateCoordinator(DataUpdateCoordinator[Timeline]): """Coordinator for calendar RPC calls.""" def __init__( self, hass: HomeAssistant, - calendar_service: GoogleCalendarService, + sync: CalendarEventSyncManager, name: str, - calendar_id: str, - search: str | None, ) -> None: """Create the Calendar event device.""" super().__init__( @@ -241,38 +258,18 @@ class CalendarUpdateCoordinator(DataUpdateCoordinator): name=name, update_interval=MIN_TIME_BETWEEN_UPDATES, ) - self.calendar_service = calendar_service - self.calendar_id = calendar_id - self._search = search + self.sync = sync - async def async_get_events( - self, start_date: datetime, end_date: datetime - ) -> list[Event]: - """Get all events in a specific time frame.""" - request = ListEventsRequest( - calendar_id=self.calendar_id, - start_time=start_date, - end_time=end_date, - search=self._search, - ) - result_items = [] - try: - result = await self.calendar_service.async_list_events(request) - async for result_page in result: - result_items.extend(result_page.items) - except ApiException as err: - self.async_set_update_error(err) - raise HomeAssistantError(str(err)) from err - return result_items - - async def _async_update_data(self) -> list[Event]: + async def _async_update_data(self) -> Timeline: """Fetch data from API endpoint.""" - request = ListEventsRequest(calendar_id=self.calendar_id, search=self._search) try: - result = await self.calendar_service.async_list_events(request) + await self.sync.run() except ApiException as err: raise UpdateFailed(f"Error communicating with API: {err}") from err - return result.items + + return await self.sync.store_service.async_get_timeline( + dt_util.DEFAULT_TIME_ZONE + ) class GoogleCalendarEntity(CoordinatorEntity, CalendarEntity): @@ -341,16 +338,28 @@ class GoogleCalendarEntity(CoordinatorEntity, CalendarEntity): async def async_added_to_hass(self) -> None: """When entity is added to hass.""" await super().async_added_to_hass() + # We do not ask for an update with async_add_entities() - # because it will update disabled entities - await self.coordinator.async_request_refresh() - self._apply_coordinator_update() + # because it will update disabled entities. This is started as a + # task to let if sync in the background without blocking startup + async def refresh() -> None: + await self.coordinator.async_request_refresh() + self._apply_coordinator_update() + + asyncio.create_task(refresh()) async def async_get_events( self, hass: HomeAssistant, start_date: datetime, end_date: datetime ) -> list[CalendarEvent]: """Get all events in a specific time frame.""" - result_items = await self.coordinator.async_get_events(start_date, end_date) + if not (timeline := self.coordinator.data): + raise HomeAssistantError( + "Unable to get events: Sync from server has not completed" + ) + result_items = timeline.overlapping( + dt_util.as_local(start_date), + dt_util.as_local(end_date), + ) return [ _get_calendar_event(event) for event in filter(self._event_filter, result_items) @@ -358,13 +367,21 @@ class GoogleCalendarEntity(CoordinatorEntity, CalendarEntity): def _apply_coordinator_update(self) -> None: """Copy state from the coordinator to this entity.""" - events = self.coordinator.data - api_event = next(filter(self._event_filter, iter(events)), None) - self._event = _get_calendar_event(api_event) if api_event else None - if self._event: + if (timeline := self.coordinator.data) and ( + api_event := next( + filter( + self._event_filter, + timeline.active_after(dt_util.now()), + ), + None, + ) + ): + self._event = _get_calendar_event(api_event) (self._event.summary, self._offset_value) = extract_offset( self._event.summary, self._offset ) + else: + self._event = None @callback def _handle_coordinator_update(self) -> None: @@ -432,7 +449,7 @@ async def async_create_event(entity: GoogleCalendarEntity, call: ServiceCall) -> raise ValueError("Missing required fields to set start or end date/datetime") try: - await entity.coordinator.calendar_service.async_create_event( + await entity.coordinator.sync.api.async_create_event( entity.calendar_id, Event( summary=call.data[EVENT_SUMMARY], @@ -443,3 +460,4 @@ async def async_create_event(entity: GoogleCalendarEntity, call: ServiceCall) -> ) except ApiException as err: raise HomeAssistantError(str(err)) from err + entity.async_write_ha_state() diff --git a/homeassistant/components/google/const.py b/homeassistant/components/google/const.py index f07958c2e6e..6a2c1974f66 100644 --- a/homeassistant/components/google/const.py +++ b/homeassistant/components/google/const.py @@ -10,6 +10,7 @@ CONF_CALENDAR_ACCESS = "calendar_access" DATA_CALENDARS = "calendars" DATA_SERVICE = "service" DATA_CONFIG = "config" +DATA_STORE = "store" class FeatureAccess(Enum): diff --git a/homeassistant/components/google/store.py b/homeassistant/components/google/store.py new file mode 100644 index 00000000000..c4d9e4c3e9c --- /dev/null +++ b/homeassistant/components/google/store.py @@ -0,0 +1,53 @@ +"""Google Calendar local storage.""" + +from __future__ import annotations + +import logging +from typing import Any + +from gcal_sync.store import CalendarStore + +from homeassistant.core import HomeAssistant +from homeassistant.helpers.storage import Store + +from .const import DOMAIN + +_LOGGER = logging.getLogger(__name__) + +STORAGE_KEY_FORMAT = "{domain}.{entry_id}" +STORAGE_VERSION = 1 +# Buffer writes every few minutes (plus guaranteed to be written at shutdown) +STORAGE_SAVE_DELAY_SECONDS = 120 + + +class LocalCalendarStore(CalendarStore): + """Storage for local persistence of calendar and event data.""" + + def __init__(self, hass: HomeAssistant, entry_id: str) -> None: + """Initialize LocalCalendarStore.""" + self._store = Store[dict[str, Any]]( + hass, + STORAGE_VERSION, + STORAGE_KEY_FORMAT.format(domain=DOMAIN, entry_id=entry_id), + private=True, + ) + self._data: dict[str, Any] | None = None + + async def async_load(self) -> dict[str, Any] | None: + """Load data.""" + if self._data is None: + self._data = await self._store.async_load() or {} + return self._data + + async def async_save(self, data: dict[str, Any]) -> None: + """Save data.""" + self._data = data + + def provide_data() -> dict: + return data + + self._store.async_delay_save(provide_data, STORAGE_SAVE_DELAY_SECONDS) + + async def async_remove(self) -> None: + """Remove data.""" + await self._store.async_remove() diff --git a/tests/components/google/conftest.py b/tests/components/google/conftest.py index e6b7c26ca84..2f5efd829bf 100644 --- a/tests/components/google/conftest.py +++ b/tests/components/google/conftest.py @@ -206,9 +206,13 @@ def mock_events_list( ) -> None: if calendar_id is None: calendar_id = CALENDAR_ID + resp = { + **response, + "nextSyncToken": "sync-token", + } aioclient_mock.get( f"{API_BASE_URL}/calendars/{calendar_id}/events", - json=response, + json=resp, exc=exc, ) return @@ -236,9 +240,13 @@ def mock_calendars_list( """Fixture to construct a fake calendar list API response.""" def _result(response: dict[str, Any], exc: ClientError | None = None) -> None: + resp = { + **response, + "nextSyncToken": "sync-token", + } aioclient_mock.get( f"{API_BASE_URL}/users/me/calendarList", - json=response, + json=resp, exc=exc, ) return diff --git a/tests/components/google/test_calendar.py b/tests/components/google/test_calendar.py index 778e4a843eb..3bd584f4c6f 100644 --- a/tests/components/google/test_calendar.py +++ b/tests/components/google/test_calendar.py @@ -2,7 +2,6 @@ from __future__ import annotations -import copy import datetime from http import HTTPStatus from typing import Any @@ -10,7 +9,6 @@ from unittest.mock import patch import urllib from aiohttp.client_exceptions import ClientError -from gcal_sync.auth import API_BASE_URL import pytest from homeassistant.components.google.const import DOMAIN @@ -28,7 +26,6 @@ from .conftest import ( ) from tests.common import async_fire_time_changed -from tests.test_util.aiohttp import AiohttpClientMockResponse TEST_ENTITY = TEST_API_ENTITY TEST_ENTITY_NAME = TEST_API_ENTITY_NAME @@ -76,6 +73,11 @@ def mock_test_setup( return +def get_events_url(entity: str, start: str, end: str) -> str: + """Create a url to get events during the specified time range.""" + return f"/api/calendars/{entity}?start={urllib.parse.quote(start)}&end={urllib.parse.quote(end)}" + + def upcoming() -> dict[str, Any]: """Create a test event with an arbitrary start/end time fetched from the api url.""" now = dt_util.now() @@ -90,7 +92,7 @@ def upcoming_event_url(entity: str = TEST_ENTITY) -> str: now = dt_util.now() start = (now - datetime.timedelta(minutes=60)).isoformat() end = (now + datetime.timedelta(minutes=60)).isoformat() - return f"/api/calendars/{entity}?start={urllib.parse.quote(start)}&end={urllib.parse.quote(end)}" + return get_events_url(entity, start, end) async def test_all_day_event(hass, mock_events_list_items, component_setup): @@ -406,14 +408,12 @@ async def test_http_event_api_failure( aioclient_mock, ): """Test the Rest API response during a calendar failure.""" - mock_events_list({}) + mock_events_list({}, exc=ClientError()) + assert await component_setup() client = await hass_client() - aioclient_mock.clear_requests() - mock_events_list({}, exc=ClientError()) - response = await client.get(upcoming_event_url()) assert response.status == HTTPStatus.INTERNAL_SERVER_ERROR @@ -472,66 +472,6 @@ async def test_http_api_all_day_event( } -@pytest.mark.freeze_time("2022-03-27 12:05:00+00:00") -async def test_http_api_event_paging( - hass, hass_client, aioclient_mock, component_setup -): - """Test paging through results from the server.""" - hass.config.set_time_zone("Asia/Baghdad") - - responses = [ - { - "nextPageToken": "page-token", - "items": [ - { - **TEST_EVENT, - "summary": "event 1", - **upcoming(), - } - ], - }, - { - "items": [ - { - **TEST_EVENT, - "summary": "event 2", - **upcoming(), - } - ], - }, - ] - - def next_response(response_list): - results = copy.copy(response_list) - - async def get(method, url, data): - return AiohttpClientMockResponse(method, url, json=results.pop(0)) - - return get - - # Setup response for initial entity load - aioclient_mock.get( - f"{API_BASE_URL}/calendars/{CALENDAR_ID}/events", - side_effect=next_response(responses), - ) - assert await component_setup() - - # Setup response for API request - aioclient_mock.clear_requests() - aioclient_mock.get( - f"{API_BASE_URL}/calendars/{CALENDAR_ID}/events", - side_effect=next_response(responses), - ) - - client = await hass_client() - response = await client.get(upcoming_event_url()) - assert response.status == HTTPStatus.OK - events = await response.json() - assert len(events) == 2 - assert events[0]["summary"] == "event 1" - assert events[1]["summary"] == "event 2" - - @pytest.mark.parametrize( "calendars_config_ignore_availability,transparency,expect_visible_event", [ @@ -781,3 +721,58 @@ async def test_invalid_unique_id_cleanup( entity_registry, config_entry.entry_id ) assert not registry_entries + + +@pytest.mark.parametrize( + "time_zone,event_order", + [ + ("America/Los_Angeles", ["One", "Two", "All Day Event"]), + ("America/Regina", ["One", "Two", "All Day Event"]), + ("UTC", ["One", "All Day Event", "Two"]), + ("Asia/Tokyo", ["All Day Event", "One", "Two"]), + ], +) +async def test_all_day_iter_order( + hass, + hass_client, + mock_events_list_items, + component_setup, + time_zone, + event_order, +): + """Test the sort order of an all day events depending on the time zone.""" + hass.config.set_time_zone(time_zone) + mock_events_list_items( + [ + { + **TEST_EVENT, + "id": "event-id-3", + "summary": "All Day Event", + "start": {"date": "2022-10-08"}, + "end": {"date": "2022-10-09"}, + }, + { + **TEST_EVENT, + "id": "event-id-1", + "summary": "One", + "start": {"dateTime": "2022-10-07T23:00:00+00:00"}, + "end": {"dateTime": "2022-10-07T23:30:00+00:00"}, + }, + { + **TEST_EVENT, + "id": "event-id-2", + "summary": "Two", + "start": {"dateTime": "2022-10-08T01:00:00+00:00"}, + "end": {"dateTime": "2022-10-08T02:00:00+00:00"}, + }, + ] + ) + assert await component_setup() + + client = await hass_client() + response = await client.get( + get_events_url(TEST_ENTITY, "2022-10-06T00:00:00Z", "2022-10-09T00:00:00Z") + ) + assert response.status == HTTPStatus.OK + events = await response.json() + assert [event["summary"] for event in events] == event_order diff --git a/tests/components/google/test_init.py b/tests/components/google/test_init.py index 613aa6dbb70..5e7696eec68 100644 --- a/tests/components/google/test_init.py +++ b/tests/components/google/test_init.py @@ -818,3 +818,24 @@ async def test_assign_unique_id_failure( assert config_entry.state is config_entry_status assert config_entry.unique_id is None + + +async def test_remove_entry( + hass: HomeAssistant, + mock_calendars_list: ApiResult, + component_setup: ComponentSetup, + test_api_calendar: dict[str, Any], + mock_events_list: ApiResult, +) -> None: + """Test load and remove of a ConfigEntry.""" + mock_calendars_list({"items": [test_api_calendar]}) + mock_events_list({}) + assert await component_setup() + + entries = hass.config_entries.async_entries(DOMAIN) + assert len(entries) == 1 + entry = entries[0] + assert entry.state is ConfigEntryState.LOADED + + assert await hass.config_entries.async_remove(entry.entry_id) + assert entry.state == ConfigEntryState.NOT_LOADED