From 096e814929ebd05017a05685e191feb7bab6494e Mon Sep 17 00:00:00 2001 From: mkmer Date: Thu, 6 Apr 2023 04:44:13 -0400 Subject: [PATCH 001/110] Handle Uncaught exceptions in async_update Honeywell (#90746) --- homeassistant/components/honeywell/climate.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/honeywell/climate.py b/homeassistant/components/honeywell/climate.py index e9dae1e2074..bde70e6bb0c 100644 --- a/homeassistant/components/honeywell/climate.py +++ b/homeassistant/components/honeywell/climate.py @@ -1,9 +1,11 @@ """Support for Honeywell (US) Total Connect Comfort climate systems.""" from __future__ import annotations +import asyncio import datetime from typing import Any +from aiohttp import ClientConnectionError import aiosomecomfort from homeassistant.components.climate import ( @@ -421,10 +423,7 @@ class HoneywellUSThermostat(ClimateEntity): try: await self._device.refresh() self._attr_available = True - except ( - aiosomecomfort.SomeComfortError, - OSError, - ): + except aiosomecomfort.SomeComfortError: try: await self._data.client.login() @@ -433,5 +432,12 @@ class HoneywellUSThermostat(ClimateEntity): await self.hass.async_create_task( self.hass.config_entries.async_reload(self._data.entry_id) ) - except aiosomecomfort.SomeComfortError: + except ( + aiosomecomfort.SomeComfortError, + ClientConnectionError, + asyncio.TimeoutError, + ): self._attr_available = False + + except (ClientConnectionError, asyncio.TimeoutError): + self._attr_available = False From 833b95722e5255091ec0b9f2d0ce0b44f3064579 Mon Sep 17 00:00:00 2001 From: saschaabraham <34794615+saschaabraham@users.noreply.github.com> Date: Thu, 6 Apr 2023 10:31:43 +0200 Subject: [PATCH 002/110] Bump fritzconnection to 1.12.0 (#90799) --- homeassistant/components/fritz/manifest.json | 2 +- homeassistant/components/fritzbox_callmonitor/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/fritz/manifest.json b/homeassistant/components/fritz/manifest.json index 1008426558f..b117218e23d 100644 --- a/homeassistant/components/fritz/manifest.json +++ b/homeassistant/components/fritz/manifest.json @@ -7,7 +7,7 @@ "documentation": "https://www.home-assistant.io/integrations/fritz", "iot_class": "local_polling", "loggers": ["fritzconnection"], - "requirements": ["fritzconnection==1.11.0", "xmltodict==0.13.0"], + "requirements": ["fritzconnection==1.12.0", "xmltodict==0.13.0"], "ssdp": [ { "st": "urn:schemas-upnp-org:device:fritzbox:1" diff --git a/homeassistant/components/fritzbox_callmonitor/manifest.json b/homeassistant/components/fritzbox_callmonitor/manifest.json index 9f1078b9b9d..cde955caa1e 100644 --- a/homeassistant/components/fritzbox_callmonitor/manifest.json +++ b/homeassistant/components/fritzbox_callmonitor/manifest.json @@ -7,5 +7,5 @@ "integration_type": "device", "iot_class": "local_polling", "loggers": ["fritzconnection"], - "requirements": ["fritzconnection==1.11.0"] + "requirements": ["fritzconnection==1.12.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index b92bf742c93..6738ca09cd1 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -748,7 +748,7 @@ freesms==0.2.0 # homeassistant.components.fritz # homeassistant.components.fritzbox_callmonitor -fritzconnection==1.11.0 +fritzconnection==1.12.0 # homeassistant.components.google_translate gTTS==2.2.4 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 6ce0b64e691..186cb7fb93f 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -570,7 +570,7 @@ freebox-api==1.1.0 # homeassistant.components.fritz # homeassistant.components.fritzbox_callmonitor -fritzconnection==1.11.0 +fritzconnection==1.12.0 # homeassistant.components.google_translate gTTS==2.2.4 From 968a4e48185a108984544bd6f809b0456e6b23c9 Mon Sep 17 00:00:00 2001 From: Tom Harris Date: Wed, 5 Apr 2023 20:53:44 -0400 Subject: [PATCH 003/110] Fix issue with Insteon All-Link Database loading (#90858) Bump to 1.4.1 --- homeassistant/components/insteon/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/insteon/manifest.json b/homeassistant/components/insteon/manifest.json index af9396399af..d9c2380de0f 100644 --- a/homeassistant/components/insteon/manifest.json +++ b/homeassistant/components/insteon/manifest.json @@ -17,7 +17,7 @@ "iot_class": "local_push", "loggers": ["pyinsteon", "pypubsub"], "requirements": [ - "pyinsteon==1.4.0", + "pyinsteon==1.4.1", "insteon-frontend-home-assistant==0.3.4" ], "usb": [ diff --git a/requirements_all.txt b/requirements_all.txt index 6738ca09cd1..56723565570 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1684,7 +1684,7 @@ pyialarm==2.2.0 pyicloud==1.0.0 # homeassistant.components.insteon -pyinsteon==1.4.0 +pyinsteon==1.4.1 # homeassistant.components.intesishome pyintesishome==1.8.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 186cb7fb93f..985a1c628f6 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1218,7 +1218,7 @@ pyialarm==2.2.0 pyicloud==1.0.0 # homeassistant.components.insteon -pyinsteon==1.4.0 +pyinsteon==1.4.1 # homeassistant.components.ipma pyipma==3.0.6 From c663d8754b2a44c2282c505ddddc262b78784ca3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Apr 2023 14:53:19 -1000 Subject: [PATCH 004/110] Generate a seperate log message per dumped object for profiler.dump_log_objects (#90867) Since some objects are very large we can generate overly large log messages ``` Event data for system_log_event exceed maximum size of 32768 bytes. This can cause database performance issues; Event data will not be stored ``` Reported in https://ptb.discord.com/channels/330944238910963714/427516175237382144/1093069996101472306 --- homeassistant/components/profiler/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/profiler/__init__.py b/homeassistant/components/profiler/__init__.py index 95ce69aed4a..f558b7301c5 100644 --- a/homeassistant/components/profiler/__init__.py +++ b/homeassistant/components/profiler/__init__.py @@ -164,11 +164,12 @@ async def async_setup_entry( # noqa: C901 obj_type = call.data[CONF_TYPE] - _LOGGER.critical( - "%s objects in memory: %s", - obj_type, - [_safe_repr(obj) for obj in objgraph.by_type(obj_type)], - ) + for obj in objgraph.by_type(obj_type): + _LOGGER.critical( + "%s object in memory: %s", + obj_type, + _safe_repr(obj), + ) persistent_notification.create( hass, From b4e2dd4e063549da5cdfe940e0dafc7cae17bdd9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Apr 2023 08:03:47 -1000 Subject: [PATCH 005/110] Add constraint for websockets to <11.0 (#90868) --- homeassistant/package_constraints.txt | 5 +++++ script/gen_requirements_all.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 731b57e4cf6..a1867bef133 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -157,3 +157,8 @@ uamqp==1.6.0;python_version<'3.11' # faust-cchardet: Ensure we have a version we can build wheels # 2.1.18 is the first version that works with our wheel builder faust-cchardet>=2.1.18 + +# websockets 11.0 is missing files in the source distribution +# which break wheel builds +# https://github.com/aaugustin/websockets/issues/1329 +websockets<11.0 diff --git a/script/gen_requirements_all.py b/script/gen_requirements_all.py index 564d0e2eb00..b31674d04e8 100755 --- a/script/gen_requirements_all.py +++ b/script/gen_requirements_all.py @@ -162,6 +162,11 @@ uamqp==1.6.0;python_version<'3.11' # faust-cchardet: Ensure we have a version we can build wheels # 2.1.18 is the first version that works with our wheel builder faust-cchardet>=2.1.18 + +# websockets 11.0 is missing files in the source distribution +# which break wheel builds +# https://github.com/aaugustin/websockets/issues/1329 +websockets<11.0 """ IGNORE_PRE_COMMIT_HOOK_ID = ( From c8ee45b53ca1a048fcb6e36a20318d1d0c7d0d6a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Apr 2023 14:46:56 -1000 Subject: [PATCH 006/110] Add MariaDB deadlock retry wrapper to database timestamp column migrations (#90880) Add deadlock retry wrapper to timestamp column migrations fixes #90819 --- homeassistant/components/recorder/migration.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py index 23382a9aeb3..7fee3d16e8d 100644 --- a/homeassistant/components/recorder/migration.py +++ b/homeassistant/components/recorder/migration.py @@ -913,7 +913,7 @@ def _apply_update( # noqa: C901 _create_index(session_maker, "events", "ix_events_event_type_time_fired_ts") _create_index(session_maker, "states", "ix_states_entity_id_last_updated_ts") _create_index(session_maker, "states", "ix_states_last_updated_ts") - _migrate_columns_to_timestamp(session_maker, engine) + _migrate_columns_to_timestamp(instance, session_maker, engine) elif new_version == 32: # Migration is done in two steps to ensure we can start using # the new columns before we wipe the old ones. @@ -966,7 +966,7 @@ def _apply_update( # noqa: C901 "ix_statistics_short_term_statistic_id_start_ts", ) try: - _migrate_statistics_columns_to_timestamp(session_maker, engine) + _migrate_statistics_columns_to_timestamp(instance, session_maker, engine) except IntegrityError as ex: _LOGGER.error( "Statistics table contains duplicate entries: %s; " @@ -979,7 +979,7 @@ def _apply_update( # noqa: C901 # and try again with session_scope(session=session_maker()) as session: delete_statistics_duplicates(instance, hass, session) - _migrate_statistics_columns_to_timestamp(session_maker, engine) + _migrate_statistics_columns_to_timestamp(instance, session_maker, engine) # Log at error level to ensure the user sees this message in the log # since we logged the error above. _LOGGER.error( @@ -1195,8 +1195,9 @@ def _wipe_old_string_time_columns( session.commit() +@database_job_retry_wrapper("Migrate columns to timestamp", 3) def _migrate_columns_to_timestamp( - session_maker: Callable[[], Session], engine: Engine + instance: Recorder, session_maker: Callable[[], Session], engine: Engine ) -> None: """Migrate columns to use timestamp.""" # Migrate all data in Events.time_fired to Events.time_fired_ts @@ -1283,8 +1284,9 @@ def _migrate_columns_to_timestamp( ) +@database_job_retry_wrapper("Migrate statistics columns to timestamp", 3) def _migrate_statistics_columns_to_timestamp( - session_maker: Callable[[], Session], engine: Engine + instance: Recorder, session_maker: Callable[[], Session], engine: Engine ) -> None: """Migrate statistics columns to use timestamp.""" # Migrate all data in statistics.start to statistics.start_ts From f341d0787e4626092b4d19547f633776aabb9287 Mon Sep 17 00:00:00 2001 From: Michael <35783820+mib1185@users.noreply.github.com> Date: Thu, 6 Apr 2023 02:52:37 +0200 Subject: [PATCH 007/110] Migrate entity unique ids in PI-Hole (#90883) * migrate entity unique ids * Update homeassistant/components/pi_hole/__init__.py --------- Co-authored-by: Paulus Schoutsen --- homeassistant/components/pi_hole/__init__.py | 36 ++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/pi_hole/__init__.py b/homeassistant/components/pi_hole/__init__.py index 49f1697adc6..96cdd7ab105 100644 --- a/homeassistant/components/pi_hole/__init__.py +++ b/homeassistant/components/pi_hole/__init__.py @@ -16,9 +16,9 @@ from homeassistant.const import ( CONF_VERIFY_SSL, Platform, ) -from homeassistant.core import HomeAssistant +from homeassistant.core import HomeAssistant, callback from homeassistant.exceptions import ConfigEntryAuthFailed -from homeassistant.helpers import config_validation as cv +from homeassistant.helpers import config_validation as cv, entity_registry as er from homeassistant.helpers.aiohttp_client import async_get_clientsession from homeassistant.helpers.entity import DeviceInfo from homeassistant.helpers.update_coordinator import ( @@ -64,6 +64,38 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: _LOGGER.debug("Setting up %s integration with host %s", DOMAIN, host) + name_to_key = { + "Core Update Available": "core_update_available", + "Web Update Available": "web_update_available", + "FTL Update Available": "ftl_update_available", + "Status": "status", + "Ads Blocked Today": "ads_blocked_today", + "Ads Percentage Blocked Today": "ads_percentage_today", + "Seen Clients": "clients_ever_seen", + "DNS Queries Today": "dns_queries_today", + "Domains Blocked": "domains_being_blocked", + "DNS Queries Cached": "queries_cached", + "DNS Queries Forwarded": "queries_forwarded", + "DNS Unique Clients": "unique_clients", + "DNS Unique Domains": "unique_domains", + } + + @callback + def update_unique_id( + entity_entry: er.RegistryEntry, + ) -> dict[str, str] | None: + """Update unique ID of entity entry.""" + unique_id_parts = entity_entry.unique_id.split("/") + if len(unique_id_parts) == 2 and unique_id_parts[1] in name_to_key: + name = unique_id_parts[1] + new_unique_id = entity_entry.unique_id.replace(name, name_to_key[name]) + _LOGGER.debug("Migrate %s to %s", entity_entry.unique_id, new_unique_id) + return {"new_unique_id": new_unique_id} + + return None + + await er.async_migrate_entries(hass, entry.entry_id, update_unique_id) + session = async_get_clientsession(hass, verify_tls) api = Hole( host, From 513a13f36907b669793712b4710c265efe985caa Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Apr 2023 22:33:00 -1000 Subject: [PATCH 008/110] Fix missing bluetooth client wrapper in bleak_retry_connector (#90885) --- homeassistant/components/bluetooth/usage.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/bluetooth/usage.py b/homeassistant/components/bluetooth/usage.py index b751559e7a4..d89f0b5b684 100644 --- a/homeassistant/components/bluetooth/usage.py +++ b/homeassistant/components/bluetooth/usage.py @@ -10,9 +10,10 @@ from .wrappers import HaBleakClientWrapper, HaBleakScannerWrapper ORIGINAL_BLEAK_SCANNER = bleak.BleakScanner ORIGINAL_BLEAK_CLIENT = bleak.BleakClient -ORIGINAL_BLEAK_RETRY_CONNECTOR_CLIENT = ( +ORIGINAL_BLEAK_RETRY_CONNECTOR_CLIENT_WITH_SERVICE_CACHE = ( bleak_retry_connector.BleakClientWithServiceCache ) +ORIGINAL_BLEAK_RETRY_CONNECTOR_CLIENT = bleak_retry_connector.BleakClient def install_multiple_bleak_catcher() -> None: @@ -23,6 +24,7 @@ def install_multiple_bleak_catcher() -> None: bleak.BleakScanner = HaBleakScannerWrapper # type: ignore[misc, assignment] bleak.BleakClient = HaBleakClientWrapper # type: ignore[misc] bleak_retry_connector.BleakClientWithServiceCache = HaBleakClientWithServiceCache # type: ignore[misc,assignment] # noqa: E501 + bleak_retry_connector.BleakClient = HaBleakClientWrapper # type: ignore[misc] # noqa: E501 def uninstall_multiple_bleak_catcher() -> None: @@ -30,6 +32,9 @@ def uninstall_multiple_bleak_catcher() -> None: bleak.BleakScanner = ORIGINAL_BLEAK_SCANNER # type: ignore[misc] bleak.BleakClient = ORIGINAL_BLEAK_CLIENT # type: ignore[misc] bleak_retry_connector.BleakClientWithServiceCache = ( # type: ignore[misc] + ORIGINAL_BLEAK_RETRY_CONNECTOR_CLIENT_WITH_SERVICE_CACHE + ) + bleak_retry_connector.BleakClient = ( # type: ignore[misc] ORIGINAL_BLEAK_RETRY_CONNECTOR_CLIENT ) From 42b7ed115fd8ebb0dd96827ce32e6e4766d93fa9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Apr 2023 16:19:43 -1000 Subject: [PATCH 009/110] Bump ulid-transform 0.6.0 (#90888) * Bump ulid-transform 0.6.0 changelog: https://github.com/bdraco/ulid-transform/compare/v0.5.1...v0.6.0 to find the source of the invalid ulids in https://github.com/home-assistant/core/issues/90887 --- homeassistant/package_constraints.txt | 2 +- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index a1867bef133..69f46be2fac 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -46,7 +46,7 @@ requests==2.28.2 scapy==2.5.0 sqlalchemy==2.0.7 typing-extensions>=4.5.0,<5.0 -ulid-transform==0.5.1 +ulid-transform==0.6.0 voluptuous-serialize==2.6.0 voluptuous==0.13.1 yarl==1.8.1 diff --git a/pyproject.toml b/pyproject.toml index fecbea62be9..36e36872bcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ "pyyaml==6.0", "requests==2.28.2", "typing-extensions>=4.5.0,<5.0", - "ulid-transform==0.5.1", + "ulid-transform==0.6.0", "voluptuous==0.13.1", "voluptuous-serialize==2.6.0", "yarl==1.8.1", diff --git a/requirements.txt b/requirements.txt index 84726cb49d9..43bb2d6d37c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ python-slugify==4.0.1 pyyaml==6.0 requests==2.28.2 typing-extensions>=4.5.0,<5.0 -ulid-transform==0.5.1 +ulid-transform==0.6.0 voluptuous==0.13.1 voluptuous-serialize==2.6.0 yarl==1.8.1 From 0b3012071e5dc722c733f542f32f4bbfc33263f6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Apr 2023 22:34:54 -1000 Subject: [PATCH 010/110] Guard against invalid ULIDs in contexts while recording events (#90889) --- .../components/recorder/models/context.py | 17 ++++++++++-- tests/components/recorder/test_init.py | 27 ++++++++++++++++++- tests/components/recorder/test_models.py | 26 ++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/recorder/models/context.py b/homeassistant/components/recorder/models/context.py index dbd9383bdeb..f722c519833 100644 --- a/homeassistant/components/recorder/models/context.py +++ b/homeassistant/components/recorder/models/context.py @@ -3,23 +3,36 @@ from __future__ import annotations from contextlib import suppress from functools import lru_cache +import logging from uuid import UUID from homeassistant.util.ulid import bytes_to_ulid, ulid_to_bytes +_LOGGER = logging.getLogger(__name__) + def ulid_to_bytes_or_none(ulid: str | None) -> bytes | None: """Convert an ulid to bytes.""" if ulid is None: return None - return ulid_to_bytes(ulid) + try: + return ulid_to_bytes(ulid) + except ValueError as ex: + _LOGGER.error("Error converting ulid %s to bytes: %s", ulid, ex, exc_info=True) + return None def bytes_to_ulid_or_none(_bytes: bytes | None) -> str | None: """Convert bytes to a ulid.""" if _bytes is None: return None - return bytes_to_ulid(_bytes) + try: + return bytes_to_ulid(_bytes) + except ValueError as ex: + _LOGGER.error( + "Error converting bytes %s to ulid: %s", _bytes, ex, exc_info=True + ) + return None @lru_cache(maxsize=16) diff --git a/tests/components/recorder/test_init.py b/tests/components/recorder/test_init.py index 2bad346b937..337aced4880 100644 --- a/tests/components/recorder/test_init.py +++ b/tests/components/recorder/test_init.py @@ -66,7 +66,7 @@ from homeassistant.const import ( STATE_LOCKED, STATE_UNLOCKED, ) -from homeassistant.core import CoreState, Event, HomeAssistant, callback +from homeassistant.core import Context, CoreState, Event, HomeAssistant, callback from homeassistant.helpers import entity_registry as er, recorder as recorder_helper from homeassistant.setup import async_setup_component, setup_component from homeassistant.util import dt as dt_util @@ -854,6 +854,31 @@ def test_saving_event_with_oversized_data( assert json_loads(events["test_event_too_big"]) == {} +def test_saving_event_invalid_context_ulid( + hass_recorder: Callable[..., HomeAssistant], caplog: pytest.LogCaptureFixture +) -> None: + """Test we handle invalid manually injected context ids.""" + hass = hass_recorder() + event_data = {"test_attr": 5, "test_attr_10": "nice"} + hass.bus.fire("test_event", event_data, context=Context(id="invalid")) + wait_recording_done(hass) + events = {} + + with session_scope(hass=hass) as session: + for _, data, event_type in ( + session.query(Events.event_id, EventData.shared_data, EventTypes.event_type) + .outerjoin(EventData, Events.data_id == EventData.data_id) + .outerjoin(EventTypes, Events.event_type_id == EventTypes.event_type_id) + .where(EventTypes.event_type.in_(["test_event"])) + ): + events[event_type] = data + + assert "invalid" in caplog.text + + assert len(events) == 1 + assert json_loads(events["test_event"]) == event_data + + def test_recorder_setup_failure(hass: HomeAssistant) -> None: """Test some exceptions.""" recorder_helper.async_initialize_recorder(hass) diff --git a/tests/components/recorder/test_models.py b/tests/components/recorder/test_models.py index df8ffa0d348..f1162815b9e 100644 --- a/tests/components/recorder/test_models.py +++ b/tests/components/recorder/test_models.py @@ -14,9 +14,11 @@ from homeassistant.components.recorder.db_schema import ( ) from homeassistant.components.recorder.models import ( LazyState, + bytes_to_ulid_or_none, process_datetime_to_timestamp, process_timestamp, process_timestamp_to_utc_isoformat, + ulid_to_bytes_or_none, ) from homeassistant.const import EVENT_STATE_CHANGED import homeassistant.core as ha @@ -415,3 +417,27 @@ async def test_process_datetime_to_timestamp_mirrors_utc_isoformat_behavior( process_datetime_to_timestamp(datetime_hst_timezone) == dt_util.parse_datetime("2016-07-09T21:00:00+00:00").timestamp() ) + + +def test_ulid_to_bytes_or_none(caplog: pytest.LogCaptureFixture) -> None: + """Test ulid_to_bytes_or_none.""" + + assert ( + ulid_to_bytes_or_none("01EYQZJXZ5Z1Z1Z1Z1Z1Z1Z1Z1") + == b"\x01w\xaf\xf9w\xe5\xf8~\x1f\x87\xe1\xf8~\x1f\x87\xe1" + ) + assert ulid_to_bytes_or_none("invalid") is None + assert "invalid" in caplog.text + assert ulid_to_bytes_or_none(None) is None + + +def test_bytes_to_ulid_or_none(caplog: pytest.LogCaptureFixture) -> None: + """Test bytes_to_ulid_or_none.""" + + assert ( + bytes_to_ulid_or_none(b"\x01w\xaf\xf9w\xe5\xf8~\x1f\x87\xe1\xf8~\x1f\x87\xe1") + == "01EYQZJXZ5Z1Z1Z1Z1Z1Z1Z1Z1" + ) + assert bytes_to_ulid_or_none(b"invalid") is None + assert "invalid" in caplog.text + assert bytes_to_ulid_or_none(None) is None From 70d1e733f6be4bcee90372e8cf3649c7a85cd95c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Apr 2023 22:34:13 -1000 Subject: [PATCH 011/110] Fix entity_id migration query failing with MySQL 8.0.30 (#90895) --- homeassistant/components/recorder/queries.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/recorder/queries.py b/homeassistant/components/recorder/queries.py index f983224e212..454c71f6dc5 100644 --- a/homeassistant/components/recorder/queries.py +++ b/homeassistant/components/recorder/queries.py @@ -730,7 +730,8 @@ def batch_cleanup_entity_ids() -> StatementLambdaElement: lambda: update(States) .where( States.state_id.in_( - select(States.state_id).join( + select(States.state_id) + .join( states_with_entity_ids := select( States.state_id.label("state_id_with_entity_id") ) @@ -739,6 +740,8 @@ def batch_cleanup_entity_ids() -> StatementLambdaElement: .subquery(), States.state_id == states_with_entity_ids.c.state_id_with_entity_id, ) + .alias("states_with_entity_ids") + .select() ) ) .values(entity_id=None) From cb06541fda0ead3a0c845468e927a6e94e0fa60d Mon Sep 17 00:00:00 2001 From: Aaron Bach Date: Thu, 6 Apr 2023 02:32:35 -0600 Subject: [PATCH 012/110] Bump `simplisafe-python` to 2023.04.0 (#90896) Co-authored-by: J. Nick Koston --- homeassistant/components/simplisafe/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/simplisafe/manifest.json b/homeassistant/components/simplisafe/manifest.json index 9a5a391240a..184e1f306e0 100644 --- a/homeassistant/components/simplisafe/manifest.json +++ b/homeassistant/components/simplisafe/manifest.json @@ -13,5 +13,5 @@ "integration_type": "hub", "iot_class": "cloud_polling", "loggers": ["simplipy"], - "requirements": ["simplisafe-python==2022.12.0"] + "requirements": ["simplisafe-python==2023.04.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index 56723565570..f58036f2c7e 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2343,7 +2343,7 @@ simplehound==0.3 simplepush==2.1.1 # homeassistant.components.simplisafe -simplisafe-python==2022.12.0 +simplisafe-python==2023.04.0 # homeassistant.components.sisyphus sisyphus-control==3.1.2 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 985a1c628f6..3bf69ae4935 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1670,7 +1670,7 @@ simplehound==0.3 simplepush==2.1.1 # homeassistant.components.simplisafe -simplisafe-python==2022.12.0 +simplisafe-python==2023.04.0 # homeassistant.components.slack slackclient==2.5.0 From 849000d5ac43db7d16f8f413fbb8eddc3ce96f86 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Apr 2023 22:30:29 -1000 Subject: [PATCH 013/110] Bump aiodiscover to 1.4.16 (#90903) --- homeassistant/components/dhcp/manifest.json | 2 +- homeassistant/package_constraints.txt | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/dhcp/manifest.json b/homeassistant/components/dhcp/manifest.json index 2e1d758746d..e65966fbaa2 100644 --- a/homeassistant/components/dhcp/manifest.json +++ b/homeassistant/components/dhcp/manifest.json @@ -7,5 +7,5 @@ "iot_class": "local_push", "loggers": ["aiodiscover", "dnspython", "pyroute2", "scapy"], "quality_scale": "internal", - "requirements": ["scapy==2.5.0", "aiodiscover==1.4.15"] + "requirements": ["scapy==2.5.0", "aiodiscover==1.4.16"] } diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 69f46be2fac..8d8f65b584b 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -1,7 +1,7 @@ PyJWT==2.6.0 PyNaCl==1.5.0 PyTurboJPEG==1.6.7 -aiodiscover==1.4.15 +aiodiscover==1.4.16 aiohttp==3.8.4 aiohttp_cors==0.7.0 astral==2.2 diff --git a/requirements_all.txt b/requirements_all.txt index f58036f2c7e..affc7c8156d 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -137,7 +137,7 @@ aiobafi6==0.8.0 aiobotocore==2.1.0 # homeassistant.components.dhcp -aiodiscover==1.4.15 +aiodiscover==1.4.16 # homeassistant.components.dnsip # homeassistant.components.minecraft_server diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 3bf69ae4935..98584ae7d3e 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -127,7 +127,7 @@ aiobafi6==0.8.0 aiobotocore==2.1.0 # homeassistant.components.dhcp -aiodiscover==1.4.15 +aiodiscover==1.4.16 # homeassistant.components.dnsip # homeassistant.components.minecraft_server From e25edea815ed24567685c4b159fbf34b754befdd Mon Sep 17 00:00:00 2001 From: stickpin <630000+stickpin@users.noreply.github.com> Date: Thu, 6 Apr 2023 10:38:09 +0200 Subject: [PATCH 014/110] Return empty available programs list if an appliance is off during initial configuration (#90905) --- homeassistant/components/home_connect/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/home_connect/api.py b/homeassistant/components/home_connect/api.py index 85d8abd1cba..d0464968d4e 100644 --- a/homeassistant/components/home_connect/api.py +++ b/homeassistant/components/home_connect/api.py @@ -151,7 +151,7 @@ class DeviceWithPrograms(HomeConnectDevice): programs_available = self.appliance.get_programs_available() except (HomeConnectError, ValueError): _LOGGER.debug("Unable to fetch available programs. Probably offline") - programs_available = None + programs_available = [] return programs_available def get_program_switches(self): From 9b2e26c27092ed6d0aef7f2bd73e8acb1b32df16 Mon Sep 17 00:00:00 2001 From: Pascal Reeb Date: Thu, 6 Apr 2023 13:51:37 +0200 Subject: [PATCH 015/110] Handle NoURLAvailableError in Nuki component (#90927) * fix(nuki): handle NoURLAvailableError * only try internal URLs --- homeassistant/components/nuki/__init__.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/nuki/__init__.py b/homeassistant/components/nuki/__init__.py index ee8cc4e7e97..ef168374bd8 100644 --- a/homeassistant/components/nuki/__init__.py +++ b/homeassistant/components/nuki/__init__.py @@ -31,7 +31,7 @@ from homeassistant.helpers import ( entity_registry as er, issue_registry as ir, ) -from homeassistant.helpers.network import get_url +from homeassistant.helpers.network import NoURLAvailableError, get_url from homeassistant.helpers.update_coordinator import ( CoordinatorEntity, DataUpdateCoordinator, @@ -152,9 +152,22 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: ) webhook_url = webhook.async_generate_path(entry.entry_id) - hass_url = get_url( - hass, allow_cloud=False, allow_external=False, allow_ip=True, require_ssl=False - ) + + try: + hass_url = get_url( + hass, + allow_cloud=False, + allow_external=False, + allow_ip=True, + require_ssl=False, + ) + except NoURLAvailableError: + webhook.async_unregister(hass, entry.entry_id) + raise ConfigEntryNotReady( + f"Error registering URL for webhook {entry.entry_id}: " + "HomeAssistant URL is not available" + ) from None + url = f"{hass_url}{webhook_url}" if hass_url.startswith("https"): From f5be60038326a1830996fa68d261bb3484406cb9 Mon Sep 17 00:00:00 2001 From: Bram Kragten Date: Thu, 6 Apr 2023 19:08:52 +0200 Subject: [PATCH 016/110] Update frontend to 20230406.1 (#90951) --- homeassistant/components/frontend/manifest.json | 2 +- homeassistant/package_constraints.txt | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/frontend/manifest.json b/homeassistant/components/frontend/manifest.json index 8417870eb0a..b1fd062032f 100644 --- a/homeassistant/components/frontend/manifest.json +++ b/homeassistant/components/frontend/manifest.json @@ -20,5 +20,5 @@ "documentation": "https://www.home-assistant.io/integrations/frontend", "integration_type": "system", "quality_scale": "internal", - "requirements": ["home-assistant-frontend==20230405.0"] + "requirements": ["home-assistant-frontend==20230406.1"] } diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 8d8f65b584b..89159448a71 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -25,7 +25,7 @@ ha-av==10.0.0 hass-nabucasa==0.63.1 hassil==1.0.6 home-assistant-bluetooth==1.9.3 -home-assistant-frontend==20230405.0 +home-assistant-frontend==20230406.1 home-assistant-intents==2023.3.29 httpx==0.23.3 ifaddr==0.1.7 diff --git a/requirements_all.txt b/requirements_all.txt index affc7c8156d..96ace3d4527 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -907,7 +907,7 @@ hole==0.8.0 holidays==0.21.13 # homeassistant.components.frontend -home-assistant-frontend==20230405.0 +home-assistant-frontend==20230406.1 # homeassistant.components.conversation home-assistant-intents==2023.3.29 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 98584ae7d3e..f15598bb557 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -693,7 +693,7 @@ hole==0.8.0 holidays==0.21.13 # homeassistant.components.frontend -home-assistant-frontend==20230405.0 +home-assistant-frontend==20230406.1 # homeassistant.components.conversation home-assistant-intents==2023.3.29 From 5ffa0cba39100bedbe674c3ce2b72a5a56f6a216 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Thu, 6 Apr 2023 13:21:13 -0400 Subject: [PATCH 017/110] Bumped version to 2023.4.1 --- homeassistant/const.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/const.py b/homeassistant/const.py index 52a07324b7c..345a2a0a45f 100644 --- a/homeassistant/const.py +++ b/homeassistant/const.py @@ -8,7 +8,7 @@ from .backports.enum import StrEnum APPLICATION_NAME: Final = "HomeAssistant" MAJOR_VERSION: Final = 2023 MINOR_VERSION: Final = 4 -PATCH_VERSION: Final = "0" +PATCH_VERSION: Final = "1" __short_version__: Final = f"{MAJOR_VERSION}.{MINOR_VERSION}" __version__: Final = f"{__short_version__}.{PATCH_VERSION}" REQUIRED_PYTHON_VER: Final[tuple[int, int, int]] = (3, 10, 0) diff --git a/pyproject.toml b/pyproject.toml index 36e36872bcc..d3f7c2ef15b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "homeassistant" -version = "2023.4.0" +version = "2023.4.1" license = {text = "Apache-2.0"} description = "Open-source home automation platform running on Python 3." readme = "README.rst" From d65791027fdeb98c377373e864aa243dc9673cbe Mon Sep 17 00:00:00 2001 From: epenet <6771947+epenet@users.noreply.github.com> Date: Thu, 6 Apr 2023 18:51:02 +0200 Subject: [PATCH 018/110] Fix flaky test in vesync (#90921) * Fix flaky test in vesync * Move sorting to the test --- .../vesync/snapshots/test_diagnostics.ambr | 46 +++++++++---------- tests/components/vesync/test_diagnostics.py | 3 ++ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/tests/components/vesync/snapshots/test_diagnostics.ambr b/tests/components/vesync/snapshots/test_diagnostics.ambr index 33378d7ccde..2c12f9bc5f6 100644 --- a/tests/components/vesync/snapshots/test_diagnostics.ambr +++ b/tests/components/vesync/snapshots/test_diagnostics.ambr @@ -209,6 +209,29 @@ }), 'unit_of_measurement': None, }), + dict({ + 'device_class': None, + 'disabled': False, + 'disabled_by': None, + 'domain': 'sensor', + 'entity_category': None, + 'entity_id': 'sensor.fan_air_quality', + 'icon': None, + 'name': None, + 'original_device_class': None, + 'original_icon': None, + 'original_name': 'Fan Air Quality', + 'state': dict({ + 'attributes': dict({ + 'friendly_name': 'Fan Air Quality', + }), + 'entity_id': 'sensor.fan_air_quality', + 'last_changed': str, + 'last_updated': str, + 'state': 'unavailable', + }), + 'unit_of_measurement': None, + }), dict({ 'device_class': None, 'disabled': False, @@ -234,29 +257,6 @@ }), 'unit_of_measurement': '%', }), - dict({ - 'device_class': None, - 'disabled': False, - 'disabled_by': None, - 'domain': 'sensor', - 'entity_category': None, - 'entity_id': 'sensor.fan_air_quality', - 'icon': None, - 'name': None, - 'original_device_class': None, - 'original_icon': None, - 'original_name': 'Fan Air Quality', - 'state': dict({ - 'attributes': dict({ - 'friendly_name': 'Fan Air Quality', - }), - 'entity_id': 'sensor.fan_air_quality', - 'last_changed': str, - 'last_updated': str, - 'state': 'unavailable', - }), - 'unit_of_measurement': None, - }), ]), 'name': 'Fan', 'name_by_user': None, diff --git a/tests/components/vesync/test_diagnostics.py b/tests/components/vesync/test_diagnostics.py index eb802bb41b8..62365189064 100644 --- a/tests/components/vesync/test_diagnostics.py +++ b/tests/components/vesync/test_diagnostics.py @@ -85,6 +85,9 @@ async def test_async_get_device_diagnostics__single_fan( diag = await get_diagnostics_for_device(hass, hass_client, config_entry, device) assert isinstance(diag, dict) + diag["home_assistant"]["entities"] = sorted( + diag["home_assistant"]["entities"], key=lambda ent: ent["entity_id"] + ) assert diag == snapshot( matcher=path_type( { From a1c71593049688af581f7004ae6107ce13c15b24 Mon Sep 17 00:00:00 2001 From: Aaron Bach Date: Thu, 6 Apr 2023 12:33:41 -0600 Subject: [PATCH 019/110] Bump `aioambient` to 2022.10.0 (#90940) Co-authored-by: J. Nick Koston --- homeassistant/components/ambient_station/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/ambient_station/manifest.json b/homeassistant/components/ambient_station/manifest.json index bd07303df3e..9dbd4507774 100644 --- a/homeassistant/components/ambient_station/manifest.json +++ b/homeassistant/components/ambient_station/manifest.json @@ -7,5 +7,5 @@ "integration_type": "hub", "iot_class": "cloud_push", "loggers": ["aioambient"], - "requirements": ["aioambient==2021.11.0"] + "requirements": ["aioambient==2022.10.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index 96ace3d4527..a70330dec63 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -119,7 +119,7 @@ aioairq==0.2.4 aioairzone==0.5.2 # homeassistant.components.ambient_station -aioambient==2021.11.0 +aioambient==2022.10.0 # homeassistant.components.aseko_pool_live aioaseko==0.0.2 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index f15598bb557..29ec27a274b 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -109,7 +109,7 @@ aioairq==0.2.4 aioairzone==0.5.2 # homeassistant.components.ambient_station -aioambient==2021.11.0 +aioambient==2022.10.0 # homeassistant.components.aseko_pool_live aioaseko==0.0.2 From 6884b0a421790897bd006f1a2a6dd717ad677801 Mon Sep 17 00:00:00 2001 From: starkillerOG Date: Thu, 6 Apr 2023 20:35:22 +0200 Subject: [PATCH 020/110] Bump reolink-aio to 0.5.10 (#90963) * use is_doorbell instead of is_doorbell_enabled * Bump reolink-aio to 0.5.10 --- homeassistant/components/reolink/binary_sensor.py | 2 +- homeassistant/components/reolink/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/reolink/binary_sensor.py b/homeassistant/components/reolink/binary_sensor.py index 1a7649f367a..850aa110171 100644 --- a/homeassistant/components/reolink/binary_sensor.py +++ b/homeassistant/components/reolink/binary_sensor.py @@ -87,7 +87,7 @@ BINARY_SENSORS = ( icon="mdi:bell-ring-outline", icon_off="mdi:doorbell", value=lambda api, ch: api.visitor_detected(ch), - supported=lambda api, ch: api.is_doorbell_enabled(ch), + supported=lambda api, ch: api.is_doorbell(ch), ), ) diff --git a/homeassistant/components/reolink/manifest.json b/homeassistant/components/reolink/manifest.json index b8de6cd8399..73318f12be1 100644 --- a/homeassistant/components/reolink/manifest.json +++ b/homeassistant/components/reolink/manifest.json @@ -18,5 +18,5 @@ "documentation": "https://www.home-assistant.io/integrations/reolink", "iot_class": "local_push", "loggers": ["reolink_aio"], - "requirements": ["reolink-aio==0.5.9"] + "requirements": ["reolink-aio==0.5.10"] } diff --git a/requirements_all.txt b/requirements_all.txt index a70330dec63..e00d154d7c8 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2231,7 +2231,7 @@ regenmaschine==2022.11.0 renault-api==0.1.12 # homeassistant.components.reolink -reolink-aio==0.5.9 +reolink-aio==0.5.10 # homeassistant.components.python_script restrictedpython==6.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 29ec27a274b..7d8564da8bc 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1594,7 +1594,7 @@ regenmaschine==2022.11.0 renault-api==0.1.12 # homeassistant.components.reolink -reolink-aio==0.5.9 +reolink-aio==0.5.10 # homeassistant.components.python_script restrictedpython==6.0 From 5e903e04cf5fcce4fd0d575416b9fe67e18849b8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 31 Mar 2023 06:23:05 -1000 Subject: [PATCH 021/110] Avoid writing state to all esphome entities at shutdown (#90555) --- homeassistant/components/esphome/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/esphome/__init__.py b/homeassistant/components/esphome/__init__.py index 192a19e480b..58659a671f0 100644 --- a/homeassistant/components/esphome/__init__.py +++ b/homeassistant/components/esphome/__init__.py @@ -349,7 +349,12 @@ async def async_setup_entry( # noqa: C901 # the next state update of that type when the device reconnects for state_keys in entry_data.state.values(): state_keys.clear() - entry_data.async_update_device_state(hass) + if not hass.is_stopping: + # Avoid marking every esphome entity as unavailable on shutdown + # since it generates a lot of state changed events and database + # writes when we already know we're shutting down and the state + # will be cleared anyway. + entry_data.async_update_device_state(hass) async def on_connect_error(err: Exception) -> None: """Start reauth flow if appropriate connect error type.""" From c944be8215420ede2b502ef51a807ac1e106f21c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 6 Apr 2023 10:32:02 -1000 Subject: [PATCH 022/110] Fix state being cleared on disconnect with deep sleep esphome devices (#90925) * Fix state being cleared on disconnect with deep sleep esphome devices fixes #90923 * fix logic --- homeassistant/components/esphome/__init__.py | 9 ++++++--- homeassistant/components/esphome/entry_data.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/esphome/__init__.py b/homeassistant/components/esphome/__init__.py index 58659a671f0..4658893d375 100644 --- a/homeassistant/components/esphome/__init__.py +++ b/homeassistant/components/esphome/__init__.py @@ -345,10 +345,13 @@ async def async_setup_entry( # noqa: C901 disconnect_cb() entry_data.disconnect_callbacks = [] entry_data.available = False - # Clear out the states so that we will always dispatch + # Mark state as stale so that we will always dispatch # the next state update of that type when the device reconnects - for state_keys in entry_data.state.values(): - state_keys.clear() + entry_data.stale_state = { + (type(entity_state), key) + for state_dict in entry_data.state.values() + for key, entity_state in state_dict.items() + } if not hass.is_stopping: # Avoid marking every esphome entity as unavailable on shutdown # since it generates a lot of state changed events and database diff --git a/homeassistant/components/esphome/entry_data.py b/homeassistant/components/esphome/entry_data.py index d7f25f319ac..7a6027f946b 100644 --- a/homeassistant/components/esphome/entry_data.py +++ b/homeassistant/components/esphome/entry_data.py @@ -70,6 +70,10 @@ class RuntimeEntryData: client: APIClient store: Store state: dict[type[EntityState], dict[int, EntityState]] = field(default_factory=dict) + # When the disconnect callback is called, we mark all states + # as stale so we will always dispatch a state update when the + # device reconnects. This is the same format as state_subscriptions. + stale_state: set[tuple[type[EntityState], int]] = field(default_factory=set) info: dict[str, dict[int, EntityInfo]] = field(default_factory=dict) # A second list of EntityInfo objects @@ -206,9 +210,11 @@ class RuntimeEntryData: """Distribute an update of state information to the target.""" key = state.key state_type = type(state) + stale_state = self.stale_state current_state_by_type = self.state[state_type] current_state = current_state_by_type.get(key, _SENTINEL) - if current_state == state: + subscription_key = (state_type, key) + if current_state == state and subscription_key not in stale_state: _LOGGER.debug( "%s: ignoring duplicate update with and key %s: %s", self.name, @@ -222,8 +228,8 @@ class RuntimeEntryData: key, state, ) + stale_state.discard(subscription_key) current_state_by_type[key] = state - subscription_key = (state_type, key) if subscription_key in self.state_subscriptions: self.state_subscriptions[subscription_key]() From bbb571fdf8d436bdc2efc54a90a68c9f4bd9f92d Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Thu, 6 Apr 2023 13:41:38 -0700 Subject: [PATCH 023/110] Coerce previously persisted local calendars to have valid durations (#90970) --- .../components/local_calendar/calendar.py | 23 +++-- tests/components/local_calendar/conftest.py | 14 ++- .../local_calendar/test_calendar.py | 89 +++++++++++++++++++ 3 files changed, 115 insertions(+), 11 deletions(-) diff --git a/homeassistant/components/local_calendar/calendar.py b/homeassistant/components/local_calendar/calendar.py index 4b6d9444fd8..6cfcaec61d0 100644 --- a/homeassistant/components/local_calendar/calendar.py +++ b/homeassistant/components/local_calendar/calendar.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import datetime +from datetime import date, datetime, timedelta import logging from typing import Any @@ -186,14 +186,23 @@ def _parse_event(event: dict[str, Any]) -> Event: def _get_calendar_event(event: Event) -> CalendarEvent: """Return a CalendarEvent from an API event.""" + start: datetime | date + end: datetime | date + if isinstance(event.start, datetime) and isinstance(event.end, datetime): + start = dt_util.as_local(event.start) + end = dt_util.as_local(event.end) + if (end - start) <= timedelta(seconds=0): + end = start + timedelta(minutes=30) + else: + start = event.start + end = event.end + if (end - start) <= timedelta(days=0): + end = start + timedelta(days=1) + return CalendarEvent( summary=event.summary, - start=dt_util.as_local(event.start) - if isinstance(event.start, datetime) - else event.start, - end=dt_util.as_local(event.end) - if isinstance(event.end, datetime) - else event.end, + start=start, + end=end, description=event.description, uid=event.uid, rrule=event.rrule.as_rrule_str() if event.rrule else None, diff --git a/tests/components/local_calendar/conftest.py b/tests/components/local_calendar/conftest.py index b083bbac78a..7dc294087bd 100644 --- a/tests/components/local_calendar/conftest.py +++ b/tests/components/local_calendar/conftest.py @@ -26,10 +26,10 @@ TEST_ENTITY = "calendar.light_schedule" class FakeStore(LocalCalendarStore): """Mock storage implementation.""" - def __init__(self, hass: HomeAssistant, path: Path) -> None: + def __init__(self, hass: HomeAssistant, path: Path, ics_content: str) -> None: """Initialize FakeStore.""" super().__init__(hass, path) - self._content = "" + self._content = ics_content def _load(self) -> str: """Read from calendar storage.""" @@ -40,15 +40,21 @@ class FakeStore(LocalCalendarStore): self._content = ics_content +@pytest.fixture(name="ics_content", autouse=True) +def mock_ics_content() -> str: + """Fixture to allow tests to set initial ics content for the calendar store.""" + return "" + + @pytest.fixture(name="store", autouse=True) -def mock_store() -> Generator[None, None, None]: +def mock_store(ics_content: str) -> Generator[None, None, None]: """Test cleanup, remove any media storage persisted during the test.""" stores: dict[Path, FakeStore] = {} def new_store(hass: HomeAssistant, path: Path) -> FakeStore: if path not in stores: - stores[path] = FakeStore(hass, path) + stores[path] = FakeStore(hass, path, ics_content) return stores[path] with patch( diff --git a/tests/components/local_calendar/test_calendar.py b/tests/components/local_calendar/test_calendar.py index a2f13ea289d..559a2af38b3 100644 --- a/tests/components/local_calendar/test_calendar.py +++ b/tests/components/local_calendar/test_calendar.py @@ -1,6 +1,7 @@ """Tests for calendar platform of local calendar.""" import datetime +import textwrap import pytest @@ -940,3 +941,91 @@ async def test_create_event_service( "location": "Test Location", } ] + + +@pytest.mark.parametrize( + "ics_content", + [ + textwrap.dedent( + """\ + BEGIN:VCALENDAR + BEGIN:VEVENT + SUMMARY:Bastille Day Party + DTSTART:19970714 + DTEND:19970714 + END:VEVENT + END:VCALENDAR + """ + ), + textwrap.dedent( + """\ + BEGIN:VCALENDAR + BEGIN:VEVENT + SUMMARY:Bastille Day Party + DTSTART:19970714 + DTEND:19970710 + END:VEVENT + END:VCALENDAR + """ + ), + ], + ids=["no_duration", "negative"], +) +async def test_invalid_all_day_event( + ws_client: ClientFixture, + setup_integration: None, + get_events: GetEventsFn, +) -> None: + """Test all day events with invalid durations, which are coerced to be valid.""" + events = await get_events("1997-07-14T00:00:00Z", "1997-07-16T00:00:00Z") + assert list(map(event_fields, events)) == [ + { + "summary": "Bastille Day Party", + "start": {"date": "1997-07-14"}, + "end": {"date": "1997-07-15"}, + } + ] + + +@pytest.mark.parametrize( + "ics_content", + [ + textwrap.dedent( + """\ + BEGIN:VCALENDAR + BEGIN:VEVENT + SUMMARY:Bastille Day Party + DTSTART:19970714T110000 + DTEND:19970714T110000 + END:VEVENT + END:VCALENDAR + """ + ), + textwrap.dedent( + """\ + BEGIN:VCALENDAR + BEGIN:VEVENT + SUMMARY:Bastille Day Party + DTSTART:19970714T110000 + DTEND:19970710T100000 + END:VEVENT + END:VCALENDAR + """ + ), + ], + ids=["no_duration", "negative"], +) +async def test_invalid_event_duration( + ws_client: ClientFixture, + setup_integration: None, + get_events: GetEventsFn, +) -> None: + """Test events with invalid durations, which are coerced to be valid.""" + events = await get_events("1997-07-14T00:00:00Z", "1997-07-16T00:00:00Z") + assert list(map(event_fields, events)) == [ + { + "summary": "Bastille Day Party", + "start": {"dateTime": "1997-07-14T11:00:00-06:00"}, + "end": {"dateTime": "1997-07-14T11:30:00-06:00"}, + } + ] From 73a960af3426182057473c29c54db2f4281f3d08 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Thu, 6 Apr 2023 13:44:34 -0700 Subject: [PATCH 024/110] Bump gcal_sync to 4.1.3 (#90968) --- homeassistant/components/google/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/google/manifest.json b/homeassistant/components/google/manifest.json index d79cd105c83..8c5df8648e7 100644 --- a/homeassistant/components/google/manifest.json +++ b/homeassistant/components/google/manifest.json @@ -7,5 +7,5 @@ "documentation": "https://www.home-assistant.io/integrations/calendar.google/", "iot_class": "cloud_polling", "loggers": ["googleapiclient"], - "requirements": ["gcal-sync==4.1.2", "oauth2client==4.1.3"] + "requirements": ["gcal-sync==4.1.3", "oauth2client==4.1.3"] } diff --git a/requirements_all.txt b/requirements_all.txt index e00d154d7c8..e749d8240af 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -757,7 +757,7 @@ gTTS==2.2.4 gassist-text==0.0.10 # homeassistant.components.google -gcal-sync==4.1.2 +gcal-sync==4.1.3 # homeassistant.components.geniushub geniushub-client==0.7.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 7d8564da8bc..4650a036aed 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -579,7 +579,7 @@ gTTS==2.2.4 gassist-text==0.0.10 # homeassistant.components.google -gcal-sync==4.1.2 +gcal-sync==4.1.3 # homeassistant.components.geocaching geocachingapi==0.2.1 From e8a6a2e1056c431d32744e0ffc823cbe684a28a8 Mon Sep 17 00:00:00 2001 From: Jan Bouwhuis Date: Thu, 6 Apr 2023 22:46:32 +0200 Subject: [PATCH 025/110] Fix error after losing an imap connection (#90966) Cleanup first after losing an imap connection --- homeassistant/components/imap/coordinator.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/imap/coordinator.py b/homeassistant/components/imap/coordinator.py index 421cedad149..e11cf1e0baf 100644 --- a/homeassistant/components/imap/coordinator.py +++ b/homeassistant/components/imap/coordinator.py @@ -194,7 +194,11 @@ class ImapDataUpdateCoordinator(DataUpdateCoordinator[int | None]): if count else None ) - if count and last_message_id is not None: + if ( + count + and last_message_id is not None + and self._last_message_id != last_message_id + ): self._last_message_id = last_message_id await self._async_process_event(last_message_id) @@ -235,18 +239,18 @@ class ImapPollingDataUpdateCoordinator(ImapDataUpdateCoordinator): UpdateFailed, asyncio.TimeoutError, ) as ex: - self.async_set_update_error(ex) await self._cleanup() + self.async_set_update_error(ex) raise UpdateFailed() from ex except InvalidFolder as ex: _LOGGER.warning("Selected mailbox folder is invalid") - self.async_set_update_error(ex) await self._cleanup() + self.async_set_update_error(ex) raise ConfigEntryError("Selected mailbox folder is invalid.") from ex except InvalidAuth as ex: _LOGGER.warning("Username or password incorrect, starting reauthentication") - self.async_set_update_error(ex) await self._cleanup() + self.async_set_update_error(ex) raise ConfigEntryAuthFailed() from ex @@ -316,6 +320,7 @@ class ImapPushDataUpdateCoordinator(ImapDataUpdateCoordinator): self.config_entry.data[CONF_SERVER], BACKOFF_TIME, ) + await self._cleanup() await asyncio.sleep(BACKOFF_TIME) async def shutdown(self, *_) -> None: From cfd8695aaa2b9b41a1207a1d8d956cae4e2aebda Mon Sep 17 00:00:00 2001 From: Steven Rollason <2099542+gadgetchnnel@users.noreply.github.com> Date: Thu, 6 Apr 2023 20:06:31 +0100 Subject: [PATCH 026/110] Fix command_template sensor value_template not being used if json_attributes set (#90603) * Allow value_template to be used if json_attributes set * Set state to None if no value_template and json_attributes used * Refactor check for no value_template when json_attributes used * Updated and additional unit test * Updated to set _attr_native_value and return if value_template is None * Update unit test docstring * Updated test docstring based on feedback --- .../components/command_line/sensor.py | 5 ++++- tests/components/command_line/test_sensor.py | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/command_line/sensor.py b/homeassistant/components/command_line/sensor.py index f459e415661..b6a2b8d83fa 100644 --- a/homeassistant/components/command_line/sensor.py +++ b/homeassistant/components/command_line/sensor.py @@ -137,8 +137,11 @@ class CommandSensor(SensorEntity): _LOGGER.warning("Unable to parse output as JSON: %s", value) else: _LOGGER.warning("Empty reply found when expecting JSON data") + if self._value_template is None: + self._attr_native_value = None + return - elif self._value_template is not None: + if self._value_template is not None: self._attr_native_value = ( self._value_template.async_render_with_possible_json_value( value, diff --git a/tests/components/command_line/test_sensor.py b/tests/components/command_line/test_sensor.py index 4643891691f..188f4aac062 100644 --- a/tests/components/command_line/test_sensor.py +++ b/tests/components/command_line/test_sensor.py @@ -169,6 +169,28 @@ async def test_update_with_json_attrs(hass: HomeAssistant) -> None: ) entity_state = hass.states.get("sensor.test") assert entity_state + assert entity_state.state == "unknown" + assert entity_state.attributes["key"] == "some_json_value" + assert entity_state.attributes["another_key"] == "another_json_value" + assert entity_state.attributes["key_three"] == "value_three" + + +async def test_update_with_json_attrs_and_value_template(hass: HomeAssistant) -> None: + """Test json_attributes can be used together with value_template.""" + await setup_test_entities( + hass, + { + "command": ( + 'echo { \\"key\\": \\"some_json_value\\", \\"another_key\\": ' + '\\"another_json_value\\", \\"key_three\\": \\"value_three\\" }' + ), + "json_attributes": ["key", "another_key", "key_three"], + "value_template": '{{ value_json["key"] }}', + }, + ) + entity_state = hass.states.get("sensor.test") + assert entity_state + assert entity_state.state == "some_json_value" assert entity_state.attributes["key"] == "some_json_value" assert entity_state.attributes["another_key"] == "another_json_value" assert entity_state.attributes["key_three"] == "value_three" From 2bf51a033b577094418c0340913357118f8d6e22 Mon Sep 17 00:00:00 2001 From: Heikki Partanen Date: Thu, 6 Apr 2023 23:54:18 +0300 Subject: [PATCH 027/110] Fix verisure autolock (#90960) Fix verisure autolock #90959 --- homeassistant/components/verisure/lock.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/verisure/lock.py b/homeassistant/components/verisure/lock.py index d13005b265d..53646c1e435 100644 --- a/homeassistant/components/verisure/lock.py +++ b/homeassistant/components/verisure/lock.py @@ -188,9 +188,10 @@ class VerisureDoorlock(CoordinatorEntity[VerisureDataUpdateCoordinator], LockEnt def disable_autolock(self) -> None: """Disable autolock on a doorlock.""" try: - self.coordinator.verisure.set_lock_config( + command = self.coordinator.verisure.set_autolock_enabled( self.serial_number, auto_lock_enabled=False ) + self.coordinator.verisure.request(command) LOGGER.debug("Disabling autolock on %s", self.serial_number) except VerisureError as ex: LOGGER.error("Could not disable autolock, %s", ex) @@ -198,9 +199,10 @@ class VerisureDoorlock(CoordinatorEntity[VerisureDataUpdateCoordinator], LockEnt def enable_autolock(self) -> None: """Enable autolock on a doorlock.""" try: - self.coordinator.verisure.set_lock_config( + command = self.coordinator.verisure.set_autolock_enabled( self.serial_number, auto_lock_enabled=True ) + self.coordinator.verisure.request(command) LOGGER.debug("Enabling autolock on %s", self.serial_number) except VerisureError as ex: LOGGER.error("Could not enable autolock, %s", ex) From be65d4f33e81c766e498426c17b71d184d78b403 Mon Sep 17 00:00:00 2001 From: PatrickGlesner <34370149+PatrickGlesner@users.noreply.github.com> Date: Fri, 7 Apr 2023 12:13:39 +0200 Subject: [PATCH 028/110] Fix NMBS AttributeError (#90525) * Fix NMBS AttributeError (Issue #90505) * Set and use API_FAILURE * Configure the logger to track API failures * Remove broad exceptions and rewite logging --- homeassistant/components/nmbs/sensor.py | 35 ++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/homeassistant/components/nmbs/sensor.py b/homeassistant/components/nmbs/sensor.py index 8fb227140a1..7fe40af3b69 100644 --- a/homeassistant/components/nmbs/sensor.py +++ b/homeassistant/components/nmbs/sensor.py @@ -22,6 +22,8 @@ import homeassistant.util.dt as dt_util _LOGGER = logging.getLogger(__name__) +API_FAILURE = -1 + DEFAULT_NAME = "NMBS" DEFAULT_ICON = "mdi:train" @@ -162,16 +164,19 @@ class NMBSLiveBoard(SensorEntity): """Set the state equal to the next departure.""" liveboard = self._api_client.get_liveboard(self._station) - if ( - liveboard is None - or liveboard.get("departures") is None - or liveboard.get("departures").get("number") is None - or liveboard.get("departures").get("number") == "0" - or liveboard.get("departures").get("departure") is None - ): + if liveboard == API_FAILURE: + _LOGGER.warning("API failed in NMBSLiveBoard") return - next_departure = liveboard["departures"]["departure"][0] + if not (departures := liveboard.get("departures")): + _LOGGER.warning("API returned invalid departures: %r", liveboard) + return + + _LOGGER.debug("API returned departures: %r", departures) + if departures["number"] == "0": + # No trains are scheduled + return + next_departure = departures["departure"][0] self._attrs = next_departure self._state = ( @@ -290,13 +295,19 @@ class NMBSSensor(SensorEntity): self._station_from, self._station_to ) - if connections is None or not connections.get("connection"): + if connections == API_FAILURE: + _LOGGER.warning("API failed in NMBSSensor") return - if int(connections["connection"][0]["departure"]["left"]) > 0: - next_connection = connections["connection"][1] + if not (connection := connections.get("connection")): + _LOGGER.warning("API returned invalid connection: %r", connections) + return + + _LOGGER.debug("API returned connection: %r", connection) + if int(connection[0]["departure"]["left"]) > 0: + next_connection = connection[1] else: - next_connection = connections["connection"][0] + next_connection = connection[0] self._attrs = next_connection From 6522a3ad1b4c02079dd5a77dbc161867ce66ff9c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 6 Apr 2023 18:03:33 -1000 Subject: [PATCH 029/110] Bump websockets constraint to 11.0.1+ (#90901) --- homeassistant/package_constraints.txt | 4 ++-- script/gen_requirements_all.py | 4 ++-- script/pip_check | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 89159448a71..7debda373fd 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -159,6 +159,6 @@ uamqp==1.6.0;python_version<'3.11' faust-cchardet>=2.1.18 # websockets 11.0 is missing files in the source distribution -# which break wheel builds +# which break wheel builds so we need at least 11.0.1 # https://github.com/aaugustin/websockets/issues/1329 -websockets<11.0 +websockets>=11.0.1 diff --git a/script/gen_requirements_all.py b/script/gen_requirements_all.py index b31674d04e8..585acc944dc 100755 --- a/script/gen_requirements_all.py +++ b/script/gen_requirements_all.py @@ -164,9 +164,9 @@ uamqp==1.6.0;python_version<'3.11' faust-cchardet>=2.1.18 # websockets 11.0 is missing files in the source distribution -# which break wheel builds +# which break wheel builds so we need at least 11.0.1 # https://github.com/aaugustin/websockets/issues/1329 -websockets<11.0 +websockets>=11.0.1 """ IGNORE_PRE_COMMIT_HOOK_ID = ( diff --git a/script/pip_check b/script/pip_check index cbbe7ffeeae..cbe6a3851e0 100755 --- a/script/pip_check +++ b/script/pip_check @@ -3,7 +3,7 @@ PIP_CACHE=$1 # Number of existing dependency conflicts # Update if a PR resolves one! -DEPENDENCY_CONFLICTS=3 +DEPENDENCY_CONFLICTS=4 PIP_CHECK=$(pip check --cache-dir=$PIP_CACHE) LINE_COUNT=$(echo "$PIP_CHECK" | wc -l) From dc2f2e8d3ffa7c22517c44bb966e17caa2d57193 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 7 Apr 2023 16:32:36 -1000 Subject: [PATCH 030/110] Raise an issue for legacy SQL queries that will cause full table scans (#90971) * Raise an issue for SQL queries that will cause full table scans * Raise an issue for SQL queries that will cause full table scans * Raise an issue for SQL queries that will cause full table scans * Raise an issue for SQL queries that will cause full table scans * Update homeassistant/components/sql/sensor.py Co-authored-by: Paulus Schoutsen * coverage --------- Co-authored-by: Paulus Schoutsen --- homeassistant/components/sql/sensor.py | 39 ++++++++++++++- homeassistant/components/sql/strings.json | 6 +++ tests/components/sql/__init__.py | 18 +++++++ tests/components/sql/test_sensor.py | 58 +++++++++++++++++++++-- 4 files changed, 116 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/sql/sensor.py b/homeassistant/components/sql/sensor.py index c19c2c258bc..8408b98730b 100644 --- a/homeassistant/components/sql/sensor.py +++ b/homeassistant/components/sql/sensor.py @@ -30,6 +30,7 @@ from homeassistant.const import ( ) from homeassistant.core import HomeAssistant from homeassistant.exceptions import TemplateError +from homeassistant.helpers import issue_registry as ir from homeassistant.helpers.device_registry import DeviceEntryType from homeassistant.helpers.entity import DeviceInfo from homeassistant.helpers.entity_platform import AddEntitiesCallback @@ -153,10 +154,44 @@ async def async_setup_sensor( ): return + upper_query = query_str.upper() + if use_database_executor: + redacted_query = redact_credentials(query_str) + + issue_key = unique_id if unique_id else redacted_query + # If the query has a unique id and they fix it we can dismiss the issue + # but if it doesn't have a unique id they have to ignore it instead + + if "ENTITY_ID" in upper_query and "STATES_META" not in upper_query: + _LOGGER.error( + "The query `%s` contains the keyword `entity_id` but does not " + "reference the `states_meta` table. This will cause a full table " + "scan and database instability. Please check the documentation and use " + "`states_meta.entity_id` instead", + redacted_query, + ) + + ir.async_create_issue( + hass, + DOMAIN, + f"entity_id_query_does_full_table_scan_{issue_key}", + translation_key="entity_id_query_does_full_table_scan", + translation_placeholders={"query": redacted_query}, + is_fixable=False, + severity=ir.IssueSeverity.ERROR, + ) + raise ValueError( + "Query contains entity_id but does not reference states_meta" + ) + + ir.async_delete_issue( + hass, DOMAIN, f"entity_id_query_does_full_table_scan_{issue_key}" + ) + # MSSQL uses TOP and not LIMIT - if not ("LIMIT" in query_str.upper() or "SELECT TOP" in query_str.upper()): + if not ("LIMIT" in upper_query or "SELECT TOP" in upper_query): if "mssql" in db_url: - query_str = query_str.upper().replace("SELECT", "SELECT TOP 1") + query_str = upper_query.replace("SELECT", "SELECT TOP 1") else: query_str = query_str.replace(";", "") + " LIMIT 1;" diff --git a/homeassistant/components/sql/strings.json b/homeassistant/components/sql/strings.json index 2a300f75b3e..1e7aef4ffde 100644 --- a/homeassistant/components/sql/strings.json +++ b/homeassistant/components/sql/strings.json @@ -53,5 +53,11 @@ "db_url_invalid": "[%key:component::sql::config::error::db_url_invalid%]", "query_invalid": "[%key:component::sql::config::error::query_invalid%]" } + }, + "issues": { + "entity_id_query_does_full_table_scan": { + "title": "SQL query does full table scan", + "description": "The query `{query}` contains the keyword `entity_id` but does not reference the `states_meta` table. This will cause a full table scan and database instability. Please check the documentation and use `states_meta.entity_id` instead." + } } } diff --git a/tests/components/sql/__init__.py b/tests/components/sql/__init__.py index c794f7a6b9a..c976f87f50f 100644 --- a/tests/components/sql/__init__.py +++ b/tests/components/sql/__init__.py @@ -63,6 +63,24 @@ YAML_CONFIG = { } } +YAML_CONFIG_FULL_TABLE_SCAN = { + "sql": { + CONF_NAME: "Get entity_id", + CONF_QUERY: "SELECT entity_id from states", + CONF_COLUMN_NAME: "entity_id", + CONF_UNIQUE_ID: "entity_id_12345", + } +} + + +YAML_CONFIG_FULL_TABLE_SCAN_NO_UNIQUE_ID = { + "sql": { + CONF_NAME: "Get entity_id", + CONF_QUERY: "SELECT entity_id from states", + CONF_COLUMN_NAME: "entity_id", + } +} + YAML_CONFIG_BINARY = { "sql": { CONF_DB_URL: "sqlite://", diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index 426dd9e196f..811bb3f45bf 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -11,14 +11,21 @@ from sqlalchemy.exc import SQLAlchemyError from homeassistant.components.recorder import Recorder from homeassistant.components.sensor import SensorDeviceClass, SensorStateClass -from homeassistant.components.sql.const import DOMAIN +from homeassistant.components.sql.const import CONF_QUERY, DOMAIN from homeassistant.config_entries import SOURCE_USER -from homeassistant.const import STATE_UNKNOWN +from homeassistant.const import CONF_UNIQUE_ID, STATE_UNKNOWN from homeassistant.core import HomeAssistant +from homeassistant.helpers import issue_registry as ir from homeassistant.setup import async_setup_component from homeassistant.util import dt -from . import YAML_CONFIG, YAML_CONFIG_BINARY, init_integration +from . import ( + YAML_CONFIG, + YAML_CONFIG_BINARY, + YAML_CONFIG_FULL_TABLE_SCAN, + YAML_CONFIG_FULL_TABLE_SCAN_NO_UNIQUE_ID, + init_integration, +) from tests.common import MockConfigEntry, async_fire_time_changed @@ -322,3 +329,48 @@ async def test_binary_data_from_yaml_setup( state = hass.states.get("sensor.get_binary_value") assert state.state == "0xd34324324230392032" assert state.attributes["test_attr"] == "0xd343aa" + + +async def test_issue_when_using_old_query( + recorder_mock: Recorder, hass: HomeAssistant, caplog: pytest.LogCaptureFixture +) -> None: + """Test we create an issue for an old query that will do a full table scan.""" + + assert await async_setup_component(hass, DOMAIN, YAML_CONFIG_FULL_TABLE_SCAN) + await hass.async_block_till_done() + assert "Query contains entity_id but does not reference states_meta" in caplog.text + + assert not hass.states.async_all() + issue_registry = ir.async_get(hass) + + config = YAML_CONFIG_FULL_TABLE_SCAN["sql"] + + unique_id = config[CONF_UNIQUE_ID] + + issue = issue_registry.async_get_issue( + DOMAIN, f"entity_id_query_does_full_table_scan_{unique_id}" + ) + assert issue.translation_placeholders == {"query": config[CONF_QUERY]} + + +async def test_issue_when_using_old_query_without_unique_id( + recorder_mock: Recorder, hass: HomeAssistant, caplog: pytest.LogCaptureFixture +) -> None: + """Test we create an issue for an old query that will do a full table scan.""" + + assert await async_setup_component( + hass, DOMAIN, YAML_CONFIG_FULL_TABLE_SCAN_NO_UNIQUE_ID + ) + await hass.async_block_till_done() + assert "Query contains entity_id but does not reference states_meta" in caplog.text + + assert not hass.states.async_all() + issue_registry = ir.async_get(hass) + + config = YAML_CONFIG_FULL_TABLE_SCAN_NO_UNIQUE_ID["sql"] + query = config[CONF_QUERY] + + issue = issue_registry.async_get_issue( + DOMAIN, f"entity_id_query_does_full_table_scan_{query}" + ) + assert issue.translation_placeholders == {"query": query} From 0a70a29e92296e177ac37f863a314474c2306f7f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 6 Apr 2023 15:16:45 -1000 Subject: [PATCH 031/110] Resume entity id post migration after a restart (#90973) * Restart entity id post migration after a restart If the entity migration finished and Home Assistant was restarted during the post migration it would never be resumed which means the old index and space would never be recovered * add migration resume test --- homeassistant/components/recorder/core.py | 14 + .../components/recorder/db_schema.py | 1 + .../components/recorder/migration.py | 3 +- tests/components/recorder/db_schema_32.py | 752 ++++++++++++++++++ .../components/recorder/test_v32_migration.py | 143 +++- 5 files changed, 911 insertions(+), 2 deletions(-) create mode 100644 tests/components/recorder/db_schema_32.py diff --git a/homeassistant/components/recorder/core.py b/homeassistant/components/recorder/core.py index 68b634d7235..8969b7a27e3 100644 --- a/homeassistant/components/recorder/core.py +++ b/homeassistant/components/recorder/core.py @@ -58,6 +58,7 @@ from .const import ( SupportedDialect, ) from .db_schema import ( + LEGACY_STATES_ENTITY_ID_LAST_UPDATED_INDEX, LEGACY_STATES_EVENT_ID_INDEX, SCHEMA_VERSION, TABLE_STATES, @@ -96,6 +97,7 @@ from .tasks import ( CompileMissingStatisticsTask, DatabaseLockTask, EntityIDMigrationTask, + EntityIDPostMigrationTask, EventIdMigrationTask, EventsContextIDMigrationTask, EventTask, @@ -757,6 +759,18 @@ class Recorder(threading.Thread): else: _LOGGER.debug("Activating states_meta manager as all data is migrated") self.states_meta_manager.active = True + with contextlib.suppress(SQLAlchemyError): + # If ix_states_entity_id_last_updated_ts still exists + # on the states table it means the entity id migration + # finished by the EntityIDPostMigrationTask did not + # because they restarted in the middle of it. We need + # to pick back up where we left off. + if get_index_by_name( + session, + TABLE_STATES, + LEGACY_STATES_ENTITY_ID_LAST_UPDATED_INDEX, + ): + self.queue_task(EntityIDPostMigrationTask()) if self.schema_version > LEGACY_STATES_EVENT_ID_INDEX_SCHEMA_VERSION: with contextlib.suppress(SQLAlchemyError): diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py index c2252e9f68f..4cef287deda 100644 --- a/homeassistant/components/recorder/db_schema.py +++ b/homeassistant/components/recorder/db_schema.py @@ -119,6 +119,7 @@ METADATA_ID_LAST_UPDATED_INDEX_TS = "ix_states_metadata_id_last_updated_ts" EVENTS_CONTEXT_ID_BIN_INDEX = "ix_events_context_id_bin" STATES_CONTEXT_ID_BIN_INDEX = "ix_states_context_id_bin" LEGACY_STATES_EVENT_ID_INDEX = "ix_states_event_id" +LEGACY_STATES_ENTITY_ID_LAST_UPDATED_INDEX = "ix_states_entity_id_last_updated_ts" CONTEXT_ID_BIN_MAX_LENGTH = 16 MYSQL_COLLATE = "utf8mb4_unicode_ci" diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py index 7fee3d16e8d..4b0244038e8 100644 --- a/homeassistant/components/recorder/migration.py +++ b/homeassistant/components/recorder/migration.py @@ -48,6 +48,7 @@ from .const import SupportedDialect from .db_schema import ( CONTEXT_ID_BIN_MAX_LENGTH, DOUBLE_PRECISION_TYPE_SQL, + LEGACY_STATES_ENTITY_ID_LAST_UPDATED_INDEX, LEGACY_STATES_EVENT_ID_INDEX, MYSQL_COLLATE, MYSQL_DEFAULT_CHARSET, @@ -1586,7 +1587,7 @@ def post_migrate_entity_ids(instance: Recorder) -> bool: if is_done: # Drop the old indexes since they are no longer needed - _drop_index(session_maker, "states", "ix_states_entity_id_last_updated_ts") + _drop_index(session_maker, "states", LEGACY_STATES_ENTITY_ID_LAST_UPDATED_INDEX) _LOGGER.debug("Cleanup legacy entity_ids done=%s", is_done) return is_done diff --git a/tests/components/recorder/db_schema_32.py b/tests/components/recorder/db_schema_32.py new file mode 100644 index 00000000000..c41775ed386 --- /dev/null +++ b/tests/components/recorder/db_schema_32.py @@ -0,0 +1,752 @@ +"""Models for SQLAlchemy. + +This file contains the model definitions for schema version 30. +It is used to test the schema migration logic. +""" +from __future__ import annotations + +from collections.abc import Callable +from datetime import datetime, timedelta +import logging +import time +from typing import Any, TypedDict, cast, overload + +import ciso8601 +from fnv_hash_fast import fnv1a_32 +from sqlalchemy import ( + JSON, + BigInteger, + Boolean, + Column, + DateTime, + Float, + ForeignKey, + Identity, + Index, + Integer, + LargeBinary, + SmallInteger, + String, + Text, + distinct, + type_coerce, +) +from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite +from sqlalchemy.orm import aliased, declarative_base, relationship +from sqlalchemy.orm.session import Session +from typing_extensions import Self + +from homeassistant.components.recorder.const import SupportedDialect +from homeassistant.const import ( + ATTR_ATTRIBUTION, + ATTR_RESTORED, + ATTR_SUPPORTED_FEATURES, + MAX_LENGTH_EVENT_CONTEXT_ID, + MAX_LENGTH_EVENT_EVENT_TYPE, + MAX_LENGTH_EVENT_ORIGIN, + MAX_LENGTH_STATE_ENTITY_ID, + MAX_LENGTH_STATE_STATE, +) +from homeassistant.core import Context, Event, EventOrigin, State, split_entity_id +from homeassistant.helpers import entity_registry as er +from homeassistant.helpers.json import JSON_DUMP, json_bytes +import homeassistant.util.dt as dt_util +from homeassistant.util.json import JSON_DECODE_EXCEPTIONS, json_loads + +ALL_DOMAIN_EXCLUDE_ATTRS = {ATTR_ATTRIBUTION, ATTR_RESTORED, ATTR_SUPPORTED_FEATURES} + +# SQLAlchemy Schema +# pylint: disable=invalid-name +Base = declarative_base() + +SCHEMA_VERSION = 30 + +_LOGGER = logging.getLogger(__name__) + +TABLE_EVENTS = "events" +TABLE_EVENT_DATA = "event_data" +TABLE_EVENT_TYPES = "event_types" +TABLE_STATES = "states" +TABLE_STATE_ATTRIBUTES = "state_attributes" +TABLE_STATES_META = "states_meta" +TABLE_RECORDER_RUNS = "recorder_runs" +TABLE_SCHEMA_CHANGES = "schema_changes" +TABLE_STATISTICS = "statistics" +TABLE_STATISTICS_META = "statistics_meta" +TABLE_STATISTICS_RUNS = "statistics_runs" +TABLE_STATISTICS_SHORT_TERM = "statistics_short_term" + +ALL_TABLES = [ + TABLE_STATES, + TABLE_STATE_ATTRIBUTES, + TABLE_STATES_META, + TABLE_EVENTS, + TABLE_EVENT_DATA, + TABLE_EVENT_TYPES, + TABLE_RECORDER_RUNS, + TABLE_SCHEMA_CHANGES, + TABLE_STATISTICS, + TABLE_STATISTICS_META, + TABLE_STATISTICS_RUNS, + TABLE_STATISTICS_SHORT_TERM, +] + +TABLES_TO_CHECK = [ + TABLE_STATES, + TABLE_EVENTS, + TABLE_RECORDER_RUNS, + TABLE_SCHEMA_CHANGES, +] + +LAST_UPDATED_INDEX = "ix_states_last_updated" +ENTITY_ID_LAST_UPDATED_TS_INDEX = "ix_states_entity_id_last_updated_ts" +EVENTS_CONTEXT_ID_INDEX = "ix_events_context_id" +STATES_CONTEXT_ID_INDEX = "ix_states_context_id" +CONTEXT_ID_BIN_MAX_LENGTH = 16 +EVENTS_CONTEXT_ID_BIN_INDEX = "ix_events_context_id_bin" +STATES_CONTEXT_ID_BIN_INDEX = "ix_states_context_id_bin" + + +class FAST_PYSQLITE_DATETIME(sqlite.DATETIME): # type: ignore[misc] + """Use ciso8601 to parse datetimes instead of sqlalchemy built-in regex.""" + + def result_processor(self, dialect, coltype): # type: ignore[no-untyped-def] + """Offload the datetime parsing to ciso8601.""" + return lambda value: None if value is None else ciso8601.parse_datetime(value) + + +JSON_VARIANT_CAST = Text().with_variant( + postgresql.JSON(none_as_null=True), "postgresql" +) +JSONB_VARIANT_CAST = Text().with_variant( + postgresql.JSONB(none_as_null=True), "postgresql" +) +DATETIME_TYPE = ( + DateTime(timezone=True) + .with_variant(mysql.DATETIME(timezone=True, fsp=6), "mysql") + .with_variant(FAST_PYSQLITE_DATETIME(), "sqlite") +) +DOUBLE_TYPE = ( + Float() + .with_variant(mysql.DOUBLE(asdecimal=False), "mysql") + .with_variant(oracle.DOUBLE_PRECISION(), "oracle") + .with_variant(postgresql.DOUBLE_PRECISION(), "postgresql") +) + +TIMESTAMP_TYPE = DOUBLE_TYPE + + +class UnsupportedDialect(Exception): + """The dialect or its version is not supported.""" + + +class StatisticResult(TypedDict): + """Statistic result data class. + + Allows multiple datapoints for the same statistic_id. + """ + + meta: StatisticMetaData + stat: StatisticData + + +class StatisticDataBase(TypedDict): + """Mandatory fields for statistic data class.""" + + start: datetime + + +class StatisticData(StatisticDataBase, total=False): + """Statistic data class.""" + + mean: float + min: float + max: float + last_reset: datetime | None + state: float + sum: float + + +class StatisticMetaData(TypedDict): + """Statistic meta data class.""" + + has_mean: bool + has_sum: bool + name: str | None + source: str + statistic_id: str + unit_of_measurement: str | None + + +class JSONLiteral(JSON): # type: ignore[misc] + """Teach SA how to literalize json.""" + + def literal_processor(self, dialect: str) -> Callable[[Any], str]: + """Processor to convert a value to JSON.""" + + def process(value: Any) -> str: + """Dump json.""" + return JSON_DUMP(value) + + return process + + +EVENT_ORIGIN_ORDER = [EventOrigin.local, EventOrigin.remote] +EVENT_ORIGIN_TO_IDX = {origin: idx for idx, origin in enumerate(EVENT_ORIGIN_ORDER)} + + +class Events(Base): # type: ignore[misc,valid-type] + """Event history data.""" + + __table_args__ = ( + # Used for fetching events at a specific time + # see logbook + Index("ix_events_event_type_time_fired", "event_type", "time_fired"), + Index( + EVENTS_CONTEXT_ID_BIN_INDEX, + "context_id_bin", + mysql_length=CONTEXT_ID_BIN_MAX_LENGTH, + mariadb_length=CONTEXT_ID_BIN_MAX_LENGTH, + ), + {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"}, + ) + __tablename__ = TABLE_EVENTS + event_id = Column(Integer, Identity(), primary_key=True) + event_type = Column(String(MAX_LENGTH_EVENT_EVENT_TYPE)) + event_data = Column(Text().with_variant(mysql.LONGTEXT, "mysql")) + origin = Column(String(MAX_LENGTH_EVENT_ORIGIN)) # no longer used for new rows + origin_idx = Column(SmallInteger) + time_fired = Column(DATETIME_TYPE, index=True) + time_fired_ts = Column(TIMESTAMP_TYPE, index=True) + context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True) + context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID)) + context_parent_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID)) + data_id = Column(Integer, ForeignKey("event_data.data_id"), index=True) + context_id_bin = Column( + LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH) + ) # *** Not originally in v3v320, only added for recorder to startup ok + context_user_id_bin = Column( + LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH) + ) # *** Not originally in v32, only added for recorder to startup ok + context_parent_id_bin = Column( + LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH) + ) # *** Not originally in v32, only added for recorder to startup ok + event_type_id = Column( + Integer, ForeignKey("event_types.event_type_id"), index=True + ) # *** Not originally in v32, only added for recorder to startup ok + event_data_rel = relationship("EventData") + event_type_rel = relationship("EventTypes") + + def __repr__(self) -> str: + """Return string representation of instance for debugging.""" + return ( + "" + ) + + @staticmethod + def from_event(event: Event) -> Events: + """Create an event database object from a native event.""" + return Events( + event_type=event.event_type, + event_data=None, + origin_idx=EVENT_ORIGIN_TO_IDX.get(event.origin), + time_fired=event.time_fired, + context_id=event.context.id, + context_user_id=event.context.user_id, + context_parent_id=event.context.parent_id, + ) + + def to_native(self, validate_entity_id: bool = True) -> Event | None: + """Convert to a native HA Event.""" + context = Context( + id=self.context_id, + user_id=self.context_user_id, + parent_id=self.context_parent_id, + ) + try: + return Event( + self.event_type, + json_loads(self.event_data) if self.event_data else {}, + EventOrigin(self.origin) + if self.origin + else EVENT_ORIGIN_ORDER[self.origin_idx], + process_timestamp(self.time_fired), + context=context, + ) + except JSON_DECODE_EXCEPTIONS: + # When json_loads fails + _LOGGER.exception("Error converting to event: %s", self) + return None + + +class EventData(Base): # type: ignore[misc,valid-type] + """Event data history.""" + + __table_args__ = ( + {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"}, + ) + __tablename__ = TABLE_EVENT_DATA + data_id = Column(Integer, Identity(), primary_key=True) + hash = Column(BigInteger, index=True) + # Note that this is not named attributes to avoid confusion with the states table + shared_data = Column(Text().with_variant(mysql.LONGTEXT, "mysql")) + + def __repr__(self) -> str: + """Return string representation of instance for debugging.""" + return ( + "" + ) + + @staticmethod + def from_event(event: Event) -> EventData: + """Create object from an event.""" + shared_data = json_bytes(event.data) + return EventData( + shared_data=shared_data.decode("utf-8"), + hash=EventData.hash_shared_data_bytes(shared_data), + ) + + @staticmethod + def shared_data_bytes_from_event( + event: Event, dialect: SupportedDialect | None + ) -> bytes: + """Create shared_data from an event.""" + return json_bytes(event.data) + + @staticmethod + def hash_shared_data_bytes(shared_data_bytes: bytes) -> int: + """Return the hash of json encoded shared data.""" + return cast(int, fnv1a_32(shared_data_bytes)) + + def to_native(self) -> dict[str, Any]: + """Convert to an HA state object.""" + try: + return cast(dict[str, Any], json_loads(self.shared_data)) + except JSON_DECODE_EXCEPTIONS: + _LOGGER.exception("Error converting row to event data: %s", self) + return {} + + +# *** Not originally in v32, only added for recorder to startup ok +# This is not being tested by the v32 statistics migration tests +class EventTypes(Base): # type: ignore[misc,valid-type] + """Event type history.""" + + __table_args__ = ( + {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"}, + ) + __tablename__ = TABLE_EVENT_TYPES + event_type_id = Column(Integer, Identity(), primary_key=True) + event_type = Column(String(MAX_LENGTH_EVENT_EVENT_TYPE)) + + +class States(Base): # type: ignore[misc,valid-type] + """State change history.""" + + __table_args__ = ( + # Used for fetching the state of entities at a specific time + # (get_states in history.py) + Index(ENTITY_ID_LAST_UPDATED_TS_INDEX, "entity_id", "last_updated_ts"), + Index( + STATES_CONTEXT_ID_BIN_INDEX, + "context_id_bin", + mysql_length=CONTEXT_ID_BIN_MAX_LENGTH, + mariadb_length=CONTEXT_ID_BIN_MAX_LENGTH, + ), + {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"}, + ) + __tablename__ = TABLE_STATES + state_id = Column(Integer, Identity(), primary_key=True) + entity_id = Column(String(MAX_LENGTH_STATE_ENTITY_ID)) + state = Column(String(MAX_LENGTH_STATE_STATE)) + attributes = Column( + Text().with_variant(mysql.LONGTEXT, "mysql") + ) # no longer used for new rows + event_id = Column( # no longer used for new rows + Integer, ForeignKey("events.event_id", ondelete="CASCADE"), index=True + ) + last_changed = Column(DATETIME_TYPE) + last_changed_ts = Column(TIMESTAMP_TYPE) + last_updated = Column(DATETIME_TYPE, default=dt_util.utcnow, index=True) + last_updated_ts = Column(TIMESTAMP_TYPE, default=time.time, index=True) + old_state_id = Column(Integer, ForeignKey("states.state_id"), index=True) + attributes_id = Column( + Integer, ForeignKey("state_attributes.attributes_id"), index=True + ) + context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True) + context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID)) + context_parent_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID)) + origin_idx = Column(SmallInteger) # 0 is local, 1 is remote + context_id_bin = Column( + LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH) + ) # *** Not originally in v32, only added for recorder to startup ok + context_user_id_bin = Column( + LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH) + ) # *** Not originally in v32, only added for recorder to startup ok + context_parent_id_bin = Column( + LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH) + ) # *** Not originally in v32, only added for recorder to startup ok + metadata_id = Column( + Integer, ForeignKey("states_meta.metadata_id"), index=True + ) # *** Not originally in v32, only added for recorder to startup ok + states_meta_rel = relationship("StatesMeta") + old_state = relationship("States", remote_side=[state_id]) + state_attributes = relationship("StateAttributes") + + def __repr__(self) -> str: + """Return string representation of instance for debugging.""" + return ( + f"" + ) + + @staticmethod + def from_event(event: Event) -> States: + """Create object from a state_changed event.""" + entity_id = event.data["entity_id"] + state: State | None = event.data.get("new_state") + dbstate = States( + entity_id=entity_id, + attributes=None, + context_id=event.context.id, + context_user_id=event.context.user_id, + context_parent_id=event.context.parent_id, + origin_idx=EVENT_ORIGIN_TO_IDX.get(event.origin), + ) + + # None state means the state was removed from the state machine + if state is None: + dbstate.state = "" + dbstate.last_updated = event.time_fired + dbstate.last_changed = None + return dbstate + + dbstate.state = state.state + dbstate.last_updated = state.last_updated + if state.last_updated == state.last_changed: + dbstate.last_changed = None + else: + dbstate.last_changed = state.last_changed + + return dbstate + + def to_native(self, validate_entity_id: bool = True) -> State | None: + """Convert to an HA state object.""" + context = Context( + id=self.context_id, + user_id=self.context_user_id, + parent_id=self.context_parent_id, + ) + try: + attrs = json_loads(self.attributes) if self.attributes else {} + except JSON_DECODE_EXCEPTIONS: + # When json_loads fails + _LOGGER.exception("Error converting row to state: %s", self) + return None + if self.last_changed is None or self.last_changed == self.last_updated: + last_changed = last_updated = process_timestamp(self.last_updated) + else: + last_updated = process_timestamp(self.last_updated) + last_changed = process_timestamp(self.last_changed) + return State( + self.entity_id, + self.state, + # Join the state_attributes table on attributes_id to get the attributes + # for newer states + attrs, + last_changed, + last_updated, + context=context, + validate_entity_id=validate_entity_id, + ) + + +class StateAttributes(Base): # type: ignore[misc,valid-type] + """State attribute change history.""" + + __table_args__ = ( + {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"}, + ) + __tablename__ = TABLE_STATE_ATTRIBUTES + attributes_id = Column(Integer, Identity(), primary_key=True) + hash = Column(BigInteger, index=True) + # Note that this is not named attributes to avoid confusion with the states table + shared_attrs = Column(Text().with_variant(mysql.LONGTEXT, "mysql")) + + def __repr__(self) -> str: + """Return string representation of instance for debugging.""" + return ( + f"" + ) + + @staticmethod + def from_event(event: Event) -> StateAttributes: + """Create object from a state_changed event.""" + state: State | None = event.data.get("new_state") + # None state means the state was removed from the state machine + attr_bytes = b"{}" if state is None else json_bytes(state.attributes) + dbstate = StateAttributes(shared_attrs=attr_bytes.decode("utf-8")) + dbstate.hash = StateAttributes.hash_shared_attrs_bytes(attr_bytes) + return dbstate + + @staticmethod + def shared_attrs_bytes_from_event( + event: Event, + entity_registry: er.EntityRegistry, + exclude_attrs_by_domain: dict[str, set[str]], + dialect: SupportedDialect | None, + ) -> bytes: + """Create shared_attrs from a state_changed event.""" + state: State | None = event.data.get("new_state") + # None state means the state was removed from the state machine + if state is None: + return b"{}" + domain = split_entity_id(state.entity_id)[0] + exclude_attrs = ( + exclude_attrs_by_domain.get(domain, set()) | ALL_DOMAIN_EXCLUDE_ATTRS + ) + return json_bytes( + {k: v for k, v in state.attributes.items() if k not in exclude_attrs} + ) + + @staticmethod + def hash_shared_attrs_bytes(shared_attrs_bytes: bytes) -> int: + """Return the hash of json encoded shared attributes.""" + return cast(int, fnv1a_32(shared_attrs_bytes)) + + def to_native(self) -> dict[str, Any]: + """Convert to an HA state object.""" + try: + return cast(dict[str, Any], json_loads(self.shared_attrs)) + except JSON_DECODE_EXCEPTIONS: + # When json_loads fails + _LOGGER.exception("Error converting row to state attributes: %s", self) + return {} + + +# *** Not originally in v30, only added for recorder to startup ok +# This is not being tested by the v30 statistics migration tests +class StatesMeta(Base): # type: ignore[misc,valid-type] + """Metadata for states.""" + + __table_args__ = ( + {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"}, + ) + __tablename__ = TABLE_STATES_META + metadata_id = Column(Integer, Identity(), primary_key=True) + entity_id = Column(String(MAX_LENGTH_STATE_ENTITY_ID)) + + def __repr__(self) -> str: + """Return string representation of instance for debugging.""" + return ( + "" + ) + + +class StatisticsBase: + """Statistics base class.""" + + id = Column(Integer, Identity(), primary_key=True) + created = Column(DATETIME_TYPE, default=dt_util.utcnow) + metadata_id = Column( + Integer, + ForeignKey(f"{TABLE_STATISTICS_META}.id", ondelete="CASCADE"), + index=True, + ) + start = Column(DATETIME_TYPE, index=True) + mean = Column(DOUBLE_TYPE) + min = Column(DOUBLE_TYPE) + max = Column(DOUBLE_TYPE) + last_reset = Column(DATETIME_TYPE) + state = Column(DOUBLE_TYPE) + sum = Column(DOUBLE_TYPE) + + @classmethod + def from_stats(cls, metadata_id: int, stats: StatisticData) -> Self: + """Create object from a statistics.""" + return cls( # type: ignore[call-arg,misc] + metadata_id=metadata_id, + **stats, + ) + + +class Statistics(Base, StatisticsBase): # type: ignore[misc,valid-type] + """Long term statistics.""" + + duration = timedelta(hours=1) + + __table_args__ = ( + # Used for fetching statistics for a certain entity at a specific time + Index("ix_statistics_statistic_id_start", "metadata_id", "start", unique=True), + ) + __tablename__ = TABLE_STATISTICS + + +class StatisticsShortTerm(Base, StatisticsBase): # type: ignore[misc,valid-type] + """Short term statistics.""" + + duration = timedelta(minutes=5) + + __table_args__ = ( + # Used for fetching statistics for a certain entity at a specific time + Index( + "ix_statistics_short_term_statistic_id_start", + "metadata_id", + "start", + unique=True, + ), + ) + __tablename__ = TABLE_STATISTICS_SHORT_TERM + + +class StatisticsMeta(Base): # type: ignore[misc,valid-type] + """Statistics meta data.""" + + __table_args__ = ( + {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"}, + ) + __tablename__ = TABLE_STATISTICS_META + id = Column(Integer, Identity(), primary_key=True) + statistic_id = Column(String(255), index=True, unique=True) + source = Column(String(32)) + unit_of_measurement = Column(String(255)) + has_mean = Column(Boolean) + has_sum = Column(Boolean) + name = Column(String(255)) + + @staticmethod + def from_meta(meta: StatisticMetaData) -> StatisticsMeta: + """Create object from meta data.""" + return StatisticsMeta(**meta) + + +class RecorderRuns(Base): # type: ignore[misc,valid-type] + """Representation of recorder run.""" + + __table_args__ = (Index("ix_recorder_runs_start_end", "start", "end"),) + __tablename__ = TABLE_RECORDER_RUNS + run_id = Column(Integer, Identity(), primary_key=True) + start = Column(DATETIME_TYPE, default=dt_util.utcnow) + end = Column(DATETIME_TYPE) + closed_incorrect = Column(Boolean, default=False) + created = Column(DATETIME_TYPE, default=dt_util.utcnow) + + def __repr__(self) -> str: + """Return string representation of instance for debugging.""" + end = ( + f"'{self.end.isoformat(sep=' ', timespec='seconds')}'" if self.end else None + ) + return ( + f"" + ) + + def entity_ids(self, point_in_time: datetime | None = None) -> list[str]: + """Return the entity ids that existed in this run. + + Specify point_in_time if you want to know which existed at that point + in time inside the run. + """ + session = Session.object_session(self) + + assert session is not None, "RecorderRuns need to be persisted" + + query = session.query(distinct(States.entity_id)).filter( + States.last_updated >= self.start + ) + + if point_in_time is not None: + query = query.filter(States.last_updated < point_in_time) + elif self.end is not None: + query = query.filter(States.last_updated < self.end) + + return [row[0] for row in query] + + def to_native(self, validate_entity_id: bool = True) -> RecorderRuns: + """Return self, native format is this model.""" + return self + + +class SchemaChanges(Base): # type: ignore[misc,valid-type] + """Representation of schema version changes.""" + + __tablename__ = TABLE_SCHEMA_CHANGES + change_id = Column(Integer, Identity(), primary_key=True) + schema_version = Column(Integer) + changed = Column(DATETIME_TYPE, default=dt_util.utcnow) + + def __repr__(self) -> str: + """Return string representation of instance for debugging.""" + return ( + "" + ) + + +class StatisticsRuns(Base): # type: ignore[misc,valid-type] + """Representation of statistics run.""" + + __tablename__ = TABLE_STATISTICS_RUNS + run_id = Column(Integer, Identity(), primary_key=True) + start = Column(DATETIME_TYPE, index=True) + + def __repr__(self) -> str: + """Return string representation of instance for debugging.""" + return ( + f"" + ) + + +EVENT_DATA_JSON = type_coerce( + EventData.shared_data.cast(JSONB_VARIANT_CAST), JSONLiteral(none_as_null=True) +) +OLD_FORMAT_EVENT_DATA_JSON = type_coerce( + Events.event_data.cast(JSONB_VARIANT_CAST), JSONLiteral(none_as_null=True) +) + +SHARED_ATTRS_JSON = type_coerce( + StateAttributes.shared_attrs.cast(JSON_VARIANT_CAST), JSON(none_as_null=True) +) +OLD_FORMAT_ATTRS_JSON = type_coerce( + States.attributes.cast(JSON_VARIANT_CAST), JSON(none_as_null=True) +) + +ENTITY_ID_IN_EVENT: Column = EVENT_DATA_JSON["entity_id"] +OLD_ENTITY_ID_IN_EVENT: Column = OLD_FORMAT_EVENT_DATA_JSON["entity_id"] +DEVICE_ID_IN_EVENT: Column = EVENT_DATA_JSON["device_id"] +OLD_STATE = aliased(States, name="old_state") + + +@overload +def process_timestamp(ts: None) -> None: + ... + + +@overload +def process_timestamp(ts: datetime) -> datetime: + ... + + +def process_timestamp(ts: datetime | None) -> datetime | None: + """Process a timestamp into datetime object.""" + if ts is None: + return None + if ts.tzinfo is None: + return ts.replace(tzinfo=dt_util.UTC) + + return dt_util.as_utc(ts) diff --git a/tests/components/recorder/test_v32_migration.py b/tests/components/recorder/test_v32_migration.py index 0b5389ddf7f..6e424558181 100644 --- a/tests/components/recorder/test_v32_migration.py +++ b/tests/components/recorder/test_v32_migration.py @@ -27,7 +27,7 @@ from tests.common import async_test_home_assistant ORIG_TZ = dt_util.DEFAULT_TIME_ZONE CREATE_ENGINE_TARGET = "homeassistant.components.recorder.core.create_engine" -SCHEMA_MODULE = "tests.components.recorder.db_schema_30" +SCHEMA_MODULE = "tests.components.recorder.db_schema_32" def _create_engine_test(*args, **kwargs): @@ -222,3 +222,144 @@ async def test_migrate_times( await hass.async_stop() dt_util.DEFAULT_TIME_ZONE = ORIG_TZ + + +async def test_migrate_can_resume_entity_id_post_migration( + caplog: pytest.LogCaptureFixture, tmpdir: py.path.local +) -> None: + """Test we resume the entity id post migration after a restart.""" + test_db_file = tmpdir.mkdir("sqlite").join("test_run_info.db") + dburl = f"{SQLITE_URL_PREFIX}//{test_db_file}" + + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] + now = dt_util.utcnow() + one_second_past = now - timedelta(seconds=1) + mock_state = State( + "sensor.test", + "old", + {"last_reset": now.isoformat()}, + last_changed=one_second_past, + last_updated=now, + ) + state_changed_event = Event( + EVENT_STATE_CHANGED, + { + "entity_id": "sensor.test", + "old_state": None, + "new_state": mock_state, + }, + EventOrigin.local, + time_fired=now, + ) + custom_event = Event( + "custom_event", + {"entity_id": "sensor.custom"}, + EventOrigin.local, + time_fired=now, + ) + number_of_migrations = 5 + + def _get_states_index_names(): + with session_scope(hass=hass) as session: + return inspect(session.connection()).get_indexes("states") + + with patch.object(recorder, "db_schema", old_db_schema), patch.object( + recorder.migration, "SCHEMA_VERSION", old_db_schema.SCHEMA_VERSION + ), patch.object(core, "StatesMeta", old_db_schema.StatesMeta), patch.object( + core, "EventTypes", old_db_schema.EventTypes + ), patch.object( + core, "EventData", old_db_schema.EventData + ), patch.object( + core, "States", old_db_schema.States + ), patch.object( + core, "Events", old_db_schema.Events + ), patch( + CREATE_ENGINE_TARGET, new=_create_engine_test + ), patch( + "homeassistant.components.recorder.Recorder._migrate_events_context_ids", + ), patch( + "homeassistant.components.recorder.Recorder._migrate_states_context_ids", + ), patch( + "homeassistant.components.recorder.Recorder._migrate_event_type_ids", + ), patch( + "homeassistant.components.recorder.Recorder._migrate_entity_ids", + ), patch( + "homeassistant.components.recorder.Recorder._post_migrate_entity_ids" + ), patch( + "homeassistant.components.recorder.Recorder._cleanup_legacy_states_event_ids" + ): + hass = await async_test_home_assistant(asyncio.get_running_loop()) + recorder_helper.async_initialize_recorder(hass) + assert await async_setup_component( + hass, "recorder", {"recorder": {"db_url": dburl}} + ) + await hass.async_block_till_done() + await async_wait_recording_done(hass) + await async_wait_recording_done(hass) + + def _add_data(): + with session_scope(hass=hass) as session: + session.add(old_db_schema.Events.from_event(custom_event)) + session.add(old_db_schema.States.from_event(state_changed_event)) + + await recorder.get_instance(hass).async_add_executor_job(_add_data) + await hass.async_block_till_done() + await recorder.get_instance(hass).async_block_till_done() + + states_indexes = await recorder.get_instance(hass).async_add_executor_job( + _get_states_index_names + ) + states_index_names = {index["name"] for index in states_indexes} + assert recorder.get_instance(hass).use_legacy_events_index is True + + await hass.async_stop() + await hass.async_block_till_done() + + assert "ix_states_event_id" in states_index_names + assert "ix_states_entity_id_last_updated_ts" in states_index_names + + with patch("homeassistant.components.recorder.Recorder._post_migrate_entity_ids"): + hass = await async_test_home_assistant(asyncio.get_running_loop()) + recorder_helper.async_initialize_recorder(hass) + assert await async_setup_component( + hass, "recorder", {"recorder": {"db_url": dburl}} + ) + await hass.async_block_till_done() + + # We need to wait for all the migration tasks to complete + # before we can check the database. + for _ in range(number_of_migrations): + await recorder.get_instance(hass).async_block_till_done() + await async_wait_recording_done(hass) + + states_indexes = await recorder.get_instance(hass).async_add_executor_job( + _get_states_index_names + ) + states_index_names = {index["name"] for index in states_indexes} + await hass.async_stop() + await hass.async_block_till_done() + + assert "ix_states_entity_id_last_updated_ts" in states_index_names + + hass = await async_test_home_assistant(asyncio.get_running_loop()) + recorder_helper.async_initialize_recorder(hass) + assert await async_setup_component( + hass, "recorder", {"recorder": {"db_url": dburl}} + ) + await hass.async_block_till_done() + + # We need to wait for all the migration tasks to complete + # before we can check the database. + for _ in range(number_of_migrations): + await recorder.get_instance(hass).async_block_till_done() + await async_wait_recording_done(hass) + + states_indexes = await recorder.get_instance(hass).async_add_executor_job( + _get_states_index_names + ) + states_index_names = {index["name"] for index in states_indexes} + assert "ix_states_entity_id_last_updated_ts" not in states_index_names + + await hass.async_stop() + dt_util.DEFAULT_TIME_ZONE = ORIG_TZ From 7f5f2866488c2ed6814930bac4e8fca10ca40364 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 6 Apr 2023 15:19:10 -1000 Subject: [PATCH 032/110] Bump vallox-websocket-api to 3.2.1 (#90980) unblocks https://github.com/home-assistant/core/pull/90901 which will finally fix the races in websockets --- homeassistant/components/vallox/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/vallox/manifest.json b/homeassistant/components/vallox/manifest.json index 924eb921a74..4f3fcbf9c87 100644 --- a/homeassistant/components/vallox/manifest.json +++ b/homeassistant/components/vallox/manifest.json @@ -6,5 +6,5 @@ "documentation": "https://www.home-assistant.io/integrations/vallox", "iot_class": "local_polling", "loggers": ["vallox_websocket_api"], - "requirements": ["vallox-websocket-api==3.0.0"] + "requirements": ["vallox-websocket-api==3.2.1"] } diff --git a/requirements_all.txt b/requirements_all.txt index e749d8240af..7f617809b11 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2565,7 +2565,7 @@ url-normalize==1.4.3 uvcclient==0.11.0 # homeassistant.components.vallox -vallox-websocket-api==3.0.0 +vallox-websocket-api==3.2.1 # homeassistant.components.rdw vehicle==1.0.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 4650a036aed..d60430cdce4 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1832,7 +1832,7 @@ url-normalize==1.4.3 uvcclient==0.11.0 # homeassistant.components.vallox -vallox-websocket-api==3.0.0 +vallox-websocket-api==3.2.1 # homeassistant.components.rdw vehicle==1.0.0 From 804270a79745988f991e67b500e06992c3a446fa Mon Sep 17 00:00:00 2001 From: Aaron Bach Date: Thu, 6 Apr 2023 23:22:27 -0600 Subject: [PATCH 033/110] Bump `aioambient` to 2023.04.0 (#90991) --- homeassistant/components/ambient_station/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- script/pip_check | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/ambient_station/manifest.json b/homeassistant/components/ambient_station/manifest.json index 9dbd4507774..ebd03651064 100644 --- a/homeassistant/components/ambient_station/manifest.json +++ b/homeassistant/components/ambient_station/manifest.json @@ -7,5 +7,5 @@ "integration_type": "hub", "iot_class": "cloud_push", "loggers": ["aioambient"], - "requirements": ["aioambient==2022.10.0"] + "requirements": ["aioambient==2023.04.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index 7f617809b11..0fd036286b3 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -119,7 +119,7 @@ aioairq==0.2.4 aioairzone==0.5.2 # homeassistant.components.ambient_station -aioambient==2022.10.0 +aioambient==2023.04.0 # homeassistant.components.aseko_pool_live aioaseko==0.0.2 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index d60430cdce4..e592b59fbcd 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -109,7 +109,7 @@ aioairq==0.2.4 aioairzone==0.5.2 # homeassistant.components.ambient_station -aioambient==2022.10.0 +aioambient==2023.04.0 # homeassistant.components.aseko_pool_live aioaseko==0.0.2 diff --git a/script/pip_check b/script/pip_check index cbe6a3851e0..cbbe7ffeeae 100755 --- a/script/pip_check +++ b/script/pip_check @@ -3,7 +3,7 @@ PIP_CACHE=$1 # Number of existing dependency conflicts # Update if a PR resolves one! -DEPENDENCY_CONFLICTS=4 +DEPENDENCY_CONFLICTS=3 PIP_CHECK=$(pip check --cache-dir=$PIP_CACHE) LINE_COUNT=$(echo "$PIP_CHECK" | wc -l) From 7392a5780cebe9684010cae58a8091ceb664fc02 Mon Sep 17 00:00:00 2001 From: Joost Lekkerkerker Date: Sat, 8 Apr 2023 01:02:13 +0200 Subject: [PATCH 034/110] Bump roombapy to 1.6.8 (#91012) * Update roombapy to 1.6.7 * Update roombapy to 1.6.8 --- homeassistant/components/roomba/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/roomba/manifest.json b/homeassistant/components/roomba/manifest.json index 08815cae9fb..7b437a4f8c4 100644 --- a/homeassistant/components/roomba/manifest.json +++ b/homeassistant/components/roomba/manifest.json @@ -24,5 +24,5 @@ "documentation": "https://www.home-assistant.io/integrations/roomba", "iot_class": "local_push", "loggers": ["paho_mqtt", "roombapy"], - "requirements": ["roombapy==1.6.6"] + "requirements": ["roombapy==1.6.8"] } diff --git a/requirements_all.txt b/requirements_all.txt index 0fd036286b3..830d2dc757f 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2258,7 +2258,7 @@ rocketchat-API==0.6.1 rokuecp==0.17.1 # homeassistant.components.roomba -roombapy==1.6.6 +roombapy==1.6.8 # homeassistant.components.roon roonapi==0.1.4 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index e592b59fbcd..35061f33f1e 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1609,7 +1609,7 @@ ring_doorbell==0.7.2 rokuecp==0.17.1 # homeassistant.components.roomba -roombapy==1.6.6 +roombapy==1.6.8 # homeassistant.components.roon roonapi==0.1.4 From 828a2779a018914efd31639a172287216187205b Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Sat, 8 Apr 2023 15:36:34 +0100 Subject: [PATCH 035/110] Delay utility_meter until HA has started (#91017) * increase information for end user * only warn after home assistant has started * delay utility_meter until HA has startED --- .../components/utility_meter/sensor.py | 17 ++++++--- tests/components/utility_meter/test_sensor.py | 36 +++++++++++-------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/homeassistant/components/utility_meter/sensor.py b/homeassistant/components/utility_meter/sensor.py index dad2d8dfaf3..099a82c5c0d 100644 --- a/homeassistant/components/utility_meter/sensor.py +++ b/homeassistant/components/utility_meter/sensor.py @@ -35,7 +35,7 @@ from homeassistant.helpers.event import ( async_track_point_in_time, async_track_state_change_event, ) -from homeassistant.helpers.start import async_at_start +from homeassistant.helpers.start import async_at_started from homeassistant.helpers.template import is_number from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType from homeassistant.util import slugify @@ -410,8 +410,11 @@ class UtilityMeterSensor(RestoreSensor): if (old_state_val := self._validate_state(old_state)) is not None: return new_state_val - old_state_val + _LOGGER.warning( - "Invalid state (%s > %s)", + "%s received an invalid state change coming from %s (%s > %s)", + self.name, + self._sensor_source_id, old_state.state if old_state else None, new_state_val, ) @@ -423,8 +426,14 @@ class UtilityMeterSensor(RestoreSensor): old_state: State | None = event.data.get("old_state") new_state: State = event.data.get("new_state") # type: ignore[assignment] # a state change event always has a new state + # First check if the new_state is valid (see discussion in PR #88446) if (new_state_val := self._validate_state(new_state)) is None: - _LOGGER.warning("Invalid state %s", new_state.state) + _LOGGER.warning( + "%s received an invalid new state from %s : %s", + self.name, + self._sensor_source_id, + new_state.state, + ) return if self._state is None: @@ -597,7 +606,7 @@ class UtilityMeterSensor(RestoreSensor): self.hass, [self._sensor_source_id], self.async_reading ) - self.async_on_remove(async_at_start(self.hass, async_source_tracking)) + self.async_on_remove(async_at_started(self.hass, async_source_tracking)) async def async_will_remove_from_hass(self) -> None: """Run when entity will be removed from hass.""" diff --git a/tests/components/utility_meter/test_sensor.py b/tests/components/utility_meter/test_sensor.py index d84099b4d66..8dcff8438ad 100644 --- a/tests/components/utility_meter/test_sensor.py +++ b/tests/components/utility_meter/test_sensor.py @@ -35,7 +35,7 @@ from homeassistant.const import ( ATTR_DEVICE_CLASS, ATTR_ENTITY_ID, ATTR_UNIT_OF_MEASUREMENT, - EVENT_HOMEASSISTANT_START, + EVENT_HOMEASSISTANT_STARTED, STATE_UNAVAILABLE, STATE_UNKNOWN, UnitOfEnergy, @@ -105,7 +105,7 @@ async def test_state(hass: HomeAssistant, yaml_config, config_entry_config) -> N await hass.async_block_till_done() entity_id = config_entry_config["source"] - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) await hass.async_block_till_done() hass.states.async_set( @@ -301,7 +301,7 @@ async def test_init(hass: HomeAssistant, yaml_config, config_entry_config) -> No await hass.async_block_till_done() entity_id = config_entry_config["source"] - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) await hass.async_block_till_done() state = hass.states.get("sensor.energy_bill_onpeak") @@ -346,7 +346,7 @@ async def test_unique_id( assert await async_setup_component(hass, DOMAIN, yaml_config) await hass.async_block_till_done() - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) await hass.async_block_till_done() assert len(entity_registry.entities) == 4 @@ -400,7 +400,7 @@ async def test_entity_name(hass: HomeAssistant, yaml_config, entity_id, name) -> assert await async_setup_component(hass, DOMAIN, yaml_config) await hass.async_block_till_done() - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) await hass.async_block_till_done() state = hass.states.get(entity_id) @@ -475,7 +475,8 @@ async def test_device_class( entity_id_energy = "sensor.energy" entity_id_gas = "sensor.gas" - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) + await hass.async_block_till_done() hass.states.async_set( @@ -657,7 +658,9 @@ async def test_restore_state( assert state.state == STATE_UNKNOWN # utility_meter is loaded, now set sensors according to utility_meter: - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) + await hass.async_block_till_done() state = hass.states.get("select.energy_bill") @@ -719,7 +722,8 @@ async def test_net_consumption( await hass.async_block_till_done() entity_id = config_entry_config["source"] - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) + hass.states.async_set( entity_id, 2, {ATTR_UNIT_OF_MEASUREMENT: UnitOfEnergy.KILO_WATT_HOUR} ) @@ -792,7 +796,8 @@ async def test_non_net_consumption( await hass.async_block_till_done() entity_id = config_entry_config["source"] - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) + hass.states.async_set( entity_id, 2, {ATTR_UNIT_OF_MEASUREMENT: UnitOfEnergy.KILO_WATT_HOUR} ) @@ -817,7 +822,7 @@ async def test_non_net_consumption( force_update=True, ) await hass.async_block_till_done() - assert "Invalid state " in caplog.text + assert "invalid new state " in caplog.text state = hass.states.get("sensor.energy_bill") assert state is not None @@ -882,7 +887,7 @@ async def test_delta_values( await hass.async_block_till_done() entity_id = config_entry_config["source"] - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) async_fire_time_changed(hass, now) hass.states.async_set( @@ -903,7 +908,7 @@ async def test_delta_values( force_update=True, ) await hass.async_block_till_done() - assert "Invalid state None" in caplog.text + assert "invalid new state from sensor.energy : None" in caplog.text now += timedelta(seconds=30) with freeze_time(now): @@ -992,7 +997,7 @@ async def test_non_periodically_resetting( await hass.async_block_till_done() entity_id = config_entry_config["source"] - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) async_fire_time_changed(hass, now) hass.states.async_set( @@ -1120,7 +1125,8 @@ async def test_non_periodically_resetting_meter_with_tariffs( await hass.async_block_till_done() entity_id = config_entry_config["source"] - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) + await hass.async_block_till_done() hass.states.async_set( @@ -1226,7 +1232,7 @@ async def _test_self_reset( assert await async_setup_component(hass, DOMAIN, config) await hass.async_block_till_done() - hass.bus.async_fire(EVENT_HOMEASSISTANT_START) + hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED) entity_id = config[DOMAIN]["energy_bill"]["source"] async_fire_time_changed(hass, now) From 07421927eca05f9ad811b16cd4e786732c17d496 Mon Sep 17 00:00:00 2001 From: Steven Looman Date: Sat, 8 Apr 2023 01:11:31 +0200 Subject: [PATCH 036/110] Make sure upnp-router is also initialized when first seen through an advertisement (#91037) --- homeassistant/components/upnp/manifest.json | 6 ++++++ homeassistant/generated/ssdp.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/homeassistant/components/upnp/manifest.json b/homeassistant/components/upnp/manifest.json index eb4a9b7afe4..1ffb8cfd946 100644 --- a/homeassistant/components/upnp/manifest.json +++ b/homeassistant/components/upnp/manifest.json @@ -15,6 +15,12 @@ }, { "st": "urn:schemas-upnp-org:device:InternetGatewayDevice:2" + }, + { + "nt": "urn:schemas-upnp-org:device:InternetGatewayDevice:1" + }, + { + "nt": "urn:schemas-upnp-org:device:InternetGatewayDevice:2" } ] } diff --git a/homeassistant/generated/ssdp.py b/homeassistant/generated/ssdp.py index 3f26ec8fa78..3a2097a1d30 100644 --- a/homeassistant/generated/ssdp.py +++ b/homeassistant/generated/ssdp.py @@ -305,6 +305,12 @@ SSDP = { { "st": "urn:schemas-upnp-org:device:InternetGatewayDevice:2", }, + { + "nt": "urn:schemas-upnp-org:device:InternetGatewayDevice:1", + }, + { + "nt": "urn:schemas-upnp-org:device:InternetGatewayDevice:2", + }, ], "webostv": [ { From cab88b72b8a74192eebc2e03355c3bfb6ea255de Mon Sep 17 00:00:00 2001 From: "David F. Mulcahey" Date: Fri, 7 Apr 2023 19:00:03 -0400 Subject: [PATCH 037/110] Bump ZHA quirks lib (#91054) --- homeassistant/components/zha/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/zha/manifest.json b/homeassistant/components/zha/manifest.json index bc5bf6a6d4b..7bc482681ca 100644 --- a/homeassistant/components/zha/manifest.json +++ b/homeassistant/components/zha/manifest.json @@ -23,7 +23,7 @@ "bellows==0.35.0", "pyserial==3.5", "pyserial-asyncio==0.6", - "zha-quirks==0.0.95", + "zha-quirks==0.0.96", "zigpy-deconz==0.20.0", "zigpy==0.54.0", "zigpy-xbee==0.17.0", diff --git a/requirements_all.txt b/requirements_all.txt index 830d2dc757f..9e922bd423f 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2698,7 +2698,7 @@ zeroconf==0.54.0 zeversolar==0.3.1 # homeassistant.components.zha -zha-quirks==0.0.95 +zha-quirks==0.0.96 # homeassistant.components.zhong_hong zhong_hong_hvac==1.0.9 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 35061f33f1e..86020307fec 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1932,7 +1932,7 @@ zeroconf==0.54.0 zeversolar==0.3.1 # homeassistant.components.zha -zha-quirks==0.0.95 +zha-quirks==0.0.96 # homeassistant.components.zha zigpy-deconz==0.20.0 From 6836e0b5117347e7683236dfe971667db727d5c5 Mon Sep 17 00:00:00 2001 From: "David F. Mulcahey" Date: Fri, 7 Apr 2023 21:55:37 -0400 Subject: [PATCH 038/110] Fix Smartthings acceleration sensor in ZHA (#91056) --- .../components/zha/core/channels/manufacturerspecific.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/zha/core/channels/manufacturerspecific.py b/homeassistant/components/zha/core/channels/manufacturerspecific.py index e312f398b54..b880a338a42 100644 --- a/homeassistant/components/zha/core/channels/manufacturerspecific.py +++ b/homeassistant/components/zha/core/channels/manufacturerspecific.py @@ -187,11 +187,16 @@ class SmartThingsAcceleration(ZigbeeChannel): @callback def attribute_updated(self, attrid, value): """Handle attribute updates on this cluster.""" + try: + attr_name = self._cluster.attributes[attrid].name + except KeyError: + attr_name = UNKNOWN + if attrid == self.value_attribute: self.async_send_signal( f"{self.unique_id}_{SIGNAL_ATTR_UPDATED}", attrid, - self._cluster.attributes.get(attrid, [UNKNOWN])[0], + attr_name, value, ) return @@ -200,7 +205,7 @@ class SmartThingsAcceleration(ZigbeeChannel): SIGNAL_ATTR_UPDATED, { ATTR_ATTRIBUTE_ID: attrid, - ATTR_ATTRIBUTE_NAME: self._cluster.attributes.get(attrid, [UNKNOWN])[0], + ATTR_ATTRIBUTE_NAME: attr_name, ATTR_VALUE: value, }, ) From 38a0eca223e305469d468c1128cfba190078a999 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 6 Apr 2023 18:07:17 -1000 Subject: [PATCH 039/110] Bump zeroconf to 0.55.0 (#90987) --- homeassistant/components/zeroconf/manifest.json | 2 +- homeassistant/package_constraints.txt | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/zeroconf/manifest.json b/homeassistant/components/zeroconf/manifest.json index de2fe44b630..2abe1398d7c 100644 --- a/homeassistant/components/zeroconf/manifest.json +++ b/homeassistant/components/zeroconf/manifest.json @@ -8,5 +8,5 @@ "iot_class": "local_push", "loggers": ["zeroconf"], "quality_scale": "internal", - "requirements": ["zeroconf==0.54.0"] + "requirements": ["zeroconf==0.55.0"] } diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 7debda373fd..88535c4cabf 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -50,7 +50,7 @@ ulid-transform==0.6.0 voluptuous-serialize==2.6.0 voluptuous==0.13.1 yarl==1.8.1 -zeroconf==0.54.0 +zeroconf==0.55.0 # Constrain pycryptodome to avoid vulnerability # see https://github.com/home-assistant/core/pull/16238 diff --git a/requirements_all.txt b/requirements_all.txt index 9e922bd423f..944836922a6 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2692,7 +2692,7 @@ zamg==0.2.2 zengge==0.2 # homeassistant.components.zeroconf -zeroconf==0.54.0 +zeroconf==0.55.0 # homeassistant.components.zeversolar zeversolar==0.3.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 86020307fec..afba384ec76 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1926,7 +1926,7 @@ youless-api==1.0.1 zamg==0.2.2 # homeassistant.components.zeroconf -zeroconf==0.54.0 +zeroconf==0.55.0 # homeassistant.components.zeversolar zeversolar==0.3.1 From 9dc15687b51499dafaa27bcccbab2d45094c4b4a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 7 Apr 2023 16:01:53 -1000 Subject: [PATCH 040/110] Bump zeroconf to 0.56.0 (#91060) --- homeassistant/components/zeroconf/manifest.json | 2 +- homeassistant/package_constraints.txt | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/zeroconf/manifest.json b/homeassistant/components/zeroconf/manifest.json index 2abe1398d7c..b967954849c 100644 --- a/homeassistant/components/zeroconf/manifest.json +++ b/homeassistant/components/zeroconf/manifest.json @@ -8,5 +8,5 @@ "iot_class": "local_push", "loggers": ["zeroconf"], "quality_scale": "internal", - "requirements": ["zeroconf==0.55.0"] + "requirements": ["zeroconf==0.56.0"] } diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 88535c4cabf..79ac5ac2a7b 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -50,7 +50,7 @@ ulid-transform==0.6.0 voluptuous-serialize==2.6.0 voluptuous==0.13.1 yarl==1.8.1 -zeroconf==0.55.0 +zeroconf==0.56.0 # Constrain pycryptodome to avoid vulnerability # see https://github.com/home-assistant/core/pull/16238 diff --git a/requirements_all.txt b/requirements_all.txt index 944836922a6..e5f40d058da 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2692,7 +2692,7 @@ zamg==0.2.2 zengge==0.2 # homeassistant.components.zeroconf -zeroconf==0.55.0 +zeroconf==0.56.0 # homeassistant.components.zeversolar zeversolar==0.3.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index afba384ec76..de527dba379 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1926,7 +1926,7 @@ youless-api==1.0.1 zamg==0.2.2 # homeassistant.components.zeroconf -zeroconf==0.55.0 +zeroconf==0.56.0 # homeassistant.components.zeversolar zeversolar==0.3.1 From 4fed5ad21c6895747a584e7d528d6e96fa556634 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sat, 8 Apr 2023 19:40:39 -0700 Subject: [PATCH 041/110] Make location optional in google calendar create service (#91061) --- homeassistant/components/google/__init__.py | 17 ++++---- homeassistant/components/google/calendar.py | 19 +++++---- tests/components/google/test_calendar.py | 2 - tests/components/google/test_init.py | 46 +++++++++++++++++++-- 4 files changed, 62 insertions(+), 22 deletions(-) diff --git a/homeassistant/components/google/__init__.py b/homeassistant/components/google/__init__.py index 25993760d80..e05a6f6fb97 100644 --- a/homeassistant/components/google/__init__.py +++ b/homeassistant/components/google/__init__.py @@ -285,17 +285,18 @@ async def async_setup_add_event_service( raise ValueError( "Missing required fields to set start or end date/datetime" ) - + event = Event( + summary=call.data[EVENT_SUMMARY], + description=call.data[EVENT_DESCRIPTION], + start=start, + end=end, + ) + if location := call.data.get(EVENT_LOCATION): + event.location = location try: await calendar_service.async_create_event( call.data[EVENT_CALENDAR_ID], - Event( - summary=call.data[EVENT_SUMMARY], - description=call.data[EVENT_DESCRIPTION], - location=call.data[EVENT_LOCATION], - start=start, - end=end, - ), + event, ) except ApiException as err: raise HomeAssistantError(str(err)) from err diff --git a/homeassistant/components/google/calendar.py b/homeassistant/components/google/calendar.py index 363b75c2c54..347e8444946 100644 --- a/homeassistant/components/google/calendar.py +++ b/homeassistant/components/google/calendar.py @@ -508,9 +508,10 @@ class GoogleCalendarEntity( "start": start, "end": end, EVENT_DESCRIPTION: kwargs.get(EVENT_DESCRIPTION), - EVENT_LOCATION: kwargs.get(EVENT_LOCATION), } ) + if location := kwargs.get(EVENT_LOCATION): + event.location = location if rrule := kwargs.get(EVENT_RRULE): event.recurrence = [f"{RRULE_PREFIX}{rrule}"] @@ -597,18 +598,20 @@ async def async_create_event(entity: GoogleCalendarEntity, call: ServiceCall) -> if start is None or end is None: raise ValueError("Missing required fields to set start or end date/datetime") + event = Event( + summary=call.data[EVENT_SUMMARY], + description=call.data[EVENT_DESCRIPTION], + start=start, + end=end, + ) + if location := call.data.get(EVENT_LOCATION): + event.location = location try: await cast( CalendarSyncUpdateCoordinator, entity.coordinator ).sync.api.async_create_event( entity.calendar_id, - Event( - summary=call.data[EVENT_SUMMARY], - description=call.data[EVENT_DESCRIPTION], - location=call.data[EVENT_LOCATION], - start=start, - end=end, - ), + event, ) except ApiException as err: raise HomeAssistantError(str(err)) from err diff --git a/tests/components/google/test_calendar.py b/tests/components/google/test_calendar.py index 6d0ea7c51f0..8b544a828e9 100644 --- a/tests/components/google/test_calendar.py +++ b/tests/components/google/test_calendar.py @@ -888,7 +888,6 @@ async def test_websocket_create( assert aioclient_mock.mock_calls[0][2] == { "summary": "Bastille Day Party", "description": None, - "location": None, "start": { "dateTime": "1997-07-14T11:00:00-06:00", "timeZone": "America/Regina", @@ -932,7 +931,6 @@ async def test_websocket_create_all_day( assert aioclient_mock.mock_calls[0][2] == { "summary": "Bastille Day Party", "description": None, - "location": None, "start": { "date": "1997-07-14", }, diff --git a/tests/components/google/test_init.py b/tests/components/google/test_init.py index 938dd2c28e7..17f300f58cb 100644 --- a/tests/components/google/test_init.py +++ b/tests/components/google/test_init.py @@ -94,7 +94,6 @@ def add_event_call_service( **params, "summary": TEST_EVENT_SUMMARY, "description": TEST_EVENT_DESCRIPTION, - "location": TEST_EVENT_LOCATION, }, target=target, blocking=True, @@ -486,7 +485,6 @@ async def test_add_event_date_in_x( assert aioclient_mock.mock_calls[0][2] == { "summary": TEST_EVENT_SUMMARY, "description": TEST_EVENT_DESCRIPTION, - "location": TEST_EVENT_LOCATION, "start": {"date": start_date.date().isoformat()}, "end": {"date": end_date.date().isoformat()}, } @@ -527,7 +525,6 @@ async def test_add_event_date( assert aioclient_mock.mock_calls[0][2] == { "summary": TEST_EVENT_SUMMARY, "description": TEST_EVENT_DESCRIPTION, - "location": TEST_EVENT_LOCATION, "start": {"date": today.isoformat()}, "end": {"date": end_date.isoformat()}, } @@ -568,7 +565,6 @@ async def test_add_event_date_time( assert aioclient_mock.mock_calls[0][2] == { "summary": TEST_EVENT_SUMMARY, "description": TEST_EVENT_DESCRIPTION, - "location": TEST_EVENT_LOCATION, "start": { "dateTime": start_datetime.isoformat(timespec="seconds"), "timeZone": "America/Regina", @@ -606,6 +602,48 @@ async def test_add_event_failure( ) +async def test_add_event_location( + hass: HomeAssistant, + component_setup: ComponentSetup, + mock_calendars_list: ApiResult, + test_api_calendar: dict[str, Any], + mock_insert_event: Callable[[str, dict[str, Any]], None], + mock_events_list: ApiResult, + aioclient_mock: AiohttpClientMocker, + add_event_call_service: Callable[dict[str, Any], Awaitable[None]], +) -> None: + """Test service call that sets a location field.""" + + mock_calendars_list({"items": [test_api_calendar]}) + mock_events_list({}) + assert await component_setup() + + now = utcnow() + today = now.date() + end_date = today + datetime.timedelta(days=2) + + aioclient_mock.clear_requests() + mock_insert_event( + calendar_id=CALENDAR_ID, + ) + + await add_event_call_service( + { + "start_date": today.isoformat(), + "end_date": end_date.isoformat(), + "location": TEST_EVENT_LOCATION, + }, + ) + assert len(aioclient_mock.mock_calls) == 1 + assert aioclient_mock.mock_calls[0][2] == { + "summary": TEST_EVENT_SUMMARY, + "description": TEST_EVENT_DESCRIPTION, + "location": TEST_EVENT_LOCATION, + "start": {"date": today.isoformat()}, + "end": {"date": end_date.isoformat()}, + } + + @pytest.mark.parametrize( "config_entry_token_expiry", [datetime.datetime.max.timestamp() + 1] ) From 935af6904d515b3220568d2feef9dda2add74409 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Fri, 7 Apr 2023 19:38:33 -0700 Subject: [PATCH 042/110] Bump gcal_sync to 4.1.4 (#91062) --- homeassistant/components/google/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/google/manifest.json b/homeassistant/components/google/manifest.json index 8c5df8648e7..f4177e8c300 100644 --- a/homeassistant/components/google/manifest.json +++ b/homeassistant/components/google/manifest.json @@ -7,5 +7,5 @@ "documentation": "https://www.home-assistant.io/integrations/calendar.google/", "iot_class": "cloud_polling", "loggers": ["googleapiclient"], - "requirements": ["gcal-sync==4.1.3", "oauth2client==4.1.3"] + "requirements": ["gcal-sync==4.1.4", "oauth2client==4.1.3"] } diff --git a/requirements_all.txt b/requirements_all.txt index e5f40d058da..36bbca8d59a 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -757,7 +757,7 @@ gTTS==2.2.4 gassist-text==0.0.10 # homeassistant.components.google -gcal-sync==4.1.3 +gcal-sync==4.1.4 # homeassistant.components.geniushub geniushub-client==0.7.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index de527dba379..8271da5f1a6 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -579,7 +579,7 @@ gTTS==2.2.4 gassist-text==0.0.10 # homeassistant.components.google -gcal-sync==4.1.3 +gcal-sync==4.1.4 # homeassistant.components.geocaching geocachingapi==0.2.1 From 3a137cb24c13d58c068aa2f2452ecf7303d42e14 Mon Sep 17 00:00:00 2001 From: Garrett <7310260+G-Two@users.noreply.github.com> Date: Fri, 7 Apr 2023 21:56:39 -0400 Subject: [PATCH 043/110] Bump subarulink to 0.7.6 (#91064) --- homeassistant/components/subaru/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/subaru/manifest.json b/homeassistant/components/subaru/manifest.json index 5852136ca45..9fae6ca9f73 100644 --- a/homeassistant/components/subaru/manifest.json +++ b/homeassistant/components/subaru/manifest.json @@ -6,5 +6,5 @@ "documentation": "https://www.home-assistant.io/integrations/subaru", "iot_class": "cloud_polling", "loggers": ["stdiomask", "subarulink"], - "requirements": ["subarulink==0.7.5"] + "requirements": ["subarulink==0.7.6"] } diff --git a/requirements_all.txt b/requirements_all.txt index 36bbca8d59a..8211ff0ced3 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2428,7 +2428,7 @@ streamlabswater==1.0.1 stringcase==1.2.0 # homeassistant.components.subaru -subarulink==0.7.5 +subarulink==0.7.6 # homeassistant.components.solarlog sunwatcher==0.2.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 8271da5f1a6..93f6e357e37 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1737,7 +1737,7 @@ stookwijzer==1.3.0 stringcase==1.2.0 # homeassistant.components.subaru -subarulink==0.7.5 +subarulink==0.7.6 # homeassistant.components.solarlog sunwatcher==0.2.1 From 19567e7fee2edfc7ec81b766819071f466180884 Mon Sep 17 00:00:00 2001 From: Michael Davie Date: Sat, 8 Apr 2023 15:25:58 -0400 Subject: [PATCH 044/110] Bump env_canada to v0.5.31 (#91094) --- homeassistant/components/environment_canada/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/environment_canada/manifest.json b/homeassistant/components/environment_canada/manifest.json index 8e1f17492fb..be255ee951d 100644 --- a/homeassistant/components/environment_canada/manifest.json +++ b/homeassistant/components/environment_canada/manifest.json @@ -6,5 +6,5 @@ "documentation": "https://www.home-assistant.io/integrations/environment_canada", "iot_class": "cloud_polling", "loggers": ["env_canada"], - "requirements": ["env_canada==0.5.30"] + "requirements": ["env_canada==0.5.31"] } diff --git a/requirements_all.txt b/requirements_all.txt index 8211ff0ced3..3ebeca3519d 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -661,7 +661,7 @@ enocean==0.50 enturclient==0.2.4 # homeassistant.components.environment_canada -env_canada==0.5.30 +env_canada==0.5.31 # homeassistant.components.enphase_envoy envoy_reader==0.20.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 93f6e357e37..9c65fb68c5b 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -517,7 +517,7 @@ energyzero==0.4.1 enocean==0.50 # homeassistant.components.environment_canada -env_canada==0.5.30 +env_canada==0.5.31 # homeassistant.components.enphase_envoy envoy_reader==0.20.1 From 0a8f399655a1c85d4d0332866eef096c436e615a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 8 Apr 2023 16:31:28 -1000 Subject: [PATCH 045/110] Fix context_user_id round trip when calling to_native (#91098) We do not actually use this in the history or logbook APIs so nothing broke but there was a bug here for anyone calling this directly fixes #91090 --- .../components/recorder/db_schema.py | 4 ++-- tests/components/recorder/test_models.py | 20 +++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py index 4cef287deda..617e56848d9 100644 --- a/homeassistant/components/recorder/db_schema.py +++ b/homeassistant/components/recorder/db_schema.py @@ -285,7 +285,7 @@ class Events(Base): """Convert to a native HA Event.""" context = Context( id=bytes_to_ulid_or_none(self.context_id_bin), - user_id=bytes_to_uuid_hex_or_none(self.context_user_id), + user_id=bytes_to_uuid_hex_or_none(self.context_user_id_bin), parent_id=bytes_to_ulid_or_none(self.context_parent_id_bin), ) try: @@ -509,7 +509,7 @@ class States(Base): """Convert to an HA state object.""" context = Context( id=bytes_to_ulid_or_none(self.context_id_bin), - user_id=bytes_to_uuid_hex_or_none(self.context_user_id), + user_id=bytes_to_uuid_hex_or_none(self.context_user_id_bin), parent_id=bytes_to_ulid_or_none(self.context_parent_id_bin), ) try: diff --git a/tests/components/recorder/test_models.py b/tests/components/recorder/test_models.py index f1162815b9e..c5033481f23 100644 --- a/tests/components/recorder/test_models.py +++ b/tests/components/recorder/test_models.py @@ -29,7 +29,15 @@ from homeassistant.util import dt, dt as dt_util def test_from_event_to_db_event() -> None: """Test converting event to db event.""" - event = ha.Event("test_event", {"some_data": 15}) + event = ha.Event( + "test_event", + {"some_data": 15}, + context=ha.Context( + id="01EYQZJXZ5Z1Z1Z1Z1Z1Z1Z1Z1", + parent_id="01EYQZJXZ5Z1Z1Z1Z1Z1Z1Z1Z1", + user_id="12345678901234567890123456789012", + ), + ) db_event = Events.from_event(event) dialect = SupportedDialect.MYSQL db_event.event_data = EventData.shared_data_bytes_from_event(event, dialect) @@ -39,7 +47,15 @@ def test_from_event_to_db_event() -> None: def test_from_event_to_db_state() -> None: """Test converting event to db state.""" - state = ha.State("sensor.temperature", "18") + state = ha.State( + "sensor.temperature", + "18", + context=ha.Context( + id="01EYQZJXZ5Z1Z1Z1Z1Z1Z1Z1Z1", + parent_id="01EYQZJXZ5Z1Z1Z1Z1Z1Z1Z1Z1", + user_id="12345678901234567890123456789012", + ), + ) event = ha.Event( EVENT_STATE_CHANGED, {"entity_id": "sensor.temperature", "old_state": None, "new_state": state}, From 8ababc75d47f9fde842e37065a250e87959797b9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 8 Apr 2023 16:48:01 -1000 Subject: [PATCH 046/110] Bump flux_led to 0.28.37 (#91099) changes: https://github.com/Danielhiversen/flux_led/releases/tag/0.28.37 --- homeassistant/components/flux_led/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/flux_led/manifest.json b/homeassistant/components/flux_led/manifest.json index a9b1ef61db5..a6e8183bcdb 100644 --- a/homeassistant/components/flux_led/manifest.json +++ b/homeassistant/components/flux_led/manifest.json @@ -51,5 +51,5 @@ "iot_class": "local_push", "loggers": ["flux_led"], "quality_scale": "platinum", - "requirements": ["flux_led==0.28.36"] + "requirements": ["flux_led==0.28.37"] } diff --git a/requirements_all.txt b/requirements_all.txt index 3ebeca3519d..658b1daa75f 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -725,7 +725,7 @@ fjaraskupan==2.2.0 flipr-api==1.5.0 # homeassistant.components.flux_led -flux_led==0.28.36 +flux_led==0.28.37 # homeassistant.components.homekit # homeassistant.components.recorder diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 9c65fb68c5b..f846a998c3a 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -553,7 +553,7 @@ fjaraskupan==2.2.0 flipr-api==1.5.0 # homeassistant.components.flux_led -flux_led==0.28.36 +flux_led==0.28.37 # homeassistant.components.homekit # homeassistant.components.recorder From 8f6b77235ef808b32f3690034c2d3db49c2e9c53 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 8 Apr 2023 16:32:56 -1000 Subject: [PATCH 047/110] Make the device_tracker more forgiving when passed an empty ip address string (#91101) This has come up over and over and over again fixes #87165 fixes #51980 --- .../components/device_tracker/config_entry.py | 4 +- .../device_tracker/test_config_entry.py | 53 +++++++++++++------ 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/homeassistant/components/device_tracker/config_entry.py b/homeassistant/components/device_tracker/config_entry.py index 6a7133b9483..10725cd0392 100644 --- a/homeassistant/components/device_tracker/config_entry.py +++ b/homeassistant/components/device_tracker/config_entry.py @@ -348,7 +348,7 @@ class ScannerEntity(BaseTrackerEntity): self.mac_address, self.unique_id, ) - if self.is_connected: + if self.is_connected and self.ip_address: _async_connected_device_registered( hass, self.mac_address, @@ -405,7 +405,7 @@ class ScannerEntity(BaseTrackerEntity): """Return the device state attributes.""" attr: dict[str, StateType] = {} attr.update(super().state_attributes) - if self.ip_address is not None: + if self.ip_address: attr[ATTR_IP] = self.ip_address if self.mac_address is not None: attr[ATTR_MAC] = self.mac_address diff --git a/tests/components/device_tracker/test_config_entry.py b/tests/components/device_tracker/test_config_entry.py index 7fa72ac5c13..f9c259a00f4 100644 --- a/tests/components/device_tracker/test_config_entry.py +++ b/tests/components/device_tracker/test_config_entry.py @@ -195,21 +195,6 @@ async def test_connected_device_registered( class MockDisconnectedScannerEntity(MockScannerEntity): """Mock a disconnected scanner entity.""" - @property - def mac_address(self) -> str: - return "aa:bb:cc:dd:ee:ff" - - @property - def is_connected(self) -> bool: - return True - - @property - def hostname(self) -> str: - return "connected" - - class MockConnectedScannerEntity(MockScannerEntity): - """Mock a disconnected scanner entity.""" - @property def mac_address(self) -> str: return "aa:bb:cc:dd:ee:00" @@ -222,10 +207,44 @@ async def test_connected_device_registered( def hostname(self) -> str: return "disconnected" + class MockConnectedScannerEntity(MockScannerEntity): + """Mock a disconnected scanner entity.""" + + @property + def mac_address(self) -> str: + return "aa:bb:cc:dd:ee:ff" + + @property + def is_connected(self) -> bool: + return True + + @property + def hostname(self) -> str: + return "connected" + + class MockConnectedScannerEntityBadIPAddress(MockConnectedScannerEntity): + """Mock a disconnected scanner entity.""" + + @property + def mac_address(self) -> str: + return "aa:bb:cc:dd:ee:01" + + @property + def ip_address(self) -> str: + return "" + + @property + def hostname(self) -> str: + return "connected_bad_ip" + async def async_setup_entry(hass, config_entry, async_add_entities): """Mock setup entry method.""" async_add_entities( - [MockConnectedScannerEntity(), MockDisconnectedScannerEntity()] + [ + MockConnectedScannerEntity(), + MockDisconnectedScannerEntity(), + MockConnectedScannerEntityBadIPAddress(), + ] ) return True @@ -240,7 +259,7 @@ async def test_connected_device_registered( full_name = f"{entity_platform.domain}.{config_entry.domain}" assert full_name in hass.config.components assert len(hass.states.async_entity_ids()) == 0 # should be disabled - assert len(entity_registry.entities) == 2 + assert len(entity_registry.entities) == 3 assert ( entity_registry.entities["test_domain.test_aa_bb_cc_dd_ee_ff"].config_entry_id == "super-mock-id" From 5f0553dd225df495d4078ca52652f95c9c4301af Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Sat, 8 Apr 2023 22:58:28 -0400 Subject: [PATCH 048/110] Bumped version to 2023.4.2 --- homeassistant/const.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/const.py b/homeassistant/const.py index 345a2a0a45f..097aa5f694b 100644 --- a/homeassistant/const.py +++ b/homeassistant/const.py @@ -8,7 +8,7 @@ from .backports.enum import StrEnum APPLICATION_NAME: Final = "HomeAssistant" MAJOR_VERSION: Final = 2023 MINOR_VERSION: Final = 4 -PATCH_VERSION: Final = "1" +PATCH_VERSION: Final = "2" __short_version__: Final = f"{MAJOR_VERSION}.{MINOR_VERSION}" __version__: Final = f"{__short_version__}.{PATCH_VERSION}" REQUIRED_PYTHON_VER: Final[tuple[int, int, int]] = (3, 10, 0) diff --git a/pyproject.toml b/pyproject.toml index d3f7c2ef15b..8b9a59f8052 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "homeassistant" -version = "2023.4.1" +version = "2023.4.2" license = {text = "Apache-2.0"} description = "Open-source home automation platform running on Python 3." readme = "README.rst" From e792350be61fee63bf747d882acde272698d8e79 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 8 Apr 2023 17:41:19 -1000 Subject: [PATCH 049/110] Fix fnvhash import in schema 32 test backport (#91112) --- tests/components/recorder/db_schema_32.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/recorder/db_schema_32.py b/tests/components/recorder/db_schema_32.py index c41775ed386..75d91cdf79c 100644 --- a/tests/components/recorder/db_schema_32.py +++ b/tests/components/recorder/db_schema_32.py @@ -12,7 +12,7 @@ import time from typing import Any, TypedDict, cast, overload import ciso8601 -from fnv_hash_fast import fnv1a_32 +from fnvhash import fnv1a_32 from sqlalchemy import ( JSON, BigInteger, From fa29aea68ec0644285c169db45364175242aa079 Mon Sep 17 00:00:00 2001 From: Anthony Mattas Date: Mon, 10 Apr 2023 12:37:36 -0400 Subject: [PATCH 050/110] Fix configuring Flo instances (#90990) * Update config_flow.py Used constant string for consistency * Update config_flow.py Removed code for location ID and name the integration using the username * Update manifest.json Updated codeowners * Update config_flow.py * Update config_flow.py Formatted with black * Update manifest.json Updated codeowners * Update test_config_flow.py Updated test --- homeassistant/components/flo/config_flow.py | 19 ++++++++----------- tests/components/flo/test_config_flow.py | 2 +- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/homeassistant/components/flo/config_flow.py b/homeassistant/components/flo/config_flow.py index 306ec945a3e..c34753c3295 100644 --- a/homeassistant/components/flo/config_flow.py +++ b/homeassistant/components/flo/config_flow.py @@ -9,7 +9,9 @@ from homeassistant.helpers.aiohttp_client import async_get_clientsession from .const import DOMAIN, LOGGER -DATA_SCHEMA = vol.Schema({vol.Required("username"): str, vol.Required("password"): str}) +DATA_SCHEMA = vol.Schema( + {vol.Required(CONF_USERNAME): str, vol.Required(CONF_PASSWORD): str} +) async def validate_input(hass: core.HomeAssistant, data): @@ -20,18 +22,11 @@ async def validate_input(hass: core.HomeAssistant, data): session = async_get_clientsession(hass) try: - api = await async_get_api( - data[CONF_USERNAME], data[CONF_PASSWORD], session=session - ) + await async_get_api(data[CONF_USERNAME], data[CONF_PASSWORD], session=session) except RequestError as request_error: LOGGER.error("Error connecting to the Flo API: %s", request_error) raise CannotConnect from request_error - user_info = await api.user.get_info() - a_location_id = user_info["locations"][0]["id"] - location_info = await api.location.get_info(a_location_id) - return {"title": location_info["nickname"]} - class ConfigFlow(config_entries.ConfigFlow, domain=DOMAIN): """Handle a config flow for flo.""" @@ -45,8 +40,10 @@ class ConfigFlow(config_entries.ConfigFlow, domain=DOMAIN): await self.async_set_unique_id(user_input[CONF_USERNAME]) self._abort_if_unique_id_configured() try: - info = await validate_input(self.hass, user_input) - return self.async_create_entry(title=info["title"], data=user_input) + await validate_input(self.hass, user_input) + return self.async_create_entry( + title=user_input[CONF_USERNAME], data=user_input + ) except CannotConnect: errors["base"] = "cannot_connect" diff --git a/tests/components/flo/test_config_flow.py b/tests/components/flo/test_config_flow.py index 747418c807a..703689e7c36 100644 --- a/tests/components/flo/test_config_flow.py +++ b/tests/components/flo/test_config_flow.py @@ -31,7 +31,7 @@ async def test_form(hass: HomeAssistant, aioclient_mock_fixture) -> None: ) assert result2["type"] == "create_entry" - assert result2["title"] == "Home" + assert result2["title"] == TEST_USER_ID assert result2["data"] == {"username": TEST_USER_ID, "password": TEST_PASSWORD} await hass.async_block_till_done() assert len(mock_setup_entry.mock_calls) == 1 From 34394d90c0da3b2e9691341cb9a9263b2da7f362 Mon Sep 17 00:00:00 2001 From: Pascal Reeb Date: Tue, 11 Apr 2023 11:13:52 +0200 Subject: [PATCH 051/110] Fall back to polling if webhook cannot be registered on Nuki (#91013) fix(nuki): throw warning if webhook cannot be created --- homeassistant/components/nuki/__init__.py | 159 ++++++++++++---------- homeassistant/components/nuki/helpers.py | 4 + 2 files changed, 89 insertions(+), 74 deletions(-) diff --git a/homeassistant/components/nuki/__init__.py b/homeassistant/components/nuki/__init__.py index ef168374bd8..b0bfe18614e 100644 --- a/homeassistant/components/nuki/__init__.py +++ b/homeassistant/components/nuki/__init__.py @@ -25,7 +25,6 @@ from homeassistant.const import ( Platform, ) from homeassistant.core import Event, HomeAssistant -from homeassistant.exceptions import ConfigEntryNotReady from homeassistant.helpers import ( device_registry as dr, entity_registry as er, @@ -47,7 +46,7 @@ from .const import ( DOMAIN, ERROR_STATES, ) -from .helpers import parse_id +from .helpers import NukiWebhookException, parse_id _NukiDeviceT = TypeVar("_NukiDeviceT", bound=NukiDevice) @@ -61,6 +60,87 @@ def _get_bridge_devices(bridge: NukiBridge) -> tuple[list[NukiLock], list[NukiOp return bridge.locks, bridge.openers +async def _create_webhook( + hass: HomeAssistant, entry: ConfigEntry, bridge: NukiBridge +) -> None: + # Create HomeAssistant webhook + async def handle_webhook( + hass: HomeAssistant, webhook_id: str, request: web.Request + ) -> web.Response: + """Handle webhook callback.""" + try: + data = await request.json() + except ValueError: + return web.Response(status=HTTPStatus.BAD_REQUEST) + + locks = hass.data[DOMAIN][entry.entry_id][DATA_LOCKS] + openers = hass.data[DOMAIN][entry.entry_id][DATA_OPENERS] + + devices = [x for x in locks + openers if x.nuki_id == data["nukiId"]] + if len(devices) == 1: + devices[0].update_from_callback(data) + + coordinator = hass.data[DOMAIN][entry.entry_id][DATA_COORDINATOR] + coordinator.async_set_updated_data(None) + + return web.Response(status=HTTPStatus.OK) + + webhook.async_register( + hass, DOMAIN, entry.title, entry.entry_id, handle_webhook, local_only=True + ) + + webhook_url = webhook.async_generate_path(entry.entry_id) + + try: + hass_url = get_url( + hass, + allow_cloud=False, + allow_external=False, + allow_ip=True, + require_ssl=False, + ) + except NoURLAvailableError: + webhook.async_unregister(hass, entry.entry_id) + raise NukiWebhookException( + f"Error registering URL for webhook {entry.entry_id}: " + "HomeAssistant URL is not available" + ) from None + + url = f"{hass_url}{webhook_url}" + + if hass_url.startswith("https"): + ir.async_create_issue( + hass, + DOMAIN, + "https_webhook", + is_fixable=False, + severity=ir.IssueSeverity.WARNING, + translation_key="https_webhook", + translation_placeholders={ + "base_url": hass_url, + "network_link": "https://my.home-assistant.io/redirect/network/", + }, + ) + else: + ir.async_delete_issue(hass, DOMAIN, "https_webhook") + + try: + async with async_timeout.timeout(10): + await hass.async_add_executor_job( + _register_webhook, bridge, entry.entry_id, url + ) + except InvalidCredentialsException as err: + webhook.async_unregister(hass, entry.entry_id) + raise NukiWebhookException( + f"Invalid credentials for Bridge: {err}" + ) from err + except RequestException as err: + webhook.async_unregister(hass, entry.entry_id) + raise NukiWebhookException( + f"Error communicating with Bridge: {err}" + ) from err + + def _register_webhook(bridge: NukiBridge, entry_id: str, url: str) -> bool: # Register HA URL as webhook if not already callbacks = bridge.callback_list() @@ -126,79 +206,10 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: sw_version=info["versions"]["firmwareVersion"], ) - async def handle_webhook( - hass: HomeAssistant, webhook_id: str, request: web.Request - ) -> web.Response: - """Handle webhook callback.""" - try: - data = await request.json() - except ValueError: - return web.Response(status=HTTPStatus.BAD_REQUEST) - - locks = hass.data[DOMAIN][entry.entry_id][DATA_LOCKS] - openers = hass.data[DOMAIN][entry.entry_id][DATA_OPENERS] - - devices = [x for x in locks + openers if x.nuki_id == data["nukiId"]] - if len(devices) == 1: - devices[0].update_from_callback(data) - - coordinator = hass.data[DOMAIN][entry.entry_id][DATA_COORDINATOR] - coordinator.async_set_updated_data(None) - - return web.Response(status=HTTPStatus.OK) - - webhook.async_register( - hass, DOMAIN, entry.title, entry.entry_id, handle_webhook, local_only=True - ) - - webhook_url = webhook.async_generate_path(entry.entry_id) - try: - hass_url = get_url( - hass, - allow_cloud=False, - allow_external=False, - allow_ip=True, - require_ssl=False, - ) - except NoURLAvailableError: - webhook.async_unregister(hass, entry.entry_id) - raise ConfigEntryNotReady( - f"Error registering URL for webhook {entry.entry_id}: " - "HomeAssistant URL is not available" - ) from None - - url = f"{hass_url}{webhook_url}" - - if hass_url.startswith("https"): - ir.async_create_issue( - hass, - DOMAIN, - "https_webhook", - is_fixable=False, - severity=ir.IssueSeverity.WARNING, - translation_key="https_webhook", - translation_placeholders={ - "base_url": hass_url, - "network_link": "https://my.home-assistant.io/redirect/network/", - }, - ) - else: - ir.async_delete_issue(hass, DOMAIN, "https_webhook") - - try: - async with async_timeout.timeout(10): - await hass.async_add_executor_job( - _register_webhook, bridge, entry.entry_id, url - ) - except InvalidCredentialsException as err: - webhook.async_unregister(hass, entry.entry_id) - raise ConfigEntryNotReady(f"Invalid credentials for Bridge: {err}") from err - except RequestException as err: - webhook.async_unregister(hass, entry.entry_id) - raise ConfigEntryNotReady( - f"Error communicating with Bridge: {err}" - ) from err + await _create_webhook(hass, entry, bridge) + except NukiWebhookException as err: + _LOGGER.warning("Error registering HomeAssistant webhook: %s", err) async def _stop_nuki(_: Event): """Stop and remove the Nuki webhook.""" diff --git a/homeassistant/components/nuki/helpers.py b/homeassistant/components/nuki/helpers.py index 45b7420754a..1ba8e393f54 100644 --- a/homeassistant/components/nuki/helpers.py +++ b/homeassistant/components/nuki/helpers.py @@ -13,3 +13,7 @@ class CannotConnect(exceptions.HomeAssistantError): class InvalidAuth(exceptions.HomeAssistantError): """Error to indicate there is invalid auth.""" + + +class NukiWebhookException(exceptions.HomeAssistantError): + """Error to indicate there was an issue with the webhook.""" From 2d41fe837cf4b8e7a1732cdc6d8654f9474061e4 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Mon, 10 Apr 2023 17:37:45 +0100 Subject: [PATCH 052/110] Track availability of source sensor in utility meter (#91035) * track availability of source sensor * address review comments --- homeassistant/components/utility_meter/sensor.py | 9 +++++++++ tests/components/utility_meter/test_sensor.py | 9 ++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/utility_meter/sensor.py b/homeassistant/components/utility_meter/sensor.py index 099a82c5c0d..a8a8a80c9fa 100644 --- a/homeassistant/components/utility_meter/sensor.py +++ b/homeassistant/components/utility_meter/sensor.py @@ -423,6 +423,15 @@ class UtilityMeterSensor(RestoreSensor): @callback def async_reading(self, event: Event): """Handle the sensor state changes.""" + if ( + source_state := self.hass.states.get(self._sensor_source_id) + ) is None or source_state.state == STATE_UNAVAILABLE: + self._attr_available = False + self.async_write_ha_state() + return + + self._attr_available = True + old_state: State | None = event.data.get("old_state") new_state: State = event.data.get("new_state") # type: ignore[assignment] # a state change event always has a new state diff --git a/tests/components/utility_meter/test_sensor.py b/tests/components/utility_meter/test_sensor.py index 8dcff8438ad..65892ae376a 100644 --- a/tests/components/utility_meter/test_sensor.py +++ b/tests/components/utility_meter/test_sensor.py @@ -217,20 +217,19 @@ async def test_state(hass: HomeAssistant, yaml_config, config_entry_config) -> N entity_id, "*", {ATTR_UNIT_OF_MEASUREMENT: UnitOfEnergy.KILO_WATT_HOUR} ) await hass.async_block_till_done() - state = hass.states.get("sensor.energy_bill_midpeak") + state = hass.states.get("sensor.energy_bill_offpeak") assert state is not None - assert state.state == "0.123" + assert state.state == "3" # test unavailable source hass.states.async_set( entity_id, STATE_UNAVAILABLE, - {ATTR_UNIT_OF_MEASUREMENT: UnitOfEnergy.KILO_WATT_HOUR}, ) await hass.async_block_till_done() - state = hass.states.get("sensor.energy_bill_midpeak") + state = hass.states.get("sensor.energy_bill_offpeak") assert state is not None - assert state.state == "0.123" + assert state.state == "unavailable" @pytest.mark.parametrize( From 3f6486db3e32159f53c981e7b9f504cf51ecedac Mon Sep 17 00:00:00 2001 From: Robert Hillis Date: Sun, 9 Apr 2023 16:48:37 -0400 Subject: [PATCH 053/110] Bump aiopyarr to 23.4.0 (#91110) --- homeassistant/components/lidarr/manifest.json | 2 +- homeassistant/components/radarr/manifest.json | 2 +- homeassistant/components/sonarr/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/lidarr/manifest.json b/homeassistant/components/lidarr/manifest.json index 717800e3110..30672afdf41 100644 --- a/homeassistant/components/lidarr/manifest.json +++ b/homeassistant/components/lidarr/manifest.json @@ -7,5 +7,5 @@ "integration_type": "service", "iot_class": "local_polling", "loggers": ["aiopyarr"], - "requirements": ["aiopyarr==22.11.0"] + "requirements": ["aiopyarr==23.4.0"] } diff --git a/homeassistant/components/radarr/manifest.json b/homeassistant/components/radarr/manifest.json index 420c352e8e3..ad5a4f05096 100644 --- a/homeassistant/components/radarr/manifest.json +++ b/homeassistant/components/radarr/manifest.json @@ -7,5 +7,5 @@ "integration_type": "service", "iot_class": "local_polling", "loggers": ["aiopyarr"], - "requirements": ["aiopyarr==22.11.0"] + "requirements": ["aiopyarr==23.4.0"] } diff --git a/homeassistant/components/sonarr/manifest.json b/homeassistant/components/sonarr/manifest.json index 19531b08de8..bfc2b6f787f 100644 --- a/homeassistant/components/sonarr/manifest.json +++ b/homeassistant/components/sonarr/manifest.json @@ -7,5 +7,5 @@ "iot_class": "local_polling", "loggers": ["aiopyarr"], "quality_scale": "silver", - "requirements": ["aiopyarr==22.11.0"] + "requirements": ["aiopyarr==23.4.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index 658b1daa75f..03c37383ce5 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -246,7 +246,7 @@ aiopvpc==4.1.0 # homeassistant.components.lidarr # homeassistant.components.radarr # homeassistant.components.sonarr -aiopyarr==22.11.0 +aiopyarr==23.4.0 # homeassistant.components.qnap_qsw aioqsw==0.3.2 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index f846a998c3a..7162f720717 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -227,7 +227,7 @@ aiopvpc==4.1.0 # homeassistant.components.lidarr # homeassistant.components.radarr # homeassistant.components.sonarr -aiopyarr==22.11.0 +aiopyarr==23.4.0 # homeassistant.components.qnap_qsw aioqsw==0.3.2 From 4cd00da319b93d7f504fa4b5bf08e4ea8357927d Mon Sep 17 00:00:00 2001 From: Michael Davie Date: Sun, 9 Apr 2023 13:35:43 -0400 Subject: [PATCH 054/110] Bump env_canada to 0.5.32 (#91126) --- homeassistant/components/environment_canada/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/environment_canada/manifest.json b/homeassistant/components/environment_canada/manifest.json index be255ee951d..79be96d9bf4 100644 --- a/homeassistant/components/environment_canada/manifest.json +++ b/homeassistant/components/environment_canada/manifest.json @@ -6,5 +6,5 @@ "documentation": "https://www.home-assistant.io/integrations/environment_canada", "iot_class": "cloud_polling", "loggers": ["env_canada"], - "requirements": ["env_canada==0.5.31"] + "requirements": ["env_canada==0.5.32"] } diff --git a/requirements_all.txt b/requirements_all.txt index 03c37383ce5..929e843c641 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -661,7 +661,7 @@ enocean==0.50 enturclient==0.2.4 # homeassistant.components.environment_canada -env_canada==0.5.31 +env_canada==0.5.32 # homeassistant.components.enphase_envoy envoy_reader==0.20.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 7162f720717..c7b0837a1a9 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -517,7 +517,7 @@ energyzero==0.4.1 enocean==0.50 # homeassistant.components.environment_canada -env_canada==0.5.31 +env_canada==0.5.32 # homeassistant.components.enphase_envoy envoy_reader==0.20.1 From dc777f78b860bfd969d58b8065d094b2128c2fa4 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Mon, 10 Apr 2023 07:04:42 -0700 Subject: [PATCH 055/110] Relax calendar event validation to allow existing zero duration events (#91129) Relax event valudation to allow existing zero duration events --- homeassistant/components/calendar/__init__.py | 49 ++++++++++++---- tests/components/caldav/test_calendar.py | 28 ++++++++- tests/components/calendar/test_init.py | 6 +- tests/components/google/test_calendar.py | 57 +++++++++++++++++++ 4 files changed, 126 insertions(+), 14 deletions(-) diff --git a/homeassistant/components/calendar/__init__.py b/homeassistant/components/calendar/__init__.py index 0b1c37cea5f..594964c129c 100644 --- a/homeassistant/components/calendar/__init__.py +++ b/homeassistant/components/calendar/__init__.py @@ -67,6 +67,13 @@ SCAN_INTERVAL = datetime.timedelta(seconds=60) # Don't support rrules more often than daily VALID_FREQS = {"DAILY", "WEEKLY", "MONTHLY", "YEARLY"} +# Ensure events created in Home Assistant have a positive duration +MIN_NEW_EVENT_DURATION = datetime.timedelta(seconds=1) + +# Events must have a non-negative duration e.g. Google Calendar can create zero +# duration events in the UI. +MIN_EVENT_DURATION = datetime.timedelta(seconds=0) + def _has_timezone(*keys: Any) -> Callable[[dict[str, Any]], dict[str, Any]]: """Assert that all datetime values have a timezone.""" @@ -116,17 +123,38 @@ def _as_local_timezone(*keys: Any) -> Callable[[dict[str, Any]], dict[str, Any]] return validate -def _has_duration( - start_key: str, end_key: str +def _has_min_duration( + start_key: str, end_key: str, min_duration: datetime.timedelta ) -> Callable[[dict[str, Any]], dict[str, Any]]: - """Verify that the time span between start and end is positive.""" + """Verify that the time span between start and end has a minimum duration.""" def validate(obj: dict[str, Any]) -> dict[str, Any]: - """Test that all keys in the dict are in order.""" if (start := obj.get(start_key)) and (end := obj.get(end_key)): duration = end - start - if duration.total_seconds() <= 0: - raise vol.Invalid(f"Expected positive event duration ({start}, {end})") + if duration < min_duration: + raise vol.Invalid( + f"Expected minimum event duration of {min_duration} ({start}, {end})" + ) + return obj + + return validate + + +def _has_all_day_event_duration( + start_key: str, + end_key: str, +) -> Callable[[dict[str, Any]], dict[str, Any]]: + """Modify all day events to have a duration of one day.""" + + def validate(obj: dict[str, Any]) -> dict[str, Any]: + if ( + (start := obj.get(start_key)) + and (end := obj.get(end_key)) + and not isinstance(start, datetime.datetime) + and not isinstance(end, datetime.datetime) + and start == end + ): + obj[end_key] = start + datetime.timedelta(days=1) return obj return validate @@ -204,8 +232,8 @@ CREATE_EVENT_SCHEMA = vol.All( ), _has_consistent_timezone(EVENT_START_DATETIME, EVENT_END_DATETIME), _as_local_timezone(EVENT_START_DATETIME, EVENT_END_DATETIME), - _has_duration(EVENT_START_DATE, EVENT_END_DATE), - _has_duration(EVENT_START_DATETIME, EVENT_END_DATETIME), + _has_min_duration(EVENT_START_DATE, EVENT_END_DATE, MIN_NEW_EVENT_DURATION), + _has_min_duration(EVENT_START_DATETIME, EVENT_END_DATETIME, MIN_NEW_EVENT_DURATION), ) WEBSOCKET_EVENT_SCHEMA = vol.Schema( @@ -221,7 +249,7 @@ WEBSOCKET_EVENT_SCHEMA = vol.Schema( _has_same_type(EVENT_START, EVENT_END), _has_consistent_timezone(EVENT_START, EVENT_END), _as_local_timezone(EVENT_START, EVENT_END), - _has_duration(EVENT_START, EVENT_END), + _has_min_duration(EVENT_START, EVENT_END, MIN_NEW_EVENT_DURATION), ) ) @@ -238,7 +266,8 @@ CALENDAR_EVENT_SCHEMA = vol.Schema( _has_timezone("start", "end"), _has_consistent_timezone("start", "end"), _as_local_timezone("start", "end"), - _has_duration("start", "end"), + _has_min_duration("start", "end", MIN_EVENT_DURATION), + _has_all_day_event_duration("start", "end"), ), extra=vol.ALLOW_EXTRA, ) diff --git a/tests/components/caldav/test_calendar.py b/tests/components/caldav/test_calendar.py index fc224d20685..87aec3a6f5d 100644 --- a/tests/components/caldav/test_calendar.py +++ b/tests/components/caldav/test_calendar.py @@ -254,6 +254,32 @@ DTEND;TZID=Europe/London:20221127T003000 SUMMARY:Event with a provided Timezone END:VEVENT END:VCALENDAR +""", + """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Global Corp.//CalDAV Client//EN +BEGIN:VEVENT +UID:16 +DTSTAMP:20171125T000000Z +DTSTART:20171127 +DTEND:20171128 +SUMMARY:All day event with same start and end +LOCATION:Hamburg +END:VEVENT +END:VCALENDAR +""", + """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Global Corp.//CalDAV Client//EN +BEGIN:VEVENT +UID:17 +DTSTAMP:20171125T000000Z +DTSTART:20171127T010000 +DTEND:20171127T010000 +SUMMARY:Event with no duration +LOCATION:Hamburg +END:VEVENT +END:VCALENDAR """, ] @@ -1001,7 +1027,7 @@ async def test_get_events(hass: HomeAssistant, calendar, get_api_events) -> None await hass.async_block_till_done() events = await get_api_events("calendar.private") - assert len(events) == 16 + assert len(events) == 18 assert calendar.call diff --git a/tests/components/calendar/test_init.py b/tests/components/calendar/test_init.py index 875d5bf8c13..d58932ce898 100644 --- a/tests/components/calendar/test_init.py +++ b/tests/components/calendar/test_init.py @@ -324,7 +324,7 @@ async def test_unsupported_create_event_service(hass: HomeAssistant) -> None: "end_date_time": "2022-04-01T06:00:00", }, vol.error.MultipleInvalid, - "Expected positive event duration", + "Expected minimum event duration", ), ( { @@ -332,7 +332,7 @@ async def test_unsupported_create_event_service(hass: HomeAssistant) -> None: "end_date": "2022-04-01", }, vol.error.MultipleInvalid, - "Expected positive event duration", + "Expected minimum event duration", ), ( { @@ -340,7 +340,7 @@ async def test_unsupported_create_event_service(hass: HomeAssistant) -> None: "end_date": "2022-04-01", }, vol.error.MultipleInvalid, - "Expected positive event duration", + "Expected minimum event duration", ), ], ids=[ diff --git a/tests/components/google/test_calendar.py b/tests/components/google/test_calendar.py index 8b544a828e9..7d59d80687e 100644 --- a/tests/components/google/test_calendar.py +++ b/tests/components/google/test_calendar.py @@ -1238,3 +1238,60 @@ async def test_reader_in_progress_event( "location": event["location"], "description": event["description"], } + + +async def test_all_day_event_without_duration( + hass: HomeAssistant, mock_events_list_items, component_setup +) -> None: + """Test that an all day event without a duration is adjusted to have a duration of one day.""" + week_from_today = dt_util.now().date() + datetime.timedelta(days=7) + event = { + **TEST_EVENT, + "start": {"date": week_from_today.isoformat()}, + "end": {"date": week_from_today.isoformat()}, + } + mock_events_list_items([event]) + + assert await component_setup() + + expected_end_event = week_from_today + datetime.timedelta(days=1) + + state = hass.states.get(TEST_ENTITY) + assert state.name == TEST_ENTITY_NAME + assert state.state == STATE_OFF + assert dict(state.attributes) == { + "friendly_name": TEST_ENTITY_NAME, + "message": event["summary"], + "all_day": True, + "offset_reached": False, + "start_time": week_from_today.strftime(DATE_STR_FORMAT), + "end_time": expected_end_event.strftime(DATE_STR_FORMAT), + "location": event["location"], + "description": event["description"], + "supported_features": 3, + } + + +async def test_event_without_duration( + hass: HomeAssistant, mock_events_list_items, component_setup +) -> None: + """Google calendar UI allows creating events without a duration.""" + one_hour_from_now = dt_util.now() + datetime.timedelta(minutes=30) + event = { + **TEST_EVENT, + "start": {"dateTime": one_hour_from_now.isoformat()}, + "end": {"dateTime": one_hour_from_now.isoformat()}, + } + mock_events_list_items([event]) + + assert await component_setup() + + state = hass.states.get(TEST_ENTITY) + assert state.name == TEST_ENTITY_NAME + assert state.state == STATE_OFF + # Confirm the event is parsed successfully, but we don't assert on the + # specific end date as the client library may adjust it + assert state.attributes.get("message") == event["summary"] + assert state.attributes.get("start_time") == one_hour_from_now.strftime( + DATE_STR_FORMAT + ) From 3efffe7688e7ecd700b9b25627ccd1a26c614cd1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 10 Apr 2023 04:03:21 -1000 Subject: [PATCH 056/110] Bump ulid-transform to 0.6.3 (#91133) * Bump ulid-transform to 0.6.2 changelog: https://github.com/bdraco/ulid-transform/compare/v0.6.0...v0.6.2 32bit fixes fixes #91092 * 0.6.3 --- homeassistant/package_constraints.txt | 2 +- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 79ac5ac2a7b..1568d9720ff 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -46,7 +46,7 @@ requests==2.28.2 scapy==2.5.0 sqlalchemy==2.0.7 typing-extensions>=4.5.0,<5.0 -ulid-transform==0.6.0 +ulid-transform==0.6.3 voluptuous-serialize==2.6.0 voluptuous==0.13.1 yarl==1.8.1 diff --git a/pyproject.toml b/pyproject.toml index 8b9a59f8052..f161f3837af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ "pyyaml==6.0", "requests==2.28.2", "typing-extensions>=4.5.0,<5.0", - "ulid-transform==0.6.0", + "ulid-transform==0.6.3", "voluptuous==0.13.1", "voluptuous-serialize==2.6.0", "yarl==1.8.1", diff --git a/requirements.txt b/requirements.txt index 43bb2d6d37c..fe6ffb649bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ python-slugify==4.0.1 pyyaml==6.0 requests==2.28.2 typing-extensions>=4.5.0,<5.0 -ulid-transform==0.6.0 +ulid-transform==0.6.3 voluptuous==0.13.1 voluptuous-serialize==2.6.0 yarl==1.8.1 From 2db8d70c2fa6fc255881ff3de3f21118f21a749d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 9 Apr 2023 19:45:08 -1000 Subject: [PATCH 057/110] Fix false positive in SQL sensor full table scan check (#91134) --- homeassistant/components/sql/sensor.py | 4 ++- tests/components/sql/__init__.py | 17 ++++++++++ tests/components/sql/test_sensor.py | 43 +++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/homeassistant/components/sql/sensor.py b/homeassistant/components/sql/sensor.py index 8408b98730b..b6cce467e1f 100644 --- a/homeassistant/components/sql/sensor.py +++ b/homeassistant/components/sql/sensor.py @@ -162,7 +162,9 @@ async def async_setup_sensor( # If the query has a unique id and they fix it we can dismiss the issue # but if it doesn't have a unique id they have to ignore it instead - if "ENTITY_ID" in upper_query and "STATES_META" not in upper_query: + if ( + "ENTITY_ID," in upper_query or "ENTITY_ID " in upper_query + ) and "STATES_META" not in upper_query: _LOGGER.error( "The query `%s` contains the keyword `entity_id` but does not " "reference the `states_meta` table. This will cause a full table " diff --git a/tests/components/sql/__init__.py b/tests/components/sql/__init__.py index c976f87f50f..97df7fe253e 100644 --- a/tests/components/sql/__init__.py +++ b/tests/components/sql/__init__.py @@ -81,6 +81,23 @@ YAML_CONFIG_FULL_TABLE_SCAN_NO_UNIQUE_ID = { } } +YAML_CONFIG_FULL_TABLE_SCAN_WITH_MULTIPLE_COLUMNS = { + "sql": { + CONF_NAME: "Get entity_id", + CONF_QUERY: "SELECT entity_id,state_id from states", + CONF_COLUMN_NAME: "entity_id", + } +} + +YAML_CONFIG_WITH_VIEW_THAT_CONTAINS_ENTITY_ID = { + "sql": { + CONF_NAME: "Get entity_id", + CONF_QUERY: "SELECT value from view_sensor_db_unique_entity_ids;", + CONF_COLUMN_NAME: "value", + } +} + + YAML_CONFIG_BINARY = { "sql": { CONF_DB_URL: "sqlite://", diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index 811bb3f45bf..7e289565b37 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -24,6 +24,8 @@ from . import ( YAML_CONFIG_BINARY, YAML_CONFIG_FULL_TABLE_SCAN, YAML_CONFIG_FULL_TABLE_SCAN_NO_UNIQUE_ID, + YAML_CONFIG_FULL_TABLE_SCAN_WITH_MULTIPLE_COLUMNS, + YAML_CONFIG_WITH_VIEW_THAT_CONTAINS_ENTITY_ID, init_integration, ) @@ -353,24 +355,55 @@ async def test_issue_when_using_old_query( assert issue.translation_placeholders == {"query": config[CONF_QUERY]} +@pytest.mark.parametrize( + "yaml_config", + [ + YAML_CONFIG_FULL_TABLE_SCAN_NO_UNIQUE_ID, + YAML_CONFIG_FULL_TABLE_SCAN_WITH_MULTIPLE_COLUMNS, + ], +) async def test_issue_when_using_old_query_without_unique_id( - recorder_mock: Recorder, hass: HomeAssistant, caplog: pytest.LogCaptureFixture + recorder_mock: Recorder, + hass: HomeAssistant, + caplog: pytest.LogCaptureFixture, + yaml_config: dict[str, Any], ) -> None: """Test we create an issue for an old query that will do a full table scan.""" - assert await async_setup_component( - hass, DOMAIN, YAML_CONFIG_FULL_TABLE_SCAN_NO_UNIQUE_ID - ) + assert await async_setup_component(hass, DOMAIN, yaml_config) await hass.async_block_till_done() assert "Query contains entity_id but does not reference states_meta" in caplog.text assert not hass.states.async_all() issue_registry = ir.async_get(hass) - config = YAML_CONFIG_FULL_TABLE_SCAN_NO_UNIQUE_ID["sql"] + config = yaml_config["sql"] query = config[CONF_QUERY] issue = issue_registry.async_get_issue( DOMAIN, f"entity_id_query_does_full_table_scan_{query}" ) assert issue.translation_placeholders == {"query": query} + + +async def test_no_issue_when_view_has_the_text_entity_id_in_it( + recorder_mock: Recorder, hass: HomeAssistant, caplog: pytest.LogCaptureFixture +) -> None: + """Test we do not trigger the full table scan issue for a custom view.""" + + with patch( + "homeassistant.components.sql.sensor.scoped_session", + ): + await init_integration( + hass, YAML_CONFIG_WITH_VIEW_THAT_CONTAINS_ENTITY_ID["sql"] + ) + async_fire_time_changed( + hass, + dt.utcnow() + timedelta(minutes=1), + ) + await hass.async_block_till_done() + + assert ( + "Query contains entity_id but does not reference states_meta" not in caplog.text + ) + assert hass.states.get("sensor.get_entity_id") is not None From 30d615f206c4241b477842e17dcba2123e409e2a Mon Sep 17 00:00:00 2001 From: starkillerOG Date: Mon, 10 Apr 2023 18:44:25 +0200 Subject: [PATCH 058/110] Reolink config flow fix custom port when USE_HTTPS not selected (#91137) give USE_HTTPS a default --- homeassistant/components/reolink/config_flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/reolink/config_flow.py b/homeassistant/components/reolink/config_flow.py index a29871f28dc..df5bf968ae1 100644 --- a/homeassistant/components/reolink/config_flow.py +++ b/homeassistant/components/reolink/config_flow.py @@ -178,7 +178,7 @@ class ReolinkFlowHandler(config_entries.ConfigFlow, domain=DOMAIN): data_schema = data_schema.extend( { vol.Optional(CONF_PORT): cv.positive_int, - vol.Optional(CONF_USE_HTTPS): bool, + vol.Required(CONF_USE_HTTPS, default=False): bool, } ) From c601266f9caabd1028aad5db48ddaee1324994c6 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Mon, 10 Apr 2023 09:05:08 -0700 Subject: [PATCH 059/110] Fix all day event coercion logic (#91169) --- homeassistant/components/calendar/__init__.py | 31 ++++++------------- .../components/local_calendar/calendar.py | 2 +- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/homeassistant/components/calendar/__init__.py b/homeassistant/components/calendar/__init__.py index 594964c129c..2445c054c6d 100644 --- a/homeassistant/components/calendar/__init__.py +++ b/homeassistant/components/calendar/__init__.py @@ -140,26 +140,6 @@ def _has_min_duration( return validate -def _has_all_day_event_duration( - start_key: str, - end_key: str, -) -> Callable[[dict[str, Any]], dict[str, Any]]: - """Modify all day events to have a duration of one day.""" - - def validate(obj: dict[str, Any]) -> dict[str, Any]: - if ( - (start := obj.get(start_key)) - and (end := obj.get(end_key)) - and not isinstance(start, datetime.datetime) - and not isinstance(end, datetime.datetime) - and start == end - ): - obj[end_key] = start + datetime.timedelta(days=1) - return obj - - return validate - - def _has_same_type(*keys: Any) -> Callable[[dict[str, Any]], dict[str, Any]]: """Verify that all values are of the same type.""" @@ -267,7 +247,6 @@ CALENDAR_EVENT_SCHEMA = vol.Schema( _has_consistent_timezone("start", "end"), _as_local_timezone("start", "end"), _has_min_duration("start", "end", MIN_EVENT_DURATION), - _has_all_day_event_duration("start", "end"), ), extra=vol.ALLOW_EXTRA, ) @@ -375,6 +354,16 @@ class CalendarEvent: f"Failed to validate CalendarEvent: {err}" ) from err + # It is common to set a start an end date to be the same thing for + # an all day event, but that is not a valid duration. Fix to have a + # duration of one day. + if ( + not isinstance(self.start, datetime.datetime) + and not isinstance(self.end, datetime.datetime) + and self.start == self.end + ): + self.end = self.start + datetime.timedelta(days=1) + def _event_dict_factory(obj: Iterable[tuple[str, Any]]) -> dict[str, str]: """Convert CalendarEvent dataclass items to dictionary of attributes.""" diff --git a/homeassistant/components/local_calendar/calendar.py b/homeassistant/components/local_calendar/calendar.py index 6cfcaec61d0..423be8143b8 100644 --- a/homeassistant/components/local_calendar/calendar.py +++ b/homeassistant/components/local_calendar/calendar.py @@ -196,7 +196,7 @@ def _get_calendar_event(event: Event) -> CalendarEvent: else: start = event.start end = event.end - if (end - start) <= timedelta(days=0): + if (end - start) < timedelta(days=0): end = start + timedelta(days=1) return CalendarEvent( From 48df638f5d47f8182f9e82266b4e91a041ffd048 Mon Sep 17 00:00:00 2001 From: Aidan Timson Date: Wed, 12 Apr 2023 23:36:51 +0100 Subject: [PATCH 060/110] Reduce startup time for System Bridge integration (#91171) --- .../components/system_bridge/__init__.py | 22 +++++++++++-------- .../components/system_bridge/config_flow.py | 2 +- .../components/system_bridge/coordinator.py | 5 +++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/homeassistant/components/system_bridge/__init__.py b/homeassistant/components/system_bridge/__init__.py index a8d3a4372ca..05e607d56ed 100644 --- a/homeassistant/components/system_bridge/__init__.py +++ b/homeassistant/components/system_bridge/__init__.py @@ -53,7 +53,10 @@ SERVICE_SEND_KEYPRESS = "send_keypress" SERVICE_SEND_TEXT = "send_text" -async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: +async def async_setup_entry( + hass: HomeAssistant, + entry: ConfigEntry, +) -> bool: """Set up System Bridge from a config entry.""" # Check version before initialising @@ -64,11 +67,12 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: session=async_get_clientsession(hass), ) try: - if not await version.check_supported(): - raise ConfigEntryNotReady( - "You are not running a supported version of System Bridge. Please" - f" update to {SUPPORTED_VERSION} or higher." - ) + async with async_timeout.timeout(10): + if not await version.check_supported(): + raise ConfigEntryNotReady( + "You are not running a supported version of System Bridge. Please" + f" update to {SUPPORTED_VERSION} or higher." + ) except AuthenticationException as exception: _LOGGER.error("Authentication failed for %s: %s", entry.title, exception) raise ConfigEntryAuthFailed from exception @@ -87,7 +91,7 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: entry=entry, ) try: - async with async_timeout.timeout(30): + async with async_timeout.timeout(10): await coordinator.async_get_data(MODULES) except AuthenticationException as exception: _LOGGER.error("Authentication failed for %s: %s", entry.title, exception) @@ -105,8 +109,8 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: try: # Wait for initial data - async with async_timeout.timeout(30): - while not coordinator.is_ready(): + async with async_timeout.timeout(10): + while not coordinator.is_ready: _LOGGER.debug( "Waiting for initial data from %s (%s)", entry.title, diff --git a/homeassistant/components/system_bridge/config_flow.py b/homeassistant/components/system_bridge/config_flow.py index 46914c0ea15..a73740e5dbd 100644 --- a/homeassistant/components/system_bridge/config_flow.py +++ b/homeassistant/components/system_bridge/config_flow.py @@ -55,7 +55,7 @@ async def _validate_input( data[CONF_API_KEY], ) try: - async with async_timeout.timeout(30): + async with async_timeout.timeout(15): await websocket_client.connect(session=async_get_clientsession(hass)) hass.async_create_task(websocket_client.listen()) response = await websocket_client.get_data(GetData(modules=["system"])) diff --git a/homeassistant/components/system_bridge/coordinator.py b/homeassistant/components/system_bridge/coordinator.py index 2810bcfac72..809e2a4fd50 100644 --- a/homeassistant/components/system_bridge/coordinator.py +++ b/homeassistant/components/system_bridge/coordinator.py @@ -82,6 +82,7 @@ class SystemBridgeDataUpdateCoordinator( hass, LOGGER, name=DOMAIN, update_interval=timedelta(seconds=30) ) + @property def is_ready(self) -> bool: """Return if the data is ready.""" if self.data is None: @@ -157,7 +158,7 @@ class SystemBridgeDataUpdateCoordinator( self.last_update_success = False self.async_update_listeners() except (ConnectionClosedException, ConnectionResetError) as exception: - self.logger.info( + self.logger.debug( "Websocket connection closed for %s. Will retry: %s", self.title, exception, @@ -168,7 +169,7 @@ class SystemBridgeDataUpdateCoordinator( self.last_update_success = False self.async_update_listeners() except ConnectionErrorException as exception: - self.logger.warning( + self.logger.debug( "Connection error occurred for %s. Will retry: %s", self.title, exception, From a87c78ca203bb8b14fc800ed8397af064b83b839 Mon Sep 17 00:00:00 2001 From: "David F. Mulcahey" Date: Mon, 10 Apr 2023 14:16:11 -0400 Subject: [PATCH 061/110] Cleanup ZHA from Zigpy deprecated property removal (#91180) --- homeassistant/components/zha/core/channels/lightlink.py | 2 +- homeassistant/components/zha/core/channels/security.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/zha/core/channels/lightlink.py b/homeassistant/components/zha/core/channels/lightlink.py index 2884769d10f..cd3fc00ac28 100644 --- a/homeassistant/components/zha/core/channels/lightlink.py +++ b/homeassistant/components/zha/core/channels/lightlink.py @@ -25,7 +25,7 @@ class LightLink(ZigbeeChannel): application = self._ch_pool.endpoint.device.application try: - coordinator = application.get_device(application.ieee) + coordinator = application.get_device(application.state.node_info.ieee) except KeyError: self.warning("Aborting - unable to locate required coordinator device.") return diff --git a/homeassistant/components/zha/core/channels/security.py b/homeassistant/components/zha/core/channels/security.py index 404e4a8d258..5ecce49267c 100644 --- a/homeassistant/components/zha/core/channels/security.py +++ b/homeassistant/components/zha/core/channels/security.py @@ -363,7 +363,7 @@ class IASZoneChannel(ZigbeeChannel): self.debug("started IASZoneChannel configuration") await self.bind() - ieee = self.cluster.endpoint.device.application.ieee + ieee = self.cluster.endpoint.device.application.state.node_info.ieee try: res = await self._cluster.write_attributes({"cie_addr": ieee}) From a806e070a21081c14a0bfe2666ef9d4660000e7a Mon Sep 17 00:00:00 2001 From: Aaron Bach Date: Mon, 10 Apr 2023 18:06:26 -0600 Subject: [PATCH 062/110] Bump `pytile` to 2023.04.0 (#91191) --- homeassistant/components/tile/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/tile/manifest.json b/homeassistant/components/tile/manifest.json index 78208c7a1c8..6f311fc5593 100644 --- a/homeassistant/components/tile/manifest.json +++ b/homeassistant/components/tile/manifest.json @@ -7,5 +7,5 @@ "integration_type": "hub", "iot_class": "cloud_polling", "loggers": ["pytile"], - "requirements": ["pytile==2022.02.0"] + "requirements": ["pytile==2023.04.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index 929e843c641..8281dd44bad 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2127,7 +2127,7 @@ python_opendata_transport==0.3.0 pythonegardia==1.0.40 # homeassistant.components.tile -pytile==2022.02.0 +pytile==2023.04.0 # homeassistant.components.tomorrowio pytomorrowio==0.3.5 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index c7b0837a1a9..903bb178b13 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1520,7 +1520,7 @@ python-telegram-bot==13.1 python_awair==0.2.4 # homeassistant.components.tile -pytile==2022.02.0 +pytile==2023.04.0 # homeassistant.components.tomorrowio pytomorrowio==0.3.5 From 68920a12aaff68797b765f7915f115b195202ba4 Mon Sep 17 00:00:00 2001 From: Erik Montnemery Date: Tue, 11 Apr 2023 14:33:08 +0200 Subject: [PATCH 063/110] Flush conversation name cache when an entity is renamed (#91214) --- .../components/conversation/default_agent.py | 7 +- tests/components/conversation/test_init.py | 286 +++++++++++++++++- 2 files changed, 289 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/conversation/default_agent.py b/homeassistant/components/conversation/default_agent.py index 49569f66ac0..98959320d7a 100644 --- a/homeassistant/components/conversation/default_agent.py +++ b/homeassistant/components/conversation/default_agent.py @@ -32,6 +32,7 @@ from .const import DEFAULT_EXPOSED_ATTRIBUTES, DEFAULT_EXPOSED_DOMAINS, DOMAIN _LOGGER = logging.getLogger(__name__) _DEFAULT_ERROR_TEXT = "Sorry, I couldn't understand that" +_ENTITY_REGISTRY_UPDATE_FIELDS = ["aliases", "name", "original_name"] REGEX_TYPE = type(re.compile("")) @@ -450,8 +451,10 @@ class DefaultAgent(AbstractConversationAgent): @core.callback def _async_handle_entity_registry_changed(self, event: core.Event) -> None: - """Clear names list cache when an entity changes aliases.""" - if event.data["action"] == "update" and "aliases" not in event.data["changes"]: + """Clear names list cache when an entity registry entry has changed.""" + if event.data["action"] == "update" and not any( + field in event.data["changes"] for field in _ENTITY_REGISTRY_UPDATE_FIELDS + ): return self._slot_lists = None diff --git a/tests/components/conversation/test_init.py b/tests/components/conversation/test_init.py index 55a345bd605..9b4348fa599 100644 --- a/tests/components/conversation/test_init.py +++ b/tests/components/conversation/test_init.py @@ -147,7 +147,7 @@ async def test_http_processing_intent_target_ha_agent( } -async def test_http_processing_intent_entity_added( +async def test_http_processing_intent_entity_added_removed( hass: HomeAssistant, init_components, hass_client: ClientSessionGenerator, @@ -197,7 +197,7 @@ async def test_http_processing_intent_entity_added( "conversation_id": None, } - # Add an alias + # Add an entity entity_registry.async_get_or_create( "light", "demo", "5678", suggested_object_id="late" ) @@ -293,6 +293,288 @@ async def test_http_processing_intent_entity_added( } +async def test_http_processing_intent_alias_added_removed( + hass: HomeAssistant, + init_components, + hass_client: ClientSessionGenerator, + hass_admin_user: MockUser, + entity_registry: er.EntityRegistry, +) -> None: + """Test processing intent via HTTP API with aliases added later. + + We want to ensure that adding an alias later busts the cache + so that the new alias is available. + """ + entity_registry.async_get_or_create( + "light", "demo", "1234", suggested_object_id="kitchen" + ) + hass.states.async_set("light.kitchen", "off", {"friendly_name": "kitchen light"}) + + calls = async_mock_service(hass, LIGHT_DOMAIN, "turn_on") + client = await hass_client() + resp = await client.post( + "/api/conversation/process", json={"text": "turn on kitchen light"} + ) + + assert resp.status == HTTPStatus.OK + assert len(calls) == 1 + data = await resp.json() + + assert data == { + "response": { + "response_type": "action_done", + "card": {}, + "speech": { + "plain": { + "extra_data": None, + "speech": "Turned on light", + } + }, + "language": hass.config.language, + "data": { + "targets": [], + "success": [ + {"id": "light.kitchen", "name": "kitchen light", "type": "entity"} + ], + "failed": [], + }, + }, + "conversation_id": None, + } + + # Add an alias + entity_registry.async_update_entity("light.kitchen", aliases={"late added alias"}) + + client = await hass_client() + resp = await client.post( + "/api/conversation/process", json={"text": "turn on late added alias"} + ) + + assert resp.status == HTTPStatus.OK + data = await resp.json() + + assert data == { + "response": { + "response_type": "action_done", + "card": {}, + "speech": { + "plain": { + "extra_data": None, + "speech": "Turned on light", + } + }, + "language": hass.config.language, + "data": { + "targets": [], + "success": [ + {"id": "light.kitchen", "name": "kitchen light", "type": "entity"} + ], + "failed": [], + }, + }, + "conversation_id": None, + } + + # Now remove the alieas + entity_registry.async_update_entity("light.kitchen", aliases={}) + + client = await hass_client() + resp = await client.post( + "/api/conversation/process", json={"text": "turn on late added alias"} + ) + + assert resp.status == HTTPStatus.OK + data = await resp.json() + assert data == { + "conversation_id": None, + "response": { + "card": {}, + "data": {"code": "no_intent_match"}, + "language": hass.config.language, + "response_type": "error", + "speech": { + "plain": { + "extra_data": None, + "speech": "Sorry, I couldn't understand that", + } + }, + }, + } + + +async def test_http_processing_intent_entity_renamed( + hass: HomeAssistant, + init_components, + hass_client: ClientSessionGenerator, + hass_admin_user: MockUser, + entity_registry: er.EntityRegistry, + enable_custom_integrations: None, +) -> None: + """Test processing intent via HTTP API with entities renamed later. + + We want to ensure that renaming an entity later busts the cache + so that the new name is used. + """ + platform = getattr(hass.components, "test.light") + platform.init(empty=True) + + entity = platform.MockLight("kitchen light", "on") + entity._attr_unique_id = "1234" + entity.entity_id = "light.kitchen" + platform.ENTITIES.append(entity) + assert await async_setup_component( + hass, + LIGHT_DOMAIN, + {LIGHT_DOMAIN: [{"platform": "test"}]}, + ) + + calls = async_mock_service(hass, LIGHT_DOMAIN, "turn_on") + client = await hass_client() + resp = await client.post( + "/api/conversation/process", json={"text": "turn on kitchen light"} + ) + + assert resp.status == HTTPStatus.OK + assert len(calls) == 1 + data = await resp.json() + + assert data == { + "response": { + "response_type": "action_done", + "card": {}, + "speech": { + "plain": { + "extra_data": None, + "speech": "Turned on light", + } + }, + "language": hass.config.language, + "data": { + "targets": [], + "success": [ + {"id": "light.kitchen", "name": "kitchen light", "type": "entity"} + ], + "failed": [], + }, + }, + "conversation_id": None, + } + + # Rename the entity + entity_registry.async_update_entity("light.kitchen", name="renamed light") + await hass.async_block_till_done() + + client = await hass_client() + resp = await client.post( + "/api/conversation/process", json={"text": "turn on renamed light"} + ) + + assert resp.status == HTTPStatus.OK + data = await resp.json() + + assert data == { + "response": { + "response_type": "action_done", + "card": {}, + "speech": { + "plain": { + "extra_data": None, + "speech": "Turned on light", + } + }, + "language": hass.config.language, + "data": { + "targets": [], + "success": [ + {"id": "light.kitchen", "name": "renamed light", "type": "entity"} + ], + "failed": [], + }, + }, + "conversation_id": None, + } + + client = await hass_client() + resp = await client.post( + "/api/conversation/process", json={"text": "turn on kitchen light"} + ) + + assert resp.status == HTTPStatus.OK + data = await resp.json() + assert data == { + "conversation_id": None, + "response": { + "card": {}, + "data": {"code": "no_intent_match"}, + "language": hass.config.language, + "response_type": "error", + "speech": { + "plain": { + "extra_data": None, + "speech": "Sorry, I couldn't understand that", + } + }, + }, + } + + # Now clear the custom name + entity_registry.async_update_entity("light.kitchen", name=None) + await hass.async_block_till_done() + + client = await hass_client() + resp = await client.post( + "/api/conversation/process", json={"text": "turn on kitchen light"} + ) + + assert resp.status == HTTPStatus.OK + data = await resp.json() + + assert data == { + "response": { + "response_type": "action_done", + "card": {}, + "speech": { + "plain": { + "extra_data": None, + "speech": "Turned on light", + } + }, + "language": hass.config.language, + "data": { + "targets": [], + "success": [ + {"id": "light.kitchen", "name": "kitchen light", "type": "entity"} + ], + "failed": [], + }, + }, + "conversation_id": None, + } + + client = await hass_client() + resp = await client.post( + "/api/conversation/process", json={"text": "turn on renamed light"} + ) + + assert resp.status == HTTPStatus.OK + data = await resp.json() + assert data == { + "conversation_id": None, + "response": { + "card": {}, + "data": {"code": "no_intent_match"}, + "language": hass.config.language, + "response_type": "error", + "speech": { + "plain": { + "extra_data": None, + "speech": "Sorry, I couldn't understand that", + } + }, + }, + } + + @pytest.mark.parametrize("agent_id", AGENT_ID_OPTIONS) @pytest.mark.parametrize("sentence", ("turn on kitchen", "turn kitchen on")) async def test_turn_on_intent( From 8eb75beb96dc8d7a4d36f7a0c100d24156e609f0 Mon Sep 17 00:00:00 2001 From: Bram Kragten Date: Tue, 11 Apr 2023 14:35:08 +0200 Subject: [PATCH 064/110] Update frontend to 20230411.0 (#91219) --- homeassistant/components/frontend/manifest.json | 2 +- homeassistant/package_constraints.txt | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/frontend/manifest.json b/homeassistant/components/frontend/manifest.json index b1fd062032f..a3af9f863ea 100644 --- a/homeassistant/components/frontend/manifest.json +++ b/homeassistant/components/frontend/manifest.json @@ -20,5 +20,5 @@ "documentation": "https://www.home-assistant.io/integrations/frontend", "integration_type": "system", "quality_scale": "internal", - "requirements": ["home-assistant-frontend==20230406.1"] + "requirements": ["home-assistant-frontend==20230411.0"] } diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 1568d9720ff..24732608eee 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -25,7 +25,7 @@ ha-av==10.0.0 hass-nabucasa==0.63.1 hassil==1.0.6 home-assistant-bluetooth==1.9.3 -home-assistant-frontend==20230406.1 +home-assistant-frontend==20230411.0 home-assistant-intents==2023.3.29 httpx==0.23.3 ifaddr==0.1.7 diff --git a/requirements_all.txt b/requirements_all.txt index 8281dd44bad..4fcf95c32e5 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -907,7 +907,7 @@ hole==0.8.0 holidays==0.21.13 # homeassistant.components.frontend -home-assistant-frontend==20230406.1 +home-assistant-frontend==20230411.0 # homeassistant.components.conversation home-assistant-intents==2023.3.29 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 903bb178b13..5cefc8567b5 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -693,7 +693,7 @@ hole==0.8.0 holidays==0.21.13 # homeassistant.components.frontend -home-assistant-frontend==20230406.1 +home-assistant-frontend==20230411.0 # homeassistant.components.conversation home-assistant-intents==2023.3.29 From d6574b4a2eb66eabd2d30ee228ed054c69d5fc5f Mon Sep 17 00:00:00 2001 From: Erik Montnemery Date: Tue, 11 Apr 2023 19:54:30 +0200 Subject: [PATCH 065/110] Fix switch_as_x name (#91232) --- homeassistant/components/switch_as_x/cover.py | 7 +- .../components/switch_as_x/entity.py | 21 ++- homeassistant/components/switch_as_x/fan.py | 3 +- homeassistant/components/switch_as_x/light.py | 7 +- homeassistant/components/switch_as_x/lock.py | 3 +- homeassistant/components/switch_as_x/siren.py | 7 +- tests/components/switch_as_x/test_init.py | 136 ++++++++++++++++++ 7 files changed, 176 insertions(+), 8 deletions(-) diff --git a/homeassistant/components/switch_as_x/cover.py b/homeassistant/components/switch_as_x/cover.py index b7f8e5bf971..7df3b177217 100644 --- a/homeassistant/components/switch_as_x/cover.py +++ b/homeassistant/components/switch_as_x/cover.py @@ -3,7 +3,11 @@ from __future__ import annotations from typing import Any -from homeassistant.components.cover import CoverEntity, CoverEntityFeature +from homeassistant.components.cover import ( + DOMAIN as COVER_DOMAIN, + CoverEntity, + CoverEntityFeature, +) from homeassistant.components.switch import DOMAIN as SWITCH_DOMAIN from homeassistant.config_entries import ConfigEntry from homeassistant.const import ( @@ -36,6 +40,7 @@ async def async_setup_entry( CoverSwitch( hass, config_entry.title, + COVER_DOMAIN, entity_id, config_entry.entry_id, ) diff --git a/homeassistant/components/switch_as_x/entity.py b/homeassistant/components/switch_as_x/entity.py index 8432c46f856..21a7b882442 100644 --- a/homeassistant/components/switch_as_x/entity.py +++ b/homeassistant/components/switch_as_x/entity.py @@ -23,13 +23,15 @@ class BaseEntity(Entity): """Represents a Switch as an X.""" _attr_should_poll = False + _is_new_entity: bool def __init__( self, hass: HomeAssistant, config_entry_title: str, + domain: str, switch_entity_id: str, - unique_id: str | None, + unique_id: str, ) -> None: """Initialize Switch as an X.""" registry = er.async_get(hass) @@ -41,7 +43,7 @@ class BaseEntity(Entity): name: str | None = config_entry_title if wrapped_switch: - name = wrapped_switch.name or wrapped_switch.original_name + name = wrapped_switch.original_name self._device_id = device_id if device_id and (device := device_registry.async_get(device_id)): @@ -55,6 +57,10 @@ class BaseEntity(Entity): self._attr_unique_id = unique_id self._switch_entity_id = switch_entity_id + self._is_new_entity = ( + registry.async_get_entity_id(domain, SWITCH_AS_X_DOMAIN, unique_id) is None + ) + @callback def async_state_changed_listener(self, event: Event | None = None) -> None: """Handle child updates.""" @@ -67,7 +73,7 @@ class BaseEntity(Entity): self._attr_available = True async def async_added_to_hass(self) -> None: - """Register callbacks.""" + """Register callbacks and copy the wrapped entity's custom name if set.""" @callback def _async_state_changed_listener(event: Event | None = None) -> None: @@ -93,6 +99,15 @@ class BaseEntity(Entity): {"entity_id": self._switch_entity_id}, ) + if not self._is_new_entity: + return + + wrapped_switch = registry.async_get(self._switch_entity_id) + if not wrapped_switch or wrapped_switch.name is None: + return + + registry.async_update_entity(self.entity_id, name=wrapped_switch.name) + class BaseToggleEntity(BaseEntity, ToggleEntity): """Represents a Switch as a ToggleEntity.""" diff --git a/homeassistant/components/switch_as_x/fan.py b/homeassistant/components/switch_as_x/fan.py index 87a6c387295..d8c43cfe381 100644 --- a/homeassistant/components/switch_as_x/fan.py +++ b/homeassistant/components/switch_as_x/fan.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import Any -from homeassistant.components.fan import FanEntity +from homeassistant.components.fan import DOMAIN as FAN_DOMAIN, FanEntity from homeassistant.config_entries import ConfigEntry from homeassistant.const import CONF_ENTITY_ID from homeassistant.core import HomeAssistant @@ -29,6 +29,7 @@ async def async_setup_entry( FanSwitch( hass, config_entry.title, + FAN_DOMAIN, entity_id, config_entry.entry_id, ) diff --git a/homeassistant/components/switch_as_x/light.py b/homeassistant/components/switch_as_x/light.py index 7bcdb659e9c..e6183c95d91 100644 --- a/homeassistant/components/switch_as_x/light.py +++ b/homeassistant/components/switch_as_x/light.py @@ -1,7 +1,11 @@ """Light support for switch entities.""" from __future__ import annotations -from homeassistant.components.light import ColorMode, LightEntity +from homeassistant.components.light import ( + DOMAIN as LIGHT_DOMAIN, + ColorMode, + LightEntity, +) from homeassistant.config_entries import ConfigEntry from homeassistant.const import CONF_ENTITY_ID from homeassistant.core import HomeAssistant @@ -27,6 +31,7 @@ async def async_setup_entry( LightSwitch( hass, config_entry.title, + LIGHT_DOMAIN, entity_id, config_entry.entry_id, ) diff --git a/homeassistant/components/switch_as_x/lock.py b/homeassistant/components/switch_as_x/lock.py index e3c29a1cf42..9778caf8e60 100644 --- a/homeassistant/components/switch_as_x/lock.py +++ b/homeassistant/components/switch_as_x/lock.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import Any -from homeassistant.components.lock import LockEntity +from homeassistant.components.lock import DOMAIN as LOCK_DOMAIN, LockEntity from homeassistant.components.switch import DOMAIN as SWITCH_DOMAIN from homeassistant.config_entries import ConfigEntry from homeassistant.const import ( @@ -36,6 +36,7 @@ async def async_setup_entry( LockSwitch( hass, config_entry.title, + LOCK_DOMAIN, entity_id, config_entry.entry_id, ) diff --git a/homeassistant/components/switch_as_x/siren.py b/homeassistant/components/switch_as_x/siren.py index 88ff9a322d3..c9981b17cfe 100644 --- a/homeassistant/components/switch_as_x/siren.py +++ b/homeassistant/components/switch_as_x/siren.py @@ -1,7 +1,11 @@ """Siren support for switch entities.""" from __future__ import annotations -from homeassistant.components.siren import SirenEntity, SirenEntityFeature +from homeassistant.components.siren import ( + DOMAIN as SIREN_DOMAIN, + SirenEntity, + SirenEntityFeature, +) from homeassistant.config_entries import ConfigEntry from homeassistant.const import CONF_ENTITY_ID from homeassistant.core import HomeAssistant @@ -27,6 +31,7 @@ async def async_setup_entry( SirenSwitch( hass, config_entry.title, + SIREN_DOMAIN, entity_id, config_entry.entry_id, ) diff --git a/tests/components/switch_as_x/test_init.py b/tests/components/switch_as_x/test_init.py index 2d63ce9617b..87cc291a599 100644 --- a/tests/components/switch_as_x/test_init.py +++ b/tests/components/switch_as_x/test_init.py @@ -534,7 +534,143 @@ async def test_entity_name( assert entity_entry assert entity_entry.device_id == switch_entity_entry.device_id assert entity_entry.has_entity_name is True + assert entity_entry.name is None assert entity_entry.original_name is None assert entity_entry.options == { DOMAIN: {"entity_id": switch_entity_entry.entity_id} } + + +@pytest.mark.parametrize("target_domain", PLATFORMS_TO_TEST) +async def test_custom_name_1( + hass: HomeAssistant, + target_domain: Platform, +) -> None: + """Test the source entity has a custom name.""" + registry = er.async_get(hass) + device_registry = dr.async_get(hass) + + switch_config_entry = MockConfigEntry() + + device_entry = device_registry.async_get_or_create( + config_entry_id=switch_config_entry.entry_id, + connections={(dr.CONNECTION_NETWORK_MAC, "12:34:56:AB:CD:EF")}, + name="Device name", + ) + + switch_entity_entry = registry.async_get_or_create( + "switch", + "test", + "unique", + device_id=device_entry.id, + has_entity_name=True, + original_name="Original entity name", + ) + switch_entity_entry = registry.async_update_entity( + switch_entity_entry.entity_id, + config_entry_id=switch_config_entry.entry_id, + name="Custom entity name", + ) + + # Add the config entry + switch_as_x_config_entry = MockConfigEntry( + data={}, + domain=DOMAIN, + options={ + CONF_ENTITY_ID: switch_entity_entry.id, + CONF_TARGET_DOMAIN: target_domain, + }, + title="ABC", + ) + switch_as_x_config_entry.add_to_hass(hass) + + assert await hass.config_entries.async_setup(switch_as_x_config_entry.entry_id) + await hass.async_block_till_done() + + entity_entry = registry.async_get( + f"{target_domain}.device_name_original_entity_name" + ) + assert entity_entry + assert entity_entry.device_id == switch_entity_entry.device_id + assert entity_entry.has_entity_name is True + assert entity_entry.name == "Custom entity name" + assert entity_entry.original_name == "Original entity name" + assert entity_entry.options == { + DOMAIN: {"entity_id": switch_entity_entry.entity_id} + } + + +@pytest.mark.parametrize("target_domain", PLATFORMS_TO_TEST) +async def test_custom_name_2( + hass: HomeAssistant, + target_domain: Platform, +) -> None: + """Test the source entity has a custom name. + + This tests the custom name is only copied from the source device when the config + switch_as_x config entry is setup the first time. + """ + registry = er.async_get(hass) + device_registry = dr.async_get(hass) + + switch_config_entry = MockConfigEntry() + + device_entry = device_registry.async_get_or_create( + config_entry_id=switch_config_entry.entry_id, + connections={(dr.CONNECTION_NETWORK_MAC, "12:34:56:AB:CD:EF")}, + name="Device name", + ) + + switch_entity_entry = registry.async_get_or_create( + "switch", + "test", + "unique", + device_id=device_entry.id, + has_entity_name=True, + original_name="Original entity name", + ) + switch_entity_entry = registry.async_update_entity( + switch_entity_entry.entity_id, + config_entry_id=switch_config_entry.entry_id, + name="New custom entity name", + ) + + # Add the config entry + switch_as_x_config_entry = MockConfigEntry( + data={}, + domain=DOMAIN, + options={ + CONF_ENTITY_ID: switch_entity_entry.id, + CONF_TARGET_DOMAIN: target_domain, + }, + title="ABC", + ) + switch_as_x_config_entry.add_to_hass(hass) + + switch_as_x_entity_entry = registry.async_get_or_create( + target_domain, + "switch_as_x", + switch_as_x_config_entry.entry_id, + suggested_object_id="device_name_original_entity_name", + ) + switch_as_x_entity_entry = registry.async_update_entity( + switch_as_x_entity_entry.entity_id, + config_entry_id=switch_config_entry.entry_id, + name="Old custom entity name", + ) + + assert await hass.config_entries.async_setup(switch_as_x_config_entry.entry_id) + await hass.async_block_till_done() + + entity_entry = registry.async_get( + f"{target_domain}.device_name_original_entity_name" + ) + assert entity_entry + assert entity_entry.entity_id == switch_as_x_entity_entry.entity_id + assert entity_entry.device_id == switch_entity_entry.device_id + assert entity_entry.has_entity_name is True + assert entity_entry.name == "Old custom entity name" + assert entity_entry.original_name == "Original entity name" + assert entity_entry.options == { + DOMAIN: {"entity_id": switch_entity_entry.entity_id} + } From fd53eda5c684eb3f872d09a4167cac95ac73058d Mon Sep 17 00:00:00 2001 From: codyhackw <49957005+codyhackw@users.noreply.github.com> Date: Wed, 12 Apr 2023 16:09:16 -0400 Subject: [PATCH 066/110] Update Inovelli Blue Series switch support in ZHA (#91254) Co-authored-by: David F. Mulcahey --- .../zha/core/channels/manufacturerspecific.py | 10 ++++- homeassistant/components/zha/number.py | 28 ++++++++++++ homeassistant/components/zha/select.py | 41 ++++++++++++++++- homeassistant/components/zha/switch.py | 44 ++++++++++++++----- 4 files changed, 110 insertions(+), 13 deletions(-) diff --git a/homeassistant/components/zha/core/channels/manufacturerspecific.py b/homeassistant/components/zha/core/channels/manufacturerspecific.py index b880a338a42..20848453e2a 100644 --- a/homeassistant/components/zha/core/channels/manufacturerspecific.py +++ b/homeassistant/components/zha/core/channels/manufacturerspecific.py @@ -251,14 +251,20 @@ class InovelliConfigEntityChannel(ZigbeeChannel): "active_energy_reports": True, "power_type": False, "switch_type": False, + "increased_non_neutral_output": True, "button_delay": False, "smart_bulb_mode": False, - "double_tap_up_for_max_brightness": True, - "double_tap_down_for_min_brightness": True, + "double_tap_up_enabled": True, + "double_tap_down_enabled": True, + "double_tap_up_level": True, + "double_tap_down_level": True, "led_color_when_on": True, "led_color_when_off": True, "led_intensity_when_on": True, "led_intensity_when_off": True, + "led_scaling_mode": True, + "aux_switch_scenes": True, + "binding_off_to_on_sync_level": True, "local_protection": False, "output_mode": False, "on_off_led_mode": True, diff --git a/homeassistant/components/zha/number.py b/homeassistant/components/zha/number.py index d0ec62eaf61..d35f9c3afad 100644 --- a/homeassistant/components/zha/number.py +++ b/homeassistant/components/zha/number.py @@ -835,6 +835,34 @@ class InovelliDefaultAllLEDOffIntensity( _attr_name: str = "Default all LED off intensity" +@CONFIG_DIAGNOSTIC_MATCH(channel_names=CHANNEL_INOVELLI) +class InovelliDoubleTapUpLevel( + ZHANumberConfigurationEntity, id_suffix="double_tap_up_level" +): + """Inovelli double tap up level configuration entity.""" + + _attr_entity_category = EntityCategory.CONFIG + _attr_icon: str = ICONS[16] + _attr_native_min_value: float = 2 + _attr_native_max_value: float = 254 + _zcl_attribute: str = "double_tap_up_level" + _attr_name: str = "Double tap up level" + + +@CONFIG_DIAGNOSTIC_MATCH(channel_names=CHANNEL_INOVELLI) +class InovelliDoubleTapDownLevel( + ZHANumberConfigurationEntity, id_suffix="double_tap_down_level" +): + """Inovelli double tap down level configuration entity.""" + + _attr_entity_category = EntityCategory.CONFIG + _attr_icon: str = ICONS[16] + _attr_native_min_value: float = 0 + _attr_native_max_value: float = 254 + _zcl_attribute: str = "double_tap_down_level" + _attr_name: str = "Double tap down level" + + @CONFIG_DIAGNOSTIC_MATCH(channel_names="opple_cluster", models={"aqara.feeder.acn001"}) class AqaraPetFeederServingSize(ZHANumberConfigurationEntity, id_suffix="serving_size"): """Aqara pet feeder serving size configuration entity.""" diff --git a/homeassistant/components/zha/select.py b/homeassistant/components/zha/select.py index 605c7d507c6..b352176411a 100644 --- a/homeassistant/components/zha/select.py +++ b/homeassistant/components/zha/select.py @@ -472,9 +472,10 @@ class InovelliOutputModeEntity(ZCLEnumSelectEntity, id_suffix="output_mode"): class InovelliSwitchType(types.enum8): """Inovelli output mode.""" - Load_Only = 0x00 + Single_Pole = 0x00 Three_Way_Dumb = 0x01 Three_Way_AUX = 0x02 + Single_Pole_Full_Sine = 0x03 @CONFIG_DIAGNOSTIC_MATCH( @@ -488,6 +489,44 @@ class InovelliSwitchTypeEntity(ZCLEnumSelectEntity, id_suffix="switch_type"): _attr_name: str = "Switch type" +class InovelliLedScalingMode(types.enum1): + """Inovelli led mode.""" + + VZM31SN = 0x00 + LZW31SN = 0x01 + + +@CONFIG_DIAGNOSTIC_MATCH( + channel_names=CHANNEL_INOVELLI, +) +class InovelliLedScalingModeEntity(ZCLEnumSelectEntity, id_suffix="led_scaling_mode"): + """Inovelli led mode control.""" + + _select_attr = "led_scaling_mode" + _enum = InovelliLedScalingMode + _attr_name: str = "Led scaling mode" + + +class InovelliNonNeutralOutput(types.enum1): + """Inovelli non neutral output selection.""" + + Low = 0x00 + High = 0x01 + + +@CONFIG_DIAGNOSTIC_MATCH( + channel_names=CHANNEL_INOVELLI, +) +class InovelliNonNeutralOutputEntity( + ZCLEnumSelectEntity, id_suffix="increased_non_neutral_output" +): + """Inovelli non neutral output control.""" + + _select_attr = "increased_non_neutral_output" + _enum = InovelliNonNeutralOutput + _attr_name: str = "Non neutral output" + + class AqaraFeedingMode(types.enum8): """Feeding mode.""" diff --git a/homeassistant/components/zha/switch.py b/homeassistant/components/zha/switch.py index f0e36750798..c57075a15ca 100644 --- a/homeassistant/components/zha/switch.py +++ b/homeassistant/components/zha/switch.py @@ -367,25 +367,49 @@ class InovelliSmartBulbMode(ZHASwitchConfigurationEntity, id_suffix="smart_bulb_ @CONFIG_DIAGNOSTIC_MATCH( channel_names=CHANNEL_INOVELLI, ) -class InovelliDoubleTapForFullBrightness( - ZHASwitchConfigurationEntity, id_suffix="double_tap_up_for_max_brightness" +class InovelliDoubleTapUpEnabled( + ZHASwitchConfigurationEntity, id_suffix="double_tap_up_enabled" ): - """Inovelli double tap for full brightness control.""" + """Inovelli double tap up enabled.""" - _zcl_attribute: str = "double_tap_up_for_max_brightness" - _attr_name: str = "Double tap full brightness" + _zcl_attribute: str = "double_tap_up_enabled" + _attr_name: str = "Double tap up enabled" @CONFIG_DIAGNOSTIC_MATCH( channel_names=CHANNEL_INOVELLI, ) -class InovelliDoubleTapForMinBrightness( - ZHASwitchConfigurationEntity, id_suffix="double_tap_down_for_min_brightness" +class InovelliDoubleTapDownEnabled( + ZHASwitchConfigurationEntity, id_suffix="double_tap_down_enabled" ): - """Inovelli double tap down for minimum brightness control.""" + """Inovelli double tap down enabled.""" - _zcl_attribute: str = "double_tap_down_for_min_brightness" - _attr_name: str = "Double tap minimum brightness" + _zcl_attribute: str = "double_tap_down_enabled" + _attr_name: str = "Double tap down enabled" + + +@CONFIG_DIAGNOSTIC_MATCH( + channel_names=CHANNEL_INOVELLI, +) +class InovelliAuxSwitchScenes( + ZHASwitchConfigurationEntity, id_suffix="aux_switch_scenes" +): + """Inovelli unique aux switch scenes.""" + + _zcl_attribute: str = "aux_switch_scenes" + _attr_name: str = "Aux switch scenes" + + +@CONFIG_DIAGNOSTIC_MATCH( + channel_names=CHANNEL_INOVELLI, +) +class InovelliBindingOffToOnSyncLevel( + ZHASwitchConfigurationEntity, id_suffix="binding_off_to_on_sync_level" +): + """Inovelli send move to level with on/off to bound devices.""" + + _zcl_attribute: str = "binding_off_to_on_sync_level" + _attr_name: str = "Binding off to on sync level" @CONFIG_DIAGNOSTIC_MATCH( From 64a9bfcc22ebe268ea4d8a247253c7474862a76c Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 12 Apr 2023 16:09:03 -0400 Subject: [PATCH 067/110] Bump ZHA dependencies (#91291) --- homeassistant/components/zha/manifest.json | 6 +++--- requirements_all.txt | 6 +++--- requirements_test_all.txt | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/homeassistant/components/zha/manifest.json b/homeassistant/components/zha/manifest.json index 7bc482681ca..10897c17b68 100644 --- a/homeassistant/components/zha/manifest.json +++ b/homeassistant/components/zha/manifest.json @@ -20,12 +20,12 @@ "zigpy_znp" ], "requirements": [ - "bellows==0.35.0", + "bellows==0.35.1", "pyserial==3.5", "pyserial-asyncio==0.6", - "zha-quirks==0.0.96", + "zha-quirks==0.0.97", "zigpy-deconz==0.20.0", - "zigpy==0.54.0", + "zigpy==0.54.1", "zigpy-xbee==0.17.0", "zigpy-zigate==0.10.3", "zigpy-znp==0.10.0" diff --git a/requirements_all.txt b/requirements_all.txt index 4fcf95c32e5..a91461b180b 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -422,7 +422,7 @@ beautifulsoup4==4.11.1 # beewi_smartclim==0.0.10 # homeassistant.components.zha -bellows==0.35.0 +bellows==0.35.1 # homeassistant.components.bmw_connected_drive bimmer_connected==0.13.0 @@ -2698,7 +2698,7 @@ zeroconf==0.56.0 zeversolar==0.3.1 # homeassistant.components.zha -zha-quirks==0.0.96 +zha-quirks==0.0.97 # homeassistant.components.zhong_hong zhong_hong_hvac==1.0.9 @@ -2719,7 +2719,7 @@ zigpy-zigate==0.10.3 zigpy-znp==0.10.0 # homeassistant.components.zha -zigpy==0.54.0 +zigpy==0.54.1 # homeassistant.components.zoneminder zm-py==0.5.2 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 5cefc8567b5..740086d187f 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -355,7 +355,7 @@ base36==0.1.1 beautifulsoup4==4.11.1 # homeassistant.components.zha -bellows==0.35.0 +bellows==0.35.1 # homeassistant.components.bmw_connected_drive bimmer_connected==0.13.0 @@ -1932,7 +1932,7 @@ zeroconf==0.56.0 zeversolar==0.3.1 # homeassistant.components.zha -zha-quirks==0.0.96 +zha-quirks==0.0.97 # homeassistant.components.zha zigpy-deconz==0.20.0 @@ -1947,7 +1947,7 @@ zigpy-zigate==0.10.3 zigpy-znp==0.10.0 # homeassistant.components.zha -zigpy==0.54.0 +zigpy==0.54.1 # homeassistant.components.zwave_js zwave-js-server-python==0.47.3 From a420007e80adf25ab8a0a544e21f87f1681cdb19 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Apr 2023 14:24:55 -1000 Subject: [PATCH 068/110] Restore use of local timezone for MariaDB/MySQL in SQL integration (#91313) * Use local timezone for recorder connection The fix in #90335 had an unexpected side effect of using UTC for the timezone since all recorder operations use UTC. Since only sqlite much use the database executor we can use a seperate connection pool which uses local time This also ensures that the engines are disposed of when Home Assistant is shutdown as previously we did not cleanly disconnect * coverage * fix unclean shutdown in config flow * tweaks --- homeassistant/components/recorder/__init__.py | 1 + homeassistant/components/sql/config_flow.py | 1 + homeassistant/components/sql/models.py | 16 +++++ homeassistant/components/sql/sensor.py | 64 ++++++++++++++++--- tests/components/sql/test_sensor.py | 50 +++++++++++++++ 5 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 homeassistant/components/sql/models.py diff --git a/homeassistant/components/recorder/__init__.py b/homeassistant/components/recorder/__init__.py index 750f504d096..7b43abd8dde 100644 --- a/homeassistant/components/recorder/__init__.py +++ b/homeassistant/components/recorder/__init__.py @@ -32,6 +32,7 @@ from .const import ( # noqa: F401 INTEGRATION_PLATFORM_EXCLUDE_ATTRIBUTES, INTEGRATION_PLATFORMS_LOAD_IN_RECORDER_THREAD, SQLITE_URL_PREFIX, + SupportedDialect, ) from .core import Recorder from .services import async_register_services diff --git a/homeassistant/components/sql/config_flow.py b/homeassistant/components/sql/config_flow.py index d52f2d10d0d..1c1ed6adae4 100644 --- a/homeassistant/components/sql/config_flow.py +++ b/homeassistant/components/sql/config_flow.py @@ -64,6 +64,7 @@ def validate_query(db_url: str, query: str, column: str) -> bool: if sess: sess.close() + engine.dispose() return True diff --git a/homeassistant/components/sql/models.py b/homeassistant/components/sql/models.py new file mode 100644 index 00000000000..feac9ebf20c --- /dev/null +++ b/homeassistant/components/sql/models.py @@ -0,0 +1,16 @@ +"""The sql integration models.""" +from __future__ import annotations + +from dataclasses import dataclass + +from sqlalchemy.orm import scoped_session + +from homeassistant.core import CALLBACK_TYPE + + +@dataclass(slots=True) +class SQLData: + """Data for the sql integration.""" + + shutdown_event_cancel: CALLBACK_TYPE + session_makers_by_db_url: dict[str, scoped_session] diff --git a/homeassistant/components/sql/sensor.py b/homeassistant/components/sql/sensor.py index b6cce467e1f..eb0e9c9c46b 100644 --- a/homeassistant/components/sql/sensor.py +++ b/homeassistant/components/sql/sensor.py @@ -13,7 +13,11 @@ from sqlalchemy.orm import Session, scoped_session, sessionmaker from sqlalchemy.sql.lambdas import StatementLambdaElement from sqlalchemy.util import LRUCache -from homeassistant.components.recorder import CONF_DB_URL, get_instance +from homeassistant.components.recorder import ( + CONF_DB_URL, + SupportedDialect, + get_instance, +) from homeassistant.components.sensor import ( CONF_STATE_CLASS, SensorDeviceClass, @@ -27,8 +31,9 @@ from homeassistant.const import ( CONF_UNIQUE_ID, CONF_UNIT_OF_MEASUREMENT, CONF_VALUE_TEMPLATE, + EVENT_HOMEASSISTANT_STOP, ) -from homeassistant.core import HomeAssistant +from homeassistant.core import Event, HomeAssistant, callback from homeassistant.exceptions import TemplateError from homeassistant.helpers import issue_registry as ir from homeassistant.helpers.device_registry import DeviceEntryType @@ -38,6 +43,7 @@ from homeassistant.helpers.template import Template from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType from .const import CONF_COLUMN_NAME, CONF_QUERY, DB_URL_RE, DOMAIN +from .models import SQLData from .util import resolve_db_url _LOGGER = logging.getLogger(__name__) @@ -127,6 +133,36 @@ async def async_setup_entry( ) +@callback +def _async_get_or_init_domain_data(hass: HomeAssistant) -> SQLData: + """Get or initialize domain data.""" + if DOMAIN in hass.data: + sql_data: SQLData = hass.data[DOMAIN] + return sql_data + + session_makers_by_db_url: dict[str, scoped_session] = {} + + # + # Ensure we dispose of all engines at shutdown + # to avoid unclean disconnects + # + # Shutdown all sessions in the executor since they will + # do blocking I/O + # + def _shutdown_db_engines(event: Event) -> None: + """Shutdown all database engines.""" + for sessmaker in session_makers_by_db_url.values(): + sessmaker.connection().engine.dispose() + + cancel_shutdown = hass.bus.async_listen_once( + EVENT_HOMEASSISTANT_STOP, _shutdown_db_engines + ) + + sql_data = SQLData(cancel_shutdown, session_makers_by_db_url) + hass.data[DOMAIN] = sql_data + return sql_data + + async def async_setup_sensor( hass: HomeAssistant, name: str, @@ -144,18 +180,30 @@ async def async_setup_sensor( """Set up the SQL sensor.""" instance = get_instance(hass) sessmaker: scoped_session | None - if use_database_executor := (db_url == instance.db_url): + sql_data = _async_get_or_init_domain_data(hass) + uses_recorder_db = db_url == instance.db_url + use_database_executor = False + if uses_recorder_db and instance.dialect_name == SupportedDialect.SQLITE: + use_database_executor = True assert instance.engine is not None sessmaker = scoped_session(sessionmaker(bind=instance.engine, future=True)) - elif not ( - sessmaker := await hass.async_add_executor_job( - _validate_and_get_session_maker_for_db_url, db_url - ) + # For other databases we need to create a new engine since + # we want the connection to use the default timezone and these + # database engines will use QueuePool as its only sqlite that + # needs our custom pool. If there is already a session maker + # for this db_url we can use that so we do not create a new engine + # for every sensor. + elif db_url in sql_data.session_makers_by_db_url: + sessmaker = sql_data.session_makers_by_db_url[db_url] + elif sessmaker := await hass.async_add_executor_job( + _validate_and_get_session_maker_for_db_url, db_url ): + sql_data.session_makers_by_db_url[db_url] = sessmaker + else: return upper_query = query_str.upper() - if use_database_executor: + if uses_recorder_db: redacted_query = redact_credentials(query_str) issue_key = unique_id if unique_id else redacted_query diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index 7e289565b37..cd123556daf 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -407,3 +407,53 @@ async def test_no_issue_when_view_has_the_text_entity_id_in_it( "Query contains entity_id but does not reference states_meta" not in caplog.text ) assert hass.states.get("sensor.get_entity_id") is not None + + +async def test_multiple_sensors_using_same_db( + recorder_mock: Recorder, hass: HomeAssistant +) -> None: + """Test multiple sensors using the same db.""" + config = { + "db_url": "sqlite:///", + "query": "SELECT 5 as value", + "column": "value", + "name": "Select value SQL query", + } + config2 = { + "db_url": "sqlite:///", + "query": "SELECT 5 as value", + "column": "value", + "name": "Select value SQL query 2", + } + await init_integration(hass, config) + await init_integration(hass, config2, entry_id="2") + + state = hass.states.get("sensor.select_value_sql_query") + assert state.state == "5" + assert state.attributes["value"] == 5 + + state = hass.states.get("sensor.select_value_sql_query_2") + assert state.state == "5" + assert state.attributes["value"] == 5 + + +async def test_engine_is_disposed_at_stop( + recorder_mock: Recorder, hass: HomeAssistant +) -> None: + """Test we dispose of the engine at stop.""" + config = { + "db_url": "sqlite:///", + "query": "SELECT 5 as value", + "column": "value", + "name": "Select value SQL query", + } + await init_integration(hass, config) + + state = hass.states.get("sensor.select_value_sql_query") + assert state.state == "5" + assert state.attributes["value"] == 5 + + with patch("sqlalchemy.engine.base.Engine.dispose") as mock_engine_dispose: + await hass.async_stop() + + assert mock_engine_dispose.call_count == 2 From e9f1148c0ac53f98c7938de5ee85e57b2262c9f7 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Wed, 12 Apr 2023 20:35:59 -0400 Subject: [PATCH 069/110] Bumped version to 2023.4.3 --- homeassistant/const.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/const.py b/homeassistant/const.py index 097aa5f694b..1944bf7e998 100644 --- a/homeassistant/const.py +++ b/homeassistant/const.py @@ -8,7 +8,7 @@ from .backports.enum import StrEnum APPLICATION_NAME: Final = "HomeAssistant" MAJOR_VERSION: Final = 2023 MINOR_VERSION: Final = 4 -PATCH_VERSION: Final = "2" +PATCH_VERSION: Final = "3" __short_version__: Final = f"{MAJOR_VERSION}.{MINOR_VERSION}" __version__: Final = f"{__short_version__}.{PATCH_VERSION}" REQUIRED_PYTHON_VER: Final[tuple[int, int, int]] = (3, 10, 0) diff --git a/pyproject.toml b/pyproject.toml index f161f3837af..7226e594c83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "homeassistant" -version = "2023.4.2" +version = "2023.4.3" license = {text = "Apache-2.0"} description = "Open-source home automation platform running on Python 3." readme = "README.rst" From c073cee04953f058778f4539dc4e451ad9da7829 Mon Sep 17 00:00:00 2001 From: tronikos Date: Wed, 12 Apr 2023 17:42:51 -0700 Subject: [PATCH 070/110] Google Assistant SDK: Fix broadcast command for Portuguese (#91293) Fix broadcast command for pt --- homeassistant/components/google_assistant_sdk/notify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/google_assistant_sdk/notify.py b/homeassistant/components/google_assistant_sdk/notify.py index 0c0338dd973..fa117b579a9 100644 --- a/homeassistant/components/google_assistant_sdk/notify.py +++ b/homeassistant/components/google_assistant_sdk/notify.py @@ -20,7 +20,7 @@ LANG_TO_BROADCAST_COMMAND = { "it": ("Trasmetti {0}", "Trasmetti in {1} {0}"), "ja": ("{0}とブロードキャストして", "{0}と{1}にブロードキャストして"), "ko": ("{0} 라고 방송해 줘", "{0} 라고 {1}에 방송해 줘"), - "pt": ("Transmite {0}", "Transmite para {1} {0}"), + "pt": ("Transmitir {0}", "Transmitir {0} para {1}"), } From bbf2d0e6ad01accc9e6c3a787083a5ed94d618a7 Mon Sep 17 00:00:00 2001 From: Franck Nijhof Date: Wed, 12 Apr 2023 21:32:32 +0200 Subject: [PATCH 071/110] Remove codecov from Python test requirements (#91295) --- requirements_test.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index caf29fc558a..e593f460454 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -8,7 +8,6 @@ -c homeassistant/package_constraints.txt -r requirements_test_pre_commit.txt astroid==2.15.0 -codecov==2.1.12 coverage==7.2.1 freezegun==1.2.2 mock-open==1.4.0 From 7a8159052eb89e7277cf7757b8e94f1549e47a29 Mon Sep 17 00:00:00 2001 From: Duco Sebel <74970928+DCSBL@users.noreply.github.com> Date: Thu, 13 Apr 2023 13:42:35 +0200 Subject: [PATCH 072/110] Bump python-homewizard-energy to 2.0.1 (#91097) --- homeassistant/components/homewizard/button.py | 3 +- homeassistant/components/homewizard/const.py | 4 +-- .../components/homewizard/coordinator.py | 32 +++++++++++++++++-- .../components/homewizard/manifest.json | 2 +- homeassistant/components/homewizard/number.py | 2 +- homeassistant/components/homewizard/switch.py | 10 +++--- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- tests/components/homewizard/conftest.py | 2 -- .../homewizard/fixtures/device.json | 2 +- tests/components/homewizard/generator.py | 4 --- tests/components/homewizard/test_button.py | 2 +- .../components/homewizard/test_diagnostics.py | 2 +- tests/components/homewizard/test_number.py | 10 +++--- tests/components/homewizard/test_switch.py | 8 ++--- 15 files changed, 53 insertions(+), 34 deletions(-) diff --git a/homeassistant/components/homewizard/button.py b/homeassistant/components/homewizard/button.py index 6a245a00420..665406499e1 100644 --- a/homeassistant/components/homewizard/button.py +++ b/homeassistant/components/homewizard/button.py @@ -1,4 +1,5 @@ """Support for HomeWizard buttons.""" + from homeassistant.components.button import ButtonEntity from homeassistant.config_entries import ConfigEntry from homeassistant.const import EntityCategory @@ -16,7 +17,7 @@ async def async_setup_entry( ) -> None: """Set up the Identify button.""" coordinator: HWEnergyDeviceUpdateCoordinator = hass.data[DOMAIN][entry.entry_id] - if coordinator.data.features.has_identify: + if coordinator.supports_identify(): async_add_entities([HomeWizardIdentifyButton(coordinator, entry)]) diff --git a/homeassistant/components/homewizard/const.py b/homeassistant/components/homewizard/const.py index 34c83626f86..ff065592283 100644 --- a/homeassistant/components/homewizard/const.py +++ b/homeassistant/components/homewizard/const.py @@ -4,7 +4,6 @@ from __future__ import annotations from dataclasses import dataclass from datetime import timedelta -from homewizard_energy.features import Features from homewizard_energy.models import Data, Device, State, System from homeassistant.const import Platform @@ -30,6 +29,5 @@ class DeviceResponseEntry: device: Device data: Data - features: Features - state: State | None + state: State | None = None system: System | None = None diff --git a/homeassistant/components/homewizard/coordinator.py b/homeassistant/components/homewizard/coordinator.py index 2da618eeb27..533af445c84 100644 --- a/homeassistant/components/homewizard/coordinator.py +++ b/homeassistant/components/homewizard/coordinator.py @@ -4,7 +4,9 @@ from __future__ import annotations import logging from homewizard_energy import HomeWizardEnergy +from homewizard_energy.const import SUPPORTS_IDENTIFY, SUPPORTS_STATE, SUPPORTS_SYSTEM from homewizard_energy.errors import DisabledError, RequestError +from homewizard_energy.models import Device from homeassistant.config_entries import ConfigEntry from homeassistant.core import HomeAssistant @@ -39,11 +41,12 @@ class HWEnergyDeviceUpdateCoordinator(DataUpdateCoordinator[DeviceResponseEntry] data = DeviceResponseEntry( device=await self.api.device(), data=await self.api.data(), - features=await self.api.features(), - state=await self.api.state(), ) - if data.features.has_system: + if self.supports_state(data.device): + data.state = await self.api.state() + + if self.supports_system(data.device): data.system = await self.api.system() except RequestError as ex: @@ -61,4 +64,27 @@ class HWEnergyDeviceUpdateCoordinator(DataUpdateCoordinator[DeviceResponseEntry] self.api_disabled = False + self.data = data return data + + def supports_state(self, device: Device | None = None) -> bool: + """Return True if the device supports state.""" + + if device is None: + device = self.data.device + + return device.product_type in SUPPORTS_STATE + + def supports_system(self, device: Device | None = None) -> bool: + """Return True if the device supports system.""" + if device is None: + device = self.data.device + + return device.product_type in SUPPORTS_SYSTEM + + def supports_identify(self, device: Device | None = None) -> bool: + """Return True if the device supports identify.""" + if device is None: + device = self.data.device + + return device.product_type in SUPPORTS_IDENTIFY diff --git a/homeassistant/components/homewizard/manifest.json b/homeassistant/components/homewizard/manifest.json index e05b34dbcd6..b1bbd8d0945 100644 --- a/homeassistant/components/homewizard/manifest.json +++ b/homeassistant/components/homewizard/manifest.json @@ -8,6 +8,6 @@ "iot_class": "local_polling", "loggers": ["homewizard_energy"], "quality_scale": "platinum", - "requirements": ["python-homewizard-energy==1.8.0"], + "requirements": ["python-homewizard-energy==2.0.1"], "zeroconf": ["_hwenergy._tcp.local."] } diff --git a/homeassistant/components/homewizard/number.py b/homeassistant/components/homewizard/number.py index 8d00234f976..0451aed9739 100644 --- a/homeassistant/components/homewizard/number.py +++ b/homeassistant/components/homewizard/number.py @@ -20,7 +20,7 @@ async def async_setup_entry( ) -> None: """Set up numbers for device.""" coordinator: HWEnergyDeviceUpdateCoordinator = hass.data[DOMAIN][entry.entry_id] - if coordinator.data.state: + if coordinator.supports_state(): async_add_entities([HWEnergyNumberEntity(coordinator, entry)]) diff --git a/homeassistant/components/homewizard/switch.py b/homeassistant/components/homewizard/switch.py index f6a9ed2b05f..1edb9e1e605 100644 --- a/homeassistant/components/homewizard/switch.py +++ b/homeassistant/components/homewizard/switch.py @@ -27,7 +27,7 @@ from .helpers import homewizard_exception_handler class HomeWizardEntityDescriptionMixin: """Mixin values for HomeWizard entities.""" - create_fn: Callable[[DeviceResponseEntry], bool] + create_fn: Callable[[HWEnergyDeviceUpdateCoordinator], bool] available_fn: Callable[[DeviceResponseEntry], bool] is_on_fn: Callable[[DeviceResponseEntry], bool | None] set_fn: Callable[[HomeWizardEnergy, bool], Awaitable[Any]] @@ -46,7 +46,7 @@ SWITCHES = [ HomeWizardSwitchEntityDescription( key="power_on", device_class=SwitchDeviceClass.OUTLET, - create_fn=lambda data: data.state is not None, + create_fn=lambda coordinator: coordinator.supports_state(), available_fn=lambda data: data.state is not None and not data.state.switch_lock, is_on_fn=lambda data: data.state.power_on if data.state else None, set_fn=lambda api, active: api.state_set(power_on=active), @@ -57,7 +57,7 @@ SWITCHES = [ entity_category=EntityCategory.CONFIG, icon="mdi:lock", icon_off="mdi:lock-open", - create_fn=lambda data: data.state is not None, + create_fn=lambda coordinator: coordinator.supports_state(), available_fn=lambda data: data.state is not None, is_on_fn=lambda data: data.state.switch_lock if data.state else None, set_fn=lambda api, active: api.state_set(switch_lock=active), @@ -68,7 +68,7 @@ SWITCHES = [ entity_category=EntityCategory.CONFIG, icon="mdi:cloud", icon_off="mdi:cloud-off-outline", - create_fn=lambda data: data.system is not None, + create_fn=lambda coordinator: coordinator.supports_system(), available_fn=lambda data: data.system is not None, is_on_fn=lambda data: data.system.cloud_enabled if data.system else None, set_fn=lambda api, active: api.system_set(cloud_enabled=active), @@ -91,7 +91,7 @@ async def async_setup_entry( entry=entry, ) for description in SWITCHES - if description.available_fn(coordinator.data) + if description.create_fn(coordinator) ) diff --git a/requirements_all.txt b/requirements_all.txt index a91461b180b..d5c42385e7a 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2054,7 +2054,7 @@ python-gc100==1.0.3a0 python-gitlab==1.6.0 # homeassistant.components.homewizard -python-homewizard-energy==1.8.0 +python-homewizard-energy==2.0.1 # homeassistant.components.hp_ilo python-hpilo==4.3 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 740086d187f..bee57746f9a 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1477,7 +1477,7 @@ python-ecobee-api==0.2.14 python-fullykiosk==0.0.12 # homeassistant.components.homewizard -python-homewizard-energy==1.8.0 +python-homewizard-energy==2.0.1 # homeassistant.components.izone python-izone==1.2.9 diff --git a/tests/components/homewizard/conftest.py b/tests/components/homewizard/conftest.py index b1bfb1190dc..4cfec96cb8f 100644 --- a/tests/components/homewizard/conftest.py +++ b/tests/components/homewizard/conftest.py @@ -3,7 +3,6 @@ from collections.abc import Generator import json from unittest.mock import AsyncMock, MagicMock, patch -from homewizard_energy.features import Features from homewizard_energy.models import Data, Device, State, System import pytest @@ -44,7 +43,6 @@ def mock_homewizardenergy(): "homeassistant.components.homewizard.coordinator.HomeWizardEnergy", ) as device: client = device.return_value - client.features = AsyncMock(return_value=Features("HWE-SKT", "3.01")) client.device = AsyncMock( side_effect=lambda: Device.from_dict( json.loads(load_fixture("homewizard/device.json")) diff --git a/tests/components/homewizard/fixtures/device.json b/tests/components/homewizard/fixtures/device.json index 493daa12b94..2e5be55c68e 100644 --- a/tests/components/homewizard/fixtures/device.json +++ b/tests/components/homewizard/fixtures/device.json @@ -1,5 +1,5 @@ { - "product_type": "HWE-P1", + "product_type": "HWE-SKT", "product_name": "P1 Meter", "serial": "3c39e7aabbcc", "firmware_version": "2.11", diff --git a/tests/components/homewizard/generator.py b/tests/components/homewizard/generator.py index f9bdea74fb4..6eb945334fd 100644 --- a/tests/components/homewizard/generator.py +++ b/tests/components/homewizard/generator.py @@ -2,7 +2,6 @@ from unittest.mock import AsyncMock -from homewizard_energy.features import Features from homewizard_energy.models import Data, Device @@ -29,9 +28,6 @@ def get_mock_device( mock_device.data = AsyncMock(return_value=Data.from_dict({})) mock_device.state = AsyncMock(return_value=None) mock_device.system = AsyncMock(return_value=None) - mock_device.features = AsyncMock( - return_value=Features(product_type, firmware_version) - ) mock_device.close = AsyncMock() diff --git a/tests/components/homewizard/test_button.py b/tests/components/homewizard/test_button.py index bc6584e903b..d8b8b5030b6 100644 --- a/tests/components/homewizard/test_button.py +++ b/tests/components/homewizard/test_button.py @@ -18,7 +18,7 @@ async def test_identify_button_entity_not_loaded_when_not_available( ) -> None: """Does not load button when device has no support for it.""" - api = get_mock_device(product_type="HWE-P1") + api = get_mock_device(product_type="SDM230-WIFI") with patch( "homeassistant.components.homewizard.coordinator.HomeWizardEnergy", diff --git a/tests/components/homewizard/test_diagnostics.py b/tests/components/homewizard/test_diagnostics.py index df5f5576158..64e8b0c6dfd 100644 --- a/tests/components/homewizard/test_diagnostics.py +++ b/tests/components/homewizard/test_diagnostics.py @@ -21,7 +21,7 @@ async def test_diagnostics( "data": { "device": { "product_name": "P1 Meter", - "product_type": "HWE-P1", + "product_type": "HWE-SKT", "serial": REDACTED, "api_version": "v1", "firmware_version": "2.11", diff --git a/tests/components/homewizard/test_number.py b/tests/components/homewizard/test_number.py index f78a9cdbab5..aa4ab01cfc6 100644 --- a/tests/components/homewizard/test_number.py +++ b/tests/components/homewizard/test_number.py @@ -44,7 +44,7 @@ async def test_number_loads_entities( ) -> None: """Test entity does load number when brightness is available.""" - api = get_mock_device() + api = get_mock_device(product_type="HWE-SKT") api.state = AsyncMock(return_value=State.from_dict({"brightness": 255})) with patch( @@ -81,7 +81,7 @@ async def test_brightness_level_set( ) -> None: """Test entity turns sets light level.""" - api = get_mock_device() + api = get_mock_device(product_type="HWE-SKT") api.state = AsyncMock(return_value=State.from_dict({"brightness": 255})) def state_set(brightness): @@ -157,7 +157,7 @@ async def test_brightness_level_set_catches_requesterror( ) -> None: """Test entity raises HomeAssistantError when RequestError was raised.""" - api = get_mock_device() + api = get_mock_device(product_type="HWE-SKT") api.state = AsyncMock(return_value=State.from_dict({"brightness": 255})) api.state_set = AsyncMock(side_effect=RequestError()) @@ -193,7 +193,7 @@ async def test_brightness_level_set_catches_disablederror( ) -> None: """Test entity raises HomeAssistantError when DisabledError was raised.""" - api = get_mock_device() + api = get_mock_device(product_type="HWE-SKT") api.state = AsyncMock(return_value=State.from_dict({"brightness": 255})) api.state_set = AsyncMock(side_effect=DisabledError()) @@ -229,7 +229,7 @@ async def test_brightness_level_set_catches_invalid_value( ) -> None: """Test entity raises ValueError when value was invalid.""" - api = get_mock_device() + api = get_mock_device(product_type="HWE-SKT") api.state = AsyncMock(return_value=State.from_dict({"brightness": 255})) def state_set(brightness): diff --git a/tests/components/homewizard/test_switch.py b/tests/components/homewizard/test_switch.py index 5dc91a3dc99..f55550ee825 100644 --- a/tests/components/homewizard/test_switch.py +++ b/tests/components/homewizard/test_switch.py @@ -54,7 +54,7 @@ async def test_switch_loads_entities( ) -> None: """Test entity loads smr version.""" - api = get_mock_device() + api = get_mock_device(product_type="HWE-SKT") api.state = AsyncMock( return_value=State.from_dict({"power_on": False, "switch_lock": False}) ) @@ -109,7 +109,7 @@ async def test_switch_power_on_off( ) -> None: """Test entity turns switch on and off.""" - api = get_mock_device() + api = get_mock_device(product_type="HWE-SKT") api.state = AsyncMock( return_value=State.from_dict({"power_on": False, "switch_lock": False}) ) @@ -164,7 +164,7 @@ async def test_switch_lock_power_on_off( ) -> None: """Test entity turns switch on and off.""" - api = get_mock_device() + api = get_mock_device(product_type="HWE-SKT") api.state = AsyncMock( return_value=State.from_dict({"power_on": False, "switch_lock": False}) ) @@ -228,7 +228,7 @@ async def test_switch_lock_sets_power_on_unavailable( ) -> None: """Test entity turns switch on and off.""" - api = get_mock_device() + api = get_mock_device(product_type="HWE-SKT") api.state = AsyncMock( return_value=State.from_dict({"power_on": True, "switch_lock": False}) ) From f77ce413bea43207283280c23fab4a63a5d35cd6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Apr 2023 01:38:27 -1000 Subject: [PATCH 073/110] Bump aiolifx to 0.8.10 (#91324) --- homeassistant/components/lifx/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/lifx/manifest.json b/homeassistant/components/lifx/manifest.json index 65f4e7ecefa..7f715a0d49b 100644 --- a/homeassistant/components/lifx/manifest.json +++ b/homeassistant/components/lifx/manifest.json @@ -41,7 +41,7 @@ "loggers": ["aiolifx", "aiolifx_effects", "bitstring"], "quality_scale": "platinum", "requirements": [ - "aiolifx==0.8.9", + "aiolifx==0.8.10", "aiolifx_effects==0.3.2", "aiolifx_themes==0.4.5" ] diff --git a/requirements_all.txt b/requirements_all.txt index d5c42385e7a..9e3c4f0930f 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -193,7 +193,7 @@ aiokafka==0.7.2 aiokef==0.2.16 # homeassistant.components.lifx -aiolifx==0.8.9 +aiolifx==0.8.10 # homeassistant.components.lifx aiolifx_effects==0.3.2 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index bee57746f9a..57abd29faa9 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -177,7 +177,7 @@ aioimaplib==1.0.1 aiokafka==0.7.2 # homeassistant.components.lifx -aiolifx==0.8.9 +aiolifx==0.8.10 # homeassistant.components.lifx aiolifx_effects==0.3.2 From 51c99d26b4af64cff833e725f3233f6273e92d7f Mon Sep 17 00:00:00 2001 From: Bram Kragten Date: Thu, 13 Apr 2023 13:37:37 +0200 Subject: [PATCH 074/110] Update frontend to 20230411.1 (#91344) --- homeassistant/components/frontend/manifest.json | 2 +- homeassistant/package_constraints.txt | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/frontend/manifest.json b/homeassistant/components/frontend/manifest.json index a3af9f863ea..9e0659048b0 100644 --- a/homeassistant/components/frontend/manifest.json +++ b/homeassistant/components/frontend/manifest.json @@ -20,5 +20,5 @@ "documentation": "https://www.home-assistant.io/integrations/frontend", "integration_type": "system", "quality_scale": "internal", - "requirements": ["home-assistant-frontend==20230411.0"] + "requirements": ["home-assistant-frontend==20230411.1"] } diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 24732608eee..63564979ae3 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -25,7 +25,7 @@ ha-av==10.0.0 hass-nabucasa==0.63.1 hassil==1.0.6 home-assistant-bluetooth==1.9.3 -home-assistant-frontend==20230411.0 +home-assistant-frontend==20230411.1 home-assistant-intents==2023.3.29 httpx==0.23.3 ifaddr==0.1.7 diff --git a/requirements_all.txt b/requirements_all.txt index 9e3c4f0930f..9c2fa18abdf 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -907,7 +907,7 @@ hole==0.8.0 holidays==0.21.13 # homeassistant.components.frontend -home-assistant-frontend==20230411.0 +home-assistant-frontend==20230411.1 # homeassistant.components.conversation home-assistant-intents==2023.3.29 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 57abd29faa9..914f5a37256 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -693,7 +693,7 @@ hole==0.8.0 holidays==0.21.13 # homeassistant.components.frontend -home-assistant-frontend==20230411.0 +home-assistant-frontend==20230411.1 # homeassistant.components.conversation home-assistant-intents==2023.3.29 From b5ac3ee288a8b031d1ad2625856762f2eb1a2d51 Mon Sep 17 00:00:00 2001 From: Franck Nijhof Date: Thu, 13 Apr 2023 13:50:25 +0200 Subject: [PATCH 075/110] Bumped version to 2023.4.4 --- homeassistant/const.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/const.py b/homeassistant/const.py index 1944bf7e998..845445d6b07 100644 --- a/homeassistant/const.py +++ b/homeassistant/const.py @@ -8,7 +8,7 @@ from .backports.enum import StrEnum APPLICATION_NAME: Final = "HomeAssistant" MAJOR_VERSION: Final = 2023 MINOR_VERSION: Final = 4 -PATCH_VERSION: Final = "3" +PATCH_VERSION: Final = "4" __short_version__: Final = f"{MAJOR_VERSION}.{MINOR_VERSION}" __version__: Final = f"{__short_version__}.{PATCH_VERSION}" REQUIRED_PYTHON_VER: Final[tuple[int, int, int]] = (3, 10, 0) diff --git a/pyproject.toml b/pyproject.toml index 7226e594c83..ea02d089a30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "homeassistant" -version = "2023.4.3" +version = "2023.4.4" license = {text = "Apache-2.0"} description = "Open-source home automation platform running on Python 3." readme = "README.rst" From 47398f03dd001482ccac4c68d631cc2a25650249 Mon Sep 17 00:00:00 2001 From: Meow Date: Sat, 15 Apr 2023 21:41:34 +0200 Subject: [PATCH 076/110] Add SetSynchronizationPoint fallback to onvif (#86400) Co-authored-by: J. Nick Koston --- homeassistant/components/onvif/event.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/homeassistant/components/onvif/event.py b/homeassistant/components/onvif/event.py index 84d75bf8048..e9fc89d6ef6 100644 --- a/homeassistant/components/onvif/event.py +++ b/homeassistant/components/onvif/event.py @@ -20,11 +20,9 @@ from .models import Event from .parsers import PARSERS UNHANDLED_TOPICS: set[str] = set() -SUBSCRIPTION_ERRORS = ( - Fault, - asyncio.TimeoutError, - TransportError, -) + +SUBSCRIPTION_ERRORS = (Fault, asyncio.TimeoutError, TransportError) +SET_SYNCHRONIZATION_POINT_ERRORS = (*SUBSCRIPTION_ERRORS, TypeError) def _stringify_onvif_error(error: Exception) -> str: @@ -99,7 +97,7 @@ class EventManager: # Initialize events pullpoint = self.device.create_pullpoint_service() - with suppress(*SUBSCRIPTION_ERRORS): + with suppress(*SET_SYNCHRONIZATION_POINT_ERRORS): await pullpoint.SetSynchronizationPoint() response = await pullpoint.PullMessages( {"MessageLimit": 100, "Timeout": dt.timedelta(seconds=5)} From 2bda40d35252f8822eb192724e680036b977b080 Mon Sep 17 00:00:00 2001 From: Mark Adkins Date: Sun, 16 Apr 2023 08:04:18 -0400 Subject: [PATCH 077/110] Fix SharkIQ token expiration (#89357) --- .../components/sharkiq/update_coordinator.py | 10 +++++++++- tests/components/sharkiq/test_vacuum.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/sharkiq/update_coordinator.py b/homeassistant/components/sharkiq/update_coordinator.py index 2afeb574f92..87f5aafe7a4 100644 --- a/homeassistant/components/sharkiq/update_coordinator.py +++ b/homeassistant/components/sharkiq/update_coordinator.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio +from datetime import datetime, timedelta from async_timeout import timeout from sharkiq import ( @@ -60,6 +61,13 @@ class SharkIqUpdateCoordinator(DataUpdateCoordinator[bool]): async def _async_update_data(self) -> bool: """Update data device by device.""" try: + if self.ayla_api.token_expiring_soon: + await self.ayla_api.async_refresh_auth() + elif datetime.now() > self.ayla_api.auth_expiration - timedelta( + seconds=600 + ): + await self.ayla_api.async_refresh_auth() + all_vacuums = await self.ayla_api.async_list_devices() self._online_dsns = { v["dsn"] @@ -78,7 +86,7 @@ class SharkIqUpdateCoordinator(DataUpdateCoordinator[bool]): LOGGER.debug("Bad auth state. Attempting re-auth", exc_info=err) raise ConfigEntryAuthFailed from err except Exception as err: - LOGGER.exception("Unexpected error updating SharkIQ") + LOGGER.exception("Unexpected error updating SharkIQ. Attempting re-auth") raise UpdateFailed(err) from err return True diff --git a/tests/components/sharkiq/test_vacuum.py b/tests/components/sharkiq/test_vacuum.py index aa43f324bba..2305b117c3e 100644 --- a/tests/components/sharkiq/test_vacuum.py +++ b/tests/components/sharkiq/test_vacuum.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Iterable from copy import deepcopy +from datetime import datetime, timedelta import enum from typing import Any from unittest.mock import patch @@ -72,9 +73,14 @@ EXPECTED_FEATURES = ( class MockAyla(AylaApi): """Mocked AylaApi that doesn't do anything.""" + desired_expiry = False + async def async_sign_in(self): """Instead of signing in, just return.""" + async def async_refresh_auth(self): + """Instead of refreshing auth, just return.""" + async def async_list_devices(self) -> list[dict]: """Return the device list.""" return [SHARK_DEVICE_DICT] @@ -89,6 +95,18 @@ class MockAyla(AylaApi): async def async_request(self, http_method: str, url: str, **kwargs): """Don't make an HTTP request.""" + @property + def token_expiring_soon(self) -> bool: + """Toggling Property for Token Expiration Flag.""" + # Alternate expiry flag for each test + self.desired_expiry = not self.desired_expiry + return self.desired_expiry + + @property + def auth_expiration(self) -> datetime: + """Sample expiration timestamp that is always 1200 seconds behind now().""" + return datetime.now() - timedelta(seconds=1200) + class MockShark(SharkIqVacuum): """Mocked SharkIqVacuum that won't hit the API.""" From 8feab57d598afbadd44584a7d4e24455711c308b Mon Sep 17 00:00:00 2001 From: starkillerOG Date: Sat, 15 Apr 2023 03:05:22 +0200 Subject: [PATCH 078/110] Reolink prevent ONVIF push being lost due to ConnectionResetError (#91070) * Make "Connection lost" error less likely * Handle connection loss during ONVIF event reading * tweak * fix styling * catch asyncio.CancelledError from request.text() * missing () * re-raise cancelation for proper cleanup * Simplify * Also set webhook_reachable if connection lost * fix styntax * Send HTTP_OK directly after data read done * protect agains garbage collection * Protect shielded task (inner) not shielded future (outer) * fix black * Make sure exceptions are logged * fix spelling * fix black * fix spelling * Simplify using hass.async_create_task * clarify comment * Eleborate comment * Update homeassistant/components/reolink/host.py Co-authored-by: J. Nick Koston * Apply suggestions from bdraco --------- Co-authored-by: J. Nick Koston --- homeassistant/components/reolink/host.py | 66 ++++++++++++++++++------ 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/homeassistant/components/reolink/host.py b/homeassistant/components/reolink/host.py index f7810746481..1710219cfb3 100644 --- a/homeassistant/components/reolink/host.py +++ b/homeassistant/components/reolink/host.py @@ -357,34 +357,70 @@ class ReolinkHost: async def handle_webhook( self, hass: HomeAssistant, webhook_id: str, request: Request - ): + ) -> None: """Shield the incoming webhook callback from cancellation.""" - await asyncio.shield(self.handle_webhook_shielded(hass, webhook_id, request)) - - async def handle_webhook_shielded( - self, hass: HomeAssistant, webhook_id: str, request: Request - ): - """Handle incoming webhook from Reolink for inbound messages and calls.""" - + shielded_future = asyncio.shield( + self._handle_webhook(hass, webhook_id, request) + ) _LOGGER.debug("Webhook '%s' called", webhook_id) if not self._webhook_reachable.is_set(): self._webhook_reachable.set() + ir.async_delete_issue(self._hass, DOMAIN, "webhook_url") + await shielded_future - if not request.body_exists: - _LOGGER.debug("Webhook '%s' triggered without payload", webhook_id) + async def _handle_webhook( + self, hass: HomeAssistant, webhook_id: str, request: Request + ) -> None: + """Handle incoming webhook from Reolink for inbound messages and calls.""" + try: + data = await request.text() + except ConnectionResetError: + # We lost the connection before reading the message, fallback to polling + # No need for a background task here as we already know the connection is lost + _LOGGER.debug( + "Webhook '%s' called, but lost connection before reading message, issuing poll", + webhook_id, + ) + if not await self._api.get_motion_state_all_ch(): + _LOGGER.error( + "Could not poll motion state after losing connection during receiving ONVIF event" + ) + return + async_dispatcher_send(hass, f"{webhook_id}_all", {}) return - data = await request.text() if not data: _LOGGER.debug( "Webhook '%s' triggered with unknown payload: %s", webhook_id, data ) return - channels = await self._api.ONVIF_event_callback(data) + # We received the data but we want handle_webhook to return as soon as possible + # so we process the data in the background + hass.async_create_background_task( + self._process_webhook_data(hass, webhook_id, data), + "Process Reolink webhook", + ) + + async def _process_webhook_data( + self, hass: HomeAssistant, webhook_id: str, data: str + ) -> None: + """Process the data from the webhook.""" + # This task is executed in the background so we need to catch exceptions + # and log them + try: + channels = await self._api.ONVIF_event_callback(data) + except Exception as ex: # pylint: disable=broad-except + _LOGGER.exception( + "Error processing ONVIF event for Reolink %s: %s", + self._api.nvr_name, + ex, + ) + return if channels is None: async_dispatcher_send(hass, f"{webhook_id}_all", {}) - else: - for channel in channels: - async_dispatcher_send(hass, f"{webhook_id}_{channel}", {}) + return + + for channel in channels: + async_dispatcher_send(hass, f"{webhook_id}_{channel}", {}) From 0e3f462bfb03b6b381198970ad62fa6d17820705 Mon Sep 17 00:00:00 2001 From: epenet <6771947+epenet@users.noreply.github.com> Date: Thu, 13 Apr 2023 19:33:38 +0200 Subject: [PATCH 079/110] Add missing mock in sharkiq tests (#91325) --- tests/components/sharkiq/test_vacuum.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/components/sharkiq/test_vacuum.py b/tests/components/sharkiq/test_vacuum.py index 2305b117c3e..cfd62c9deaf 100644 --- a/tests/components/sharkiq/test_vacuum.py +++ b/tests/components/sharkiq/test_vacuum.py @@ -81,6 +81,9 @@ class MockAyla(AylaApi): async def async_refresh_auth(self): """Instead of refreshing auth, just return.""" + async def async_sign_out(self): + """Instead of signing out, just return.""" + async def async_list_devices(self) -> list[dict]: """Return the device list.""" return [SHARK_DEVICE_DICT] From 5f7b447d7a91465af045fb3959dd243c6f56c3c5 Mon Sep 17 00:00:00 2001 From: rich-kettlewell <122128709+rich-kettlewell@users.noreply.github.com> Date: Thu, 13 Apr 2023 18:49:07 +0100 Subject: [PATCH 080/110] Tado set_water_heater_timer should use water_heater domain (#91364) --- homeassistant/components/tado/services.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/tado/services.yaml b/homeassistant/components/tado/services.yaml index 3c5a830698d..211ae4cd1ff 100644 --- a/homeassistant/components/tado/services.yaml +++ b/homeassistant/components/tado/services.yaml @@ -41,7 +41,7 @@ set_water_heater_timer: target: entity: integration: tado - domain: climate + domain: water_heater fields: time_period: name: Time period From e29d5a135689df9d9933cf6e44893ecde2ed948e Mon Sep 17 00:00:00 2001 From: Aidan Timson Date: Sat, 15 Apr 2023 14:49:05 +0100 Subject: [PATCH 081/110] Fix listener running in foreground for System Bridge integration (#91391) Co-authored-by: J. Nick Koston --- homeassistant/components/system_bridge/coordinator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/system_bridge/coordinator.py b/homeassistant/components/system_bridge/coordinator.py index 809e2a4fd50..adb88efd5ec 100644 --- a/homeassistant/components/system_bridge/coordinator.py +++ b/homeassistant/components/system_bridge/coordinator.py @@ -188,7 +188,10 @@ class SystemBridgeDataUpdateCoordinator( session=async_get_clientsession(self.hass), ) - self.hass.async_create_task(self._listen_for_data()) + self.hass.async_create_background_task( + self._listen_for_data(), + name="System Bridge WebSocket Listener", + ) await self.websocket_client.register_data_listener( RegisterDataListener(modules=MODULES) From afe3fd5ec04d78449c12a79c9a7d1843526fa252 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Apr 2023 20:37:12 -1000 Subject: [PATCH 082/110] Bump onvif-zeep-async to 1.2.5 (#91399) --- homeassistant/components/onvif/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/onvif/manifest.json b/homeassistant/components/onvif/manifest.json index ef4497fa284..9095bb2620e 100644 --- a/homeassistant/components/onvif/manifest.json +++ b/homeassistant/components/onvif/manifest.json @@ -7,5 +7,5 @@ "documentation": "https://www.home-assistant.io/integrations/onvif", "iot_class": "local_push", "loggers": ["onvif", "wsdiscovery", "zeep"], - "requirements": ["onvif-zeep-async==1.2.3", "WSDiscovery==2.0.0"] + "requirements": ["onvif-zeep-async==1.2.5", "WSDiscovery==2.0.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index 9c2fa18abdf..651982d4722 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1260,7 +1260,7 @@ ondilo==0.2.0 onkyo-eiscp==1.2.7 # homeassistant.components.onvif -onvif-zeep-async==1.2.3 +onvif-zeep-async==1.2.5 # homeassistant.components.opengarage open-garage==0.2.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 914f5a37256..6a8e16b9147 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -938,7 +938,7 @@ omnilogic==0.4.5 ondilo==0.2.0 # homeassistant.components.onvif -onvif-zeep-async==1.2.3 +onvif-zeep-async==1.2.5 # homeassistant.components.opengarage open-garage==0.2.0 From 2b9cc39d2bb447ce1751eaf1fed16ebda23720de Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Sat, 15 Apr 2023 09:48:34 -0400 Subject: [PATCH 083/110] Fix attribute reporting config failures in ZHA (#91403) --- .../components/zha/core/channels/base.py | 37 ++++++------ tests/components/zha/test_channels.py | 56 +++++++++++++++++++ 2 files changed, 76 insertions(+), 17 deletions(-) diff --git a/homeassistant/components/zha/core/channels/base.py b/homeassistant/components/zha/core/channels/base.py index 6d4899be37c..48f69ffbf2d 100644 --- a/homeassistant/components/zha/core/channels/base.py +++ b/homeassistant/components/zha/core/channels/base.py @@ -49,8 +49,8 @@ _LOGGER = logging.getLogger(__name__) class AttrReportConfig(TypedDict, total=True): """Configuration to report for the attributes.""" - # Could be either an attribute name or attribute id - attr: str | int + # An attribute name + attr: str # The config for the attribute reporting configuration consists of a tuple for # (minimum_reported_time_interval_s, maximum_reported_time_interval_s, value_delta) config: tuple[int, int, int | float] @@ -130,15 +130,13 @@ class ZigbeeChannel(LogMixin): unique_id = ch_pool.unique_id.replace("-", ":") self._unique_id = f"{unique_id}:0x{cluster.cluster_id:04x}" if not hasattr(self, "_value_attribute") and self.REPORT_CONFIG: - attr = self.REPORT_CONFIG[0].get("attr") - if isinstance(attr, str): - attribute: ZCLAttributeDef = self.cluster.attributes_by_name.get(attr) - if attribute is not None: - self.value_attribute = attribute.id - else: - self.value_attribute = None + attr_def: ZCLAttributeDef | None = self.cluster.attributes_by_name.get( + self.REPORT_CONFIG[0]["attr"] + ) + if attr_def is not None: + self.value_attribute = attr_def.id else: - self.value_attribute = attr + self.value_attribute = None self._status = ChannelStatus.CREATED self._cluster.add_listener(self) self.data_cache: dict[str, Enum] = {} @@ -233,7 +231,12 @@ class ZigbeeChannel(LogMixin): for attr_report in self.REPORT_CONFIG: attr, config = attr_report["attr"], attr_report["config"] - attr_name = self.cluster.attributes.get(attr, [attr])[0] + + try: + attr_name = self.cluster.find_attribute(attr).name + except KeyError: + attr_name = attr + event_data[attr_name] = { "min": config[0], "max": config[1], @@ -282,7 +285,7 @@ class ZigbeeChannel(LogMixin): ) def _configure_reporting_status( - self, attrs: dict[int | str, tuple[int, int, float | int]], res: list | tuple + self, attrs: dict[str, tuple[int, int, float | int]], res: list | tuple ) -> None: """Parse configure reporting result.""" if isinstance(res, (Exception, ConfigureReportingResponseRecord)): @@ -304,14 +307,14 @@ class ZigbeeChannel(LogMixin): return failed = [ - self.cluster.attributes.get(r.attrid, [r.attrid])[0] - for r in res - if r.status != Status.SUCCESS + self.cluster.find_attribute(record.attrid).name + for record in res + if record.status != Status.SUCCESS ] - attributes = {self.cluster.attributes.get(r, [r])[0] for r in attrs} + self.debug( "Successfully configured reporting for '%s' on '%s' cluster", - attributes - set(failed), + set(attrs) - set(failed), self.name, ) self.debug( diff --git a/tests/components/zha/test_channels.py b/tests/components/zha/test_channels.py index 9c43a76ea85..b8542433e7c 100644 --- a/tests/components/zha/test_channels.py +++ b/tests/components/zha/test_channels.py @@ -5,9 +5,12 @@ from unittest import mock from unittest.mock import AsyncMock, patch import pytest +import zigpy.endpoint import zigpy.profiles.zha import zigpy.types as t +from zigpy.zcl import foundation import zigpy.zcl.clusters +import zigpy.zdo.types as zdo_t import homeassistant.components.zha.core.channels as zha_channels import homeassistant.components.zha.core.channels.base as base_channels @@ -726,3 +729,56 @@ async def test_cluster_no_ep_attribute(m1, zha_device_mock) -> None: pools = {pool.id: pool for pool in channels.pools} assert "1:0x042e" in pools[1].all_channels assert pools[1].all_channels["1:0x042e"].name + + +async def test_configure_reporting(hass: HomeAssistant) -> None: + """Test setting up a channel and configuring attribute reporting in two batches.""" + + class TestZigbeeChannel(base_channels.ZigbeeChannel): + BIND = True + REPORT_CONFIG = ( + # By name + base_channels.AttrReportConfig(attr="current_x", config=(1, 60, 1)), + base_channels.AttrReportConfig(attr="current_hue", config=(1, 60, 2)), + base_channels.AttrReportConfig(attr="color_temperature", config=(1, 60, 3)), + base_channels.AttrReportConfig(attr="current_y", config=(1, 60, 4)), + ) + + mock_ep = mock.AsyncMock(spec_set=zigpy.endpoint.Endpoint) + mock_ep.device.zdo = AsyncMock() + + cluster = zigpy.zcl.clusters.lighting.Color(mock_ep) + cluster.bind = AsyncMock( + spec_set=cluster.bind, + return_value=[zdo_t.Status.SUCCESS], # ZDOCmd.Bind_rsp + ) + cluster.configure_reporting_multiple = AsyncMock( + spec_set=cluster.configure_reporting_multiple, + return_value=[ + foundation.ConfigureReportingResponseRecord( + status=foundation.Status.SUCCESS + ) + ], + ) + + ch_pool = mock.AsyncMock(spec_set=zha_channels.ChannelPool) + ch_pool.skip_configuration = False + + channel = TestZigbeeChannel(cluster, ch_pool) + await channel.async_configure() + + # Since we request reporting for five attributes, we need to make two calls (3 + 1) + assert cluster.configure_reporting_multiple.mock_calls == [ + mock.call( + { + "current_x": (1, 60, 1), + "current_hue": (1, 60, 2), + "color_temperature": (1, 60, 3), + } + ), + mock.call( + { + "current_y": (1, 60, 4), + } + ), + ] From bf389440dc928d22d0cdb494616d8c8a0b76c694 Mon Sep 17 00:00:00 2001 From: Erik Montnemery Date: Fri, 14 Apr 2023 11:48:03 +0200 Subject: [PATCH 084/110] Save Thread dataset store when changing preferred dataset (#91411) --- .../components/thread/dataset_store.py | 27 ++++++++++++++----- .../components/thread/websocket_api.py | 5 ++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/homeassistant/components/thread/dataset_store.py b/homeassistant/components/thread/dataset_store.py index 786ea55b34f..bcadf3ca5fb 100644 --- a/homeassistant/components/thread/dataset_store.py +++ b/homeassistant/components/thread/dataset_store.py @@ -82,7 +82,7 @@ class DatasetStore: """Initialize the dataset store.""" self.hass = hass self.datasets: dict[str, DatasetEntry] = {} - self.preferred_dataset: str | None = None + self._preferred_dataset: str | None = None self._store: Store[dict[str, Any]] = Store( hass, STORAGE_VERSION_MAJOR, @@ -103,14 +103,14 @@ class DatasetStore: entry = DatasetEntry(source=source, tlv=tlv) self.datasets[entry.id] = entry # Set to preferred if there is no preferred dataset - if self.preferred_dataset is None: - self.preferred_dataset = entry.id + if self._preferred_dataset is None: + self._preferred_dataset = entry.id self.async_schedule_save() @callback def async_delete(self, dataset_id: str) -> None: """Delete dataset.""" - if self.preferred_dataset == dataset_id: + if self._preferred_dataset == dataset_id: raise DatasetPreferredError("attempt to remove preferred dataset") del self.datasets[dataset_id] self.async_schedule_save() @@ -120,6 +120,21 @@ class DatasetStore: """Get dataset by id.""" return self.datasets.get(dataset_id) + @property + @callback + def preferred_dataset(self) -> str | None: + """Get the id of the preferred dataset.""" + return self._preferred_dataset + + @preferred_dataset.setter + @callback + def preferred_dataset(self, dataset_id: str) -> None: + """Set the preferred dataset.""" + if dataset_id not in self.datasets: + raise KeyError("unknown dataset") + self._preferred_dataset = dataset_id + self.async_schedule_save() + async def async_load(self) -> None: """Load the datasets.""" data = await self._store.async_load() @@ -139,7 +154,7 @@ class DatasetStore: preferred_dataset = data["preferred_dataset"] self.datasets = datasets - self.preferred_dataset = preferred_dataset + self._preferred_dataset = preferred_dataset @callback def async_schedule_save(self) -> None: @@ -151,7 +166,7 @@ class DatasetStore: """Return data of datasets to store in a file.""" data: dict[str, Any] = {} data["datasets"] = [dataset.to_json() for dataset in self.datasets.values()] - data["preferred_dataset"] = self.preferred_dataset + data["preferred_dataset"] = self._preferred_dataset return data diff --git a/homeassistant/components/thread/websocket_api.py b/homeassistant/components/thread/websocket_api.py index aca0d5e5d96..60941426b7e 100644 --- a/homeassistant/components/thread/websocket_api.py +++ b/homeassistant/components/thread/websocket_api.py @@ -65,13 +65,14 @@ async def ws_set_preferred_dataset( dataset_id = msg["dataset_id"] store = await dataset_store.async_get_store(hass) - if not (store.async_get(dataset_id)): + try: + store.preferred_dataset = dataset_id + except KeyError: connection.send_error( msg["id"], websocket_api.const.ERR_NOT_FOUND, "unknown dataset" ) return - store.preferred_dataset = dataset_id connection.send_result(msg["id"]) From 89b1d5bb684bd971067d62acdd594c469432ba96 Mon Sep 17 00:00:00 2001 From: Michael Davie Date: Sat, 15 Apr 2023 15:44:07 -0400 Subject: [PATCH 085/110] Bump env_canada to v0.5.33 (#91468) --- homeassistant/components/environment_canada/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/environment_canada/manifest.json b/homeassistant/components/environment_canada/manifest.json index 79be96d9bf4..17e0ed6e2ac 100644 --- a/homeassistant/components/environment_canada/manifest.json +++ b/homeassistant/components/environment_canada/manifest.json @@ -6,5 +6,5 @@ "documentation": "https://www.home-assistant.io/integrations/environment_canada", "iot_class": "cloud_polling", "loggers": ["env_canada"], - "requirements": ["env_canada==0.5.32"] + "requirements": ["env_canada==0.5.33"] } diff --git a/requirements_all.txt b/requirements_all.txt index 651982d4722..064fc3c2eb6 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -661,7 +661,7 @@ enocean==0.50 enturclient==0.2.4 # homeassistant.components.environment_canada -env_canada==0.5.32 +env_canada==0.5.33 # homeassistant.components.enphase_envoy envoy_reader==0.20.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 6a8e16b9147..4f1b57002c9 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -517,7 +517,7 @@ energyzero==0.4.1 enocean==0.50 # homeassistant.components.environment_canada -env_canada==0.5.32 +env_canada==0.5.33 # homeassistant.components.enphase_envoy envoy_reader==0.20.1 From b06d624d439a47f85a37e5a7edb80a29b4eae210 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 15 Apr 2023 09:34:07 -1000 Subject: [PATCH 086/110] Fix creating onvif pull point subscriptions when InitialTerminationTime is required (#91470) * Fix creating onvif pull point subscriptions when InitialTerminationTime is required fixes #85902 * Bump again because I got it wrong the first time.. this is why retest is good --- homeassistant/components/onvif/event.py | 20 +++++++++++++------- homeassistant/components/onvif/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/homeassistant/components/onvif/event.py b/homeassistant/components/onvif/event.py index e9fc89d6ef6..5bc2a8248fc 100644 --- a/homeassistant/components/onvif/event.py +++ b/homeassistant/components/onvif/event.py @@ -32,6 +32,15 @@ def _stringify_onvif_error(error: Exception) -> str: return str(error) +def _get_next_termination_time() -> str: + """Get next termination time.""" + return ( + (dt_util.utcnow() + dt.timedelta(days=1)) + .isoformat(timespec="seconds") + .replace("+00:00", "Z") + ) + + class EventManager: """ONVIF Event Manager.""" @@ -84,7 +93,9 @@ class EventManager: async def async_start(self) -> bool: """Start polling events.""" - if not await self.device.create_pullpoint_subscription(): + if not await self.device.create_pullpoint_subscription( + {"InitialTerminationTime": _get_next_termination_time()} + ): return False # Create subscription manager @@ -171,16 +182,11 @@ class EventManager: if not self._subscription: return - termination_time = ( - (dt_util.utcnow() + dt.timedelta(days=1)) - .isoformat(timespec="seconds") - .replace("+00:00", "Z") - ) with suppress(*SUBSCRIPTION_ERRORS): # The first time we renew, we may get a Fault error so we # suppress it. The subscription will be restarted in # async_restart later. - await self._subscription.Renew(termination_time) + await self._subscription.Renew(_get_next_termination_time()) def async_schedule_pull(self) -> None: """Schedule async_pull_messages to run.""" diff --git a/homeassistant/components/onvif/manifest.json b/homeassistant/components/onvif/manifest.json index 9095bb2620e..04e26980921 100644 --- a/homeassistant/components/onvif/manifest.json +++ b/homeassistant/components/onvif/manifest.json @@ -7,5 +7,5 @@ "documentation": "https://www.home-assistant.io/integrations/onvif", "iot_class": "local_push", "loggers": ["onvif", "wsdiscovery", "zeep"], - "requirements": ["onvif-zeep-async==1.2.5", "WSDiscovery==2.0.0"] + "requirements": ["onvif-zeep-async==1.2.7", "WSDiscovery==2.0.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index 064fc3c2eb6..83ce86e49f2 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1260,7 +1260,7 @@ ondilo==0.2.0 onkyo-eiscp==1.2.7 # homeassistant.components.onvif -onvif-zeep-async==1.2.5 +onvif-zeep-async==1.2.7 # homeassistant.components.opengarage open-garage==0.2.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 4f1b57002c9..af6012e493d 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -938,7 +938,7 @@ omnilogic==0.4.5 ondilo==0.2.0 # homeassistant.components.onvif -onvif-zeep-async==1.2.5 +onvif-zeep-async==1.2.7 # homeassistant.components.opengarage open-garage==0.2.0 From 00a86757faf524814de7d2f5ab786193cb9deb3c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Apr 2023 01:56:10 -1000 Subject: [PATCH 087/110] Bump onvif-zeep-async to 1.2.11 (#91472) --- homeassistant/components/onvif/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/onvif/manifest.json b/homeassistant/components/onvif/manifest.json index 04e26980921..aa06d9c028d 100644 --- a/homeassistant/components/onvif/manifest.json +++ b/homeassistant/components/onvif/manifest.json @@ -7,5 +7,5 @@ "documentation": "https://www.home-assistant.io/integrations/onvif", "iot_class": "local_push", "loggers": ["onvif", "wsdiscovery", "zeep"], - "requirements": ["onvif-zeep-async==1.2.7", "WSDiscovery==2.0.0"] + "requirements": ["onvif-zeep-async==1.2.11", "WSDiscovery==2.0.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index 83ce86e49f2..71b961ce954 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1260,7 +1260,7 @@ ondilo==0.2.0 onkyo-eiscp==1.2.7 # homeassistant.components.onvif -onvif-zeep-async==1.2.7 +onvif-zeep-async==1.2.11 # homeassistant.components.opengarage open-garage==0.2.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index af6012e493d..3e75724010e 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -938,7 +938,7 @@ omnilogic==0.4.5 ondilo==0.2.0 # homeassistant.components.onvif -onvif-zeep-async==1.2.7 +onvif-zeep-async==1.2.11 # homeassistant.components.opengarage open-garage==0.2.0 From 5321c60058a0de4ac7ed7eec5d28eb3d72914e30 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Apr 2023 02:06:30 -1000 Subject: [PATCH 088/110] Handle a few more transient onvif errors (#91473) --- homeassistant/components/onvif/__init__.py | 22 ++++++- homeassistant/components/onvif/device.py | 71 ++++++++-------------- 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/homeassistant/components/onvif/__init__.py b/homeassistant/components/onvif/__init__.py index 77a5c6d1bd8..438a313c3ca 100644 --- a/homeassistant/components/onvif/__init__.py +++ b/homeassistant/components/onvif/__init__.py @@ -1,5 +1,7 @@ """The ONVIF integration.""" +from httpx import RequestError from onvif.exceptions import ONVIFAuthError, ONVIFError, ONVIFTimeoutError +from zeep.exceptions import Fault from homeassistant.components.ffmpeg import CONF_EXTRA_ARGUMENTS from homeassistant.components.stream import CONF_RTSP_TRANSPORT, RTSP_TRANSPORTS @@ -27,9 +29,25 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: device = ONVIFDevice(hass, entry) - if not await device.async_setup(): + try: + await device.async_setup() + except RequestError as err: await device.device.close() - return False + raise ConfigEntryNotReady( + f"Could not connect to camera {device.device.host}:{device.device.port}: {err}" + ) from err + except Fault as err: + await device.device.close() + # We do no know if the credentials are wrong or the camera is + # still booting up, so we will retry later + raise ConfigEntryNotReady( + f"Could not connect to camera, verify credentials are correct: {err}" + ) from err + except ONVIFError as err: + await device.device.close() + raise ConfigEntryNotReady( + f"Could not setup camera {device.device.host}:{device.device.port}: {err}" + ) from err if not device.available: raise ConfigEntryNotReady() diff --git a/homeassistant/components/onvif/device.py b/homeassistant/components/onvif/device.py index 1556ae8a1fe..d5baa829d05 100644 --- a/homeassistant/components/onvif/device.py +++ b/homeassistant/components/onvif/device.py @@ -83,7 +83,7 @@ class ONVIFDevice: """Return the password of this device.""" return self.config_entry.data[CONF_PASSWORD] - async def async_setup(self) -> bool: + async def async_setup(self) -> None: """Set up the device.""" self.device = get_device( self.hass, @@ -94,57 +94,34 @@ class ONVIFDevice: ) # Get all device info - try: - await self.device.update_xaddrs() - await self.async_check_date_and_time() + await self.device.update_xaddrs() + await self.async_check_date_and_time() - # Create event manager - assert self.config_entry.unique_id - self.events = EventManager( - self.hass, self.device, self.config_entry.unique_id - ) + # Create event manager + assert self.config_entry.unique_id + self.events = EventManager(self.hass, self.device, self.config_entry.unique_id) - # Fetch basic device info and capabilities - self.info = await self.async_get_device_info() - LOGGER.debug("Camera %s info = %s", self.name, self.info) - self.capabilities = await self.async_get_capabilities() - LOGGER.debug("Camera %s capabilities = %s", self.name, self.capabilities) - self.profiles = await self.async_get_profiles() - LOGGER.debug("Camera %s profiles = %s", self.name, self.profiles) + # Fetch basic device info and capabilities + self.info = await self.async_get_device_info() + LOGGER.debug("Camera %s info = %s", self.name, self.info) + self.capabilities = await self.async_get_capabilities() + LOGGER.debug("Camera %s capabilities = %s", self.name, self.capabilities) + self.profiles = await self.async_get_profiles() + LOGGER.debug("Camera %s profiles = %s", self.name, self.profiles) - # No camera profiles to add - if not self.profiles: - return False + # No camera profiles to add + if not self.profiles: + raise ONVIFError("No camera profiles found") - if self.capabilities.ptz: - self.device.create_ptz_service() + if self.capabilities.ptz: + self.device.create_ptz_service() - # Determine max resolution from profiles - self.max_resolution = max( - profile.video.resolution.width - for profile in self.profiles - if profile.video.encoding == "H264" - ) - except RequestError as err: - LOGGER.warning( - "Couldn't connect to camera '%s', but will retry later. Error: %s", - self.name, - err, - ) - self.available = False - await self.device.close() - except Fault as err: - LOGGER.error( - ( - "Couldn't connect to camera '%s', please verify " - "that the credentials are correct. Error: %s" - ), - self.name, - err, - ) - return False - - return True + # Determine max resolution from profiles + self.max_resolution = max( + profile.video.resolution.width + for profile in self.profiles + if profile.video.encoding == "H264" + ) async def async_stop(self, event=None): """Shut it all down.""" From 572f2cc1672918c030d204d923b86080cd333254 Mon Sep 17 00:00:00 2001 From: starkillerOG Date: Mon, 17 Apr 2023 10:48:39 +0200 Subject: [PATCH 089/110] Reolink ONVIF move read to primary callback (#91478) * Move read to primary callback * fix styling * Do not raise on ConnectionResetError * Split request.text() to .read() and decode("utf-8") --- homeassistant/components/reolink/host.py | 95 +++++++++++++----------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/homeassistant/components/reolink/host.py b/homeassistant/components/reolink/host.py index 1710219cfb3..6ddfa733d8d 100644 --- a/homeassistant/components/reolink/host.py +++ b/homeassistant/components/reolink/host.py @@ -358,58 +358,65 @@ class ReolinkHost: async def handle_webhook( self, hass: HomeAssistant, webhook_id: str, request: Request ) -> None: - """Shield the incoming webhook callback from cancellation.""" - shielded_future = asyncio.shield( - self._handle_webhook(hass, webhook_id, request) - ) + """Read the incoming webhook from Reolink for inbound messages and schedule processing.""" _LOGGER.debug("Webhook '%s' called", webhook_id) + data: bytes | None = None + try: + data = await request.read() + if not data: + _LOGGER.debug( + "Webhook '%s' triggered with unknown payload: %s", webhook_id, data + ) + except ConnectionResetError: + _LOGGER.debug( + "Webhook '%s' called, but lost connection before reading message " + "(ConnectionResetError), issuing poll", + webhook_id, + ) + return + except aiohttp.ClientResponseError: + _LOGGER.debug( + "Webhook '%s' called, but could not read the message, issuing poll", + webhook_id, + ) + return + except asyncio.CancelledError: + _LOGGER.debug( + "Webhook '%s' called, but lost connection before reading message " + "(CancelledError), issuing poll", + webhook_id, + ) + raise + finally: + # We want handle_webhook to return as soon as possible + # so we process the data in the background, this also shields from cancellation + hass.async_create_background_task( + self._process_webhook_data(hass, webhook_id, data), + "Process Reolink webhook", + ) + + async def _process_webhook_data( + self, hass: HomeAssistant, webhook_id: str, data: bytes | None + ) -> None: + """Process the data from the Reolink webhook.""" + # This task is executed in the background so we need to catch exceptions + # and log them if not self._webhook_reachable.is_set(): self._webhook_reachable.set() ir.async_delete_issue(self._hass, DOMAIN, "webhook_url") - await shielded_future - async def _handle_webhook( - self, hass: HomeAssistant, webhook_id: str, request: Request - ) -> None: - """Handle incoming webhook from Reolink for inbound messages and calls.""" try: - data = await request.text() - except ConnectionResetError: - # We lost the connection before reading the message, fallback to polling - # No need for a background task here as we already know the connection is lost - _LOGGER.debug( - "Webhook '%s' called, but lost connection before reading message, issuing poll", - webhook_id, - ) - if not await self._api.get_motion_state_all_ch(): - _LOGGER.error( - "Could not poll motion state after losing connection during receiving ONVIF event" - ) + if not data: + if not await self._api.get_motion_state_all_ch(): + _LOGGER.error( + "Could not poll motion state after losing connection during receiving ONVIF event" + ) + return + async_dispatcher_send(hass, f"{webhook_id}_all", {}) return - async_dispatcher_send(hass, f"{webhook_id}_all", {}) - return - if not data: - _LOGGER.debug( - "Webhook '%s' triggered with unknown payload: %s", webhook_id, data - ) - return - - # We received the data but we want handle_webhook to return as soon as possible - # so we process the data in the background - hass.async_create_background_task( - self._process_webhook_data(hass, webhook_id, data), - "Process Reolink webhook", - ) - - async def _process_webhook_data( - self, hass: HomeAssistant, webhook_id: str, data: str - ) -> None: - """Process the data from the webhook.""" - # This task is executed in the background so we need to catch exceptions - # and log them - try: - channels = await self._api.ONVIF_event_callback(data) + message = data.decode("utf-8") + channels = await self._api.ONVIF_event_callback(message) except Exception as ex: # pylint: disable=broad-except _LOGGER.exception( "Error processing ONVIF event for Reolink %s: %s", From 489a6e766b56be705becaed1a76e2d3365e3ea67 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Apr 2023 02:05:10 -1000 Subject: [PATCH 090/110] Fix onvif failing to reload (#91482) --- homeassistant/components/onvif/__init__.py | 16 ++++++---------- homeassistant/components/onvif/device.py | 2 ++ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/homeassistant/components/onvif/__init__.py b/homeassistant/components/onvif/__init__.py index 438a313c3ca..45fd04049ad 100644 --- a/homeassistant/components/onvif/__init__.py +++ b/homeassistant/components/onvif/__init__.py @@ -57,15 +57,15 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: hass.data[DOMAIN][entry.unique_id] = device - platforms = [Platform.BUTTON, Platform.CAMERA] + device.platforms = [Platform.BUTTON, Platform.CAMERA] if device.capabilities.events: - platforms += [Platform.BINARY_SENSOR, Platform.SENSOR] + device.platforms += [Platform.BINARY_SENSOR, Platform.SENSOR] if device.capabilities.imaging: - platforms += [Platform.SWITCH] + device.platforms += [Platform.SWITCH] - await hass.config_entries.async_forward_entry_setups(entry, platforms) + await hass.config_entries.async_forward_entry_setups(entry, device.platforms) entry.async_on_unload( hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, device.async_stop) @@ -77,16 +77,12 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: """Unload a config entry.""" - device = hass.data[DOMAIN][entry.unique_id] - platforms = ["camera"] + device: ONVIFDevice = hass.data[DOMAIN][entry.unique_id] if device.capabilities.events and device.events.started: - platforms += [Platform.BINARY_SENSOR, Platform.SENSOR] await device.events.async_stop() - if device.capabilities.imaging: - platforms += [Platform.SWITCH] - return await hass.config_entries.async_unload_platforms(entry, platforms) + return await hass.config_entries.async_unload_platforms(entry, device.platforms) async def _get_snapshot_auth(device): diff --git a/homeassistant/components/onvif/device.py b/homeassistant/components/onvif/device.py index d5baa829d05..a9f8625521e 100644 --- a/homeassistant/components/onvif/device.py +++ b/homeassistant/components/onvif/device.py @@ -20,6 +20,7 @@ from homeassistant.const import ( CONF_PASSWORD, CONF_PORT, CONF_USERNAME, + Platform, ) from homeassistant.core import HomeAssistant import homeassistant.util.dt as dt_util @@ -55,6 +56,7 @@ class ONVIFDevice: self.capabilities: Capabilities = Capabilities() self.profiles: list[Profile] = [] self.max_resolution: int = 0 + self.platforms: list[Platform] = [] self._dt_diff_seconds: float = 0 From 5ddc18f8ed4cbc783f06be420f8d6615067063a3 Mon Sep 17 00:00:00 2001 From: Ben Morton Date: Sun, 16 Apr 2023 20:32:51 +0100 Subject: [PATCH 091/110] Resolve issue with switchbot blind tilt devices getting stuck in opening/closing state (#91495) --- homeassistant/components/switchbot/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/switchbot/manifest.json b/homeassistant/components/switchbot/manifest.json index 31ce20bea3f..c90a1a64289 100644 --- a/homeassistant/components/switchbot/manifest.json +++ b/homeassistant/components/switchbot/manifest.json @@ -40,5 +40,5 @@ "documentation": "https://www.home-assistant.io/integrations/switchbot", "iot_class": "local_push", "loggers": ["switchbot"], - "requirements": ["PySwitchbot==0.37.5"] + "requirements": ["PySwitchbot==0.37.6"] } diff --git a/requirements_all.txt b/requirements_all.txt index 71b961ce954..fb1aee74e76 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -40,7 +40,7 @@ PyRMVtransport==0.3.3 PySocks==1.7.1 # homeassistant.components.switchbot -PySwitchbot==0.37.5 +PySwitchbot==0.37.6 # homeassistant.components.transport_nsw PyTransportNSW==0.1.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 3e75724010e..62831521f0b 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -36,7 +36,7 @@ PyRMVtransport==0.3.3 PySocks==1.7.1 # homeassistant.components.switchbot -PySwitchbot==0.37.5 +PySwitchbot==0.37.6 # homeassistant.components.transport_nsw PyTransportNSW==0.1.1 From 273e1fd2be122756e71ce1059c77b17b8b158252 Mon Sep 17 00:00:00 2001 From: rappenze Date: Mon, 17 Apr 2023 11:09:11 +0200 Subject: [PATCH 092/110] Fix state mapping in fibaro climate (#91505) --- homeassistant/components/fibaro/climate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/homeassistant/components/fibaro/climate.py b/homeassistant/components/fibaro/climate.py index 5f34e0d67dd..f4b1cd0c1f5 100644 --- a/homeassistant/components/fibaro/climate.py +++ b/homeassistant/components/fibaro/climate.py @@ -95,6 +95,7 @@ HA_OPMODES_HVAC = { HVACMode.COOL: 2, HVACMode.AUTO: 3, HVACMode.FAN_ONLY: 6, + HVACMode.DRY: 8, } TARGET_TEMP_ACTIONS = ( From 559ce6a275d5fa1c72178f0b7a0563d478d83274 Mon Sep 17 00:00:00 2001 From: Christopher Bailey Date: Sun, 16 Apr 2023 23:50:06 -0400 Subject: [PATCH 093/110] Bump unifiprotect to 4.8.1 (#91522) --- homeassistant/components/unifiprotect/config_flow.py | 3 +++ homeassistant/components/unifiprotect/manifest.json | 2 +- homeassistant/components/unifiprotect/utils.py | 3 +++ requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- tests/components/unifiprotect/conftest.py | 3 +++ 6 files changed, 12 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/unifiprotect/config_flow.py b/homeassistant/components/unifiprotect/config_flow.py index 571922d8651..1ca030ce48e 100644 --- a/homeassistant/components/unifiprotect/config_flow.py +++ b/homeassistant/components/unifiprotect/config_flow.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Mapping import logging +from pathlib import Path from typing import Any from aiohttp import CookieJar @@ -28,6 +29,7 @@ from homeassistant.helpers.aiohttp_client import ( async_create_clientsession, async_get_clientsession, ) +from homeassistant.helpers.storage import STORAGE_DIR from homeassistant.helpers.typing import DiscoveryInfoType from homeassistant.loader import async_get_integration from homeassistant.util.network import is_ip_address @@ -248,6 +250,7 @@ class ProtectFlowHandler(config_entries.ConfigFlow, domain=DOMAIN): username=user_input[CONF_USERNAME], password=user_input[CONF_PASSWORD], verify_ssl=verify_ssl, + cache_dir=Path(self.hass.config.path(STORAGE_DIR, "unifiprotect_cache")), ) errors = {} diff --git a/homeassistant/components/unifiprotect/manifest.json b/homeassistant/components/unifiprotect/manifest.json index 77570a1cb38..d229d8f71fe 100644 --- a/homeassistant/components/unifiprotect/manifest.json +++ b/homeassistant/components/unifiprotect/manifest.json @@ -41,7 +41,7 @@ "iot_class": "local_push", "loggers": ["pyunifiprotect", "unifi_discovery"], "quality_scale": "platinum", - "requirements": ["pyunifiprotect==4.7.0", "unifi-discovery==1.1.7"], + "requirements": ["pyunifiprotect==4.8.1", "unifi-discovery==1.1.7"], "ssdp": [ { "manufacturer": "Ubiquiti Networks", diff --git a/homeassistant/components/unifiprotect/utils.py b/homeassistant/components/unifiprotect/utils.py index f58bb14eb41..3152213cce8 100644 --- a/homeassistant/components/unifiprotect/utils.py +++ b/homeassistant/components/unifiprotect/utils.py @@ -4,6 +4,7 @@ from __future__ import annotations from collections.abc import Generator, Iterable import contextlib from enum import Enum +from pathlib import Path import socket from typing import Any @@ -27,6 +28,7 @@ from homeassistant.const import ( ) from homeassistant.core import HomeAssistant, callback from homeassistant.helpers.aiohttp_client import async_create_clientsession +from homeassistant.helpers.storage import STORAGE_DIR from .const import ( CONF_ALL_UPDATES, @@ -142,4 +144,5 @@ def async_create_api_client( override_connection_host=entry.options.get(CONF_OVERRIDE_CHOST, False), ignore_stats=not entry.options.get(CONF_ALL_UPDATES, False), ignore_unadopted=False, + cache_dir=Path(hass.config.path(STORAGE_DIR, "unifiprotect_cache")), ) diff --git a/requirements_all.txt b/requirements_all.txt index fb1aee74e76..2421e7303c9 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2150,7 +2150,7 @@ pytrafikverket==0.2.3 pyudev==0.23.2 # homeassistant.components.unifiprotect -pyunifiprotect==4.7.0 +pyunifiprotect==4.8.1 # homeassistant.components.uptimerobot pyuptimerobot==22.2.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 62831521f0b..dd116a13753 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1540,7 +1540,7 @@ pytrafikverket==0.2.3 pyudev==0.23.2 # homeassistant.components.unifiprotect -pyunifiprotect==4.7.0 +pyunifiprotect==4.8.1 # homeassistant.components.uptimerobot pyuptimerobot==22.2.0 diff --git a/tests/components/unifiprotect/conftest.py b/tests/components/unifiprotect/conftest.py index d66ed0ea060..fcfac60fa71 100644 --- a/tests/components/unifiprotect/conftest.py +++ b/tests/components/unifiprotect/conftest.py @@ -7,6 +7,8 @@ from datetime import datetime, timedelta from functools import partial from ipaddress import IPv4Address import json +from pathlib import Path +from tempfile import gettempdir from typing import Any from unittest.mock import AsyncMock, Mock, patch @@ -105,6 +107,7 @@ def mock_ufp_client(bootstrap: Bootstrap): client.bootstrap = bootstrap client._bootstrap = bootstrap client.api_path = "/api" + client.cache_dir = Path(gettempdir()) / "ufp_cache" # functionality from API client tests actually need client._stream_response = partial(ProtectApiClient._stream_response, client) client.get_camera_video = partial(ProtectApiClient.get_camera_video, client) From 940861e2be6c95985e2dab6df0ff17d21b19665f Mon Sep 17 00:00:00 2001 From: Franck Nijhof Date: Mon, 17 Apr 2023 15:37:08 +0200 Subject: [PATCH 094/110] Bumped version to 2023.4.5 --- homeassistant/const.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/const.py b/homeassistant/const.py index 845445d6b07..712e8ff0eb6 100644 --- a/homeassistant/const.py +++ b/homeassistant/const.py @@ -8,7 +8,7 @@ from .backports.enum import StrEnum APPLICATION_NAME: Final = "HomeAssistant" MAJOR_VERSION: Final = 2023 MINOR_VERSION: Final = 4 -PATCH_VERSION: Final = "4" +PATCH_VERSION: Final = "5" __short_version__: Final = f"{MAJOR_VERSION}.{MINOR_VERSION}" __version__: Final = f"{__short_version__}.{PATCH_VERSION}" REQUIRED_PYTHON_VER: Final[tuple[int, int, int]] = (3, 10, 0) diff --git a/pyproject.toml b/pyproject.toml index ea02d089a30..b2bcdb011b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "homeassistant" -version = "2023.4.4" +version = "2023.4.5" license = {text = "Apache-2.0"} description = "Open-source home automation platform running on Python 3." readme = "README.rst" From 03c517b066fce832fd224d228d7ad9268e0c248f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Apr 2023 03:26:41 -1000 Subject: [PATCH 095/110] Add a guard against selecting all entities in `state_changes_during_period` (#91585) Add a guard against selecting all entities in state_changes_during_period This cannot happen in `dev` because we require entity ids --- homeassistant/components/recorder/history/modern.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/recorder/history/modern.py b/homeassistant/components/recorder/history/modern.py index f7d08c6bba8..44950b8fe71 100644 --- a/homeassistant/components/recorder/history/modern.py +++ b/homeassistant/components/recorder/history/modern.py @@ -374,6 +374,8 @@ def state_changes_during_period( if entity_id: instance = recorder.get_instance(hass) metadata_id = instance.states_meta_manager.get(entity_id, session, False) + if metadata_id is None: + return {} entity_id_to_metadata_id = {entity_id: metadata_id} stmt = _state_changed_during_period_stmt( start_time, @@ -394,7 +396,7 @@ def state_changes_during_period( states, start_time, entity_ids, - entity_id_to_metadata_id, + entity_id_to_metadata_id, # type: ignore[arg-type] include_start_time_state=include_start_time_state, ), ) From 71f0f53ddc53457ebb5eb32b4b1a35db3dafbc97 Mon Sep 17 00:00:00 2001 From: Aaron Godfrey Date: Thu, 13 Apr 2023 21:12:58 -0700 Subject: [PATCH 096/110] Fix tasks with no due date from not triggering `on` calendar state. (#91196) Fix tasks with no due date. Prior to this change we were setting the start date/time to utc rather than the user's timezone. --- homeassistant/components/todoist/calendar.py | 2 +- tests/components/todoist/test_calendar.py | 54 ++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/todoist/calendar.py b/homeassistant/components/todoist/calendar.py index c3e8f61fcc8..ea5aab15344 100644 --- a/homeassistant/components/todoist/calendar.py +++ b/homeassistant/components/todoist/calendar.py @@ -446,7 +446,7 @@ class TodoistProjectData: LABELS: [], OVERDUE: False, PRIORITY: data.priority, - START: dt.utcnow(), + START: dt.now(), SUMMARY: data.content, } diff --git a/tests/components/todoist/test_calendar.py b/tests/components/todoist/test_calendar.py index 4f792b3cc01..d967c3dc035 100644 --- a/tests/components/todoist/test_calendar.py +++ b/tests/components/todoist/test_calendar.py @@ -25,6 +25,14 @@ from homeassistant.util import dt from tests.typing import ClientSessionGenerator +@pytest.fixture(autouse=True) +def set_time_zone(hass: HomeAssistant): + """Set the time zone for the tests.""" + # Set our timezone to CST/Regina so we can check calculations + # This keeps UTC-6 all year round + hass.config.set_time_zone("America/Regina") + + @pytest.fixture(name="task") def mock_task() -> Task: """Mock a todoist Task instance.""" @@ -132,6 +140,52 @@ async def test_update_entity_for_custom_project_with_labels_on( assert state.state == "on" +@patch("homeassistant.components.todoist.calendar.TodoistAPIAsync") +async def test_update_entity_for_custom_project_no_due_date_on( + todoist_api, hass: HomeAssistant, api +) -> None: + """Test that a task without an explicit due date is considered to be in an on state.""" + task_wo_due_date = Task( + assignee_id=None, + assigner_id=None, + comment_count=0, + is_completed=False, + content="No due date task", + created_at="2023-04-11T00:25:25.589971Z", + creator_id="1", + description="", + due=None, + id="123", + labels=["Label1"], + order=10, + parent_id=None, + priority=1, + project_id="12345", + section_id=None, + url="https://todoist.com/showTask?id=123", + sync_id=None, + ) + api.get_tasks.return_value = [task_wo_due_date] + todoist_api.return_value = api + + assert await setup.async_setup_component( + hass, + "calendar", + { + "calendar": { + "platform": DOMAIN, + CONF_TOKEN: "token", + "custom_projects": [{"name": "All projects", "labels": ["Label1"]}], + } + }, + ) + await hass.async_block_till_done() + + await async_update_entity(hass, "calendar.all_projects") + state = hass.states.get("calendar.all_projects") + assert state.state == "on" + + @patch("homeassistant.components.todoist.calendar.TodoistAPIAsync") async def test_failed_coordinator_update(todoist_api, hass: HomeAssistant, api) -> None: """Test a failed data coordinator update is handled correctly.""" From c544da74260627c1ce6eed44cf39fd77e32a718b Mon Sep 17 00:00:00 2001 From: Tom Harris Date: Wed, 19 Apr 2023 08:41:32 -0400 Subject: [PATCH 097/110] Fix Insteon thermostat issue (#91568) * Bump pyinsteon * Bump pyinsteon * Bump pyinsteon --- homeassistant/components/insteon/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/insteon/manifest.json b/homeassistant/components/insteon/manifest.json index d9c2380de0f..08adce918c1 100644 --- a/homeassistant/components/insteon/manifest.json +++ b/homeassistant/components/insteon/manifest.json @@ -17,7 +17,7 @@ "iot_class": "local_push", "loggers": ["pyinsteon", "pypubsub"], "requirements": [ - "pyinsteon==1.4.1", + "pyinsteon==1.4.2", "insteon-frontend-home-assistant==0.3.4" ], "usb": [ diff --git a/requirements_all.txt b/requirements_all.txt index 2421e7303c9..8af07350c5d 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1684,7 +1684,7 @@ pyialarm==2.2.0 pyicloud==1.0.0 # homeassistant.components.insteon -pyinsteon==1.4.1 +pyinsteon==1.4.2 # homeassistant.components.intesishome pyintesishome==1.8.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index dd116a13753..900861f5172 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1218,7 +1218,7 @@ pyialarm==2.2.0 pyicloud==1.0.0 # homeassistant.components.insteon -pyinsteon==1.4.1 +pyinsteon==1.4.2 # homeassistant.components.ipma pyipma==3.0.6 From 6b02892c285704874742f6bcebb28204a66addf4 Mon Sep 17 00:00:00 2001 From: Duco Sebel <74970928+DCSBL@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:14:28 +0200 Subject: [PATCH 098/110] Handle UnsupportedError in HomeWizard (#91608) * Handle UnsupportedEror * Make error message more clear * Remove debug line, whoops --- .../components/homewizard/coordinator.py | 23 ++++++++--- tests/components/homewizard/test_switch.py | 38 ++++++++++++++++++- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/homeassistant/components/homewizard/coordinator.py b/homeassistant/components/homewizard/coordinator.py index 533af445c84..fb89989b2a5 100644 --- a/homeassistant/components/homewizard/coordinator.py +++ b/homeassistant/components/homewizard/coordinator.py @@ -5,7 +5,7 @@ import logging from homewizard_energy import HomeWizardEnergy from homewizard_energy.const import SUPPORTS_IDENTIFY, SUPPORTS_STATE, SUPPORTS_SYSTEM -from homewizard_energy.errors import DisabledError, RequestError +from homewizard_energy.errors import DisabledError, RequestError, UnsupportedError from homewizard_energy.models import Device from homeassistant.config_entries import ConfigEntry @@ -24,6 +24,8 @@ class HWEnergyDeviceUpdateCoordinator(DataUpdateCoordinator[DeviceResponseEntry] api: HomeWizardEnergy api_disabled: bool = False + _unsupported_error: bool = False + def __init__( self, hass: HomeAssistant, @@ -43,11 +45,22 @@ class HWEnergyDeviceUpdateCoordinator(DataUpdateCoordinator[DeviceResponseEntry] data=await self.api.data(), ) - if self.supports_state(data.device): - data.state = await self.api.state() + try: + if self.supports_state(data.device): + data.state = await self.api.state() - if self.supports_system(data.device): - data.system = await self.api.system() + if self.supports_system(data.device): + data.system = await self.api.system() + + except UnsupportedError as ex: + # Old firmware, ignore + if not self._unsupported_error: + self._unsupported_error = True + _LOGGER.warning( + "%s is running an outdated firmware version (%s). Contact HomeWizard support to update your device", + self.entry.title, + ex, + ) except RequestError as ex: raise UpdateFailed(ex) from ex diff --git a/tests/components/homewizard/test_switch.py b/tests/components/homewizard/test_switch.py index f55550ee825..6a2623e964f 100644 --- a/tests/components/homewizard/test_switch.py +++ b/tests/components/homewizard/test_switch.py @@ -1,7 +1,7 @@ """Test the update coordinator for HomeWizard.""" from unittest.mock import AsyncMock, patch -from homewizard_energy.errors import DisabledError, RequestError +from homewizard_energy.errors import DisabledError, RequestError, UnsupportedError from homewizard_energy.models import State, System import pytest @@ -507,3 +507,39 @@ async def test_switch_handles_disablederror( {"entity_id": "switch.product_name_aabbccddeeff_cloud_connection"}, blocking=True, ) + + +async def test_switch_handles_unsupportedrrror( + hass: HomeAssistant, mock_config_entry_data, mock_config_entry +) -> None: + """Test entity raises HomeAssistantError when Disabled was raised.""" + + api = get_mock_device(product_type="HWE-SKT", firmware_version="3.02") + api.state = AsyncMock(side_effect=UnsupportedError()) + api.system = AsyncMock(side_effect=UnsupportedError()) + + with patch( + "homeassistant.components.homewizard.coordinator.HomeWizardEnergy", + return_value=api, + ): + entry = mock_config_entry + entry.data = mock_config_entry_data + entry.add_to_hass(hass) + + await hass.config_entries.async_setup(entry.entry_id) + await hass.async_block_till_done() + + assert ( + hass.states.get("switch.product_name_aabbccddeeff_cloud_connection").state + == STATE_UNAVAILABLE + ) + + assert ( + hass.states.get("switch.product_name_aabbccddeeff_switch_lock").state + == STATE_UNAVAILABLE + ) + + assert ( + hass.states.get("switch.product_name_aabbccddeeff").state + == STATE_UNAVAILABLE + ) From 38de9765dfcad2125c236d9b2da6418102ba1f82 Mon Sep 17 00:00:00 2001 From: epenet <6771947+epenet@users.noreply.github.com> Date: Tue, 18 Apr 2023 19:33:09 +0200 Subject: [PATCH 099/110] Bump renault-api to 0.1.13 (#91609) --- homeassistant/components/renault/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/renault/manifest.json b/homeassistant/components/renault/manifest.json index 9fade49b4b4..5f2670fb170 100644 --- a/homeassistant/components/renault/manifest.json +++ b/homeassistant/components/renault/manifest.json @@ -8,5 +8,5 @@ "iot_class": "cloud_polling", "loggers": ["renault_api"], "quality_scale": "platinum", - "requirements": ["renault-api==0.1.12"] + "requirements": ["renault-api==0.1.13"] } diff --git a/requirements_all.txt b/requirements_all.txt index 8af07350c5d..e61c2e922d5 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2228,7 +2228,7 @@ raspyrfm-client==1.2.8 regenmaschine==2022.11.0 # homeassistant.components.renault -renault-api==0.1.12 +renault-api==0.1.13 # homeassistant.components.reolink reolink-aio==0.5.10 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 900861f5172..2434743e222 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1591,7 +1591,7 @@ radiotherm==2.1.0 regenmaschine==2022.11.0 # homeassistant.components.renault -renault-api==0.1.12 +renault-api==0.1.13 # homeassistant.components.reolink reolink-aio==0.5.10 From 36d2accb5bead9d0a826b390f3ad91df543e2130 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Apr 2023 02:40:04 -1000 Subject: [PATCH 100/110] Handle long format context UUIDs during migration (#91657) In https://github.com/home-assistant/core/issues/91514 is was discovered these exist in older versions --- .../components/recorder/migration.py | 3 +- .../recorder/test_migration_from_schema_32.py | 775 ++++++++++++++++++ 2 files changed, 776 insertions(+), 2 deletions(-) create mode 100644 tests/components/recorder/test_migration_from_schema_32.py diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py index 4b0244038e8..8b50b419f1b 100644 --- a/homeassistant/components/recorder/migration.py +++ b/homeassistant/components/recorder/migration.py @@ -1364,10 +1364,9 @@ def _context_id_to_bytes(context_id: str | None) -> bytes | None: # ULIDs that filled the column to the max length # so we need to catch the ValueError and return # None if it happens - if len(context_id) == 32: - return UUID(context_id).bytes if len(context_id) == 26: return ulid_to_bytes(context_id) + return UUID(context_id).bytes return None diff --git a/tests/components/recorder/test_migration_from_schema_32.py b/tests/components/recorder/test_migration_from_schema_32.py new file mode 100644 index 00000000000..01c086e119a --- /dev/null +++ b/tests/components/recorder/test_migration_from_schema_32.py @@ -0,0 +1,775 @@ +"""The tests for the recorder filter matching the EntityFilter component.""" +# pylint: disable=invalid-name +import importlib +import sys +from unittest.mock import patch +import uuid + +import pytest +from sqlalchemy import create_engine, inspect +from sqlalchemy.orm import Session + +from homeassistant.components import recorder +from homeassistant.components.recorder import core, migration, statistics +from homeassistant.components.recorder.db_schema import ( + Events, + EventTypes, + States, + StatesMeta, +) +from homeassistant.components.recorder.queries import select_event_type_ids +from homeassistant.components.recorder.tasks import ( + EntityIDMigrationTask, + EntityIDPostMigrationTask, + EventsContextIDMigrationTask, + EventTypeIDMigrationTask, + StatesContextIDMigrationTask, +) +from homeassistant.components.recorder.util import session_scope +from homeassistant.core import HomeAssistant +import homeassistant.util.dt as dt_util +from homeassistant.util.ulid import bytes_to_ulid + +from .common import async_recorder_block_till_done, async_wait_recording_done + +from tests.typing import RecorderInstanceGenerator + +CREATE_ENGINE_TARGET = "homeassistant.components.recorder.core.create_engine" +SCHEMA_MODULE = "tests.components.recorder.db_schema_32" +ORIG_TZ = dt_util.DEFAULT_TIME_ZONE + + +def _create_engine_test(*args, **kwargs): + """Test version of create_engine that initializes with old schema. + + This simulates an existing db with the old schema. + """ + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] + engine = create_engine(*args, **kwargs) + old_db_schema.Base.metadata.create_all(engine) + with Session(engine) as session: + session.add( + recorder.db_schema.StatisticsRuns(start=statistics.get_start_time()) + ) + session.add( + recorder.db_schema.SchemaChanges( + schema_version=old_db_schema.SCHEMA_VERSION + ) + ) + session.commit() + return engine + + +@pytest.fixture(autouse=True) +def db_schema_32(): + """Fixture to initialize the db with the old schema.""" + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] + + with patch.object(recorder, "db_schema", old_db_schema), patch.object( + recorder.migration, "SCHEMA_VERSION", old_db_schema.SCHEMA_VERSION + ), patch.object(core, "StatesMeta", old_db_schema.StatesMeta), patch.object( + core, "EventTypes", old_db_schema.EventTypes + ), patch.object( + core, "EventData", old_db_schema.EventData + ), patch.object( + core, "States", old_db_schema.States + ), patch.object( + core, "Events", old_db_schema.Events + ), patch.object( + core, "StateAttributes", old_db_schema.StateAttributes + ), patch.object( + core, "EntityIDMigrationTask", core.RecorderTask + ), patch( + CREATE_ENGINE_TARGET, new=_create_engine_test + ): + yield + + +@pytest.fixture(name="legacy_recorder_mock") +async def legacy_recorder_mock_fixture(recorder_mock): + """Fixture for legacy recorder mock.""" + with patch.object(recorder_mock.states_meta_manager, "active", False): + yield recorder_mock + + +@pytest.mark.parametrize("enable_migrate_context_ids", [True]) +async def test_migrate_events_context_ids( + async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant +) -> None: + """Test we can migrate old uuid context ids and ulid context ids to binary format.""" + instance = await async_setup_recorder_instance(hass) + await async_wait_recording_done(hass) + + test_uuid = uuid.uuid4() + uuid_hex = test_uuid.hex + uuid_bin = test_uuid.bytes + + def _insert_events(): + with session_scope(hass=hass) as session: + session.add_all( + ( + Events( + event_type="old_uuid_context_id_event", + event_data=None, + origin_idx=0, + time_fired=None, + time_fired_ts=1677721632.452529, + context_id=uuid_hex, + context_id_bin=None, + context_user_id=None, + context_user_id_bin=None, + context_parent_id=None, + context_parent_id_bin=None, + ), + Events( + event_type="empty_context_id_event", + event_data=None, + origin_idx=0, + time_fired=None, + time_fired_ts=1677721632.552529, + context_id=None, + context_id_bin=None, + context_user_id=None, + context_user_id_bin=None, + context_parent_id=None, + context_parent_id_bin=None, + ), + Events( + event_type="ulid_context_id_event", + event_data=None, + origin_idx=0, + time_fired=None, + time_fired_ts=1677721632.552529, + context_id="01ARZ3NDEKTSV4RRFFQ69G5FAV", + context_id_bin=None, + context_user_id="9400facee45711eaa9308bfd3d19e474", + context_user_id_bin=None, + context_parent_id="01ARZ3NDEKTSV4RRFFQ69G5FA2", + context_parent_id_bin=None, + ), + Events( + event_type="invalid_context_id_event", + event_data=None, + origin_idx=0, + time_fired=None, + time_fired_ts=1677721632.552529, + context_id="invalid", + context_id_bin=None, + context_user_id=None, + context_user_id_bin=None, + context_parent_id=None, + context_parent_id_bin=None, + ), + Events( + event_type="garbage_context_id_event", + event_data=None, + origin_idx=0, + time_fired=None, + time_fired_ts=1677721632.552529, + context_id="adapt_lgt:b'5Cf*':interval:b'0R'", + context_id_bin=None, + context_user_id=None, + context_user_id_bin=None, + context_parent_id=None, + context_parent_id_bin=None, + ), + ) + ) + + await instance.async_add_executor_job(_insert_events) + + await async_wait_recording_done(hass) + # This is a threadsafe way to add a task to the recorder + instance.queue_task(EventsContextIDMigrationTask()) + await async_recorder_block_till_done(hass) + + def _object_as_dict(obj): + return {c.key: getattr(obj, c.key) for c in inspect(obj).mapper.column_attrs} + + def _fetch_migrated_events(): + with session_scope(hass=hass) as session: + events = ( + session.query(Events) + .filter( + Events.event_type.in_( + [ + "old_uuid_context_id_event", + "empty_context_id_event", + "ulid_context_id_event", + "invalid_context_id_event", + "garbage_context_id_event", + ] + ) + ) + .all() + ) + assert len(events) == 5 + return {event.event_type: _object_as_dict(event) for event in events} + + events_by_type = await instance.async_add_executor_job(_fetch_migrated_events) + + old_uuid_context_id_event = events_by_type["old_uuid_context_id_event"] + assert old_uuid_context_id_event["context_id"] is None + assert old_uuid_context_id_event["context_user_id"] is None + assert old_uuid_context_id_event["context_parent_id"] is None + assert old_uuid_context_id_event["context_id_bin"] == uuid_bin + assert old_uuid_context_id_event["context_user_id_bin"] is None + assert old_uuid_context_id_event["context_parent_id_bin"] is None + + empty_context_id_event = events_by_type["empty_context_id_event"] + assert empty_context_id_event["context_id"] is None + assert empty_context_id_event["context_user_id"] is None + assert empty_context_id_event["context_parent_id"] is None + assert empty_context_id_event["context_id_bin"] == b"\x00" * 16 + assert empty_context_id_event["context_user_id_bin"] is None + assert empty_context_id_event["context_parent_id_bin"] is None + + ulid_context_id_event = events_by_type["ulid_context_id_event"] + assert ulid_context_id_event["context_id"] is None + assert ulid_context_id_event["context_user_id"] is None + assert ulid_context_id_event["context_parent_id"] is None + assert ( + bytes_to_ulid(ulid_context_id_event["context_id_bin"]) + == "01ARZ3NDEKTSV4RRFFQ69G5FAV" + ) + assert ( + ulid_context_id_event["context_user_id_bin"] + == b"\x94\x00\xfa\xce\xe4W\x11\xea\xa90\x8b\xfd=\x19\xe4t" + ) + assert ( + bytes_to_ulid(ulid_context_id_event["context_parent_id_bin"]) + == "01ARZ3NDEKTSV4RRFFQ69G5FA2" + ) + + invalid_context_id_event = events_by_type["invalid_context_id_event"] + assert invalid_context_id_event["context_id"] is None + assert invalid_context_id_event["context_user_id"] is None + assert invalid_context_id_event["context_parent_id"] is None + assert invalid_context_id_event["context_id_bin"] == b"\x00" * 16 + assert invalid_context_id_event["context_user_id_bin"] is None + assert invalid_context_id_event["context_parent_id_bin"] is None + + garbage_context_id_event = events_by_type["garbage_context_id_event"] + assert garbage_context_id_event["context_id"] is None + assert garbage_context_id_event["context_user_id"] is None + assert garbage_context_id_event["context_parent_id"] is None + assert garbage_context_id_event["context_id_bin"] == b"\x00" * 16 + assert garbage_context_id_event["context_user_id_bin"] is None + assert garbage_context_id_event["context_parent_id_bin"] is None + + +@pytest.mark.parametrize("enable_migrate_context_ids", [True]) +async def test_migrate_states_context_ids( + async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant +) -> None: + """Test we can migrate old uuid context ids and ulid context ids to binary format.""" + instance = await async_setup_recorder_instance(hass) + await async_wait_recording_done(hass) + + test_uuid = uuid.uuid4() + uuid_hex = test_uuid.hex + uuid_bin = test_uuid.bytes + + def _insert_events(): + with session_scope(hass=hass) as session: + session.add_all( + ( + States( + entity_id="state.old_uuid_context_id", + last_updated_ts=1677721632.452529, + context_id=uuid_hex, + context_id_bin=None, + context_user_id=None, + context_user_id_bin=None, + context_parent_id=None, + context_parent_id_bin=None, + ), + States( + entity_id="state.empty_context_id", + last_updated_ts=1677721632.552529, + context_id=None, + context_id_bin=None, + context_user_id=None, + context_user_id_bin=None, + context_parent_id=None, + context_parent_id_bin=None, + ), + States( + entity_id="state.ulid_context_id", + last_updated_ts=1677721632.552529, + context_id="01ARZ3NDEKTSV4RRFFQ69G5FAV", + context_id_bin=None, + context_user_id="9400facee45711eaa9308bfd3d19e474", + context_user_id_bin=None, + context_parent_id="01ARZ3NDEKTSV4RRFFQ69G5FA2", + context_parent_id_bin=None, + ), + States( + entity_id="state.invalid_context_id", + last_updated_ts=1677721632.552529, + context_id="invalid", + context_id_bin=None, + context_user_id=None, + context_user_id_bin=None, + context_parent_id=None, + context_parent_id_bin=None, + ), + States( + entity_id="state.garbage_context_id", + last_updated_ts=1677721632.552529, + context_id="adapt_lgt:b'5Cf*':interval:b'0R'", + context_id_bin=None, + context_user_id=None, + context_user_id_bin=None, + context_parent_id=None, + context_parent_id_bin=None, + ), + States( + entity_id="state.human_readable_uuid_context_id", + last_updated_ts=1677721632.552529, + context_id="0ae29799-ee4e-4f45-8116-f582d7d3ee65", + context_id_bin=None, + context_user_id="0ae29799-ee4e-4f45-8116-f582d7d3ee65", + context_user_id_bin=None, + context_parent_id="0ae29799-ee4e-4f45-8116-f582d7d3ee65", + context_parent_id_bin=None, + ), + ) + ) + + await instance.async_add_executor_job(_insert_events) + + await async_wait_recording_done(hass) + # This is a threadsafe way to add a task to the recorder + instance.queue_task(StatesContextIDMigrationTask()) + await async_recorder_block_till_done(hass) + + def _object_as_dict(obj): + return {c.key: getattr(obj, c.key) for c in inspect(obj).mapper.column_attrs} + + def _fetch_migrated_states(): + with session_scope(hass=hass) as session: + events = ( + session.query(States) + .filter( + States.entity_id.in_( + [ + "state.old_uuid_context_id", + "state.empty_context_id", + "state.ulid_context_id", + "state.invalid_context_id", + "state.garbage_context_id", + "state.human_readable_uuid_context_id", + ] + ) + ) + .all() + ) + assert len(events) == 6 + return {state.entity_id: _object_as_dict(state) for state in events} + + states_by_entity_id = await instance.async_add_executor_job(_fetch_migrated_states) + + old_uuid_context_id = states_by_entity_id["state.old_uuid_context_id"] + assert old_uuid_context_id["context_id"] is None + assert old_uuid_context_id["context_user_id"] is None + assert old_uuid_context_id["context_parent_id"] is None + assert old_uuid_context_id["context_id_bin"] == uuid_bin + assert old_uuid_context_id["context_user_id_bin"] is None + assert old_uuid_context_id["context_parent_id_bin"] is None + + empty_context_id = states_by_entity_id["state.empty_context_id"] + assert empty_context_id["context_id"] is None + assert empty_context_id["context_user_id"] is None + assert empty_context_id["context_parent_id"] is None + assert empty_context_id["context_id_bin"] == b"\x00" * 16 + assert empty_context_id["context_user_id_bin"] is None + assert empty_context_id["context_parent_id_bin"] is None + + ulid_context_id = states_by_entity_id["state.ulid_context_id"] + assert ulid_context_id["context_id"] is None + assert ulid_context_id["context_user_id"] is None + assert ulid_context_id["context_parent_id"] is None + assert ( + bytes_to_ulid(ulid_context_id["context_id_bin"]) == "01ARZ3NDEKTSV4RRFFQ69G5FAV" + ) + assert ( + ulid_context_id["context_user_id_bin"] + == b"\x94\x00\xfa\xce\xe4W\x11\xea\xa90\x8b\xfd=\x19\xe4t" + ) + assert ( + bytes_to_ulid(ulid_context_id["context_parent_id_bin"]) + == "01ARZ3NDEKTSV4RRFFQ69G5FA2" + ) + + invalid_context_id = states_by_entity_id["state.invalid_context_id"] + assert invalid_context_id["context_id"] is None + assert invalid_context_id["context_user_id"] is None + assert invalid_context_id["context_parent_id"] is None + assert invalid_context_id["context_id_bin"] == b"\x00" * 16 + assert invalid_context_id["context_user_id_bin"] is None + assert invalid_context_id["context_parent_id_bin"] is None + + garbage_context_id = states_by_entity_id["state.garbage_context_id"] + assert garbage_context_id["context_id"] is None + assert garbage_context_id["context_user_id"] is None + assert garbage_context_id["context_parent_id"] is None + assert garbage_context_id["context_id_bin"] == b"\x00" * 16 + assert garbage_context_id["context_user_id_bin"] is None + assert garbage_context_id["context_parent_id_bin"] is None + + human_readable_uuid_context_id = states_by_entity_id[ + "state.human_readable_uuid_context_id" + ] + assert human_readable_uuid_context_id["context_id"] is None + assert human_readable_uuid_context_id["context_user_id"] is None + assert human_readable_uuid_context_id["context_parent_id"] is None + assert ( + human_readable_uuid_context_id["context_id_bin"] + == b"\n\xe2\x97\x99\xeeNOE\x81\x16\xf5\x82\xd7\xd3\xeee" + ) + assert ( + human_readable_uuid_context_id["context_user_id_bin"] + == b"\n\xe2\x97\x99\xeeNOE\x81\x16\xf5\x82\xd7\xd3\xeee" + ) + assert ( + human_readable_uuid_context_id["context_parent_id_bin"] + == b"\n\xe2\x97\x99\xeeNOE\x81\x16\xf5\x82\xd7\xd3\xeee" + ) + + +@pytest.mark.parametrize("enable_migrate_event_type_ids", [True]) +async def test_migrate_event_type_ids( + async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant +) -> None: + """Test we can migrate event_types to the EventTypes table.""" + instance = await async_setup_recorder_instance(hass) + await async_wait_recording_done(hass) + + def _insert_events(): + with session_scope(hass=hass) as session: + session.add_all( + ( + Events( + event_type="event_type_one", + origin_idx=0, + time_fired_ts=1677721632.452529, + ), + Events( + event_type="event_type_one", + origin_idx=0, + time_fired_ts=1677721632.552529, + ), + Events( + event_type="event_type_two", + origin_idx=0, + time_fired_ts=1677721632.552529, + ), + ) + ) + + await instance.async_add_executor_job(_insert_events) + + await async_wait_recording_done(hass) + # This is a threadsafe way to add a task to the recorder + instance.queue_task(EventTypeIDMigrationTask()) + await async_recorder_block_till_done(hass) + + def _fetch_migrated_events(): + with session_scope(hass=hass, read_only=True) as session: + events = ( + session.query(Events.event_id, Events.time_fired, EventTypes.event_type) + .filter( + Events.event_type_id.in_( + select_event_type_ids( + ( + "event_type_one", + "event_type_two", + ) + ) + ) + ) + .outerjoin(EventTypes, Events.event_type_id == EventTypes.event_type_id) + .all() + ) + assert len(events) == 3 + result = {} + for event in events: + result.setdefault(event.event_type, []).append( + { + "event_id": event.event_id, + "time_fired": event.time_fired, + "event_type": event.event_type, + } + ) + return result + + events_by_type = await instance.async_add_executor_job(_fetch_migrated_events) + assert len(events_by_type["event_type_one"]) == 2 + assert len(events_by_type["event_type_two"]) == 1 + + +@pytest.mark.parametrize("enable_migrate_entity_ids", [True]) +async def test_migrate_entity_ids( + async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant +) -> None: + """Test we can migrate entity_ids to the StatesMeta table.""" + instance = await async_setup_recorder_instance(hass) + await async_wait_recording_done(hass) + + def _insert_states(): + with session_scope(hass=hass) as session: + session.add_all( + ( + States( + entity_id="sensor.one", + state="one_1", + last_updated_ts=1.452529, + ), + States( + entity_id="sensor.two", + state="two_2", + last_updated_ts=2.252529, + ), + States( + entity_id="sensor.two", + state="two_1", + last_updated_ts=3.152529, + ), + ) + ) + + await instance.async_add_executor_job(_insert_states) + + await async_wait_recording_done(hass) + # This is a threadsafe way to add a task to the recorder + instance.queue_task(EntityIDMigrationTask()) + await async_recorder_block_till_done(hass) + + def _fetch_migrated_states(): + with session_scope(hass=hass, read_only=True) as session: + states = ( + session.query( + States.state, + States.metadata_id, + States.last_updated_ts, + StatesMeta.entity_id, + ) + .outerjoin(StatesMeta, States.metadata_id == StatesMeta.metadata_id) + .all() + ) + assert len(states) == 3 + result = {} + for state in states: + result.setdefault(state.entity_id, []).append( + { + "state_id": state.entity_id, + "last_updated_ts": state.last_updated_ts, + "state": state.state, + } + ) + return result + + states_by_entity_id = await instance.async_add_executor_job(_fetch_migrated_states) + assert len(states_by_entity_id["sensor.two"]) == 2 + assert len(states_by_entity_id["sensor.one"]) == 1 + + +@pytest.mark.parametrize("enable_migrate_entity_ids", [True]) +async def test_post_migrate_entity_ids( + async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant +) -> None: + """Test we can migrate entity_ids to the StatesMeta table.""" + instance = await async_setup_recorder_instance(hass) + await async_wait_recording_done(hass) + + def _insert_events(): + with session_scope(hass=hass) as session: + session.add_all( + ( + States( + entity_id="sensor.one", + state="one_1", + last_updated_ts=1.452529, + ), + States( + entity_id="sensor.two", + state="two_2", + last_updated_ts=2.252529, + ), + States( + entity_id="sensor.two", + state="two_1", + last_updated_ts=3.152529, + ), + ) + ) + + await instance.async_add_executor_job(_insert_events) + + await async_wait_recording_done(hass) + # This is a threadsafe way to add a task to the recorder + instance.queue_task(EntityIDPostMigrationTask()) + await async_recorder_block_till_done(hass) + + def _fetch_migrated_states(): + with session_scope(hass=hass, read_only=True) as session: + states = session.query( + States.state, + States.entity_id, + ).all() + assert len(states) == 3 + return {state.state: state.entity_id for state in states} + + states_by_state = await instance.async_add_executor_job(_fetch_migrated_states) + assert states_by_state["one_1"] is None + assert states_by_state["two_2"] is None + assert states_by_state["two_1"] is None + + +@pytest.mark.parametrize("enable_migrate_entity_ids", [True]) +async def test_migrate_null_entity_ids( + async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant +) -> None: + """Test we can migrate entity_ids to the StatesMeta table.""" + instance = await async_setup_recorder_instance(hass) + await async_wait_recording_done(hass) + + def _insert_states(): + with session_scope(hass=hass) as session: + session.add( + States( + entity_id="sensor.one", + state="one_1", + last_updated_ts=1.452529, + ), + ) + session.add_all( + States( + entity_id=None, + state="empty", + last_updated_ts=time + 1.452529, + ) + for time in range(1000) + ) + session.add( + States( + entity_id="sensor.one", + state="one_1", + last_updated_ts=2.452529, + ), + ) + + await instance.async_add_executor_job(_insert_states) + + await async_wait_recording_done(hass) + # This is a threadsafe way to add a task to the recorder + instance.queue_task(EntityIDMigrationTask()) + await async_recorder_block_till_done(hass) + await async_recorder_block_till_done(hass) + + def _fetch_migrated_states(): + with session_scope(hass=hass, read_only=True) as session: + states = ( + session.query( + States.state, + States.metadata_id, + States.last_updated_ts, + StatesMeta.entity_id, + ) + .outerjoin(StatesMeta, States.metadata_id == StatesMeta.metadata_id) + .all() + ) + assert len(states) == 1002 + result = {} + for state in states: + result.setdefault(state.entity_id, []).append( + { + "state_id": state.entity_id, + "last_updated_ts": state.last_updated_ts, + "state": state.state, + } + ) + return result + + states_by_entity_id = await instance.async_add_executor_job(_fetch_migrated_states) + assert len(states_by_entity_id[migration._EMPTY_ENTITY_ID]) == 1000 + assert len(states_by_entity_id["sensor.one"]) == 2 + + +@pytest.mark.parametrize("enable_migrate_event_type_ids", [True]) +async def test_migrate_null_event_type_ids( + async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant +) -> None: + """Test we can migrate event_types to the EventTypes table when the event_type is NULL.""" + instance = await async_setup_recorder_instance(hass) + await async_wait_recording_done(hass) + + def _insert_events(): + with session_scope(hass=hass) as session: + session.add( + Events( + event_type="event_type_one", + origin_idx=0, + time_fired_ts=1.452529, + ), + ) + session.add_all( + Events( + event_type=None, + origin_idx=0, + time_fired_ts=time + 1.452529, + ) + for time in range(1000) + ) + session.add( + Events( + event_type="event_type_one", + origin_idx=0, + time_fired_ts=2.452529, + ), + ) + + await instance.async_add_executor_job(_insert_events) + + await async_wait_recording_done(hass) + # This is a threadsafe way to add a task to the recorder + + instance.queue_task(EventTypeIDMigrationTask()) + await async_recorder_block_till_done(hass) + await async_recorder_block_till_done(hass) + + def _fetch_migrated_events(): + with session_scope(hass=hass, read_only=True) as session: + events = ( + session.query(Events.event_id, Events.time_fired, EventTypes.event_type) + .filter( + Events.event_type_id.in_( + select_event_type_ids( + ( + "event_type_one", + migration._EMPTY_EVENT_TYPE, + ) + ) + ) + ) + .outerjoin(EventTypes, Events.event_type_id == EventTypes.event_type_id) + .all() + ) + assert len(events) == 1002 + result = {} + for event in events: + result.setdefault(event.event_type, []).append( + { + "event_id": event.event_id, + "time_fired": event.time_fired, + "event_type": event.event_type, + } + ) + return result + + events_by_type = await instance.async_add_executor_job(_fetch_migrated_events) + assert len(events_by_type["event_type_one"]) == 2 + assert len(events_by_type[migration._EMPTY_EVENT_TYPE]) == 1000 From fc4e8e5e7b66dfddc757f6024b9a8253c6196d43 Mon Sep 17 00:00:00 2001 From: Shay Levy Date: Wed, 19 Apr 2023 20:26:45 +0300 Subject: [PATCH 101/110] Bump aioshelly to 5.3.2 (#91679) --- homeassistant/components/shelly/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/shelly/manifest.json b/homeassistant/components/shelly/manifest.json index de085925549..39a1427346d 100644 --- a/homeassistant/components/shelly/manifest.json +++ b/homeassistant/components/shelly/manifest.json @@ -9,7 +9,7 @@ "iot_class": "local_push", "loggers": ["aioshelly"], "quality_scale": "platinum", - "requirements": ["aioshelly==5.3.1"], + "requirements": ["aioshelly==5.3.2"], "zeroconf": [ { "type": "_http._tcp.local.", diff --git a/requirements_all.txt b/requirements_all.txt index e61c2e922d5..05dc2280c15 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -267,7 +267,7 @@ aiosenseme==0.6.1 aiosenz==1.0.0 # homeassistant.components.shelly -aioshelly==5.3.1 +aioshelly==5.3.2 # homeassistant.components.skybell aioskybell==22.7.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 2434743e222..7cdc39abba9 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -248,7 +248,7 @@ aiosenseme==0.6.1 aiosenz==1.0.0 # homeassistant.components.shelly -aioshelly==5.3.1 +aioshelly==5.3.2 # homeassistant.components.skybell aioskybell==22.7.0 From 591ffe23400b6620efc4615fe682a4d0fd6d8568 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Apr 2023 15:56:07 -1000 Subject: [PATCH 102/110] Fallback to generating a new ULID on migraiton if context is missing or invalid (#91704) * Fallback to generating a new ULID on migraiton if context is missing or invalid It was discovered that postgresql will do a full scan if there is a low cardinality on the index because of missing context ids. We will now generate a ULID for the timestamp of the row if the context data is missing or invalid fixes #91514 * tests * tweak * tweak * preen --- .../components/recorder/migration.py | 19 ++-- homeassistant/components/recorder/queries.py | 2 + .../recorder/test_migration_from_schema_32.py | 93 ++++++++++++++----- 3 files changed, 83 insertions(+), 31 deletions(-) diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py index 8b50b419f1b..c487f0b70d7 100644 --- a/homeassistant/components/recorder/migration.py +++ b/homeassistant/components/recorder/migration.py @@ -6,6 +6,7 @@ import contextlib from dataclasses import dataclass, replace as dataclass_replace from datetime import timedelta import logging +from time import time from typing import TYPE_CHECKING, cast from uuid import UUID @@ -26,7 +27,7 @@ from sqlalchemy.sql.expression import true from homeassistant.core import HomeAssistant from homeassistant.util.enum import try_parse_enum -from homeassistant.util.ulid import ulid_to_bytes +from homeassistant.util.ulid import ulid_at_time, ulid_to_bytes from .auto_repairs.events.schema import ( correct_db_schema as events_correct_db_schema, @@ -92,7 +93,6 @@ if TYPE_CHECKING: from . import Recorder LIVE_MIGRATION_MIN_SCHEMA_VERSION = 0 -_EMPTY_CONTEXT_ID = b"\x00" * 16 _EMPTY_ENTITY_ID = "missing.entity_id" _EMPTY_EVENT_TYPE = "missing_event_type" @@ -1370,6 +1370,11 @@ def _context_id_to_bytes(context_id: str | None) -> bytes | None: return None +def _generate_ulid_bytes_at_time(timestamp: float | None) -> bytes: + """Generate a ulid with a specific timestamp.""" + return ulid_to_bytes(ulid_at_time(timestamp or time())) + + @retryable_database_job("migrate states context_ids to binary format") def migrate_states_context_ids(instance: Recorder) -> bool: """Migrate states context_ids to use binary format.""" @@ -1384,13 +1389,14 @@ def migrate_states_context_ids(instance: Recorder) -> bool: { "state_id": state_id, "context_id": None, - "context_id_bin": _to_bytes(context_id) or _EMPTY_CONTEXT_ID, + "context_id_bin": _to_bytes(context_id) + or _generate_ulid_bytes_at_time(last_updated_ts), "context_user_id": None, "context_user_id_bin": _to_bytes(context_user_id), "context_parent_id": None, "context_parent_id_bin": _to_bytes(context_parent_id), } - for state_id, context_id, context_user_id, context_parent_id in states + for state_id, last_updated_ts, context_id, context_user_id, context_parent_id in states ], ) # If there is more work to do return False @@ -1418,13 +1424,14 @@ def migrate_events_context_ids(instance: Recorder) -> bool: { "event_id": event_id, "context_id": None, - "context_id_bin": _to_bytes(context_id) or _EMPTY_CONTEXT_ID, + "context_id_bin": _to_bytes(context_id) + or _generate_ulid_bytes_at_time(time_fired_ts), "context_user_id": None, "context_user_id_bin": _to_bytes(context_user_id), "context_parent_id": None, "context_parent_id_bin": _to_bytes(context_parent_id), } - for event_id, context_id, context_user_id, context_parent_id in events + for event_id, time_fired_ts, context_id, context_user_id, context_parent_id in events ], ) # If there is more work to do return False diff --git a/homeassistant/components/recorder/queries.py b/homeassistant/components/recorder/queries.py index 454c71f6dc5..f8a1b769d87 100644 --- a/homeassistant/components/recorder/queries.py +++ b/homeassistant/components/recorder/queries.py @@ -690,6 +690,7 @@ def find_events_context_ids_to_migrate() -> StatementLambdaElement: return lambda_stmt( lambda: select( Events.event_id, + Events.time_fired_ts, Events.context_id, Events.context_user_id, Events.context_parent_id, @@ -788,6 +789,7 @@ def find_states_context_ids_to_migrate() -> StatementLambdaElement: return lambda_stmt( lambda: select( States.state_id, + States.last_updated_ts, States.context_id, States.context_user_id, States.context_parent_id, diff --git a/tests/components/recorder/test_migration_from_schema_32.py b/tests/components/recorder/test_migration_from_schema_32.py index 01c086e119a..f76cf318008 100644 --- a/tests/components/recorder/test_migration_from_schema_32.py +++ b/tests/components/recorder/test_migration_from_schema_32.py @@ -5,6 +5,7 @@ import sys from unittest.mock import patch import uuid +from freezegun import freeze_time import pytest from sqlalchemy import create_engine, inspect from sqlalchemy.orm import Session @@ -28,7 +29,7 @@ from homeassistant.components.recorder.tasks import ( from homeassistant.components.recorder.util import session_scope from homeassistant.core import HomeAssistant import homeassistant.util.dt as dt_util -from homeassistant.util.ulid import bytes_to_ulid +from homeassistant.util.ulid import bytes_to_ulid, ulid_at_time, ulid_to_bytes from .common import async_recorder_block_till_done, async_wait_recording_done @@ -115,7 +116,7 @@ async def test_migrate_events_context_ids( event_data=None, origin_idx=0, time_fired=None, - time_fired_ts=1677721632.452529, + time_fired_ts=1877721632.452529, context_id=uuid_hex, context_id_bin=None, context_user_id=None, @@ -128,7 +129,7 @@ async def test_migrate_events_context_ids( event_data=None, origin_idx=0, time_fired=None, - time_fired_ts=1677721632.552529, + time_fired_ts=1877721632.552529, context_id=None, context_id_bin=None, context_user_id=None, @@ -141,7 +142,7 @@ async def test_migrate_events_context_ids( event_data=None, origin_idx=0, time_fired=None, - time_fired_ts=1677721632.552529, + time_fired_ts=1877721632.552529, context_id="01ARZ3NDEKTSV4RRFFQ69G5FAV", context_id_bin=None, context_user_id="9400facee45711eaa9308bfd3d19e474", @@ -154,7 +155,7 @@ async def test_migrate_events_context_ids( event_data=None, origin_idx=0, time_fired=None, - time_fired_ts=1677721632.552529, + time_fired_ts=1877721632.552529, context_id="invalid", context_id_bin=None, context_user_id=None, @@ -167,7 +168,20 @@ async def test_migrate_events_context_ids( event_data=None, origin_idx=0, time_fired=None, - time_fired_ts=1677721632.552529, + time_fired_ts=1277721632.552529, + context_id="adapt_lgt:b'5Cf*':interval:b'0R'", + context_id_bin=None, + context_user_id=None, + context_user_id_bin=None, + context_parent_id=None, + context_parent_id_bin=None, + ), + Events( + event_type="event_with_garbage_context_id_no_time_fired_ts", + event_data=None, + origin_idx=0, + time_fired=None, + time_fired_ts=None, context_id="adapt_lgt:b'5Cf*':interval:b'0R'", context_id_bin=None, context_user_id=None, @@ -181,9 +195,12 @@ async def test_migrate_events_context_ids( await instance.async_add_executor_job(_insert_events) await async_wait_recording_done(hass) - # This is a threadsafe way to add a task to the recorder - instance.queue_task(EventsContextIDMigrationTask()) - await async_recorder_block_till_done(hass) + now = dt_util.utcnow() + expected_ulid_fallback_start = ulid_to_bytes(ulid_at_time(now.timestamp()))[0:6] + with freeze_time(now): + # This is a threadsafe way to add a task to the recorder + instance.queue_task(EventsContextIDMigrationTask()) + await async_recorder_block_till_done(hass) def _object_as_dict(obj): return {c.key: getattr(obj, c.key) for c in inspect(obj).mapper.column_attrs} @@ -200,12 +217,13 @@ async def test_migrate_events_context_ids( "ulid_context_id_event", "invalid_context_id_event", "garbage_context_id_event", + "event_with_garbage_context_id_no_time_fired_ts", ] ) ) .all() ) - assert len(events) == 5 + assert len(events) == 6 return {event.event_type: _object_as_dict(event) for event in events} events_by_type = await instance.async_add_executor_job(_fetch_migrated_events) @@ -222,7 +240,9 @@ async def test_migrate_events_context_ids( assert empty_context_id_event["context_id"] is None assert empty_context_id_event["context_user_id"] is None assert empty_context_id_event["context_parent_id"] is None - assert empty_context_id_event["context_id_bin"] == b"\x00" * 16 + assert empty_context_id_event["context_id_bin"].startswith( + b"\x01\xb50\xeeO(" + ) # 6 bytes of timestamp + random assert empty_context_id_event["context_user_id_bin"] is None assert empty_context_id_event["context_parent_id_bin"] is None @@ -247,7 +267,9 @@ async def test_migrate_events_context_ids( assert invalid_context_id_event["context_id"] is None assert invalid_context_id_event["context_user_id"] is None assert invalid_context_id_event["context_parent_id"] is None - assert invalid_context_id_event["context_id_bin"] == b"\x00" * 16 + assert invalid_context_id_event["context_id_bin"].startswith( + b"\x01\xb50\xeeO(" + ) # 6 bytes of timestamp + random assert invalid_context_id_event["context_user_id_bin"] is None assert invalid_context_id_event["context_parent_id_bin"] is None @@ -255,10 +277,26 @@ async def test_migrate_events_context_ids( assert garbage_context_id_event["context_id"] is None assert garbage_context_id_event["context_user_id"] is None assert garbage_context_id_event["context_parent_id"] is None - assert garbage_context_id_event["context_id_bin"] == b"\x00" * 16 + assert garbage_context_id_event["context_id_bin"].startswith( + b"\x01)~$\xdf(" + ) # 6 bytes of timestamp + random assert garbage_context_id_event["context_user_id_bin"] is None assert garbage_context_id_event["context_parent_id_bin"] is None + event_with_garbage_context_id_no_time_fired_ts = events_by_type[ + "event_with_garbage_context_id_no_time_fired_ts" + ] + assert event_with_garbage_context_id_no_time_fired_ts["context_id"] is None + assert event_with_garbage_context_id_no_time_fired_ts["context_user_id"] is None + assert event_with_garbage_context_id_no_time_fired_ts["context_parent_id"] is None + assert event_with_garbage_context_id_no_time_fired_ts["context_id_bin"].startswith( + expected_ulid_fallback_start + ) # 6 bytes of timestamp + random + assert event_with_garbage_context_id_no_time_fired_ts["context_user_id_bin"] is None + assert ( + event_with_garbage_context_id_no_time_fired_ts["context_parent_id_bin"] is None + ) + @pytest.mark.parametrize("enable_migrate_context_ids", [True]) async def test_migrate_states_context_ids( @@ -272,13 +310,13 @@ async def test_migrate_states_context_ids( uuid_hex = test_uuid.hex uuid_bin = test_uuid.bytes - def _insert_events(): + def _insert_states(): with session_scope(hass=hass) as session: session.add_all( ( States( entity_id="state.old_uuid_context_id", - last_updated_ts=1677721632.452529, + last_updated_ts=1477721632.452529, context_id=uuid_hex, context_id_bin=None, context_user_id=None, @@ -288,7 +326,7 @@ async def test_migrate_states_context_ids( ), States( entity_id="state.empty_context_id", - last_updated_ts=1677721632.552529, + last_updated_ts=1477721632.552529, context_id=None, context_id_bin=None, context_user_id=None, @@ -298,7 +336,7 @@ async def test_migrate_states_context_ids( ), States( entity_id="state.ulid_context_id", - last_updated_ts=1677721632.552529, + last_updated_ts=1477721632.552529, context_id="01ARZ3NDEKTSV4RRFFQ69G5FAV", context_id_bin=None, context_user_id="9400facee45711eaa9308bfd3d19e474", @@ -308,7 +346,7 @@ async def test_migrate_states_context_ids( ), States( entity_id="state.invalid_context_id", - last_updated_ts=1677721632.552529, + last_updated_ts=1477721632.552529, context_id="invalid", context_id_bin=None, context_user_id=None, @@ -318,7 +356,7 @@ async def test_migrate_states_context_ids( ), States( entity_id="state.garbage_context_id", - last_updated_ts=1677721632.552529, + last_updated_ts=1477721632.552529, context_id="adapt_lgt:b'5Cf*':interval:b'0R'", context_id_bin=None, context_user_id=None, @@ -328,7 +366,7 @@ async def test_migrate_states_context_ids( ), States( entity_id="state.human_readable_uuid_context_id", - last_updated_ts=1677721632.552529, + last_updated_ts=1477721632.552529, context_id="0ae29799-ee4e-4f45-8116-f582d7d3ee65", context_id_bin=None, context_user_id="0ae29799-ee4e-4f45-8116-f582d7d3ee65", @@ -339,10 +377,9 @@ async def test_migrate_states_context_ids( ) ) - await instance.async_add_executor_job(_insert_events) + await instance.async_add_executor_job(_insert_states) await async_wait_recording_done(hass) - # This is a threadsafe way to add a task to the recorder instance.queue_task(StatesContextIDMigrationTask()) await async_recorder_block_till_done(hass) @@ -384,7 +421,9 @@ async def test_migrate_states_context_ids( assert empty_context_id["context_id"] is None assert empty_context_id["context_user_id"] is None assert empty_context_id["context_parent_id"] is None - assert empty_context_id["context_id_bin"] == b"\x00" * 16 + assert empty_context_id["context_id_bin"].startswith( + b"\x01X\x0f\x12\xaf(" + ) # 6 bytes of timestamp + random assert empty_context_id["context_user_id_bin"] is None assert empty_context_id["context_parent_id_bin"] is None @@ -408,7 +447,9 @@ async def test_migrate_states_context_ids( assert invalid_context_id["context_id"] is None assert invalid_context_id["context_user_id"] is None assert invalid_context_id["context_parent_id"] is None - assert invalid_context_id["context_id_bin"] == b"\x00" * 16 + assert invalid_context_id["context_id_bin"].startswith( + b"\x01X\x0f\x12\xaf(" + ) # 6 bytes of timestamp + random assert invalid_context_id["context_user_id_bin"] is None assert invalid_context_id["context_parent_id_bin"] is None @@ -416,7 +457,9 @@ async def test_migrate_states_context_ids( assert garbage_context_id["context_id"] is None assert garbage_context_id["context_user_id"] is None assert garbage_context_id["context_parent_id"] is None - assert garbage_context_id["context_id_bin"] == b"\x00" * 16 + assert garbage_context_id["context_id_bin"].startswith( + b"\x01X\x0f\x12\xaf(" + ) # 6 bytes of timestamp + random assert garbage_context_id["context_user_id_bin"] is None assert garbage_context_id["context_parent_id_bin"] is None From e19279fda5de396a56aa13129b8beac3087ad807 Mon Sep 17 00:00:00 2001 From: Teemu R Date: Thu, 20 Apr 2023 20:57:45 +0200 Subject: [PATCH 103/110] Bump python-songpal dependency (#91708) --- homeassistant/components/songpal/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/songpal/manifest.json b/homeassistant/components/songpal/manifest.json index d41cc2ad587..aa1157e8d0b 100644 --- a/homeassistant/components/songpal/manifest.json +++ b/homeassistant/components/songpal/manifest.json @@ -7,7 +7,7 @@ "iot_class": "local_push", "loggers": ["songpal"], "quality_scale": "gold", - "requirements": ["python-songpal==0.15.1"], + "requirements": ["python-songpal==0.15.2"], "ssdp": [ { "st": "urn:schemas-sony-com:service:ScalarWebAPI:1", diff --git a/requirements_all.txt b/requirements_all.txt index 05dc2280c15..40b76ffa0dc 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2106,7 +2106,7 @@ python-ripple-api==0.0.3 python-smarttub==0.0.33 # homeassistant.components.songpal -python-songpal==0.15.1 +python-songpal==0.15.2 # homeassistant.components.tado python-tado==0.12.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 7cdc39abba9..bb01f69d501 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1508,7 +1508,7 @@ python-picnic-api==1.1.0 python-smarttub==0.0.33 # homeassistant.components.songpal -python-songpal==0.15.1 +python-songpal==0.15.2 # homeassistant.components.tado python-tado==0.12.0 From 8363183943accd8f437292e9b5af48ce1f163d91 Mon Sep 17 00:00:00 2001 From: Jan Bouwhuis Date: Fri, 21 Apr 2023 09:00:48 +0200 Subject: [PATCH 104/110] Do not wait for mqtt at startup mqtt_statestream (#91721) --- homeassistant/components/mqtt_statestream/__init__.py | 9 --------- tests/components/mqtt_statestream/test_init.py | 11 +++++++++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/homeassistant/components/mqtt_statestream/__init__.py b/homeassistant/components/mqtt_statestream/__init__.py index 01425737543..aa4c2c628b4 100644 --- a/homeassistant/components/mqtt_statestream/__init__.py +++ b/homeassistant/components/mqtt_statestream/__init__.py @@ -41,15 +41,6 @@ _LOGGER = logging.getLogger(__name__) async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool: """Set up the MQTT state feed.""" - # Make sure MQTT is available and the entry is loaded - if not hass.config_entries.async_entries( - mqtt.DOMAIN - ) or not await hass.config_entries.async_wait_component( - hass.config_entries.async_entries(mqtt.DOMAIN)[0] - ): - _LOGGER.error("MQTT integration is not available") - return False - conf: ConfigType = config[DOMAIN] publish_filter = convert_include_exclude_filter(conf) base_topic: str = conf[CONF_BASE_TOPIC] diff --git a/tests/components/mqtt_statestream/test_init.py b/tests/components/mqtt_statestream/test_init.py index 130d874cc50..c5c91a97eea 100644 --- a/tests/components/mqtt_statestream/test_init.py +++ b/tests/components/mqtt_statestream/test_init.py @@ -96,12 +96,19 @@ async def test_setup_and_stop_waits_for_ha( mqtt_mock.async_publish.assert_not_called() +@pytest.mark.xfail() async def test_startup_no_mqtt( hass: HomeAssistant, caplog: pytest.LogCaptureFixture ) -> None: """Test startup without MQTT support.""" - assert not await add_statestream(hass, base_topic="pub") - assert "MQTT integration is not available" in caplog.text + e_id = "fake.entity" + + assert await add_statestream(hass, base_topic="pub") + # Set a state of an entity + mock_state_change_event(hass, State(e_id, "on")) + await hass.async_block_till_done() + await hass.async_block_till_done() + assert "MQTT is not enabled" in caplog.text async def test_setup_succeeds_with_attributes( From 64f8059f0027941dc0bc3c545c65e82aec4fea47 Mon Sep 17 00:00:00 2001 From: Nathan Spencer Date: Thu, 20 Apr 2023 12:29:35 -0600 Subject: [PATCH 105/110] Bump pylitterbot to 2023.4.0 (#91759) --- homeassistant/components/litterrobot/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- tests/components/litterrobot/test_sensor.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/litterrobot/manifest.json b/homeassistant/components/litterrobot/manifest.json index 0b162ee2e56..d3dcf77f324 100644 --- a/homeassistant/components/litterrobot/manifest.json +++ b/homeassistant/components/litterrobot/manifest.json @@ -12,5 +12,5 @@ "integration_type": "hub", "iot_class": "cloud_push", "loggers": ["pylitterbot"], - "requirements": ["pylitterbot==2023.1.2"] + "requirements": ["pylitterbot==2023.4.0"] } diff --git a/requirements_all.txt b/requirements_all.txt index 40b76ffa0dc..423219243c6 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1753,7 +1753,7 @@ pylibrespot-java==0.1.1 pylitejet==0.5.0 # homeassistant.components.litterrobot -pylitterbot==2023.1.2 +pylitterbot==2023.4.0 # homeassistant.components.lutron_caseta pylutron-caseta==0.18.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index bb01f69d501..2f9d3163cbf 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1269,7 +1269,7 @@ pylibrespot-java==0.1.1 pylitejet==0.5.0 # homeassistant.components.litterrobot -pylitterbot==2023.1.2 +pylitterbot==2023.4.0 # homeassistant.components.lutron_caseta pylutron-caseta==0.18.1 diff --git a/tests/components/litterrobot/test_sensor.py b/tests/components/litterrobot/test_sensor.py index 9586e7cdbfc..adb44d59bff 100644 --- a/tests/components/litterrobot/test_sensor.py +++ b/tests/components/litterrobot/test_sensor.py @@ -101,5 +101,5 @@ async def test_feeder_robot_sensor( """Tests Feeder-Robot sensors.""" await setup_integration(hass, mock_account_with_feederrobot, PLATFORM_DOMAIN) sensor = hass.states.get("sensor.test_food_level") - assert sensor.state == "20" + assert sensor.state == "10" assert sensor.attributes["unit_of_measurement"] == PERCENTAGE From 26b28001c5a1ccff30ea5610613605a803c2acf3 Mon Sep 17 00:00:00 2001 From: Stephan Uhle Date: Fri, 21 Apr 2023 08:16:32 +0200 Subject: [PATCH 106/110] Bump pysml to 0.0.10 (#91773) --- homeassistant/components/edl21/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/edl21/manifest.json b/homeassistant/components/edl21/manifest.json index f6363473def..034d3fcae2e 100644 --- a/homeassistant/components/edl21/manifest.json +++ b/homeassistant/components/edl21/manifest.json @@ -7,5 +7,5 @@ "integration_type": "hub", "iot_class": "local_push", "loggers": ["sml"], - "requirements": ["pysml==0.0.9"] + "requirements": ["pysml==0.0.10"] } diff --git a/requirements_all.txt b/requirements_all.txt index 423219243c6..8d23f37514b 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1976,7 +1976,7 @@ pysmartthings==0.7.6 pysmarty==0.8 # homeassistant.components.edl21 -pysml==0.0.9 +pysml==0.0.10 # homeassistant.components.snmp pysnmplib==5.0.21 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 2f9d3163cbf..04292652fc5 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1438,7 +1438,7 @@ pysmartapp==0.3.3 pysmartthings==0.7.6 # homeassistant.components.edl21 -pysml==0.0.9 +pysml==0.0.10 # homeassistant.components.snmp pysnmplib==5.0.21 From 30da629285928421ecc6ef9494696b5121ddea44 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Fri, 21 Apr 2023 07:25:52 -0700 Subject: [PATCH 107/110] Relax the constraint that events must have a consistent timezone for start/end (#91788) --- homeassistant/components/calendar/__init__.py | 1 - tests/components/google/test_calendar.py | 34 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/calendar/__init__.py b/homeassistant/components/calendar/__init__.py index 2445c054c6d..aedfafbf368 100644 --- a/homeassistant/components/calendar/__init__.py +++ b/homeassistant/components/calendar/__init__.py @@ -244,7 +244,6 @@ CALENDAR_EVENT_SCHEMA = vol.Schema( }, _has_same_type("start", "end"), _has_timezone("start", "end"), - _has_consistent_timezone("start", "end"), _as_local_timezone("start", "end"), _has_min_duration("start", "end", MIN_EVENT_DURATION), ), diff --git a/tests/components/google/test_calendar.py b/tests/components/google/test_calendar.py index 7d59d80687e..d6431700fca 100644 --- a/tests/components/google/test_calendar.py +++ b/tests/components/google/test_calendar.py @@ -1295,3 +1295,37 @@ async def test_event_without_duration( assert state.attributes.get("start_time") == one_hour_from_now.strftime( DATE_STR_FORMAT ) + + +async def test_event_differs_timezone( + hass: HomeAssistant, mock_events_list_items, component_setup +) -> None: + """Test a case where the event has a different start/end timezone.""" + one_hour_from_now = dt_util.now() + datetime.timedelta(minutes=30) + end_event = one_hour_from_now + datetime.timedelta(hours=8) + event = { + **TEST_EVENT, + "start": { + "dateTime": one_hour_from_now.isoformat(), + "timeZone": "America/Regina", + }, + "end": {"dateTime": end_event.isoformat(), "timeZone": "UTC"}, + } + mock_events_list_items([event]) + + assert await component_setup() + + state = hass.states.get(TEST_ENTITY) + assert state.name == TEST_ENTITY_NAME + assert state.state == STATE_OFF + assert dict(state.attributes) == { + "friendly_name": TEST_ENTITY_NAME, + "message": event["summary"], + "all_day": False, + "offset_reached": False, + "start_time": one_hour_from_now.strftime(DATE_STR_FORMAT), + "end_time": end_event.strftime(DATE_STR_FORMAT), + "location": event["location"], + "description": event["description"], + "supported_features": 3, + } From deb55a74da85538b07e7116ab0a1d9391c00f8cd Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Fri, 21 Apr 2023 10:21:20 -0400 Subject: [PATCH 108/110] Disallow uploading files to bypass the media dirs (#91817) --- homeassistant/components/media_source/local_source.py | 8 +++++++- tests/components/media_source/test_local_source.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/media_source/local_source.py b/homeassistant/components/media_source/local_source.py index d211b878b99..c29794ae8d7 100644 --- a/homeassistant/components/media_source/local_source.py +++ b/homeassistant/components/media_source/local_source.py @@ -48,7 +48,10 @@ class LocalSource(MediaSource): @callback def async_full_path(self, source_dir_id: str, location: str) -> Path: """Return full path.""" - return Path(self.hass.config.media_dirs[source_dir_id], location) + base_path = self.hass.config.media_dirs[source_dir_id] + full_path = Path(base_path, location) + full_path.relative_to(base_path) + return full_path @callback def async_parse_identifier(self, item: MediaSourceItem) -> tuple[str, str]: @@ -65,6 +68,9 @@ class LocalSource(MediaSource): except ValueError as err: raise Unresolvable("Invalid path.") from err + if Path(location).is_absolute(): + raise Unresolvable("Invalid path.") + return source_dir_id, location async def async_resolve_media(self, item: MediaSourceItem) -> PlayMedia: diff --git a/tests/components/media_source/test_local_source.py b/tests/components/media_source/test_local_source.py index 585f92c7a0f..cf50e967558 100644 --- a/tests/components/media_source/test_local_source.py +++ b/tests/components/media_source/test_local_source.py @@ -132,9 +132,13 @@ async def test_upload_view( hass: HomeAssistant, hass_client: ClientSessionGenerator, temp_dir, + tmpdir, hass_admin_user: MockUser, ) -> None: """Allow uploading media.""" + # We need a temp dir that's not under tempdir fixture + extra_media_dir = tmpdir + hass.config.media_dirs["another_path"] = temp_dir img = (Path(__file__).parent.parent / "image_upload/logo.png").read_bytes() @@ -167,6 +171,8 @@ async def test_upload_view( "media-source://media_source/test_dir/..", # Domain != media_source "media-source://nest/test_dir/.", + # Other directory + f"media-source://media_source/another_path///{extra_media_dir}/", # Completely something else "http://bla", ): @@ -178,7 +184,7 @@ async def test_upload_view( }, ) - assert res.status == 400 + assert res.status == 400, bad_id assert not (Path(temp_dir) / "bad-source-id.png").is_file() # Test invalid POST data From f1c4605fba7229d7b75f70bf9f1ab0601bfe3a1c Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Fri, 21 Apr 2023 14:58:46 -0400 Subject: [PATCH 109/110] Bumped version to 2023.4.6 --- homeassistant/const.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/const.py b/homeassistant/const.py index 712e8ff0eb6..2fc41b74376 100644 --- a/homeassistant/const.py +++ b/homeassistant/const.py @@ -8,7 +8,7 @@ from .backports.enum import StrEnum APPLICATION_NAME: Final = "HomeAssistant" MAJOR_VERSION: Final = 2023 MINOR_VERSION: Final = 4 -PATCH_VERSION: Final = "5" +PATCH_VERSION: Final = "6" __short_version__: Final = f"{MAJOR_VERSION}.{MINOR_VERSION}" __version__: Final = f"{__short_version__}.{PATCH_VERSION}" REQUIRED_PYTHON_VER: Final[tuple[int, int, int]] = (3, 10, 0) diff --git a/pyproject.toml b/pyproject.toml index b2bcdb011b5..26b90151610 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "homeassistant" -version = "2023.4.5" +version = "2023.4.6" license = {text = "Apache-2.0"} description = "Open-source home automation platform running on Python 3." readme = "README.rst" From d58f62cb5e59ad6a22b14b143a06370f28ad38d1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 21 Apr 2023 18:12:21 -0500 Subject: [PATCH 110/110] Remove old migration tests that have been replaced (#91842) These tests were moved to test_migration_from_schema_32.py in `dev` and have changed. Remove the old tests as they are no longer correct --- tests/components/recorder/test_migrate.py | 322 +--------------------- 1 file changed, 1 insertion(+), 321 deletions(-) diff --git a/tests/components/recorder/test_migrate.py b/tests/components/recorder/test_migrate.py index b75d536d152..c9e49697585 100644 --- a/tests/components/recorder/test_migrate.py +++ b/tests/components/recorder/test_migrate.py @@ -6,10 +6,9 @@ import sqlite3 import sys import threading from unittest.mock import Mock, PropertyMock, call, patch -import uuid import pytest -from sqlalchemy import create_engine, inspect, text +from sqlalchemy import create_engine, text from sqlalchemy.exc import ( DatabaseError, InternalError, @@ -35,15 +34,12 @@ from homeassistant.components.recorder.queries import select_event_type_ids from homeassistant.components.recorder.tasks import ( EntityIDMigrationTask, EntityIDPostMigrationTask, - EventsContextIDMigrationTask, EventTypeIDMigrationTask, - StatesContextIDMigrationTask, ) from homeassistant.components.recorder.util import session_scope from homeassistant.core import HomeAssistant from homeassistant.helpers import recorder as recorder_helper import homeassistant.util.dt as dt_util -from homeassistant.util.ulid import bytes_to_ulid from .common import ( async_recorder_block_till_done, @@ -603,322 +599,6 @@ def test_raise_if_exception_missing_empty_cause_str() -> None: migration.raise_if_exception_missing_str(programming_exc, ["not present"]) -@pytest.mark.parametrize("enable_migrate_context_ids", [True]) -async def test_migrate_events_context_ids( - async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant -) -> None: - """Test we can migrate old uuid context ids and ulid context ids to binary format.""" - instance = await async_setup_recorder_instance(hass) - await async_wait_recording_done(hass) - - test_uuid = uuid.uuid4() - uuid_hex = test_uuid.hex - uuid_bin = test_uuid.bytes - - def _insert_events(): - with session_scope(hass=hass) as session: - session.add_all( - ( - Events( - event_type="old_uuid_context_id_event", - event_data=None, - origin_idx=0, - time_fired=None, - time_fired_ts=1677721632.452529, - context_id=uuid_hex, - context_id_bin=None, - context_user_id=None, - context_user_id_bin=None, - context_parent_id=None, - context_parent_id_bin=None, - ), - Events( - event_type="empty_context_id_event", - event_data=None, - origin_idx=0, - time_fired=None, - time_fired_ts=1677721632.552529, - context_id=None, - context_id_bin=None, - context_user_id=None, - context_user_id_bin=None, - context_parent_id=None, - context_parent_id_bin=None, - ), - Events( - event_type="ulid_context_id_event", - event_data=None, - origin_idx=0, - time_fired=None, - time_fired_ts=1677721632.552529, - context_id="01ARZ3NDEKTSV4RRFFQ69G5FAV", - context_id_bin=None, - context_user_id="9400facee45711eaa9308bfd3d19e474", - context_user_id_bin=None, - context_parent_id="01ARZ3NDEKTSV4RRFFQ69G5FA2", - context_parent_id_bin=None, - ), - Events( - event_type="invalid_context_id_event", - event_data=None, - origin_idx=0, - time_fired=None, - time_fired_ts=1677721632.552529, - context_id="invalid", - context_id_bin=None, - context_user_id=None, - context_user_id_bin=None, - context_parent_id=None, - context_parent_id_bin=None, - ), - Events( - event_type="garbage_context_id_event", - event_data=None, - origin_idx=0, - time_fired=None, - time_fired_ts=1677721632.552529, - context_id="adapt_lgt:b'5Cf*':interval:b'0R'", - context_id_bin=None, - context_user_id=None, - context_user_id_bin=None, - context_parent_id=None, - context_parent_id_bin=None, - ), - ) - ) - - await instance.async_add_executor_job(_insert_events) - - await async_wait_recording_done(hass) - # This is a threadsafe way to add a task to the recorder - instance.queue_task(EventsContextIDMigrationTask()) - await async_recorder_block_till_done(hass) - - def _object_as_dict(obj): - return {c.key: getattr(obj, c.key) for c in inspect(obj).mapper.column_attrs} - - def _fetch_migrated_events(): - with session_scope(hass=hass) as session: - events = ( - session.query(Events) - .filter( - Events.event_type.in_( - [ - "old_uuid_context_id_event", - "empty_context_id_event", - "ulid_context_id_event", - "invalid_context_id_event", - "garbage_context_id_event", - ] - ) - ) - .all() - ) - assert len(events) == 5 - return {event.event_type: _object_as_dict(event) for event in events} - - events_by_type = await instance.async_add_executor_job(_fetch_migrated_events) - - old_uuid_context_id_event = events_by_type["old_uuid_context_id_event"] - assert old_uuid_context_id_event["context_id"] is None - assert old_uuid_context_id_event["context_user_id"] is None - assert old_uuid_context_id_event["context_parent_id"] is None - assert old_uuid_context_id_event["context_id_bin"] == uuid_bin - assert old_uuid_context_id_event["context_user_id_bin"] is None - assert old_uuid_context_id_event["context_parent_id_bin"] is None - - empty_context_id_event = events_by_type["empty_context_id_event"] - assert empty_context_id_event["context_id"] is None - assert empty_context_id_event["context_user_id"] is None - assert empty_context_id_event["context_parent_id"] is None - assert empty_context_id_event["context_id_bin"] == b"\x00" * 16 - assert empty_context_id_event["context_user_id_bin"] is None - assert empty_context_id_event["context_parent_id_bin"] is None - - ulid_context_id_event = events_by_type["ulid_context_id_event"] - assert ulid_context_id_event["context_id"] is None - assert ulid_context_id_event["context_user_id"] is None - assert ulid_context_id_event["context_parent_id"] is None - assert ( - bytes_to_ulid(ulid_context_id_event["context_id_bin"]) - == "01ARZ3NDEKTSV4RRFFQ69G5FAV" - ) - assert ( - ulid_context_id_event["context_user_id_bin"] - == b"\x94\x00\xfa\xce\xe4W\x11\xea\xa90\x8b\xfd=\x19\xe4t" - ) - assert ( - bytes_to_ulid(ulid_context_id_event["context_parent_id_bin"]) - == "01ARZ3NDEKTSV4RRFFQ69G5FA2" - ) - - invalid_context_id_event = events_by_type["invalid_context_id_event"] - assert invalid_context_id_event["context_id"] is None - assert invalid_context_id_event["context_user_id"] is None - assert invalid_context_id_event["context_parent_id"] is None - assert invalid_context_id_event["context_id_bin"] == b"\x00" * 16 - assert invalid_context_id_event["context_user_id_bin"] is None - assert invalid_context_id_event["context_parent_id_bin"] is None - - garbage_context_id_event = events_by_type["garbage_context_id_event"] - assert garbage_context_id_event["context_id"] is None - assert garbage_context_id_event["context_user_id"] is None - assert garbage_context_id_event["context_parent_id"] is None - assert garbage_context_id_event["context_id_bin"] == b"\x00" * 16 - assert garbage_context_id_event["context_user_id_bin"] is None - assert garbage_context_id_event["context_parent_id_bin"] is None - - -@pytest.mark.parametrize("enable_migrate_context_ids", [True]) -async def test_migrate_states_context_ids( - async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant -) -> None: - """Test we can migrate old uuid context ids and ulid context ids to binary format.""" - instance = await async_setup_recorder_instance(hass) - await async_wait_recording_done(hass) - - test_uuid = uuid.uuid4() - uuid_hex = test_uuid.hex - uuid_bin = test_uuid.bytes - - def _insert_events(): - with session_scope(hass=hass) as session: - session.add_all( - ( - States( - entity_id="state.old_uuid_context_id", - last_updated_ts=1677721632.452529, - context_id=uuid_hex, - context_id_bin=None, - context_user_id=None, - context_user_id_bin=None, - context_parent_id=None, - context_parent_id_bin=None, - ), - States( - entity_id="state.empty_context_id", - last_updated_ts=1677721632.552529, - context_id=None, - context_id_bin=None, - context_user_id=None, - context_user_id_bin=None, - context_parent_id=None, - context_parent_id_bin=None, - ), - States( - entity_id="state.ulid_context_id", - last_updated_ts=1677721632.552529, - context_id="01ARZ3NDEKTSV4RRFFQ69G5FAV", - context_id_bin=None, - context_user_id="9400facee45711eaa9308bfd3d19e474", - context_user_id_bin=None, - context_parent_id="01ARZ3NDEKTSV4RRFFQ69G5FA2", - context_parent_id_bin=None, - ), - States( - entity_id="state.invalid_context_id", - last_updated_ts=1677721632.552529, - context_id="invalid", - context_id_bin=None, - context_user_id=None, - context_user_id_bin=None, - context_parent_id=None, - context_parent_id_bin=None, - ), - States( - entity_id="state.garbage_context_id", - last_updated_ts=1677721632.552529, - context_id="adapt_lgt:b'5Cf*':interval:b'0R'", - context_id_bin=None, - context_user_id=None, - context_user_id_bin=None, - context_parent_id=None, - context_parent_id_bin=None, - ), - ) - ) - - await instance.async_add_executor_job(_insert_events) - - await async_wait_recording_done(hass) - # This is a threadsafe way to add a task to the recorder - instance.queue_task(StatesContextIDMigrationTask()) - await async_recorder_block_till_done(hass) - - def _object_as_dict(obj): - return {c.key: getattr(obj, c.key) for c in inspect(obj).mapper.column_attrs} - - def _fetch_migrated_states(): - with session_scope(hass=hass) as session: - events = ( - session.query(States) - .filter( - States.entity_id.in_( - [ - "state.old_uuid_context_id", - "state.empty_context_id", - "state.ulid_context_id", - "state.invalid_context_id", - "state.garbage_context_id", - ] - ) - ) - .all() - ) - assert len(events) == 5 - return {state.entity_id: _object_as_dict(state) for state in events} - - states_by_entity_id = await instance.async_add_executor_job(_fetch_migrated_states) - - old_uuid_context_id = states_by_entity_id["state.old_uuid_context_id"] - assert old_uuid_context_id["context_id"] is None - assert old_uuid_context_id["context_user_id"] is None - assert old_uuid_context_id["context_parent_id"] is None - assert old_uuid_context_id["context_id_bin"] == uuid_bin - assert old_uuid_context_id["context_user_id_bin"] is None - assert old_uuid_context_id["context_parent_id_bin"] is None - - empty_context_id = states_by_entity_id["state.empty_context_id"] - assert empty_context_id["context_id"] is None - assert empty_context_id["context_user_id"] is None - assert empty_context_id["context_parent_id"] is None - assert empty_context_id["context_id_bin"] == b"\x00" * 16 - assert empty_context_id["context_user_id_bin"] is None - assert empty_context_id["context_parent_id_bin"] is None - - ulid_context_id = states_by_entity_id["state.ulid_context_id"] - assert ulid_context_id["context_id"] is None - assert ulid_context_id["context_user_id"] is None - assert ulid_context_id["context_parent_id"] is None - assert ( - bytes_to_ulid(ulid_context_id["context_id_bin"]) == "01ARZ3NDEKTSV4RRFFQ69G5FAV" - ) - assert ( - ulid_context_id["context_user_id_bin"] - == b"\x94\x00\xfa\xce\xe4W\x11\xea\xa90\x8b\xfd=\x19\xe4t" - ) - assert ( - bytes_to_ulid(ulid_context_id["context_parent_id_bin"]) - == "01ARZ3NDEKTSV4RRFFQ69G5FA2" - ) - - invalid_context_id = states_by_entity_id["state.invalid_context_id"] - assert invalid_context_id["context_id"] is None - assert invalid_context_id["context_user_id"] is None - assert invalid_context_id["context_parent_id"] is None - assert invalid_context_id["context_id_bin"] == b"\x00" * 16 - assert invalid_context_id["context_user_id_bin"] is None - assert invalid_context_id["context_parent_id_bin"] is None - - garbage_context_id = states_by_entity_id["state.garbage_context_id"] - assert garbage_context_id["context_id"] is None - assert garbage_context_id["context_user_id"] is None - assert garbage_context_id["context_parent_id"] is None - assert garbage_context_id["context_id_bin"] == b"\x00" * 16 - assert garbage_context_id["context_user_id_bin"] is None - assert garbage_context_id["context_parent_id_bin"] is None - - @pytest.mark.parametrize("enable_migrate_event_type_ids", [True]) async def test_migrate_event_type_ids( async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant