diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 6d947f51aca..7636d628e41 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -56,7 +56,7 @@ jobs: echo "CI_BUILD=1" echo "ENABLE_HEADLESS=1" - # Use C-Extension for sqlalchemy + # Use C-Extension for SQLAlchemy echo "REQUIRE_SQLALCHEMY_CEXT=1" ) > .env_file @@ -186,7 +186,7 @@ jobs: wheels-key: ${{ secrets.WHEELS_KEY }} env-file: true apk: "bluez-dev;libffi-dev;openssl-dev;glib-dev;eudev-dev;libxml2-dev;libxslt-dev;libpng-dev;libjpeg-turbo-dev;tiff-dev;cups-dev;gmp-dev;mpfr-dev;mpc1-dev;ffmpeg-dev;gammu-dev;yaml-dev;openblas-dev;fftw-dev;lapack-dev;gfortran;blas-dev;eigen-dev;freetype-dev;glew-dev;harfbuzz-dev;hdf5-dev;libdc1394-dev;libtbb-dev;mesa-dev;openexr-dev;openjpeg-dev;uchardet-dev" - skip-binary: aiohttp;grpcio;sqlalchemy;protobuf + skip-binary: aiohttp;grpcio;SQLAlchemy;protobuf constraints: "homeassistant/package_constraints.txt" requirements-diff: "requirements_diff.txt" requirements: "requirements_all.txtaa" @@ -200,7 +200,7 @@ jobs: wheels-key: ${{ secrets.WHEELS_KEY }} env-file: true apk: "bluez-dev;libffi-dev;openssl-dev;glib-dev;eudev-dev;libxml2-dev;libxslt-dev;libpng-dev;libjpeg-turbo-dev;tiff-dev;cups-dev;gmp-dev;mpfr-dev;mpc1-dev;ffmpeg-dev;gammu-dev;yaml-dev;openblas-dev;fftw-dev;lapack-dev;gfortran;blas-dev;eigen-dev;freetype-dev;glew-dev;harfbuzz-dev;hdf5-dev;libdc1394-dev;libtbb-dev;mesa-dev;openexr-dev;openjpeg-dev;uchardet-dev" - skip-binary: aiohttp;grpcio;sqlalchemy;protobuf + skip-binary: aiohttp;grpcio;SQLAlchemy;protobuf constraints: "homeassistant/package_constraints.txt" requirements-diff: "requirements_diff.txt" requirements: "requirements_all.txtab" @@ -214,7 +214,7 @@ jobs: wheels-key: ${{ secrets.WHEELS_KEY }} env-file: true apk: "bluez-dev;libffi-dev;openssl-dev;glib-dev;eudev-dev;libxml2-dev;libxslt-dev;libpng-dev;libjpeg-turbo-dev;tiff-dev;cups-dev;gmp-dev;mpfr-dev;mpc1-dev;ffmpeg-dev;gammu-dev;yaml-dev;openblas-dev;fftw-dev;lapack-dev;gfortran;blas-dev;eigen-dev;freetype-dev;glew-dev;harfbuzz-dev;hdf5-dev;libdc1394-dev;libtbb-dev;mesa-dev;openexr-dev;openjpeg-dev;uchardet-dev" - skip-binary: aiohttp;grpcio;sqlalchemy;protobuf + skip-binary: aiohttp;grpcio;SQLAlchemy;protobuf constraints: "homeassistant/package_constraints.txt" requirements-diff: "requirements_diff.txt" requirements: "requirements_all.txtac" diff --git a/homeassistant/components/recorder/history/legacy.py b/homeassistant/components/recorder/history/legacy.py index 191c74ac0d4..2e1b02a8b64 100644 --- a/homeassistant/components/recorder/history/legacy.py +++ b/homeassistant/components/recorder/history/legacy.py @@ -50,7 +50,7 @@ _BASE_STATES = ( States.last_changed_ts, States.last_updated_ts, ) -_BASE_STATES_NO_LAST_CHANGED = ( # type: ignore[var-annotated] +_BASE_STATES_NO_LAST_CHANGED = ( States.entity_id, States.state, literal(value=None).label("last_changed_ts"), diff --git a/homeassistant/components/recorder/manifest.json b/homeassistant/components/recorder/manifest.json index 63b19cdb3bf..f40797fe38c 100644 --- a/homeassistant/components/recorder/manifest.json +++ b/homeassistant/components/recorder/manifest.json @@ -7,7 +7,7 @@ "iot_class": "local_push", "quality_scale": "internal", "requirements": [ - "SQLAlchemy==2.0.15", + "SQLAlchemy==2.0.21", "fnv-hash-fast==0.4.1", "psutil-home-assistant==0.0.1" ] diff --git a/homeassistant/components/sql/manifest.json b/homeassistant/components/sql/manifest.json index 44de8fc6923..7424807c804 100644 --- a/homeassistant/components/sql/manifest.json +++ b/homeassistant/components/sql/manifest.json @@ -5,5 +5,5 @@ "config_flow": true, "documentation": "https://www.home-assistant.io/integrations/sql", "iot_class": "local_polling", - "requirements": ["SQLAlchemy==2.0.15"] + "requirements": ["SQLAlchemy==2.0.21"] } diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 77327fc6c80..b1ad7f7a3c5 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -46,7 +46,7 @@ pyudev==0.23.2 PyYAML==6.0.1 requests==2.31.0 scapy==2.5.0 -SQLAlchemy==2.0.15 +SQLAlchemy==2.0.21 typing-extensions>=4.8.0,<5.0 ulid-transform==0.8.1 voluptuous-serialize==2.6.0 diff --git a/requirements_all.txt b/requirements_all.txt index 0ef33a05476..8eb3b324064 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -129,7 +129,7 @@ RtmAPI==0.7.2 # homeassistant.components.recorder # homeassistant.components.sql -SQLAlchemy==2.0.15 +SQLAlchemy==2.0.21 # homeassistant.components.travisci TravisPy==0.3.5 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 610e875d2d3..d6d9bb9242e 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -116,7 +116,7 @@ RtmAPI==0.7.2 # homeassistant.components.recorder # homeassistant.components.sql -SQLAlchemy==2.0.15 +SQLAlchemy==2.0.21 # homeassistant.components.onvif WSDiscovery==2.0.0 diff --git a/tests/components/recorder/test_migration_from_schema_32.py b/tests/components/recorder/test_migration_from_schema_32.py index cdf930fde26..e007d2408dd 100644 --- a/tests/components/recorder/test_migration_from_schema_32.py +++ b/tests/components/recorder/test_migration_from_schema_32.py @@ -39,6 +39,12 @@ SCHEMA_MODULE = "tests.components.recorder.db_schema_32" ORIG_TZ = dt_util.DEFAULT_TIME_ZONE +async def _async_wait_migration_done(hass: HomeAssistant) -> None: + """Wait for the migration to be done.""" + await recorder.get_instance(hass).async_block_till_done() + await async_recorder_block_till_done(hass) + + def _create_engine_test(*args, **kwargs): """Test version of create_engine that initializes with old schema. @@ -101,6 +107,8 @@ async def test_migrate_events_context_ids( """Test we can migrate old uuid context ids and ulid context ids to binary format.""" instance = await async_setup_recorder_instance(hass) await async_wait_recording_done(hass) + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] test_uuid = uuid.uuid4() uuid_hex = test_uuid.hex @@ -110,7 +118,7 @@ async def test_migrate_events_context_ids( with session_scope(hass=hass) as session: session.add_all( ( - Events( + old_db_schema.Events( event_type="old_uuid_context_id_event", event_data=None, origin_idx=0, @@ -123,7 +131,7 @@ async def test_migrate_events_context_ids( context_parent_id=None, context_parent_id_bin=None, ), - Events( + old_db_schema.Events( event_type="empty_context_id_event", event_data=None, origin_idx=0, @@ -136,7 +144,7 @@ async def test_migrate_events_context_ids( context_parent_id=None, context_parent_id_bin=None, ), - Events( + old_db_schema.Events( event_type="ulid_context_id_event", event_data=None, origin_idx=0, @@ -149,7 +157,7 @@ async def test_migrate_events_context_ids( context_parent_id="01ARZ3NDEKTSV4RRFFQ69G5FA2", context_parent_id_bin=None, ), - Events( + old_db_schema.Events( event_type="invalid_context_id_event", event_data=None, origin_idx=0, @@ -162,7 +170,7 @@ async def test_migrate_events_context_ids( context_parent_id=None, context_parent_id_bin=None, ), - Events( + old_db_schema.Events( event_type="garbage_context_id_event", event_data=None, origin_idx=0, @@ -175,7 +183,7 @@ async def test_migrate_events_context_ids( context_parent_id=None, context_parent_id_bin=None, ), - Events( + old_db_schema.Events( event_type="event_with_garbage_context_id_no_time_fired_ts", event_data=None, origin_idx=0, @@ -196,10 +204,12 @@ async def test_migrate_events_context_ids( await async_wait_recording_done(hass) now = dt_util.utcnow() expected_ulid_fallback_start = ulid_to_bytes(ulid_at_time(now.timestamp()))[0:6] + await _async_wait_migration_done(hass) + with freeze_time(now): # This is a threadsafe way to add a task to the recorder instance.queue_task(EventsContextIDMigrationTask()) - await async_recorder_block_till_done(hass) + await _async_wait_migration_done(hass) def _object_as_dict(obj): return {c.key: getattr(obj, c.key) for c in inspect(obj).mapper.column_attrs} @@ -304,6 +314,8 @@ async def test_migrate_states_context_ids( """Test we can migrate old uuid context ids and ulid context ids to binary format.""" instance = await async_setup_recorder_instance(hass) await async_wait_recording_done(hass) + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] test_uuid = uuid.uuid4() uuid_hex = test_uuid.hex @@ -313,7 +325,7 @@ async def test_migrate_states_context_ids( with session_scope(hass=hass) as session: session.add_all( ( - States( + old_db_schema.States( entity_id="state.old_uuid_context_id", last_updated_ts=1477721632.452529, context_id=uuid_hex, @@ -323,7 +335,7 @@ async def test_migrate_states_context_ids( context_parent_id=None, context_parent_id_bin=None, ), - States( + old_db_schema.States( entity_id="state.empty_context_id", last_updated_ts=1477721632.552529, context_id=None, @@ -333,7 +345,7 @@ async def test_migrate_states_context_ids( context_parent_id=None, context_parent_id_bin=None, ), - States( + old_db_schema.States( entity_id="state.ulid_context_id", last_updated_ts=1477721632.552529, context_id="01ARZ3NDEKTSV4RRFFQ69G5FAV", @@ -343,7 +355,7 @@ async def test_migrate_states_context_ids( context_parent_id="01ARZ3NDEKTSV4RRFFQ69G5FA2", context_parent_id_bin=None, ), - States( + old_db_schema.States( entity_id="state.invalid_context_id", last_updated_ts=1477721632.552529, context_id="invalid", @@ -353,7 +365,7 @@ async def test_migrate_states_context_ids( context_parent_id=None, context_parent_id_bin=None, ), - States( + old_db_schema.States( entity_id="state.garbage_context_id", last_updated_ts=1477721632.552529, context_id="adapt_lgt:b'5Cf*':interval:b'0R'", @@ -363,7 +375,7 @@ async def test_migrate_states_context_ids( context_parent_id=None, context_parent_id_bin=None, ), - States( + old_db_schema.States( entity_id="state.human_readable_uuid_context_id", last_updated_ts=1477721632.552529, context_id="0ae29799-ee4e-4f45-8116-f582d7d3ee65", @@ -380,7 +392,7 @@ async def test_migrate_states_context_ids( await async_wait_recording_done(hass) instance.queue_task(StatesContextIDMigrationTask()) - await async_recorder_block_till_done(hass) + await _async_wait_migration_done(hass) def _object_as_dict(obj): return {c.key: getattr(obj, c.key) for c in inspect(obj).mapper.column_attrs} @@ -489,22 +501,24 @@ async def test_migrate_event_type_ids( """Test we can migrate event_types to the EventTypes table.""" instance = await async_setup_recorder_instance(hass) await async_wait_recording_done(hass) + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] def _insert_events(): with session_scope(hass=hass) as session: session.add_all( ( - Events( + old_db_schema.Events( event_type="event_type_one", origin_idx=0, time_fired_ts=1677721632.452529, ), - Events( + old_db_schema.Events( event_type="event_type_one", origin_idx=0, time_fired_ts=1677721632.552529, ), - Events( + old_db_schema.Events( event_type="event_type_two", origin_idx=0, time_fired_ts=1677721632.552529, @@ -517,7 +531,7 @@ async def test_migrate_event_type_ids( await async_wait_recording_done(hass) # This is a threadsafe way to add a task to the recorder instance.queue_task(EventTypeIDMigrationTask()) - await async_recorder_block_till_done(hass) + await _async_wait_migration_done(hass) def _fetch_migrated_events(): with session_scope(hass=hass, read_only=True) as session: @@ -570,22 +584,24 @@ async def test_migrate_entity_ids( """Test we can migrate entity_ids to the StatesMeta table.""" instance = await async_setup_recorder_instance(hass) await async_wait_recording_done(hass) + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] def _insert_states(): with session_scope(hass=hass) as session: session.add_all( ( - States( + old_db_schema.States( entity_id="sensor.one", state="one_1", last_updated_ts=1.452529, ), - States( + old_db_schema.States( entity_id="sensor.two", state="two_2", last_updated_ts=2.252529, ), - States( + old_db_schema.States( entity_id="sensor.two", state="two_1", last_updated_ts=3.152529, @@ -595,10 +611,10 @@ async def test_migrate_entity_ids( await instance.async_add_executor_job(_insert_states) - await async_wait_recording_done(hass) + await _async_wait_migration_done(hass) # This is a threadsafe way to add a task to the recorder instance.queue_task(EntityIDMigrationTask()) - await async_recorder_block_till_done(hass) + await _async_wait_migration_done(hass) def _fetch_migrated_states(): with session_scope(hass=hass, read_only=True) as session: @@ -636,22 +652,24 @@ async def test_post_migrate_entity_ids( """Test we can migrate entity_ids to the StatesMeta table.""" instance = await async_setup_recorder_instance(hass) await async_wait_recording_done(hass) + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] def _insert_events(): with session_scope(hass=hass) as session: session.add_all( ( - States( + old_db_schema.States( entity_id="sensor.one", state="one_1", last_updated_ts=1.452529, ), - States( + old_db_schema.States( entity_id="sensor.two", state="two_2", last_updated_ts=2.252529, ), - States( + old_db_schema.States( entity_id="sensor.two", state="two_1", last_updated_ts=3.152529, @@ -661,10 +679,10 @@ async def test_post_migrate_entity_ids( await instance.async_add_executor_job(_insert_events) - await async_wait_recording_done(hass) + await _async_wait_migration_done(hass) # This is a threadsafe way to add a task to the recorder instance.queue_task(EntityIDPostMigrationTask()) - await async_recorder_block_till_done(hass) + await _async_wait_migration_done(hass) def _fetch_migrated_states(): with session_scope(hass=hass, read_only=True) as session: @@ -688,18 +706,20 @@ async def test_migrate_null_entity_ids( """Test we can migrate entity_ids to the StatesMeta table.""" instance = await async_setup_recorder_instance(hass) await async_wait_recording_done(hass) + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] def _insert_states(): with session_scope(hass=hass) as session: session.add( - States( + old_db_schema.States( entity_id="sensor.one", state="one_1", last_updated_ts=1.452529, ), ) session.add_all( - States( + old_db_schema.States( entity_id=None, state="empty", last_updated_ts=time + 1.452529, @@ -707,7 +727,7 @@ async def test_migrate_null_entity_ids( for time in range(1000) ) session.add( - States( + old_db_schema.States( entity_id="sensor.one", state="one_1", last_updated_ts=2.452529, @@ -716,11 +736,10 @@ async def test_migrate_null_entity_ids( await instance.async_add_executor_job(_insert_states) - await async_wait_recording_done(hass) + await _async_wait_migration_done(hass) # This is a threadsafe way to add a task to the recorder instance.queue_task(EntityIDMigrationTask()) - await async_recorder_block_till_done(hass) - await async_recorder_block_till_done(hass) + await _async_wait_migration_done(hass) def _fetch_migrated_states(): with session_scope(hass=hass, read_only=True) as session: @@ -758,18 +777,20 @@ async def test_migrate_null_event_type_ids( """Test we can migrate event_types to the EventTypes table when the event_type is NULL.""" instance = await async_setup_recorder_instance(hass) await async_wait_recording_done(hass) + importlib.import_module(SCHEMA_MODULE) + old_db_schema = sys.modules[SCHEMA_MODULE] def _insert_events(): with session_scope(hass=hass) as session: session.add( - Events( + old_db_schema.Events( event_type="event_type_one", origin_idx=0, time_fired_ts=1.452529, ), ) session.add_all( - Events( + old_db_schema.Events( event_type=None, origin_idx=0, time_fired_ts=time + 1.452529, @@ -777,7 +798,7 @@ async def test_migrate_null_event_type_ids( for time in range(1000) ) session.add( - Events( + old_db_schema.Events( event_type="event_type_one", origin_idx=0, time_fired_ts=2.452529, @@ -786,12 +807,10 @@ async def test_migrate_null_event_type_ids( await instance.async_add_executor_job(_insert_events) - await async_wait_recording_done(hass) + await _async_wait_migration_done(hass) # This is a threadsafe way to add a task to the recorder - instance.queue_task(EventTypeIDMigrationTask()) - await async_recorder_block_till_done(hass) - await async_recorder_block_till_done(hass) + await _async_wait_migration_done(hass) def _fetch_migrated_events(): with session_scope(hass=hass, read_only=True) as session: