From 5284837c8f6c45f67512b0fa3828eb12e0b43342 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 31 Jan 2023 13:42:07 -0600 Subject: [PATCH] Add a repair issue when using MariaDB is affected by MDEV-25020 (#87040) closes https://github.com/home-assistant/core/issues/83787 --- .../components/recorder/strings.json | 6 + .../components/recorder/translations/en.json | 6 + homeassistant/components/recorder/util.py | 72 ++++++++++- tests/components/recorder/test_util.py | 115 +++++++++++++++++- 4 files changed, 195 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/recorder/strings.json b/homeassistant/components/recorder/strings.json index 9b616372adf..7af67f10e25 100644 --- a/homeassistant/components/recorder/strings.json +++ b/homeassistant/components/recorder/strings.json @@ -7,5 +7,11 @@ "database_engine": "Database Engine", "database_version": "Database Version" } + }, + "issues": { + "maria_db_range_index_regression": { + "title": "Update MariaDB to {min_version} or later resolve a significant performance issue", + "description": "Older versions of MariaDB suffer from a significant performance regression when retrieving history data or purging the database. Update to MariaDB version {min_version} or later and restart Home Assistant. If you are using the MariaDB core add-on, make sure to update it to the latest version." + } } } diff --git a/homeassistant/components/recorder/translations/en.json b/homeassistant/components/recorder/translations/en.json index c9ceffc7397..30c17b854ca 100644 --- a/homeassistant/components/recorder/translations/en.json +++ b/homeassistant/components/recorder/translations/en.json @@ -1,4 +1,10 @@ { + "issues": { + "maria_db_range_index_regression": { + "description": "Older versions of MariaDB suffer from a significant performance regression when retrieving history data or purging the database. Update to MariaDB version {min_version} or later and restart Home Assistant. If you are using the MariaDB core add-on, make sure to update it to the latest version.", + "title": "Update MariaDB to {min_version} or later resolve a significant performance issue" + } + }, "system_health": { "info": { "current_recorder_run": "Current Run Start Time", diff --git a/homeassistant/components/recorder/util.py b/homeassistant/components/recorder/util.py index fbd1b0c3c93..05246c7848c 100644 --- a/homeassistant/components/recorder/util.py +++ b/homeassistant/components/recorder/util.py @@ -25,11 +25,11 @@ from sqlalchemy.orm.session import Session from sqlalchemy.sql.lambdas import StatementLambdaElement import voluptuous as vol -from homeassistant.core import HomeAssistant -from homeassistant.helpers import config_validation as cv +from homeassistant.core import HomeAssistant, callback +from homeassistant.helpers import config_validation as cv, issue_registry as ir import homeassistant.util.dt as dt_util -from .const import DATA_INSTANCE, SQLITE_URL_PREFIX, SupportedDialect +from .const import DATA_INSTANCE, DOMAIN, SQLITE_URL_PREFIX, SupportedDialect from .db_schema import ( TABLE_RECORDER_RUNS, TABLE_SCHEMA_CHANGES, @@ -51,9 +51,35 @@ QUERY_RETRY_WAIT = 0.1 SQLITE3_POSTFIXES = ["", "-wal", "-shm"] DEFAULT_YIELD_STATES_ROWS = 32768 +# Our minimum versions for each database +# +# Older MariaDB suffers https://jira.mariadb.org/browse/MDEV-25020 +# which is fixed in 10.5.17, 10.6.9, 10.7.5, 10.8.4 +# MIN_VERSION_MARIA_DB = AwesomeVersion( "10.3.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER ) +RECOMMENDED_MIN_VERSION_MARIA_DB = AwesomeVersion( + "10.5.17", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER +) +MARIA_DB_106 = AwesomeVersion( + "10.6.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER +) +RECOMMENDED_MIN_VERSION_MARIA_DB_106 = AwesomeVersion( + "10.6.9", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER +) +MARIA_DB_107 = AwesomeVersion( + "10.7.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER +) +RECOMMENDED_MIN_VERSION_MARIA_DB_107 = AwesomeVersion( + "10.7.5", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER +) +MARIA_DB_108 = AwesomeVersion( + "10.8.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER +) +RECOMMENDED_MIN_VERSION_MARIA_DB_108 = AwesomeVersion( + "10.8.4", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER +) MIN_VERSION_MYSQL = AwesomeVersion( "8.0.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER ) @@ -410,6 +436,34 @@ def build_mysqldb_conv() -> dict: return {**conversions, FIELD_TYPE.DATETIME: _datetime_or_none} +@callback +def _async_create_mariadb_range_index_regression_issue( + hass: HomeAssistant, version: AwesomeVersion +) -> None: + """Create an issue for the index range regression in older MariaDB. + + The range scan issue was fixed in MariaDB 10.5.17, 10.6.9, 10.7.5, 10.8.4 and later. + """ + if version >= MARIA_DB_108: + min_version = RECOMMENDED_MIN_VERSION_MARIA_DB_108 + elif version >= MARIA_DB_107: + min_version = RECOMMENDED_MIN_VERSION_MARIA_DB_107 + elif version >= MARIA_DB_106: + min_version = RECOMMENDED_MIN_VERSION_MARIA_DB_106 + else: + min_version = RECOMMENDED_MIN_VERSION_MARIA_DB + ir.async_create_issue( + hass, + DOMAIN, + "maria_db_range_index_regression", + is_fixable=False, + severity=ir.IssueSeverity.CRITICAL, + learn_more_url="https://jira.mariadb.org/browse/MDEV-25020", + translation_key="maria_db_range_index_regression", + translation_placeholders={"min_version": str(min_version)}, + ) + + def setup_connection_for_dialect( instance: Recorder, dialect_name: str, @@ -466,6 +520,18 @@ def setup_connection_for_dialect( _fail_unsupported_version( version or version_string, "MariaDB", MIN_VERSION_MARIA_DB ) + if version and ( + (version < RECOMMENDED_MIN_VERSION_MARIA_DB) + or (MARIA_DB_106 <= version < RECOMMENDED_MIN_VERSION_MARIA_DB_106) + or (MARIA_DB_107 <= version < RECOMMENDED_MIN_VERSION_MARIA_DB_107) + or (MARIA_DB_108 <= version < RECOMMENDED_MIN_VERSION_MARIA_DB_108) + ): + instance.hass.add_job( + _async_create_mariadb_range_index_regression_issue, + instance.hass, + version, + ) + else: if not version or version < MIN_VERSION_MYSQL: _fail_unsupported_version( diff --git a/tests/components/recorder/test_util.py b/tests/components/recorder/test_util.py index 35fefad970f..c133dd90fdb 100644 --- a/tests/components/recorder/test_util.py +++ b/tests/components/recorder/test_util.py @@ -14,7 +14,7 @@ from sqlalchemy.sql.lambdas import StatementLambdaElement from homeassistant.components import recorder from homeassistant.components.recorder import history, util -from homeassistant.components.recorder.const import SQLITE_URL_PREFIX +from homeassistant.components.recorder.const import DOMAIN, SQLITE_URL_PREFIX from homeassistant.components.recorder.db_schema import RecorderRuns from homeassistant.components.recorder.models import UnsupportedDialect from homeassistant.components.recorder.util import ( @@ -25,6 +25,7 @@ from homeassistant.components.recorder.util import ( ) from homeassistant.const import EVENT_HOMEASSISTANT_STOP from homeassistant.core import HomeAssistant +from homeassistant.helpers.issue_registry import async_get as async_get_issue_registry from homeassistant.util import dt as dt_util from .common import corrupt_db_file, run_information_with_session, wait_recording_done @@ -549,6 +550,118 @@ def test_warn_unsupported_dialect(caplog, dialect, message): assert message in caplog.text +@pytest.mark.parametrize( + "mysql_version,min_version", + [ + ( + "10.5.16-MariaDB", + "10.5.17", + ), + ( + "10.6.8-MariaDB", + "10.6.9", + ), + ( + "10.7.1-MariaDB", + "10.7.5", + ), + ( + "10.8.0-MariaDB", + "10.8.4", + ), + ], +) +async def test_issue_for_mariadb_with_MDEV_25020( + hass, caplog, mysql_version, min_version +): + """Test we create an issue for MariaDB versions affected. + + See https://jira.mariadb.org/browse/MDEV-25020. + """ + instance_mock = MagicMock() + instance_mock.hass = hass + execute_args = [] + close_mock = MagicMock() + + def execute_mock(statement): + nonlocal execute_args + execute_args.append(statement) + + def fetchall_mock(): + nonlocal execute_args + if execute_args[-1] == "SELECT VERSION()": + return [[mysql_version]] + return None + + def _make_cursor_mock(*_): + return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock) + + dbapi_connection = MagicMock(cursor=_make_cursor_mock) + + await hass.async_add_executor_job( + util.setup_connection_for_dialect, + instance_mock, + "mysql", + dbapi_connection, + True, + ) + await hass.async_block_till_done() + + registry = async_get_issue_registry(hass) + issue = registry.async_get_issue(DOMAIN, "maria_db_range_index_regression") + assert issue is not None + assert issue.translation_placeholders == {"min_version": min_version} + + +@pytest.mark.parametrize( + "mysql_version", + [ + "10.5.17-MariaDB", + "10.6.9-MariaDB", + "10.7.5-MariaDB", + "10.8.4-MariaDB", + "10.9.1-MariaDB", + ], +) +async def test_no_issue_for_mariadb_with_MDEV_25020(hass, caplog, mysql_version): + """Test we do not create an issue for MariaDB versions not affected. + + See https://jira.mariadb.org/browse/MDEV-25020. + """ + instance_mock = MagicMock() + instance_mock.hass = hass + execute_args = [] + close_mock = MagicMock() + + def execute_mock(statement): + nonlocal execute_args + execute_args.append(statement) + + def fetchall_mock(): + nonlocal execute_args + if execute_args[-1] == "SELECT VERSION()": + return [[mysql_version]] + return None + + def _make_cursor_mock(*_): + return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock) + + dbapi_connection = MagicMock(cursor=_make_cursor_mock) + + await hass.async_add_executor_job( + util.setup_connection_for_dialect, + instance_mock, + "mysql", + dbapi_connection, + True, + ) + await hass.async_block_till_done() + + registry = async_get_issue_registry(hass) + issue = registry.async_get_issue(DOMAIN, "maria_db_range_index_regression") + assert issue is None + + def test_basic_sanity_check(hass_recorder, recorder_db_url): """Test the basic sanity checks with a missing table.""" if recorder_db_url.startswith("mysql://"):