From e9ba5f3b4bd5cfff4ac43c9914c0ba4908c0c435 Mon Sep 17 00:00:00 2001 From: Erik Montnemery Date: Tue, 26 Oct 2021 13:41:59 +0200 Subject: [PATCH] Warn when recorder connects to an unsupported database (#58161) --- homeassistant/components/recorder/util.py | 127 ++++++++++- tests/components/recorder/test_util.py | 257 +++++++++++++++++++++- 2 files changed, 371 insertions(+), 13 deletions(-) diff --git a/homeassistant/components/recorder/util.py b/homeassistant/components/recorder/util.py index 8277f86e9f9..8658c7b3677 100644 --- a/homeassistant/components/recorder/util.py +++ b/homeassistant/components/recorder/util.py @@ -7,9 +7,15 @@ from datetime import timedelta import functools import logging import os +import re import time from typing import TYPE_CHECKING +from awesomeversion import ( + AwesomeVersion, + AwesomeVersionException, + AwesomeVersionStrategy, +) from sqlalchemy import text from sqlalchemy.exc import OperationalError, SQLAlchemyError from sqlalchemy.orm.session import Session @@ -39,6 +45,14 @@ RETRIES = 3 QUERY_RETRY_WAIT = 0.1 SQLITE3_POSTFIXES = ["", "-wal", "-shm"] +MIN_VERSION_MARIA_DB = AwesomeVersion("10.3.0", AwesomeVersionStrategy.SIMPLEVER) +MIN_VERSION_MARIA_DB_ROWNUM = AwesomeVersion("10.2.0", AwesomeVersionStrategy.SIMPLEVER) +MIN_VERSION_MYSQL = AwesomeVersion("8.0.0", AwesomeVersionStrategy.SIMPLEVER) +MIN_VERSION_MYSQL_ROWNUM = AwesomeVersion("5.8.0", AwesomeVersionStrategy.SIMPLEVER) +MIN_VERSION_PGSQL = AwesomeVersion(120000, AwesomeVersionStrategy.BUILDVER) +MIN_VERSION_SQLITE = AwesomeVersion("3.32.1", AwesomeVersionStrategy.SIMPLEVER) +MIN_VERSION_SQLITE_ROWNUM = AwesomeVersion("3.25.0", AwesomeVersionStrategy.SIMPLEVER) + # This is the maximum time after the recorder ends the session # before we no longer consider startup to be a "restart" and we # should do a check on the sqlite3 database. @@ -275,6 +289,55 @@ def query_on_connection(dbapi_connection, statement): return result +def _warn_unsupported_dialect(dialect): + """Warn about unsupported database version.""" + _LOGGER.warning( + "Database %s is not supported; Home Assistant supports %s. " + "Starting with Home Assistant 2022.2 this will prevent the recorder from " + "starting. Please migrate your database to a supported software before then", + dialect, + "MariaDB ≥ 10.3, MySQL ≥ 8.0, PostgreSQL ≥ 12, SQLite ≥ 3.32.1", + ) + + +def _warn_unsupported_version(server_version, dialect, minimum_version): + """Warn about unsupported database version.""" + _LOGGER.warning( + "Version %s of %s is not supported; minimum supported version is %s. " + "Starting with Home Assistant 2022.2 this will prevent the recorder from " + "starting. Please upgrade your database software before then", + server_version, + dialect, + minimum_version, + ) + + +def _extract_version_from_server_response(server_response): + """Attempt to extract version from server response.""" + try: + return AwesomeVersion( + server_response, + ensure_strategy=AwesomeVersionStrategy.SIMPLEVER, + find_first_match=True, + ) + except AwesomeVersionException: + return None + + +def _pgsql_numerical_version_to_string(version_num): + """Convert numerical PostgreSQL version to string.""" + if version_num < 100000: + major = version_num // 10000 + minor = version_num % 10000 // 100 + patch = version_num % 100 + return f"{major}.{minor}.{patch}" + + # version 10+ + major = version_num // 10000 + patch = version_num % 10000 + return f"{major}.{patch}" + + def setup_connection_for_dialect( instance, dialect_name, dbapi_connection, first_connection ): @@ -292,12 +355,17 @@ def setup_connection_for_dialect( # instead of every time we open the sqlite connection # as its persistent and isn't free to call every time. result = query_on_connection(dbapi_connection, "SELECT sqlite_version()") - version = result[0][0] - major, minor, _patch = version.split(".", 2) - if int(major) == 3 and int(minor) < 25: + version_string = result[0][0] + version = _extract_version_from_server_response(version_string) + + if version and version < MIN_VERSION_SQLITE_ROWNUM: instance._db_supports_row_number = ( # pylint: disable=[protected-access] False ) + if not version or version < MIN_VERSION_SQLITE: + _warn_unsupported_version( + version or version_string, "SQLite", MIN_VERSION_SQLITE + ) # approximately 8MiB of memory execute_on_connection(dbapi_connection, "PRAGMA cache_size = -8192") @@ -305,18 +373,55 @@ def setup_connection_for_dialect( # enable support for foreign keys execute_on_connection(dbapi_connection, "PRAGMA foreign_keys=ON") - if dialect_name == "mysql": + elif dialect_name == "mysql": execute_on_connection(dbapi_connection, "SET session wait_timeout=28800") if first_connection: result = query_on_connection(dbapi_connection, "SELECT VERSION()") - version = result[0][0] - major, minor, _patch = version.split(".", 2) - if (int(major) == 5 and int(minor) < 8) or ( - int(major) == 10 and int(minor) < 2 - ): - instance._db_supports_row_number = ( # pylint: disable=[protected-access] - False + version_string = result[0][0] + version = _extract_version_from_server_response(version_string) + is_maria_db = re.search("MariaDb", version_string, re.IGNORECASE) + + if is_maria_db: + if version and version < MIN_VERSION_MARIA_DB_ROWNUM: + instance._db_supports_row_number = ( # pylint: disable=[protected-access] + False + ) + if not version or version < MIN_VERSION_MARIA_DB: + _warn_unsupported_version( + version or version_string, "MariaDB", MIN_VERSION_MARIA_DB + ) + else: + if version and version < MIN_VERSION_MYSQL_ROWNUM: + instance._db_supports_row_number = ( # pylint: disable=[protected-access] + False + ) + if not version or version < MIN_VERSION_MYSQL: + _warn_unsupported_version( + version or version_string, "MySQL", MIN_VERSION_MYSQL + ) + + elif dialect_name == "postgresql": + if first_connection: + # server_version_num was added in 2006 + result = query_on_connection(dbapi_connection, "SHOW server_version_num") + version_string = result[0][0] + try: + version = AwesomeVersion( + version_string, AwesomeVersionStrategy.BUILDVER ) + except AwesomeVersionException: + version = None + if not version or version < MIN_VERSION_PGSQL: + if version: + version_string = _pgsql_numerical_version_to_string(int(version)) + _warn_unsupported_version( + version_string, + "PostgreSQL", + _pgsql_numerical_version_to_string(int(MIN_VERSION_PGSQL)), + ) + + else: + _warn_unsupported_dialect(dialect_name) def end_incomplete_runs(session, start_time): diff --git a/tests/components/recorder/test_util.py b/tests/components/recorder/test_util.py index ff690e24279..ced614ee2fa 100644 --- a/tests/components/recorder/test_util.py +++ b/tests/components/recorder/test_util.py @@ -125,8 +125,8 @@ async def test_last_run_was_recently_clean(hass): @pytest.mark.parametrize( "mysql_version, db_supports_row_number", [ - ("10.2.0", True), - ("10.1.0", False), + ("10.2.0-MariaDB", True), + ("10.1.0-MariaDB", False), ("5.8.0", True), ("5.7.0", False), ], @@ -207,6 +207,259 @@ def test_setup_connection_for_dialect_sqlite(sqlite_version, db_supports_row_num assert instance_mock._db_supports_row_number == db_supports_row_number +@pytest.mark.parametrize( + "mysql_version,message", + [ + ( + "10.2.0-MariaDB", + "Version 10.2.0 of MariaDB is not supported; minimum supported version is 10.3.0.", + ), + ( + "5.7.26-0ubuntu0.18.04.1", + "Version 5.7.26 of MySQL is not supported; minimum supported version is 8.0.0.", + ), + ( + "some_random_response", + "Version some_random_response of MySQL is not supported; minimum supported version is 8.0.0.", + ), + ], +) +def test_warn_outdated_mysql(caplog, mysql_version, message): + """Test setting up the connection for an outdated mysql version.""" + instance_mock = MagicMock(_db_supports_row_number=True) + execute_args = [] + close_mock = MagicMock() + + def execute_mock(statement): + nonlocal execute_args + execute_args.append(statement) + + def fetchall_mock(): + nonlocal execute_args + if execute_args[-1] == "SELECT VERSION()": + return [[mysql_version]] + return None + + def _make_cursor_mock(*_): + return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock) + + dbapi_connection = MagicMock(cursor=_make_cursor_mock) + + util.setup_connection_for_dialect(instance_mock, "mysql", dbapi_connection, True) + + assert message in caplog.text + + +@pytest.mark.parametrize( + "mysql_version", + [ + ("10.3.0"), + ("8.0.0"), + ], +) +def test_supported_mysql(caplog, mysql_version): + """Test setting up the connection for a supported mysql version.""" + instance_mock = MagicMock(_db_supports_row_number=True) + execute_args = [] + close_mock = MagicMock() + + def execute_mock(statement): + nonlocal execute_args + execute_args.append(statement) + + def fetchall_mock(): + nonlocal execute_args + if execute_args[-1] == "SELECT VERSION()": + return [[mysql_version]] + return None + + def _make_cursor_mock(*_): + return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock) + + dbapi_connection = MagicMock(cursor=_make_cursor_mock) + + util.setup_connection_for_dialect(instance_mock, "mysql", dbapi_connection, True) + + assert "minimum supported version" not in caplog.text + + +@pytest.mark.parametrize( + "pgsql_version,message", + [ + ( + "110013", + "Version 11.13 of PostgreSQL is not supported; minimum supported version is 12.0.", + ), + ( + "90210", + "Version 9.2.10 of PostgreSQL is not supported; minimum supported version is 12.0.", + ), + ( + "unexpected", + "Version unexpected of PostgreSQL is not supported; minimum supported version is 12.0.", + ), + ], +) +def test_warn_outdated_pgsql(caplog, pgsql_version, message): + """Test setting up the connection for an outdated PostgreSQL version.""" + instance_mock = MagicMock(_db_supports_row_number=True) + execute_args = [] + close_mock = MagicMock() + + def execute_mock(statement): + nonlocal execute_args + execute_args.append(statement) + + def fetchall_mock(): + nonlocal execute_args + if execute_args[-1] == "SHOW server_version_num": + return [[pgsql_version]] + return None + + def _make_cursor_mock(*_): + return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock) + + dbapi_connection = MagicMock(cursor=_make_cursor_mock) + + util.setup_connection_for_dialect( + instance_mock, "postgresql", dbapi_connection, True + ) + + assert message in caplog.text + + +@pytest.mark.parametrize( + "pgsql_version", + [ + (130000), + ], +) +def test_supported_pgsql(caplog, pgsql_version): + """Test setting up the connection for a supported PostgreSQL version.""" + instance_mock = MagicMock(_db_supports_row_number=True) + execute_args = [] + close_mock = MagicMock() + + def execute_mock(statement): + nonlocal execute_args + execute_args.append(statement) + + def fetchall_mock(): + nonlocal execute_args + if execute_args[-1] == "SHOW server_version_num": + return [[pgsql_version]] + return None + + def _make_cursor_mock(*_): + return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock) + + dbapi_connection = MagicMock(cursor=_make_cursor_mock) + + util.setup_connection_for_dialect( + instance_mock, "postgresql", dbapi_connection, True + ) + + assert "minimum supported version" not in caplog.text + + +@pytest.mark.parametrize( + "sqlite_version,message", + [ + ( + "3.32.0", + "Version 3.32.0 of SQLite is not supported; minimum supported version is 3.32.1.", + ), + ( + "3.31.0", + "Version 3.31.0 of SQLite is not supported; minimum supported version is 3.32.1.", + ), + ( + "2.0.0", + "Version 2.0.0 of SQLite is not supported; minimum supported version is 3.32.1.", + ), + ( + "dogs", + "Version dogs of SQLite is not supported; minimum supported version is 3.32.1.", + ), + ], +) +def test_warn_outdated_sqlite(caplog, sqlite_version, message): + """Test setting up the connection for an outdated sqlite version.""" + instance_mock = MagicMock(_db_supports_row_number=True) + execute_args = [] + close_mock = MagicMock() + + def execute_mock(statement): + nonlocal execute_args + execute_args.append(statement) + + def fetchall_mock(): + nonlocal execute_args + if execute_args[-1] == "SELECT sqlite_version()": + return [[sqlite_version]] + return None + + def _make_cursor_mock(*_): + return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock) + + dbapi_connection = MagicMock(cursor=_make_cursor_mock) + + util.setup_connection_for_dialect(instance_mock, "sqlite", dbapi_connection, True) + + assert message in caplog.text + + +@pytest.mark.parametrize( + "sqlite_version", + [ + ("3.32.1"), + ("3.33.0"), + ], +) +def test_supported_sqlite(caplog, sqlite_version): + """Test setting up the connection for a supported sqlite version.""" + instance_mock = MagicMock(_db_supports_row_number=True) + execute_args = [] + close_mock = MagicMock() + + def execute_mock(statement): + nonlocal execute_args + execute_args.append(statement) + + def fetchall_mock(): + nonlocal execute_args + if execute_args[-1] == "SELECT sqlite_version()": + return [[sqlite_version]] + return None + + def _make_cursor_mock(*_): + return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock) + + dbapi_connection = MagicMock(cursor=_make_cursor_mock) + + util.setup_connection_for_dialect(instance_mock, "sqlite", dbapi_connection, True) + + assert "minimum supported version" not in caplog.text + + +@pytest.mark.parametrize( + "dialect,message", + [ + ("mssql", "Database mssql is not supported"), + ("oracle", "Database oracle is not supported"), + ("some_db", "Database some_db is not supported"), + ], +) +def test_warn_unsupported_dialect(caplog, dialect, message): + """Test setting up the connection for an outdated sqlite version.""" + instance_mock = MagicMock() + dbapi_connection = MagicMock() + + util.setup_connection_for_dialect(instance_mock, dialect, dbapi_connection, True) + + assert message in caplog.text + + def test_basic_sanity_check(hass_recorder): """Test the basic sanity checks with a missing table.""" hass = hass_recorder()