mirror of
https://github.com/home-assistant/core.git
synced 2025-07-23 21:27:38 +00:00
Add ability to disable the sqlite3 quick_check (#39479)
This commit is contained in:
parent
225becc89a
commit
557684c3ce
@ -35,7 +35,7 @@ from homeassistant.helpers.typing import ConfigType
|
|||||||
import homeassistant.util.dt as dt_util
|
import homeassistant.util.dt as dt_util
|
||||||
|
|
||||||
from . import migration, purge
|
from . import migration, purge
|
||||||
from .const import DATA_INSTANCE, DOMAIN, SQLITE_URL_PREFIX
|
from .const import CONF_DB_INTEGRITY_CHECK, DATA_INSTANCE, DOMAIN, SQLITE_URL_PREFIX
|
||||||
from .models import Base, Events, RecorderRuns, States
|
from .models import Base, Events, RecorderRuns, States
|
||||||
from .util import session_scope, validate_or_move_away_sqlite_database
|
from .util import session_scope, validate_or_move_away_sqlite_database
|
||||||
|
|
||||||
@ -55,6 +55,7 @@ SERVICE_PURGE_SCHEMA = vol.Schema(
|
|||||||
|
|
||||||
DEFAULT_URL = "sqlite:///{hass_config_path}"
|
DEFAULT_URL = "sqlite:///{hass_config_path}"
|
||||||
DEFAULT_DB_FILE = "home-assistant_v2.db"
|
DEFAULT_DB_FILE = "home-assistant_v2.db"
|
||||||
|
DEFAULT_DB_INTEGRITY_CHECK = True
|
||||||
DEFAULT_DB_MAX_RETRIES = 10
|
DEFAULT_DB_MAX_RETRIES = 10
|
||||||
DEFAULT_DB_RETRY_WAIT = 3
|
DEFAULT_DB_RETRY_WAIT = 3
|
||||||
KEEPALIVE_TIME = 30
|
KEEPALIVE_TIME = 30
|
||||||
@ -99,6 +100,9 @@ CONFIG_SCHEMA = vol.Schema(
|
|||||||
vol.Optional(
|
vol.Optional(
|
||||||
CONF_DB_RETRY_WAIT, default=DEFAULT_DB_RETRY_WAIT
|
CONF_DB_RETRY_WAIT, default=DEFAULT_DB_RETRY_WAIT
|
||||||
): cv.positive_int,
|
): cv.positive_int,
|
||||||
|
vol.Optional(
|
||||||
|
CONF_DB_INTEGRITY_CHECK, default=DEFAULT_DB_INTEGRITY_CHECK
|
||||||
|
): cv.boolean,
|
||||||
}
|
}
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@ -156,6 +160,7 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
|
|||||||
commit_interval = conf[CONF_COMMIT_INTERVAL]
|
commit_interval = conf[CONF_COMMIT_INTERVAL]
|
||||||
db_max_retries = conf[CONF_DB_MAX_RETRIES]
|
db_max_retries = conf[CONF_DB_MAX_RETRIES]
|
||||||
db_retry_wait = conf[CONF_DB_RETRY_WAIT]
|
db_retry_wait = conf[CONF_DB_RETRY_WAIT]
|
||||||
|
db_integrity_check = conf[CONF_DB_INTEGRITY_CHECK]
|
||||||
|
|
||||||
db_url = conf.get(CONF_DB_URL)
|
db_url = conf.get(CONF_DB_URL)
|
||||||
if not db_url:
|
if not db_url:
|
||||||
@ -172,6 +177,7 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
|
|||||||
db_retry_wait=db_retry_wait,
|
db_retry_wait=db_retry_wait,
|
||||||
entity_filter=entity_filter,
|
entity_filter=entity_filter,
|
||||||
exclude_t=exclude_t,
|
exclude_t=exclude_t,
|
||||||
|
db_integrity_check=db_integrity_check,
|
||||||
)
|
)
|
||||||
instance.async_initialize()
|
instance.async_initialize()
|
||||||
instance.start()
|
instance.start()
|
||||||
@ -204,6 +210,7 @@ class Recorder(threading.Thread):
|
|||||||
db_retry_wait: int,
|
db_retry_wait: int,
|
||||||
entity_filter: Callable[[str], bool],
|
entity_filter: Callable[[str], bool],
|
||||||
exclude_t: List[str],
|
exclude_t: List[str],
|
||||||
|
db_integrity_check: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Initialize the recorder."""
|
"""Initialize the recorder."""
|
||||||
threading.Thread.__init__(self, name="Recorder")
|
threading.Thread.__init__(self, name="Recorder")
|
||||||
@ -217,6 +224,7 @@ class Recorder(threading.Thread):
|
|||||||
self.db_url = uri
|
self.db_url = uri
|
||||||
self.db_max_retries = db_max_retries
|
self.db_max_retries = db_max_retries
|
||||||
self.db_retry_wait = db_retry_wait
|
self.db_retry_wait = db_retry_wait
|
||||||
|
self.db_integrity_check = db_integrity_check
|
||||||
self.async_db_ready = asyncio.Future()
|
self.async_db_ready = asyncio.Future()
|
||||||
self.engine: Any = None
|
self.engine: Any = None
|
||||||
self.run_info: Any = None
|
self.run_info: Any = None
|
||||||
@ -547,7 +555,9 @@ class Recorder(threading.Thread):
|
|||||||
# On systems with very large databases and
|
# On systems with very large databases and
|
||||||
# very slow disk or cpus, this can take a while.
|
# very slow disk or cpus, this can take a while.
|
||||||
#
|
#
|
||||||
validate_or_move_away_sqlite_database(self.db_url)
|
validate_or_move_away_sqlite_database(
|
||||||
|
self.db_url, self.db_integrity_check
|
||||||
|
)
|
||||||
|
|
||||||
if self.engine is not None:
|
if self.engine is not None:
|
||||||
self.engine.dispose()
|
self.engine.dispose()
|
||||||
|
@ -3,3 +3,5 @@
|
|||||||
DATA_INSTANCE = "recorder_instance"
|
DATA_INSTANCE = "recorder_instance"
|
||||||
SQLITE_URL_PREFIX = "sqlite://"
|
SQLITE_URL_PREFIX = "sqlite://"
|
||||||
DOMAIN = "recorder"
|
DOMAIN = "recorder"
|
||||||
|
|
||||||
|
CONF_DB_INTEGRITY_CHECK = "db_integrity_check"
|
||||||
|
@ -9,7 +9,7 @@ from sqlalchemy.exc import OperationalError, SQLAlchemyError
|
|||||||
|
|
||||||
import homeassistant.util.dt as dt_util
|
import homeassistant.util.dt as dt_util
|
||||||
|
|
||||||
from .const import DATA_INSTANCE, SQLITE_URL_PREFIX
|
from .const import CONF_DB_INTEGRITY_CHECK, DATA_INSTANCE, SQLITE_URL_PREFIX
|
||||||
from .models import ALL_TABLES, process_timestamp
|
from .models import ALL_TABLES, process_timestamp
|
||||||
|
|
||||||
_LOGGER = logging.getLogger(__name__)
|
_LOGGER = logging.getLogger(__name__)
|
||||||
@ -21,7 +21,7 @@ SQLITE3_POSTFIXES = ["", "-wal", "-shm"]
|
|||||||
# This is the maximum time after the recorder ends the session
|
# This is the maximum time after the recorder ends the session
|
||||||
# before we no longer consider startup to be a "restart" and we
|
# before we no longer consider startup to be a "restart" and we
|
||||||
# should do a check on the sqlite3 database.
|
# should do a check on the sqlite3 database.
|
||||||
MAX_RESTART_TIME = timedelta(minutes=6)
|
MAX_RESTART_TIME = timedelta(minutes=10)
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
@ -110,7 +110,7 @@ def execute(qry, to_native=False, validate_entity_ids=True):
|
|||||||
time.sleep(QUERY_RETRY_WAIT)
|
time.sleep(QUERY_RETRY_WAIT)
|
||||||
|
|
||||||
|
|
||||||
def validate_or_move_away_sqlite_database(dburl: str) -> bool:
|
def validate_or_move_away_sqlite_database(dburl: str, db_integrity_check: bool) -> bool:
|
||||||
"""Ensure that the database is valid or move it away."""
|
"""Ensure that the database is valid or move it away."""
|
||||||
dbpath = dburl[len(SQLITE_URL_PREFIX) :]
|
dbpath = dburl[len(SQLITE_URL_PREFIX) :]
|
||||||
|
|
||||||
@ -118,7 +118,7 @@ def validate_or_move_away_sqlite_database(dburl: str) -> bool:
|
|||||||
# Database does not exist yet, this is OK
|
# Database does not exist yet, this is OK
|
||||||
return True
|
return True
|
||||||
|
|
||||||
if not validate_sqlite_database(dbpath):
|
if not validate_sqlite_database(dbpath, db_integrity_check):
|
||||||
_move_away_broken_database(dbpath)
|
_move_away_broken_database(dbpath)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@ -154,13 +154,13 @@ def basic_sanity_check(cursor):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def validate_sqlite_database(dbpath: str) -> bool:
|
def validate_sqlite_database(dbpath: str, db_integrity_check: bool) -> bool:
|
||||||
"""Run a quick check on an sqlite database to see if it is corrupt."""
|
"""Run a quick check on an sqlite database to see if it is corrupt."""
|
||||||
import sqlite3 # pylint: disable=import-outside-toplevel
|
import sqlite3 # pylint: disable=import-outside-toplevel
|
||||||
|
|
||||||
try:
|
try:
|
||||||
conn = sqlite3.connect(dbpath)
|
conn = sqlite3.connect(dbpath)
|
||||||
run_checks_on_open_db(dbpath, conn.cursor())
|
run_checks_on_open_db(dbpath, conn.cursor(), db_integrity_check)
|
||||||
conn.close()
|
conn.close()
|
||||||
except sqlite3.DatabaseError:
|
except sqlite3.DatabaseError:
|
||||||
_LOGGER.exception("The database at %s is corrupt or malformed.", dbpath)
|
_LOGGER.exception("The database at %s is corrupt or malformed.", dbpath)
|
||||||
@ -169,7 +169,7 @@ def validate_sqlite_database(dbpath: str) -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def run_checks_on_open_db(dbpath, cursor):
|
def run_checks_on_open_db(dbpath, cursor, db_integrity_check):
|
||||||
"""Run checks that will generate a sqlite3 exception if there is corruption."""
|
"""Run checks that will generate a sqlite3 exception if there is corruption."""
|
||||||
if basic_sanity_check(cursor) and last_run_was_recently_clean(cursor):
|
if basic_sanity_check(cursor) and last_run_was_recently_clean(cursor):
|
||||||
_LOGGER.debug(
|
_LOGGER.debug(
|
||||||
@ -177,6 +177,16 @@ def run_checks_on_open_db(dbpath, cursor):
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if not db_integrity_check:
|
||||||
|
# Always warn so when it does fail they remember it has
|
||||||
|
# been manually disabled
|
||||||
|
_LOGGER.warning(
|
||||||
|
"The quick_check on the sqlite3 database at %s was skipped because %s was disabled",
|
||||||
|
dbpath,
|
||||||
|
CONF_DB_INTEGRITY_CHECK,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
_LOGGER.debug(
|
_LOGGER.debug(
|
||||||
"A quick_check is being performed on the sqlite3 database at %s", dbpath
|
"A quick_check is being performed on the sqlite3 database at %s", dbpath
|
||||||
)
|
)
|
||||||
|
@ -304,6 +304,7 @@ def test_recorder_setup_failure():
|
|||||||
db_retry_wait=3,
|
db_retry_wait=3,
|
||||||
entity_filter=CONFIG_SCHEMA({DOMAIN: {}}),
|
entity_filter=CONFIG_SCHEMA({DOMAIN: {}}),
|
||||||
exclude_t=[],
|
exclude_t=[],
|
||||||
|
db_integrity_check=False,
|
||||||
)
|
)
|
||||||
rec.start()
|
rec.start()
|
||||||
rec.join()
|
rec.join()
|
||||||
|
@ -69,29 +69,77 @@ def test_recorder_bad_execute(hass_recorder):
|
|||||||
assert e_mock.call_count == 2
|
assert e_mock.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
def test_validate_or_move_away_sqlite_database(hass, tmpdir, caplog):
|
def test_validate_or_move_away_sqlite_database_with_integrity_check(
|
||||||
"""Ensure a malformed sqlite database is moved away."""
|
hass, tmpdir, caplog
|
||||||
|
):
|
||||||
|
"""Ensure a malformed sqlite database is moved away.
|
||||||
|
|
||||||
|
A quick_check is run here
|
||||||
|
"""
|
||||||
|
|
||||||
|
db_integrity_check = True
|
||||||
|
|
||||||
test_dir = tmpdir.mkdir("test_validate_or_move_away_sqlite_database")
|
test_dir = tmpdir.mkdir("test_validate_or_move_away_sqlite_database")
|
||||||
test_db_file = f"{test_dir}/broken.db"
|
test_db_file = f"{test_dir}/broken.db"
|
||||||
dburl = f"{SQLITE_URL_PREFIX}{test_db_file}"
|
dburl = f"{SQLITE_URL_PREFIX}{test_db_file}"
|
||||||
|
|
||||||
util.validate_sqlite_database(test_db_file) is True
|
util.validate_sqlite_database(test_db_file, db_integrity_check) is True
|
||||||
|
|
||||||
assert os.path.exists(test_db_file) is True
|
assert os.path.exists(test_db_file) is True
|
||||||
assert util.validate_or_move_away_sqlite_database(dburl) is False
|
assert (
|
||||||
|
util.validate_or_move_away_sqlite_database(dburl, db_integrity_check) is False
|
||||||
|
)
|
||||||
|
|
||||||
_corrupt_db_file(test_db_file)
|
_corrupt_db_file(test_db_file)
|
||||||
|
|
||||||
assert util.validate_sqlite_database(dburl) is False
|
assert util.validate_sqlite_database(dburl, db_integrity_check) is False
|
||||||
|
|
||||||
assert util.validate_or_move_away_sqlite_database(dburl) is False
|
assert (
|
||||||
|
util.validate_or_move_away_sqlite_database(dburl, db_integrity_check) is False
|
||||||
|
)
|
||||||
|
|
||||||
assert "corrupt or malformed" in caplog.text
|
assert "corrupt or malformed" in caplog.text
|
||||||
|
|
||||||
assert util.validate_sqlite_database(dburl) is False
|
assert util.validate_sqlite_database(dburl, db_integrity_check) is False
|
||||||
|
|
||||||
assert util.validate_or_move_away_sqlite_database(dburl) is True
|
assert util.validate_or_move_away_sqlite_database(dburl, db_integrity_check) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_or_move_away_sqlite_database_without_integrity_check(
|
||||||
|
hass, tmpdir, caplog
|
||||||
|
):
|
||||||
|
"""Ensure a malformed sqlite database is moved away.
|
||||||
|
|
||||||
|
The quick_check is skipped, but we can still find
|
||||||
|
corruption if the whole database is unreadable
|
||||||
|
"""
|
||||||
|
|
||||||
|
db_integrity_check = False
|
||||||
|
|
||||||
|
test_dir = tmpdir.mkdir("test_validate_or_move_away_sqlite_database")
|
||||||
|
test_db_file = f"{test_dir}/broken.db"
|
||||||
|
dburl = f"{SQLITE_URL_PREFIX}{test_db_file}"
|
||||||
|
|
||||||
|
util.validate_sqlite_database(test_db_file, db_integrity_check) is True
|
||||||
|
|
||||||
|
assert os.path.exists(test_db_file) is True
|
||||||
|
assert (
|
||||||
|
util.validate_or_move_away_sqlite_database(dburl, db_integrity_check) is False
|
||||||
|
)
|
||||||
|
|
||||||
|
_corrupt_db_file(test_db_file)
|
||||||
|
|
||||||
|
assert util.validate_sqlite_database(dburl, db_integrity_check) is False
|
||||||
|
|
||||||
|
assert (
|
||||||
|
util.validate_or_move_away_sqlite_database(dburl, db_integrity_check) is False
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "corrupt or malformed" in caplog.text
|
||||||
|
|
||||||
|
assert util.validate_sqlite_database(dburl, db_integrity_check) is False
|
||||||
|
|
||||||
|
assert util.validate_or_move_away_sqlite_database(dburl, db_integrity_check) is True
|
||||||
|
|
||||||
|
|
||||||
def test_last_run_was_recently_clean(hass_recorder):
|
def test_last_run_was_recently_clean(hass_recorder):
|
||||||
@ -134,25 +182,32 @@ def test_combined_checks(hass_recorder):
|
|||||||
"""Run Checks on the open database."""
|
"""Run Checks on the open database."""
|
||||||
hass = hass_recorder()
|
hass = hass_recorder()
|
||||||
|
|
||||||
|
db_integrity_check = False
|
||||||
|
|
||||||
cursor = hass.data[DATA_INSTANCE].engine.raw_connection().cursor()
|
cursor = hass.data[DATA_INSTANCE].engine.raw_connection().cursor()
|
||||||
|
|
||||||
assert util.run_checks_on_open_db("fake_db_path", cursor) is None
|
assert (
|
||||||
|
util.run_checks_on_open_db("fake_db_path", cursor, db_integrity_check) is None
|
||||||
|
)
|
||||||
|
|
||||||
# We are patching recorder.util here in order
|
# We are patching recorder.util here in order
|
||||||
# to avoid creating the full database on disk
|
# to avoid creating the full database on disk
|
||||||
with patch("homeassistant.components.recorder.util.last_run_was_recently_clean"):
|
with patch("homeassistant.components.recorder.util.last_run_was_recently_clean"):
|
||||||
assert util.run_checks_on_open_db("fake_db_path", cursor) is None
|
assert (
|
||||||
|
util.run_checks_on_open_db("fake_db_path", cursor, db_integrity_check)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"homeassistant.components.recorder.util.last_run_was_recently_clean",
|
"homeassistant.components.recorder.util.last_run_was_recently_clean",
|
||||||
side_effect=sqlite3.DatabaseError,
|
side_effect=sqlite3.DatabaseError,
|
||||||
), pytest.raises(sqlite3.DatabaseError):
|
), pytest.raises(sqlite3.DatabaseError):
|
||||||
util.run_checks_on_open_db("fake_db_path", cursor)
|
util.run_checks_on_open_db("fake_db_path", cursor, db_integrity_check)
|
||||||
|
|
||||||
cursor.execute("DROP TABLE events;")
|
cursor.execute("DROP TABLE events;")
|
||||||
|
|
||||||
with pytest.raises(sqlite3.DatabaseError):
|
with pytest.raises(sqlite3.DatabaseError):
|
||||||
util.run_checks_on_open_db("fake_db_path", cursor)
|
util.run_checks_on_open_db("fake_db_path", cursor, db_integrity_check)
|
||||||
|
|
||||||
|
|
||||||
def _corrupt_db_file(test_db_file):
|
def _corrupt_db_file(test_db_file):
|
||||||
|
Loading…
x
Reference in New Issue
Block a user