From 337b8d279eabc4e2eb52e5b449ab89d773de6b21 Mon Sep 17 00:00:00 2001 From: moinmoin-sh <53935853+moinmoin-sh@users.noreply.github.com> Date: Sat, 28 Nov 2020 19:42:29 +0100 Subject: [PATCH] Ensure MariaDB/MySQL can be purged and handle states being deleted out from under the recorder (#43610) * MariaDB doesn't purge #42402 This addresses home-assistant#42402 Relationships within table "states" and between tables "states" and "events " home-assistant#40467 prevent the purge from working correctly. The database increases w/o any purge. This proposal sets related indices to NULL and permits deleting of rows. Further explanations can be found here home-assistant#42402 This proposal also allows to purge the tables "events" and "states" in any order. * Update models.py Corrected for Black style requirements * Update homeassistant/components/recorder/models.py Co-authored-by: J. Nick Koston * Add the options to foreign key constraints * purge old states when database gets deleted out from under us * pylint Co-authored-by: J. Nick Koston --- homeassistant/components/recorder/__init__.py | 8 ++++ .../components/recorder/migration.py | 40 ++++++++++++++++++- homeassistant/components/recorder/models.py | 12 ++++-- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/homeassistant/components/recorder/__init__.py b/homeassistant/components/recorder/__init__.py index 18d364315b7..0f8a5ae7f8f 100644 --- a/homeassistant/components/recorder/__init__.py +++ b/homeassistant/components/recorder/__init__.py @@ -514,6 +514,14 @@ class Recorder(threading.Thread): self.event_session.expunge(dbstate) self._pending_expunge = [] self.event_session.commit() + except exc.IntegrityError as err: + _LOGGER.error( + "Integrity error executing query (database likely deleted out from under us): %s", + err, + ) + self.event_session.rollback() + self._old_states = {} + raise except Exception as err: _LOGGER.error("Error executing query: %s", err) self.event_session.rollback() diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py index e88852e4a5a..c633c114b46 100644 --- a/homeassistant/components/recorder/migration.py +++ b/homeassistant/components/recorder/migration.py @@ -1,12 +1,13 @@ """Schema migration helpers.""" import logging -from sqlalchemy import Table, text +from sqlalchemy import ForeignKeyConstraint, MetaData, Table, text from sqlalchemy.engine import reflection from sqlalchemy.exc import InternalError, OperationalError, SQLAlchemyError +from sqlalchemy.schema import AddConstraint, DropConstraint from .const import DOMAIN -from .models import SCHEMA_VERSION, Base, SchemaChanges +from .models import SCHEMA_VERSION, TABLE_STATES, Base, SchemaChanges from .util import session_scope _LOGGER = logging.getLogger(__name__) @@ -205,6 +206,39 @@ def _add_columns(engine, table_name, columns_def): ) +def _update_states_table_with_foreign_key_options(engine): + """Add the options to foreign key constraints.""" + inspector = reflection.Inspector.from_engine(engine) + alters = [] + for foreign_key in inspector.get_foreign_keys(TABLE_STATES): + if foreign_key["name"] and not foreign_key["options"]: + alters.append( + { + "old_fk": ForeignKeyConstraint((), (), name=foreign_key["name"]), + "columns": foreign_key["constrained_columns"], + } + ) + + if not alters: + return + + states_key_constraints = Base.metadata.tables[TABLE_STATES].foreign_key_constraints + old_states_table = Table( # noqa: F841 pylint: disable=unused-variable + TABLE_STATES, MetaData(), *[alter["old_fk"] for alter in alters] + ) + + for alter in alters: + try: + engine.execute(DropConstraint(alter["old_fk"])) + for fkc in states_key_constraints: + if fkc.column_keys == alter["columns"]: + engine.execute(AddConstraint(fkc)) + except (InternalError, OperationalError): + _LOGGER.exception( + "Could not update foreign options in %s table", TABLE_STATES + ) + + def _apply_update(engine, new_version, old_version): """Perform operations to bring schema up to date.""" if new_version == 1: @@ -277,6 +311,8 @@ def _apply_update(engine, new_version, old_version): _drop_index(engine, "states", "ix_states_entity_id") _create_index(engine, "events", "ix_events_event_type_time_fired") _drop_index(engine, "events", "ix_events_event_type") + elif new_version == 10: + _update_states_table_with_foreign_key_options(engine) else: raise ValueError(f"No schema migration defined for version {new_version}") diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py index 4756ac13ce3..6c8b6050a9a 100644 --- a/homeassistant/components/recorder/models.py +++ b/homeassistant/components/recorder/models.py @@ -25,7 +25,7 @@ import homeassistant.util.dt as dt_util # pylint: disable=invalid-name Base = declarative_base() -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 _LOGGER = logging.getLogger(__name__) @@ -36,7 +36,7 @@ TABLE_STATES = "states" TABLE_RECORDER_RUNS = "recorder_runs" TABLE_SCHEMA_CHANGES = "schema_changes" -ALL_TABLES = [TABLE_EVENTS, TABLE_STATES, TABLE_RECORDER_RUNS, TABLE_SCHEMA_CHANGES] +ALL_TABLES = [TABLE_STATES, TABLE_EVENTS, TABLE_RECORDER_RUNS, TABLE_SCHEMA_CHANGES] class Events(Base): # type: ignore @@ -102,11 +102,15 @@ class States(Base): # type: ignore entity_id = Column(String(255)) state = Column(String(255)) attributes = Column(Text) - event_id = Column(Integer, ForeignKey("events.event_id"), index=True) + event_id = Column( + Integer, ForeignKey("events.event_id", ondelete="CASCADE"), index=True + ) last_changed = Column(DateTime(timezone=True), default=dt_util.utcnow) last_updated = Column(DateTime(timezone=True), default=dt_util.utcnow, index=True) created = Column(DateTime(timezone=True), default=dt_util.utcnow) - old_state_id = Column(Integer, ForeignKey("states.state_id")) + old_state_id = Column( + Integer, ForeignKey("states.state_id", ondelete="SET NULL"), index=True + ) event = relationship("Events", uselist=False) old_state = relationship("States", remote_side=[state_id])