diff --git a/homeassistant/components/recorder/__init__.py b/homeassistant/components/recorder/__init__.py index 091bff8445f..e2a5f6b9a6d 100644 --- a/homeassistant/components/recorder/__init__.py +++ b/homeassistant/components/recorder/__init__.py @@ -54,6 +54,7 @@ from .util import ( dburl_to_path, end_incomplete_runs, move_away_broken_database, + perodic_db_cleanups, session_scope, setup_connection_for_dialect, validate_or_move_away_sqlite_database, @@ -278,6 +279,10 @@ class PurgeTask(NamedTuple): apply_filter: bool +class PerodicCleanupTask: + """An object to insert into the recorder to trigger cleanup tasks when auto purge is disabled.""" + + class StatisticsTask(NamedTuple): """An object to insert into the recorder queue to run a statistics task.""" @@ -484,9 +489,15 @@ class Recorder(threading.Thread): self.async_recorder_ready.set() @callback - def async_purge(self, now): + def async_nightly_tasks(self, now): """Trigger the purge.""" - self.queue.put(PurgeTask(self.keep_days, repack=False, apply_filter=False)) + if self.auto_purge: + # Purge will schedule the perodic cleanups + # after it completes to ensure it does not happen + # until after the database is vacuumed + self.queue.put(PurgeTask(self.keep_days, repack=False, apply_filter=False)) + else: + self.queue.put(PerodicCleanupTask()) @callback def async_hourly_statistics(self, now): @@ -496,11 +507,10 @@ class Recorder(threading.Thread): def _async_setup_periodic_tasks(self): """Prepare periodic tasks.""" - if self.auto_purge: - # Purge every night at 4:12am - async_track_time_change( - self.hass, self.async_purge, hour=4, minute=12, second=0 - ) + # Run nightly tasks at 4:12am + async_track_time_change( + self.hass, self.async_nightly_tasks, hour=4, minute=12, second=0 + ) # Compile hourly statistics every hour at *:12 async_track_time_change( self.hass, self.async_hourly_statistics, minute=12, second=0 @@ -646,6 +656,10 @@ class Recorder(threading.Thread): def _run_purge(self, keep_days, repack, apply_filter): """Purge the database.""" if purge.purge_old_data(self, keep_days, repack, apply_filter): + # We always need to do the db cleanups after a purge + # is finished to ensure the WAL checkpoint and other + # tasks happen after a vacuum. + perodic_db_cleanups(self) return # Schedule a new purge task if this one didn't finish self.queue.put(PurgeTask(keep_days, repack, apply_filter)) @@ -662,6 +676,9 @@ class Recorder(threading.Thread): if isinstance(event, PurgeTask): self._run_purge(event.keep_days, event.repack, event.apply_filter) return + if isinstance(event, PerodicCleanupTask): + perodic_db_cleanups(self) + return if isinstance(event, StatisticsTask): self._run_statistics(event.start) return diff --git a/homeassistant/components/recorder/util.py b/homeassistant/components/recorder/util.py index 186aad4fe9e..db9fb46425b 100644 --- a/homeassistant/components/recorder/util.py +++ b/homeassistant/components/recorder/util.py @@ -318,3 +318,15 @@ def retryable_database_job(description: str): return wrapper return decorator + + +def perodic_db_cleanups(instance: Recorder): + """Run any database cleanups that need to happen perodiclly. + + These cleanups will happen nightly or after any purge. + """ + + if instance.engine.dialect.name == "sqlite": + # Execute sqlite to create a wal checkpoint and free up disk space + _LOGGER.debug("WAL checkpoint") + instance.engine.execute("PRAGMA wal_checkpoint(TRUNCATE);") diff --git a/tests/components/recorder/test_init.py b/tests/components/recorder/test_init.py index 540764b2ed0..bb334599c26 100644 --- a/tests/components/recorder/test_init.py +++ b/tests/components/recorder/test_init.py @@ -8,6 +8,7 @@ from sqlalchemy.exc import DatabaseError, OperationalError, SQLAlchemyError from homeassistant.components import recorder from homeassistant.components.recorder import ( + CONF_AUTO_PURGE, CONF_DB_URL, CONFIG_SCHEMA, DOMAIN, @@ -610,30 +611,73 @@ def test_auto_purge(hass_recorder): with patch( "homeassistant.components.recorder.purge.purge_old_data", return_value=True - ) as purge_old_data: + ) as purge_old_data, patch( + "homeassistant.components.recorder.perodic_db_cleanups" + ) as perodic_db_cleanups: # Advance one day, and the purge task should run test_time = test_time + timedelta(days=1) run_tasks_at_time(hass, test_time) assert len(purge_old_data.mock_calls) == 1 + assert len(perodic_db_cleanups.mock_calls) == 1 purge_old_data.reset_mock() + perodic_db_cleanups.reset_mock() # Advance one day, and the purge task should run again test_time = test_time + timedelta(days=1) run_tasks_at_time(hass, test_time) assert len(purge_old_data.mock_calls) == 1 + assert len(perodic_db_cleanups.mock_calls) == 1 purge_old_data.reset_mock() + perodic_db_cleanups.reset_mock() # Advance less than one full day. The alarm should not yet fire. test_time = test_time + timedelta(hours=23) run_tasks_at_time(hass, test_time) assert len(purge_old_data.mock_calls) == 0 + assert len(perodic_db_cleanups.mock_calls) == 0 # Advance to the next day and fire the alarm again test_time = test_time + timedelta(hours=1) run_tasks_at_time(hass, test_time) assert len(purge_old_data.mock_calls) == 1 + assert len(perodic_db_cleanups.mock_calls) == 1 + + dt_util.set_default_time_zone(original_tz) + + +def test_auto_purge_disabled(hass_recorder): + """Test periodic db cleanup still run when auto purge is disabled.""" + hass = hass_recorder({CONF_AUTO_PURGE: False}) + + original_tz = dt_util.DEFAULT_TIME_ZONE + + tz = dt_util.get_time_zone("Europe/Copenhagen") + dt_util.set_default_time_zone(tz) + + # Purging is scheduled to happen at 4:12am every day. We want + # to verify that when auto purge is disabled perodic db cleanups + # are still scheduled + # + # The clock is started at 4:15am then advanced forward below + now = dt_util.utcnow() + test_time = datetime(now.year + 2, 1, 1, 4, 15, 0, tzinfo=tz) + run_tasks_at_time(hass, test_time) + + with patch( + "homeassistant.components.recorder.purge.purge_old_data", return_value=True + ) as purge_old_data, patch( + "homeassistant.components.recorder.perodic_db_cleanups" + ) as perodic_db_cleanups: + # Advance one day, and the purge task should run + test_time = test_time + timedelta(days=1) + run_tasks_at_time(hass, test_time) + assert len(purge_old_data.mock_calls) == 0 + assert len(perodic_db_cleanups.mock_calls) == 1 + + purge_old_data.reset_mock() + perodic_db_cleanups.reset_mock() dt_util.set_default_time_zone(original_tz) diff --git a/tests/components/recorder/test_util.py b/tests/components/recorder/test_util.py index 0a9f90be83e..5ba206a751f 100644 --- a/tests/components/recorder/test_util.py +++ b/tests/components/recorder/test_util.py @@ -269,3 +269,11 @@ def test_end_incomplete_runs(hass_recorder, caplog): assert run_info.end == now_without_tz assert "Ended unfinished session" in caplog.text + + +def test_perodic_db_cleanups(hass_recorder): + """Test perodic db cleanups.""" + hass = hass_recorder() + with patch.object(hass.data[DATA_INSTANCE].engine, "execute") as execute_mock: + util.perodic_db_cleanups(hass.data[DATA_INSTANCE]) + assert execute_mock.call_args[0][0] == "PRAGMA wal_checkpoint(TRUNCATE);"