From c89cd6a68c127dcfa15fc3f0cb176e7c825e5535 Mon Sep 17 00:00:00 2001 From: Justyn Shull Date: Fri, 1 Apr 2016 19:47:32 -0500 Subject: [PATCH 1/4] Add 'purge_days' option to the history/recorder component Issue https://github.com/balloob/home-assistant/issues/1337 When purge_days is set under the history component, recorder.py will delete all events and states that are older than purge_days days ago. Currently, this is only done once at start up. A vacuum command is also run to free up the disk space sqlite would still use after deleting records. --- homeassistant/components/recorder.py | 38 ++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/recorder.py b/homeassistant/components/recorder.py index 05a95ee27b4..d5dd3ea1205 100644 --- a/homeassistant/components/recorder.py +++ b/homeassistant/components/recorder.py @@ -13,7 +13,7 @@ import logging import queue import sqlite3 import threading -from datetime import date, datetime +from datetime import date, datetime, timedelta import homeassistant.util.dt as dt_util from homeassistant.const import ( @@ -102,14 +102,13 @@ def setup(hass, config): """Setup the recorder.""" # pylint: disable=global-statement global _INSTANCE - - _INSTANCE = Recorder(hass) + _INSTANCE = Recorder(hass, config.get('history', {})) return True class RecorderRun(object): - """Representation of arecorder run.""" + """Representation of a recorder run.""" def __init__(self, row=None): """Initialize the recorder run.""" @@ -169,11 +168,12 @@ class Recorder(threading.Thread): """A threaded recorder class.""" # pylint: disable=too-many-instance-attributes - def __init__(self, hass): + def __init__(self, hass, config): """Initialize the recorder.""" threading.Thread.__init__(self) self.hass = hass + self.config = config self.conn = None self.queue = queue.Queue() self.quit_object = object() @@ -194,6 +194,7 @@ class Recorder(threading.Thread): """Start processing events to save.""" self._setup_connection() self._setup_run() + self._purge_old_data() while True: event = self.queue.get() @@ -475,6 +476,33 @@ class Recorder(threading.Thread): "UPDATE recorder_runs SET end=? WHERE start=?", (dt_util.utcnow(), self.recording_start)) + def _purge_old_data(self): + """Purge events and states older than purge_days ago.""" + purge_days = self.config.get('purge_days', -1) + if purge_days < 1: + _LOGGER.debug("purge_days set to %s, will not purge any old data.", + purge_days) + return + + purge_before = dt_util.utcnow() - timedelta(days=purge_days) + + _LOGGER.info("Purging events created before %s", purge_before) + deleted_rows = self.query( + sql_query="DELETE FROM events WHERE created < ?;", + data=(int(purge_before.timestamp()),), + return_value=RETURN_ROWCOUNT) + _LOGGER.debug("Deleted %s events", deleted_rows) + + _LOGGER.info("Purging states created before %s", purge_before) + deleted_rows = self.query( + sql_query="DELETE FROM states WHERE created < ?;", + data=(int(purge_before.timestamp()),), + return_value=RETURN_ROWCOUNT) + _LOGGER.debug("Deleted %s states", deleted_rows) + + # Execute sqlite vacuum command to free up space on disk + self.query("VACUUM;") + def _adapt_datetime(datetimestamp): """Turn a datetime into an integer for in the DB.""" From fd48fc5f8346010bc51326a2b5d03fb2c66819be Mon Sep 17 00:00:00 2001 From: Justyn Shull Date: Wed, 13 Apr 2016 19:37:33 -0500 Subject: [PATCH 2/4] Add CONFIG_SCHEMA to verify config. Move purge_days key name to CONF_PURGE_DAYS so it can be changed easier later. Use 'recorder' domain instead of 'history' domain. Pass purge_days config directly into Recorder object instead of passing the config object around. --- homeassistant/components/recorder.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/homeassistant/components/recorder.py b/homeassistant/components/recorder.py index d5dd3ea1205..308adb1aa44 100644 --- a/homeassistant/components/recorder.py +++ b/homeassistant/components/recorder.py @@ -13,6 +13,7 @@ import logging import queue import sqlite3 import threading +import voluptuous as vol from datetime import date, datetime, timedelta import homeassistant.util.dt as dt_util @@ -30,6 +31,14 @@ RETURN_ROWCOUNT = "rowcount" RETURN_LASTROWID = "lastrowid" RETURN_ONE_ROW = "one_row" +CONF_PURGE_DAYS = "purge_days" +CONFIG_SCHEMA = vol.Schema({ + DOMAIN: vol.All(dict, { + CONF_PURGE_DAYS: int + }) +}, extra=vol.ALLOW_EXTRA) + + _INSTANCE = None _LOGGER = logging.getLogger(__name__) @@ -102,7 +111,8 @@ def setup(hass, config): """Setup the recorder.""" # pylint: disable=global-statement global _INSTANCE - _INSTANCE = Recorder(hass, config.get('history', {})) + purge_days = config.get(DOMAIN, {}).get(CONF_PURGE_DAYS) + _INSTANCE = Recorder(hass, purge_days=purge_days) return True @@ -168,12 +178,12 @@ class Recorder(threading.Thread): """A threaded recorder class.""" # pylint: disable=too-many-instance-attributes - def __init__(self, hass, config): + def __init__(self, hass, purge_days): """Initialize the recorder.""" threading.Thread.__init__(self) self.hass = hass - self.config = config + self.purge_days = purge_days self.conn = None self.queue = queue.Queue() self.quit_object = object() @@ -478,13 +488,12 @@ class Recorder(threading.Thread): def _purge_old_data(self): """Purge events and states older than purge_days ago.""" - purge_days = self.config.get('purge_days', -1) - if purge_days < 1: + if not self.purge_days or self.purge_days < 1: _LOGGER.debug("purge_days set to %s, will not purge any old data.", - purge_days) + self.purge_days) return - purge_before = dt_util.utcnow() - timedelta(days=purge_days) + purge_before = dt_util.utcnow() - timedelta(days=self.purge_days) _LOGGER.info("Purging events created before %s", purge_before) deleted_rows = self.query( From d5ca97b1f6c3183cc64fdc518293ac6fbd0f03e9 Mon Sep 17 00:00:00 2001 From: Justyn Shull Date: Fri, 15 Apr 2016 21:02:17 -0500 Subject: [PATCH 3/4] Add tests for purging old states and events --- homeassistant/components/recorder.py | 2 +- tests/components/test_recorder.py | 119 ++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/recorder.py b/homeassistant/components/recorder.py index 308adb1aa44..c1e4b850397 100644 --- a/homeassistant/components/recorder.py +++ b/homeassistant/components/recorder.py @@ -13,8 +13,8 @@ import logging import queue import sqlite3 import threading -import voluptuous as vol from datetime import date, datetime, timedelta +import voluptuous as vol import homeassistant.util.dt as dt_util from homeassistant.const import ( diff --git a/tests/components/test_recorder.py b/tests/components/test_recorder.py index 4326134ed84..0be5ce42182 100644 --- a/tests/components/test_recorder.py +++ b/tests/components/test_recorder.py @@ -1,6 +1,8 @@ """The tests for the Recorder component.""" # pylint: disable=too-many-public-methods,protected-access import unittest +import time +import json from unittest.mock import patch from homeassistant.const import MATCH_ALL @@ -10,7 +12,7 @@ from tests.common import get_test_home_assistant class TestRecorder(unittest.TestCase): - """Test the chromecast module.""" + """Test the recorder module.""" def setUp(self): # pylint: disable=invalid-name """Setup things to be run when tests are started.""" @@ -25,6 +27,66 @@ class TestRecorder(unittest.TestCase): self.hass.stop() recorder._INSTANCE.block_till_done() + def _add_test_states(self): + """Adds multiple states to the db for testing.""" + now = int(time.time()) + five_days_ago = now - (60*60*24*5) + attributes = {'test_attr': 5, 'test_attr_10': 'nice'} + + test_states = """ + INSERT INTO states ( + entity_id, domain, state, attributes, last_changed, last_updated, + created, utc_offset, event_id) + VALUES + ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, + {five_days_ago}, {five_days_ago}, -18000, 1001), + ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, + {five_days_ago}, {five_days_ago}, -18000, 1002), + ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, + {five_days_ago}, {five_days_ago}, -18000, 1002), + ('test.recorder2', 'sensor', 'dontpurgeme', '{attr}', {now}, + {now}, {now}, -18000, 1003), + ('test.recorder2', 'sensor', 'dontpurgeme', '{attr}', {now}, + {now}, {now}, -18000, 1004); + """.format( + attr=json.dumps(attributes), + five_days_ago=five_days_ago, + now=now, + ) + + # insert test states + self.hass.pool.block_till_done() + recorder._INSTANCE.block_till_done() + recorder.query(test_states) + + def _add_test_events(self): + """Adds a few events for testing.""" + now = int(time.time()) + five_days_ago = now - (60*60*24*5) + event_data = {'test_attr': 5, 'test_attr_10': 'nice'} + + test_events = """ + INSERT INTO events ( + event_type, event_data, origin, created, time_fired, utc_offset + ) VALUES + ('EVENT_TEST_PURGE', '{event_data}', 'LOCAL', {five_days_ago}, + {five_days_ago}, -18000), + ('EVENT_TEST_PURGE', '{event_data}', 'LOCAL', {five_days_ago}, + {five_days_ago}, -18000), + ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000), + ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000), + ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000); + """.format( + event_data=json.dumps(event_data), + now=now, + five_days_ago=five_days_ago + ) + + # insert test events + self.hass.pool.block_till_done() + recorder._INSTANCE.block_till_done() + recorder.query(test_events) + def test_saving_state(self): """Test saving and restoring a state.""" entity_id = 'test.recorder' @@ -64,3 +126,58 @@ class TestRecorder(unittest.TestCase): 'SELECT * FROM events WHERE event_type = ?', (event_type, )) self.assertEqual(events, db_events) + + def test_purge_old_states(self): + """Tests deleting old states.""" + self._add_test_states() + # make sure we start with 5 states + states = recorder.query_states('SELECT * FROM states') + self.assertEqual(len(states), 5) + + # run purge_old_data() + recorder._INSTANCE.purge_days = 4 + recorder._INSTANCE._purge_old_data() + + # we should only have 2 states left after purging + states = recorder.query_states('SELECT * FROM states') + self.assertEqual(len(states), 2) + + def test_purge_old_events(self): + """Tests deleting old events.""" + self._add_test_events() + events = recorder.query_events('SELECT * FROM events WHERE ' + 'event_type LIKE "EVENT_TEST%"') + self.assertEqual(len(events), 5) + + # run purge_old_data() + recorder._INSTANCE.purge_days = 4 + recorder._INSTANCE._purge_old_data() + + # now we should only have 3 events left + events = recorder.query_events('SELECT * FROM events WHERE ' + 'event_type LIKE "EVENT_TEST%"') + self.assertEqual(len(events), 3) + + + def test_purge_disabled(self): + """Tests leaving purge_days disabled.""" + self._add_test_states() + self._add_test_events() + # make sure we start with 5 states and events + states = recorder.query_states('SELECT * FROM states') + events = recorder.query_events('SELECT * FROM events WHERE ' + 'event_type LIKE "EVENT_TEST%"') + self.assertEqual(len(states), 5) + self.assertEqual(len(events), 5) + + + # run purge_old_data() + recorder._INSTANCE.purge_days = None + recorder._INSTANCE._purge_old_data() + + # we should have all of our states still + states = recorder.query_states('SELECT * FROM states') + events = recorder.query_events('SELECT * FROM events WHERE ' + 'event_type LIKE "EVENT_TEST%"') + self.assertEqual(len(states), 5) + self.assertEqual(len(events), 5) From bf3b77e1f23a9a37554daa5e13af8abb11033659 Mon Sep 17 00:00:00 2001 From: Justyn Shull Date: Fri, 15 Apr 2016 21:18:51 -0500 Subject: [PATCH 4/4] Change sqlite queries to work with older versions of sqlite --- tests/components/test_recorder.py | 69 ++++++++++++------------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/tests/components/test_recorder.py b/tests/components/test_recorder.py index 0be5ce42182..94161520709 100644 --- a/tests/components/test_recorder.py +++ b/tests/components/test_recorder.py @@ -33,31 +33,22 @@ class TestRecorder(unittest.TestCase): five_days_ago = now - (60*60*24*5) attributes = {'test_attr': 5, 'test_attr_10': 'nice'} - test_states = """ - INSERT INTO states ( - entity_id, domain, state, attributes, last_changed, last_updated, - created, utc_offset, event_id) - VALUES - ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, - {five_days_ago}, {five_days_ago}, -18000, 1001), - ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, - {five_days_ago}, {five_days_ago}, -18000, 1002), - ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, - {five_days_ago}, {five_days_ago}, -18000, 1002), - ('test.recorder2', 'sensor', 'dontpurgeme', '{attr}', {now}, - {now}, {now}, -18000, 1003), - ('test.recorder2', 'sensor', 'dontpurgeme', '{attr}', {now}, - {now}, {now}, -18000, 1004); - """.format( - attr=json.dumps(attributes), - five_days_ago=five_days_ago, - now=now, - ) - - # insert test states self.hass.pool.block_till_done() recorder._INSTANCE.block_till_done() - recorder.query(test_states) + for event_id in range(5): + if event_id < 3: + timestamp = five_days_ago + state = 'purgeme' + else: + timestamp = now + state = 'dontpurgeme' + recorder.query("INSERT INTO states (" + "entity_id, domain, state, attributes, last_changed," + "last_updated, created, utc_offset, event_id)" + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ('test.recorder2', 'sensor', state, + json.dumps(attributes), timestamp, timestamp, + timestamp, -18000, event_id + 1000)) def _add_test_events(self): """Adds a few events for testing.""" @@ -65,27 +56,21 @@ class TestRecorder(unittest.TestCase): five_days_ago = now - (60*60*24*5) event_data = {'test_attr': 5, 'test_attr_10': 'nice'} - test_events = """ - INSERT INTO events ( - event_type, event_data, origin, created, time_fired, utc_offset - ) VALUES - ('EVENT_TEST_PURGE', '{event_data}', 'LOCAL', {five_days_ago}, - {five_days_ago}, -18000), - ('EVENT_TEST_PURGE', '{event_data}', 'LOCAL', {five_days_ago}, - {five_days_ago}, -18000), - ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000), - ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000), - ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000); - """.format( - event_data=json.dumps(event_data), - now=now, - five_days_ago=five_days_ago - ) - - # insert test events self.hass.pool.block_till_done() recorder._INSTANCE.block_till_done() - recorder.query(test_events) + for event_id in range(5): + if event_id < 2: + timestamp = five_days_ago + event_type = 'EVENT_TEST_PURGE' + else: + timestamp = now + event_type = 'EVENT_TEST' + recorder.query("INSERT INTO events" + "(event_type, event_data, origin, created," + "time_fired, utc_offset)" + "VALUES (?, ?, ?, ?, ?, ?)", + (event_type, json.dumps(event_data), 'LOCAL', + timestamp, timestamp, -18000)) def test_saving_state(self): """Test saving and restoring a state."""