From c89cd6a68c127dcfa15fc3f0cb176e7c825e5535 Mon Sep 17 00:00:00 2001 From: Justyn Shull Date: Fri, 1 Apr 2016 19:47:32 -0500 Subject: [PATCH 1/7] Add 'purge_days' option to the history/recorder component Issue https://github.com/balloob/home-assistant/issues/1337 When purge_days is set under the history component, recorder.py will delete all events and states that are older than purge_days days ago. Currently, this is only done once at start up. A vacuum command is also run to free up the disk space sqlite would still use after deleting records. --- homeassistant/components/recorder.py | 38 ++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/recorder.py b/homeassistant/components/recorder.py index 05a95ee27b4..d5dd3ea1205 100644 --- a/homeassistant/components/recorder.py +++ b/homeassistant/components/recorder.py @@ -13,7 +13,7 @@ import logging import queue import sqlite3 import threading -from datetime import date, datetime +from datetime import date, datetime, timedelta import homeassistant.util.dt as dt_util from homeassistant.const import ( @@ -102,14 +102,13 @@ def setup(hass, config): """Setup the recorder.""" # pylint: disable=global-statement global _INSTANCE - - _INSTANCE = Recorder(hass) + _INSTANCE = Recorder(hass, config.get('history', {})) return True class RecorderRun(object): - """Representation of arecorder run.""" + """Representation of a recorder run.""" def __init__(self, row=None): """Initialize the recorder run.""" @@ -169,11 +168,12 @@ class Recorder(threading.Thread): """A threaded recorder class.""" # pylint: disable=too-many-instance-attributes - def __init__(self, hass): + def __init__(self, hass, config): """Initialize the recorder.""" threading.Thread.__init__(self) self.hass = hass + self.config = config self.conn = None self.queue = queue.Queue() self.quit_object = object() @@ -194,6 +194,7 @@ class Recorder(threading.Thread): """Start processing events to save.""" self._setup_connection() self._setup_run() + self._purge_old_data() while True: event = self.queue.get() @@ -475,6 +476,33 @@ class Recorder(threading.Thread): "UPDATE recorder_runs SET end=? WHERE start=?", (dt_util.utcnow(), self.recording_start)) + def _purge_old_data(self): + """Purge events and states older than purge_days ago.""" + purge_days = self.config.get('purge_days', -1) + if purge_days < 1: + _LOGGER.debug("purge_days set to %s, will not purge any old data.", + purge_days) + return + + purge_before = dt_util.utcnow() - timedelta(days=purge_days) + + _LOGGER.info("Purging events created before %s", purge_before) + deleted_rows = self.query( + sql_query="DELETE FROM events WHERE created < ?;", + data=(int(purge_before.timestamp()),), + return_value=RETURN_ROWCOUNT) + _LOGGER.debug("Deleted %s events", deleted_rows) + + _LOGGER.info("Purging states created before %s", purge_before) + deleted_rows = self.query( + sql_query="DELETE FROM states WHERE created < ?;", + data=(int(purge_before.timestamp()),), + return_value=RETURN_ROWCOUNT) + _LOGGER.debug("Deleted %s states", deleted_rows) + + # Execute sqlite vacuum command to free up space on disk + self.query("VACUUM;") + def _adapt_datetime(datetimestamp): """Turn a datetime into an integer for in the DB.""" From fd48fc5f8346010bc51326a2b5d03fb2c66819be Mon Sep 17 00:00:00 2001 From: Justyn Shull Date: Wed, 13 Apr 2016 19:37:33 -0500 Subject: [PATCH 2/7] Add CONFIG_SCHEMA to verify config. Move purge_days key name to CONF_PURGE_DAYS so it can be changed easier later. Use 'recorder' domain instead of 'history' domain. Pass purge_days config directly into Recorder object instead of passing the config object around. --- homeassistant/components/recorder.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/homeassistant/components/recorder.py b/homeassistant/components/recorder.py index d5dd3ea1205..308adb1aa44 100644 --- a/homeassistant/components/recorder.py +++ b/homeassistant/components/recorder.py @@ -13,6 +13,7 @@ import logging import queue import sqlite3 import threading +import voluptuous as vol from datetime import date, datetime, timedelta import homeassistant.util.dt as dt_util @@ -30,6 +31,14 @@ RETURN_ROWCOUNT = "rowcount" RETURN_LASTROWID = "lastrowid" RETURN_ONE_ROW = "one_row" +CONF_PURGE_DAYS = "purge_days" +CONFIG_SCHEMA = vol.Schema({ + DOMAIN: vol.All(dict, { + CONF_PURGE_DAYS: int + }) +}, extra=vol.ALLOW_EXTRA) + + _INSTANCE = None _LOGGER = logging.getLogger(__name__) @@ -102,7 +111,8 @@ def setup(hass, config): """Setup the recorder.""" # pylint: disable=global-statement global _INSTANCE - _INSTANCE = Recorder(hass, config.get('history', {})) + purge_days = config.get(DOMAIN, {}).get(CONF_PURGE_DAYS) + _INSTANCE = Recorder(hass, purge_days=purge_days) return True @@ -168,12 +178,12 @@ class Recorder(threading.Thread): """A threaded recorder class.""" # pylint: disable=too-many-instance-attributes - def __init__(self, hass, config): + def __init__(self, hass, purge_days): """Initialize the recorder.""" threading.Thread.__init__(self) self.hass = hass - self.config = config + self.purge_days = purge_days self.conn = None self.queue = queue.Queue() self.quit_object = object() @@ -478,13 +488,12 @@ class Recorder(threading.Thread): def _purge_old_data(self): """Purge events and states older than purge_days ago.""" - purge_days = self.config.get('purge_days', -1) - if purge_days < 1: + if not self.purge_days or self.purge_days < 1: _LOGGER.debug("purge_days set to %s, will not purge any old data.", - purge_days) + self.purge_days) return - purge_before = dt_util.utcnow() - timedelta(days=purge_days) + purge_before = dt_util.utcnow() - timedelta(days=self.purge_days) _LOGGER.info("Purging events created before %s", purge_before) deleted_rows = self.query( From d5ca97b1f6c3183cc64fdc518293ac6fbd0f03e9 Mon Sep 17 00:00:00 2001 From: Justyn Shull Date: Fri, 15 Apr 2016 21:02:17 -0500 Subject: [PATCH 3/7] Add tests for purging old states and events --- homeassistant/components/recorder.py | 2 +- tests/components/test_recorder.py | 119 ++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/recorder.py b/homeassistant/components/recorder.py index 308adb1aa44..c1e4b850397 100644 --- a/homeassistant/components/recorder.py +++ b/homeassistant/components/recorder.py @@ -13,8 +13,8 @@ import logging import queue import sqlite3 import threading -import voluptuous as vol from datetime import date, datetime, timedelta +import voluptuous as vol import homeassistant.util.dt as dt_util from homeassistant.const import ( diff --git a/tests/components/test_recorder.py b/tests/components/test_recorder.py index 4326134ed84..0be5ce42182 100644 --- a/tests/components/test_recorder.py +++ b/tests/components/test_recorder.py @@ -1,6 +1,8 @@ """The tests for the Recorder component.""" # pylint: disable=too-many-public-methods,protected-access import unittest +import time +import json from unittest.mock import patch from homeassistant.const import MATCH_ALL @@ -10,7 +12,7 @@ from tests.common import get_test_home_assistant class TestRecorder(unittest.TestCase): - """Test the chromecast module.""" + """Test the recorder module.""" def setUp(self): # pylint: disable=invalid-name """Setup things to be run when tests are started.""" @@ -25,6 +27,66 @@ class TestRecorder(unittest.TestCase): self.hass.stop() recorder._INSTANCE.block_till_done() + def _add_test_states(self): + """Adds multiple states to the db for testing.""" + now = int(time.time()) + five_days_ago = now - (60*60*24*5) + attributes = {'test_attr': 5, 'test_attr_10': 'nice'} + + test_states = """ + INSERT INTO states ( + entity_id, domain, state, attributes, last_changed, last_updated, + created, utc_offset, event_id) + VALUES + ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, + {five_days_ago}, {five_days_ago}, -18000, 1001), + ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, + {five_days_ago}, {five_days_ago}, -18000, 1002), + ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, + {five_days_ago}, {five_days_ago}, -18000, 1002), + ('test.recorder2', 'sensor', 'dontpurgeme', '{attr}', {now}, + {now}, {now}, -18000, 1003), + ('test.recorder2', 'sensor', 'dontpurgeme', '{attr}', {now}, + {now}, {now}, -18000, 1004); + """.format( + attr=json.dumps(attributes), + five_days_ago=five_days_ago, + now=now, + ) + + # insert test states + self.hass.pool.block_till_done() + recorder._INSTANCE.block_till_done() + recorder.query(test_states) + + def _add_test_events(self): + """Adds a few events for testing.""" + now = int(time.time()) + five_days_ago = now - (60*60*24*5) + event_data = {'test_attr': 5, 'test_attr_10': 'nice'} + + test_events = """ + INSERT INTO events ( + event_type, event_data, origin, created, time_fired, utc_offset + ) VALUES + ('EVENT_TEST_PURGE', '{event_data}', 'LOCAL', {five_days_ago}, + {five_days_ago}, -18000), + ('EVENT_TEST_PURGE', '{event_data}', 'LOCAL', {five_days_ago}, + {five_days_ago}, -18000), + ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000), + ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000), + ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000); + """.format( + event_data=json.dumps(event_data), + now=now, + five_days_ago=five_days_ago + ) + + # insert test events + self.hass.pool.block_till_done() + recorder._INSTANCE.block_till_done() + recorder.query(test_events) + def test_saving_state(self): """Test saving and restoring a state.""" entity_id = 'test.recorder' @@ -64,3 +126,58 @@ class TestRecorder(unittest.TestCase): 'SELECT * FROM events WHERE event_type = ?', (event_type, )) self.assertEqual(events, db_events) + + def test_purge_old_states(self): + """Tests deleting old states.""" + self._add_test_states() + # make sure we start with 5 states + states = recorder.query_states('SELECT * FROM states') + self.assertEqual(len(states), 5) + + # run purge_old_data() + recorder._INSTANCE.purge_days = 4 + recorder._INSTANCE._purge_old_data() + + # we should only have 2 states left after purging + states = recorder.query_states('SELECT * FROM states') + self.assertEqual(len(states), 2) + + def test_purge_old_events(self): + """Tests deleting old events.""" + self._add_test_events() + events = recorder.query_events('SELECT * FROM events WHERE ' + 'event_type LIKE "EVENT_TEST%"') + self.assertEqual(len(events), 5) + + # run purge_old_data() + recorder._INSTANCE.purge_days = 4 + recorder._INSTANCE._purge_old_data() + + # now we should only have 3 events left + events = recorder.query_events('SELECT * FROM events WHERE ' + 'event_type LIKE "EVENT_TEST%"') + self.assertEqual(len(events), 3) + + + def test_purge_disabled(self): + """Tests leaving purge_days disabled.""" + self._add_test_states() + self._add_test_events() + # make sure we start with 5 states and events + states = recorder.query_states('SELECT * FROM states') + events = recorder.query_events('SELECT * FROM events WHERE ' + 'event_type LIKE "EVENT_TEST%"') + self.assertEqual(len(states), 5) + self.assertEqual(len(events), 5) + + + # run purge_old_data() + recorder._INSTANCE.purge_days = None + recorder._INSTANCE._purge_old_data() + + # we should have all of our states still + states = recorder.query_states('SELECT * FROM states') + events = recorder.query_events('SELECT * FROM events WHERE ' + 'event_type LIKE "EVENT_TEST%"') + self.assertEqual(len(states), 5) + self.assertEqual(len(events), 5) From bf3b77e1f23a9a37554daa5e13af8abb11033659 Mon Sep 17 00:00:00 2001 From: Justyn Shull Date: Fri, 15 Apr 2016 21:18:51 -0500 Subject: [PATCH 4/7] Change sqlite queries to work with older versions of sqlite --- tests/components/test_recorder.py | 69 ++++++++++++------------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/tests/components/test_recorder.py b/tests/components/test_recorder.py index 0be5ce42182..94161520709 100644 --- a/tests/components/test_recorder.py +++ b/tests/components/test_recorder.py @@ -33,31 +33,22 @@ class TestRecorder(unittest.TestCase): five_days_ago = now - (60*60*24*5) attributes = {'test_attr': 5, 'test_attr_10': 'nice'} - test_states = """ - INSERT INTO states ( - entity_id, domain, state, attributes, last_changed, last_updated, - created, utc_offset, event_id) - VALUES - ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, - {five_days_ago}, {five_days_ago}, -18000, 1001), - ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, - {five_days_ago}, {five_days_ago}, -18000, 1002), - ('test.recorder2', 'sensor', 'purgeme', '{attr}', {five_days_ago}, - {five_days_ago}, {five_days_ago}, -18000, 1002), - ('test.recorder2', 'sensor', 'dontpurgeme', '{attr}', {now}, - {now}, {now}, -18000, 1003), - ('test.recorder2', 'sensor', 'dontpurgeme', '{attr}', {now}, - {now}, {now}, -18000, 1004); - """.format( - attr=json.dumps(attributes), - five_days_ago=five_days_ago, - now=now, - ) - - # insert test states self.hass.pool.block_till_done() recorder._INSTANCE.block_till_done() - recorder.query(test_states) + for event_id in range(5): + if event_id < 3: + timestamp = five_days_ago + state = 'purgeme' + else: + timestamp = now + state = 'dontpurgeme' + recorder.query("INSERT INTO states (" + "entity_id, domain, state, attributes, last_changed," + "last_updated, created, utc_offset, event_id)" + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ('test.recorder2', 'sensor', state, + json.dumps(attributes), timestamp, timestamp, + timestamp, -18000, event_id + 1000)) def _add_test_events(self): """Adds a few events for testing.""" @@ -65,27 +56,21 @@ class TestRecorder(unittest.TestCase): five_days_ago = now - (60*60*24*5) event_data = {'test_attr': 5, 'test_attr_10': 'nice'} - test_events = """ - INSERT INTO events ( - event_type, event_data, origin, created, time_fired, utc_offset - ) VALUES - ('EVENT_TEST_PURGE', '{event_data}', 'LOCAL', {five_days_ago}, - {five_days_ago}, -18000), - ('EVENT_TEST_PURGE', '{event_data}', 'LOCAL', {five_days_ago}, - {five_days_ago}, -18000), - ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000), - ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000), - ('EVENT_TEST', '{event_data}', 'LOCAL', {now}, {five_days_ago}, -18000); - """.format( - event_data=json.dumps(event_data), - now=now, - five_days_ago=five_days_ago - ) - - # insert test events self.hass.pool.block_till_done() recorder._INSTANCE.block_till_done() - recorder.query(test_events) + for event_id in range(5): + if event_id < 2: + timestamp = five_days_ago + event_type = 'EVENT_TEST_PURGE' + else: + timestamp = now + event_type = 'EVENT_TEST' + recorder.query("INSERT INTO events" + "(event_type, event_data, origin, created," + "time_fired, utc_offset)" + "VALUES (?, ?, ?, ?, ?, ?)", + (event_type, json.dumps(event_data), 'LOCAL', + timestamp, timestamp, -18000)) def test_saving_state(self): """Test saving and restoring a state.""" From 93fd6fa11b30c2fa0941847c721e9e4ff3dfe938 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Thu, 12 May 2016 10:33:22 -0700 Subject: [PATCH 5/7] fixes for pep and delay start --- homeassistant/components/recorder.py | 11 ++++++++--- tests/components/test_recorder.py | 11 +++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/homeassistant/components/recorder.py b/homeassistant/components/recorder.py index f760fc882f5..0c7454ad694 100644 --- a/homeassistant/components/recorder.py +++ b/homeassistant/components/recorder.py @@ -22,6 +22,7 @@ from homeassistant.const import ( EVENT_TIME_CHANGED, MATCH_ALL) from homeassistant.core import Event, EventOrigin, State from homeassistant.remote import JSONEncoder +from homeassistant.helpers.event import track_point_in_utc_time DOMAIN = "recorder" @@ -33,8 +34,9 @@ RETURN_ONE_ROW = "one_row" CONF_PURGE_DAYS = "purge_days" CONFIG_SCHEMA = vol.Schema({ - DOMAIN: vol.All(dict, { - CONF_PURGE_DAYS: int + DOMAIN: vol.Schema({ + vol.Optional(CONF_PURGE_DAYS): vol.All(vol.Coerce(int), + vol.Range(min=1)), }) }, extra=vol.ALLOW_EXTRA) @@ -204,7 +206,10 @@ class Recorder(threading.Thread): """Start processing events to save.""" self._setup_connection() self._setup_run() - self._purge_old_data() + if self.purge_days is not None: + track_point_in_utc_time(self.hass, + lambda now: self._purge_old_data(), + dt_util.utcnow() + timedelta(minutes=5)) while True: event = self.queue.get() diff --git a/tests/components/test_recorder.py b/tests/components/test_recorder.py index cb96e079a63..b2db41efc3c 100644 --- a/tests/components/test_recorder.py +++ b/tests/components/test_recorder.py @@ -43,12 +43,13 @@ class TestRecorder(unittest.TestCase): timestamp = now state = 'dontpurgeme' recorder.query("INSERT INTO states (" - "entity_id, domain, state, attributes, last_changed," - "last_updated, created, utc_offset, event_id)" + "entity_id, domain, state, attributes," + "last_changed, last_updated, created," + "utc_offset, event_id)" "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", ('test.recorder2', 'sensor', state, - json.dumps(attributes), timestamp, timestamp, - timestamp, -18000, event_id + 1000)) + json.dumps(attributes), timestamp, timestamp, + timestamp, -18000, event_id + 1000)) def _add_test_events(self): """Adds a few events for testing.""" @@ -155,7 +156,6 @@ class TestRecorder(unittest.TestCase): 'event_type LIKE "EVENT_TEST%"') self.assertEqual(len(events), 3) - def test_purge_disabled(self): """Tests leaving purge_days disabled.""" self._add_test_states() @@ -167,7 +167,6 @@ class TestRecorder(unittest.TestCase): self.assertEqual(len(states), 5) self.assertEqual(len(events), 5) - # run purge_old_data() recorder._INSTANCE.purge_days = None recorder._INSTANCE._purge_old_data() From cba85cad8d98427a41baa294b71a7f12881e3307 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Fri, 13 May 2016 14:42:08 -0700 Subject: [PATCH 6/7] Fixes for farcy --- tests/components/test_recorder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/components/test_recorder.py b/tests/components/test_recorder.py index b2db41efc3c..3efd84cc0f4 100644 --- a/tests/components/test_recorder.py +++ b/tests/components/test_recorder.py @@ -28,7 +28,7 @@ class TestRecorder(unittest.TestCase): recorder._INSTANCE.block_till_done() def _add_test_states(self): - """Adds multiple states to the db for testing.""" + """Add multiple states to the db for testing.""" now = int(time.time()) five_days_ago = now - (60*60*24*5) attributes = {'test_attr': 5, 'test_attr_10': 'nice'} @@ -52,7 +52,7 @@ class TestRecorder(unittest.TestCase): timestamp, -18000, event_id + 1000)) def _add_test_events(self): - """Adds a few events for testing.""" + """Add a few events for testing.""" now = int(time.time()) five_days_ago = now - (60*60*24*5) event_data = {'test_attr': 5, 'test_attr_10': 'nice'} @@ -126,7 +126,7 @@ class TestRecorder(unittest.TestCase): db_event.time_fired.replace(microsecond=0) def test_purge_old_states(self): - """Tests deleting old states.""" + """Test deleting old states.""" self._add_test_states() # make sure we start with 5 states states = recorder.query_states('SELECT * FROM states') @@ -157,7 +157,7 @@ class TestRecorder(unittest.TestCase): self.assertEqual(len(events), 3) def test_purge_disabled(self): - """Tests leaving purge_days disabled.""" + """Test leaving purge_days disabled.""" self._add_test_states() self._add_test_events() # make sure we start with 5 states and events From 53d7e0730c8627a24e6ea064ded9e369f4bb3fe2 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Fri, 13 May 2016 14:43:22 -0700 Subject: [PATCH 7/7] Fixes for farcy --- tests/components/test_recorder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/test_recorder.py b/tests/components/test_recorder.py index 3efd84cc0f4..0577ab27889 100644 --- a/tests/components/test_recorder.py +++ b/tests/components/test_recorder.py @@ -141,7 +141,7 @@ class TestRecorder(unittest.TestCase): self.assertEqual(len(states), 2) def test_purge_old_events(self): - """Tests deleting old events.""" + """Test deleting old events.""" self._add_test_events() events = recorder.query_events('SELECT * FROM events WHERE ' 'event_type LIKE "EVENT_TEST%"')