jeans/core
jeans/core
1
0
Fork 0
mirror of https://github.com/home-assistant/core.git synced 2025-07-24 21:57:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update recorder. (#2549)

Browse Source 
* Update recorder.

models.py:
 - Use scoped_session in models.py to fix shutdown error
__init__.py:
 - Session _commit & retry method
 - Single session var for purge_data
 - Ensure single _INSTANCE
 - repeat purge every 2 days
 - show correct time in log_error

* _commit

* Restore models to old functionality, swap purge, remove _INSTANCE cleanup from tests, typing ignore Base class

* pylint

* Remove recorder from model unit test
This commit is contained in:
Johann Kellerman 2016-07-23 20:25:17 +02:00 committed by Paulus Schoutsen
parent d4f78e8552
commit 4cf618334c
4 changed files with 80 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
homeassistant/components/recorder/__init__.py
View File

@ -91,8 +91,12 @@ def run_information(point_in_time=None):
def setup(hass, config): def setup(hass, config):
"""Setup the recorder.""" """Setup the recorder."""
# pylint: disable=global-statement # pylint: disable=global-statement
# pylint: disable=too-many-locals
global _INSTANCE global _INSTANCE
if _INSTANCE is not None:
_LOGGER.error('Only a single instance allowed.')
return False
purge_days = config.get(DOMAIN, {}).get(CONF_PURGE_DAYS) purge_days = config.get(DOMAIN, {}).get(CONF_PURGE_DAYS)
db_url = config.get(DOMAIN, {}).get(CONF_DB_URL, None) db_url = config.get(DOMAIN, {}).get(CONF_DB_URL, None)
@ -130,7 +134,7 @@ def log_error(e, retry_wait=0, rollback=True,
if rollback: if rollback:
Session().rollback() Session().rollback()
if retry_wait: if retry_wait:
_LOGGER.info("Retrying failed query in %s seconds", QUERY_RETRY_WAIT) _LOGGER.info("Retrying in %s seconds", retry_wait)
time.sleep(retry_wait) time.sleep(retry_wait)
@ -165,8 +169,6 @@ class Recorder(threading.Thread):
from homeassistant.components.recorder.models import Events, States from homeassistant.components.recorder.models import Events, States
import sqlalchemy.exc import sqlalchemy.exc
global _INSTANCE
while True: while True:
try: try:
self._setup_connection() self._setup_connection()
@ -177,8 +179,12 @@ class Recorder(threading.Thread):
message="Error during connection setup: %s") message="Error during connection setup: %s")
if self.purge_days is not None: if self.purge_days is not None:
track_point_in_utc_time(self.hass, def purge_ticker(event):
lambda now: self._purge_old_data(), """Rerun purge every second day."""
self._purge_old_data()
track_point_in_utc_time(self.hass, purge_ticker,
dt_util.utcnow() + timedelta(days=2))
track_point_in_utc_time(self.hass, purge_ticker,
dt_util.utcnow() + timedelta(minutes=5)) dt_util.utcnow() + timedelta(minutes=5))
while True: while True:
@ -187,42 +193,26 @@ class Recorder(threading.Thread):
if event == self.quit_object: if event == self.quit_object:
self._close_run() self._close_run()
self._close_connection() self._close_connection()
# pylint: disable=global-statement
global _INSTANCE
_INSTANCE = None _INSTANCE = None
self.queue.task_done() self.queue.task_done()
return return
elif event.event_type == EVENT_TIME_CHANGED: if event.event_type == EVENT_TIME_CHANGED:
self.queue.task_done() self.queue.task_done()
continue continue
session = Session()
dbevent = Events.from_event(event) dbevent = Events.from_event(event)
session.add(dbevent) self._commit(dbevent)
for _ in range(0, RETRIES):
try:
session.commit()
break
except sqlalchemy.exc.OperationalError as e:
log_error(e, retry_wait=QUERY_RETRY_WAIT,
rollback=True)
if event.event_type != EVENT_STATE_CHANGED: if event.event_type != EVENT_STATE_CHANGED:
self.queue.task_done() self.queue.task_done()
continue continue
session = Session()
dbstate = States.from_event(event) dbstate = States.from_event(event)
dbstate.event_id = dbevent.event_id
for _ in range(0, RETRIES): self._commit(dbstate)
try:
dbstate.event_id = dbevent.event_id
session.add(dbstate)
session.commit()
break
except sqlalchemy.exc.OperationalError as e:
log_error(e, retry_wait=QUERY_RETRY_WAIT,
rollback=True)
self.queue.task_done() self.queue.task_done()
@ -269,6 +259,7 @@ class Recorder(threading.Thread):
def _close_connection(self): def _close_connection(self):
"""Close the connection.""" """Close the connection."""
# pylint: disable=global-statement
global Session global Session
self.engine.dispose() self.engine.dispose()
self.engine = None self.engine = None
@ -290,16 +281,12 @@ class Recorder(threading.Thread):
start=self.recording_start, start=self.recording_start,
created=dt_util.utcnow() created=dt_util.utcnow()
) )
session = Session() self._commit(self._run)
session.add(self._run)
session.commit()
def _close_run(self): def _close_run(self):
"""Save end time for current run.""" """Save end time for current run."""
self._run.end = dt_util.utcnow() self._run.end = dt_util.utcnow()
session = Session() self._commit(self._run)
session.add(self._run)
session.commit()
self._run = None self._run = None
def _purge_old_data(self): def _purge_old_data(self):
@ -313,17 +300,24 @@ class Recorder(threading.Thread):
purge_before = dt_util.utcnow() - timedelta(days=self.purge_days) purge_before = dt_util.utcnow() - timedelta(days=self.purge_days)
_LOGGER.info("Purging events created before %s", purge_before) def _purge_states(session):
deleted_rows = Session().query(Events).filter( deleted_rows = session.query(States) \
(Events.created < purge_before)).delete(synchronize_session=False) .filter((States.created < purge_before)) \
_LOGGER.debug("Deleted %s events", deleted_rows) .delete(synchronize_session=False)
_LOGGER.debug("Deleted %s states", deleted_rows)
_LOGGER.info("Purging states created before %s", purge_before) if self._commit(_purge_states):
deleted_rows = Session().query(States).filter( _LOGGER.info("Purged states created before %s", purge_before)
(States.created < purge_before)).delete(synchronize_session=False)
_LOGGER.debug("Deleted %s states", deleted_rows) def _purge_events(session):
deleted_rows = session.query(Events) \
.filter((Events.created < purge_before)) \
.delete(synchronize_session=False)
_LOGGER.debug("Deleted %s events", deleted_rows)
if self._commit(_purge_events):
_LOGGER.info("Purged events created before %s", purge_before)
Session().commit()
Session().expire_all() Session().expire_all()
# Execute sqlite vacuum command to free up space on disk # Execute sqlite vacuum command to free up space on disk
@ -331,6 +325,23 @@ class Recorder(threading.Thread):
_LOGGER.info("Vacuuming SQLite to free space") _LOGGER.info("Vacuuming SQLite to free space")
self.engine.execute("VACUUM") self.engine.execute("VACUUM")
@staticmethod
def _commit(work):
"""Commit & retry work: Either a model or in a function."""
import sqlalchemy.exc
session = Session()
for _ in range(0, RETRIES):
try:
if callable(work):
work(session)
else:
session.add(work)
session.commit()
return True
except sqlalchemy.exc.OperationalError as e:
log_error(e, retry_wait=QUERY_RETRY_WAIT, rollback=True)
return False
def _verify_instance(): def _verify_instance():
"""Throw error if recorder not initialized.""" """Throw error if recorder not initialized."""

6
homeassistant/components/recorder/models.py
View File

@ -20,7 +20,7 @@ Base = declarative_base()
_LOGGER = logging.getLogger(__name__) _LOGGER = logging.getLogger(__name__)
class Events(Base): class Events(Base): # type: ignore
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
"""Event history data.""" """Event history data."""
@ -55,7 +55,7 @@ class Events(Base):
return None return None
class States(Base): class States(Base): # type: ignore
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
"""State change history.""" """State change history."""
@ -114,7 +114,7 @@ class States(Base):
return None return None
class RecorderRuns(Base): class RecorderRuns(Base): # type: ignore
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
"""Representation of recorder run.""" """Representation of recorder run."""

4
tests/components/recorder/test_init.py
View File

@ -1,8 +1,8 @@
"""The tests for the Recorder component.""" """The tests for the Recorder component."""
# pylint: disable=too-many-public-methods,protected-access # pylint: disable=protected-access
import unittest
import json import json
from datetime import datetime, timedelta from datetime import datetime, timedelta
import unittest
from unittest.mock import patch from unittest.mock import patch
from homeassistant.const import MATCH_ALL from homeassistant.const import MATCH_ALL

38
tests/components/recorder/test_models.py
View File

@ -1,5 +1,4 @@
"""The tests for the Recorder component.""" """The tests for the Recorder component."""
# pylint: disable=too-many-public-methods,protected-access
import unittest import unittest
from datetime import datetime from datetime import datetime
@ -12,32 +11,35 @@ from homeassistant.util import dt
from homeassistant.components.recorder.models import ( from homeassistant.components.recorder.models import (
Base, Events, States, RecorderRuns) Base, Events, States, RecorderRuns)
engine = None ENGINE = None
Session = None SESSION = None
def setUpModule(): def setUpModule(): # pylint: disable=invalid-name
"""Set up a database to use.""" """Set up a database to use."""
global engine, Session global ENGINE
global SESSION
engine = create_engine("sqlite://") ENGINE = create_engine("sqlite://")
Base.metadata.create_all(engine) Base.metadata.create_all(ENGINE)
session_factory = sessionmaker(bind=engine) session_factory = sessionmaker(bind=ENGINE)
Session = scoped_session(session_factory) SESSION = scoped_session(session_factory)
def tearDownModule(): def tearDownModule(): # pylint: disable=invalid-name
"""Close database.""" """Close database."""
global engine, Session global ENGINE
global SESSION
engine.dispose() ENGINE.dispose()
engine = None ENGINE = None
Session = None SESSION = None
class TestEvents(unittest.TestCase): class TestEvents(unittest.TestCase):
"""Test Events model.""" """Test Events model."""
# pylint: disable=no-self-use
def test_from_event(self): def test_from_event(self):
"""Test converting event to db event.""" """Test converting event to db event."""
event = ha.Event('test_event', { event = ha.Event('test_event', {
@ -49,6 +51,8 @@ class TestEvents(unittest.TestCase):
class TestStates(unittest.TestCase): class TestStates(unittest.TestCase):
"""Test States model.""" """Test States model."""
# pylint: disable=no-self-use
def test_from_event(self): def test_from_event(self):
"""Test converting event to db state.""" """Test converting event to db state."""
state = ha.State('sensor.temperature', '18') state = ha.State('sensor.temperature', '18')
@ -78,14 +82,14 @@ class TestStates(unittest.TestCase):
class TestRecorderRuns(unittest.TestCase): class TestRecorderRuns(unittest.TestCase):
"""Test recorder run model.""" """Test recorder run model."""
def setUp(self): def setUp(self): # pylint: disable=invalid-name
"""Set up recorder runs.""" """Set up recorder runs."""
self.session = session = Session() self.session = session = SESSION()
session.query(Events).delete() session.query(Events).delete()
session.query(States).delete() session.query(States).delete()
session.query(RecorderRuns).delete() session.query(RecorderRuns).delete()
def tearDown(self): def tearDown(self): # pylint: disable=invalid-name
"""Clean up.""" """Clean up."""
self.session.rollback() self.session.rollback()