Added include and exclude functionality to history component (#3674)

* added include and exclude functionality to history component

* fixed summary lines in test method doc.

* cleanup of query filter creation

* o improved config validation
o move move IGNORE_DOMAINS to Filter.apply()
o removed config from Last5StatesView
o Filters instance is now created on setup
o config values are processed in setup and set to the Filters instance
o function _set_filters_in_query() moved to Filters class and renamed to apply()

* added unittests for more include/exclude filter combinations

* make pylint happy

homeassistant/components/history.py

@@ -7,16 +7,39 @@ https://home-assistant.io/components/history/
 from collections import defaultdict
 from datetime import timedelta
 from itertools import groupby
+import voluptuous as vol
 
+import homeassistant.helpers.config_validation as cv
 import homeassistant.util.dt as dt_util
 from homeassistant.components import recorder, script
 from homeassistant.components.frontend import register_built_in_panel
 from homeassistant.components.http import HomeAssistantView
+from homeassistant.const import ATTR_HIDDEN
 
 DOMAIN = 'history'
 DEPENDENCIES = ['recorder', 'http']
 
-SIGNIFICANT_DOMAINS = ('thermostat',)
+CONF_EXCLUDE = 'exclude'
+CONF_INCLUDE = 'include'
+CONF_ENTITIES = 'entities'
+CONF_DOMAINS = 'domains'
+
+CONFIG_SCHEMA = vol.Schema({
+    DOMAIN: vol.Schema({
+        CONF_EXCLUDE: vol.Schema({
+            vol.Optional(CONF_ENTITIES, default=[]): cv.entity_ids,
+            vol.Optional(CONF_DOMAINS, default=[]): vol.All(cv.ensure_list,
+                                                            [cv.string])
+        }),
+        CONF_INCLUDE: vol.Schema({
+            vol.Optional(CONF_ENTITIES, default=[]): cv.entity_ids,
+            vol.Optional(CONF_DOMAINS, default=[]): vol.All(cv.ensure_list,
+                                                            [cv.string])
+        })
+    }),
+}, extra=vol.ALLOW_EXTRA)
+
+SIGNIFICANT_DOMAINS = ('thermostat', 'climate')
 IGNORE_DOMAINS = ('zone', 'scene',)
 
 
@@ -32,7 +55,8 @@ def last_5_states(entity_id):
         ).order_by(states.state_id.desc()).limit(5))
 
 
-def get_significant_states(start_time, end_time=None, entity_id=None):
+def get_significant_states(start_time, end_time=None, entity_id=None,
+                           filters=None):
     """
     Return states changes during UTC period start_time - end_time.
 
@@ -40,25 +64,25 @@ def get_significant_states(start_time, end_time=None, entity_id=None):
     as well as all states from certain domains (for instance
     thermostat so that we get current temperature in our graphs).
     """
+    entity_ids = (entity_id.lower(), ) if entity_id is not None else None
     states = recorder.get_model('States')
     query = recorder.query('States').filter(
         (states.domain.in_(SIGNIFICANT_DOMAINS) |
          (states.last_changed == states.last_updated)) &
-        ((~states.domain.in_(IGNORE_DOMAINS)) &
+        (states.last_updated > start_time))
-         (states.last_updated > start_time)))
+    if filters:
+        query = filters.apply(query, entity_ids)
 
     if end_time is not None:
         query = query.filter(states.last_updated < end_time)
 
-    if entity_id is not None:
-        query = query.filter_by(entity_id=entity_id.lower())
-
     states = (
         state for state in recorder.execute(
             query.order_by(states.entity_id, states.last_updated))
-        if _is_significant(state))
+        if (_is_significant(state) and
+            not state.attributes.get(ATTR_HIDDEN, False)))
 
-    return states_to_json(states, start_time, entity_id)
+    return states_to_json(states, start_time, entity_id, filters)
 
 
 def state_changes_during_period(start_time, end_time=None, entity_id=None):
@@ -80,7 +104,7 @@ def state_changes_during_period(start_time, end_time=None, entity_id=None):
     return states_to_json(states, start_time, entity_id)
 
 
-def get_states(utc_point_in_time, entity_ids=None, run=None):
+def get_states(utc_point_in_time, entity_ids=None, run=None, filters=None):
     """Return the states at a specific point in time."""
     if run is None:
         run = recorder.run_information(utc_point_in_time)
@@ -96,12 +120,11 @@ def get_states(utc_point_in_time, entity_ids=None, run=None):
         func.max(states.state_id).label('max_state_id')
     ).filter(
         (states.created >= run.start) &
-        (states.created < utc_point_in_time)
+        (states.created < utc_point_in_time) &
-    )
+        (~states.domain.in_(IGNORE_DOMAINS)))
-
+    if filters:
-    if entity_ids is not None:
+        most_recent_state_ids = filters.apply(most_recent_state_ids,
-        most_recent_state_ids = most_recent_state_ids.filter(
+                                              entity_ids)
-            states.entity_id.in_(entity_ids))
 
     most_recent_state_ids = most_recent_state_ids.group_by(
         states.entity_id).subquery()
@@ -109,10 +132,12 @@ def get_states(utc_point_in_time, entity_ids=None, run=None):
     query = recorder.query('States').join(most_recent_state_ids, and_(
         states.state_id == most_recent_state_ids.c.max_state_id))
 
-    return recorder.execute(query)
+    for state in recorder.execute(query):
+        if not state.attributes.get(ATTR_HIDDEN, False):
+            yield state
 
 
-def states_to_json(states, start_time, entity_id):
+def states_to_json(states, start_time, entity_id, filters=None):
     """Convert SQL results into JSON friendly data structure.
 
     This takes our state list and turns it into a JSON friendly data
@@ -127,7 +152,7 @@ def states_to_json(states, start_time, entity_id):
     entity_ids = [entity_id] if entity_id is not None else None
 
     # Get the states at the start time
-    for state in get_states(start_time, entity_ids):
+    for state in get_states(start_time, entity_ids, filters=filters):
         state.last_changed = start_time
         state.last_updated = start_time
         result[state.entity_id].append(state)
@@ -140,16 +165,25 @@ def states_to_json(states, start_time, entity_id):
 
 def get_state(utc_point_in_time, entity_id, run=None):
     """Return a state at a specific point in time."""
-    states = get_states(utc_point_in_time, (entity_id,), run)
+    states = list(get_states(utc_point_in_time, (entity_id,), run))
-
     return states[0] if states else None
 
 
 # pylint: disable=unused-argument
 def setup(hass, config):
     """Setup the history hooks."""
-    hass.wsgi.register_view(Last5StatesView)
+    filters = Filters()
-    hass.wsgi.register_view(HistoryPeriodView)
+    exclude = config[DOMAIN].get(CONF_EXCLUDE)
+    if exclude:
+        filters.excluded_entities = exclude[CONF_ENTITIES]
+        filters.excluded_domains = exclude[CONF_DOMAINS]
+    include = config[DOMAIN].get(CONF_INCLUDE)
+    if include:
+        filters.included_entities = include[CONF_ENTITIES]
+        filters.included_domains = include[CONF_DOMAINS]
+
+    hass.wsgi.register_view(Last5StatesView(hass))
+    hass.wsgi.register_view(HistoryPeriodView(hass, filters))
     register_built_in_panel(hass, 'history', 'History', 'mdi:poll-box')
 
     return True
@@ -161,6 +195,10 @@ class Last5StatesView(HomeAssistantView):
     url = '/api/history/entity/<entity:entity_id>/recent_states'
     name = 'api:history:entity-recent-states'
 
+    def __init__(self, hass):
+        """Initilalize the history last 5 states view."""
+        super().__init__(hass)
+
     def get(self, request, entity_id):
         """Retrieve last 5 states of entity."""
         return self.json(last_5_states(entity_id))
@@ -173,6 +211,11 @@ class HistoryPeriodView(HomeAssistantView):
     name = 'api:history:view-period'
     extra_urls = ['/api/history/period/<datetime:datetime>']
 
+    def __init__(self, hass, filters):
+        """Initilalize the history period view."""
+        super().__init__(hass)
+        self.filters = filters
+
     def get(self, request, datetime=None):
         """Return history over a period of time."""
         one_day = timedelta(days=1)
@@ -185,8 +228,68 @@ class HistoryPeriodView(HomeAssistantView):
         end_time = start_time + one_day
         entity_id = request.args.get('filter_entity_id')
 
-        return self.json(
+        return self.json(get_significant_states(
-            get_significant_states(start_time, end_time, entity_id).values())
+            start_time, end_time, entity_id, self.filters).values())
+
+
+# pylint: disable=too-few-public-methods
+class Filters(object):
+    """Container for the configured include and exclude filters."""
+
+    def __init__(self):
+        """Initialise the include and exclude filters."""
+        self.excluded_entities = []
+        self.excluded_domains = []
+        self.included_entities = []
+        self.included_domains = []
+
+    def apply(self, query, entity_ids=None):
+        """Apply the Include/exclude filter on domains and entities on query.
+
+        Following rules apply:
+        * only the include section is configured - just query the specified
+          entities or domains.
+        * only the exclude section is configured - filter the specified
+          entities and domains from all the entities in the system.
+        * if include and exclude is defined - select the entities specified in
+          the include and filter out the ones from the exclude list.
+        """
+        states = recorder.get_model('States')
+        # specific entities requested - do not in/exclude anything
+        if entity_ids is not None:
+            return query.filter(states.entity_id.in_(entity_ids))
+        query = query.filter(~states.domain.in_(IGNORE_DOMAINS))
+
+        filter_query = None
+        # filter if only excluded domain is configured
+        if self.excluded_domains and not self.included_domains:
+            filter_query = ~states.domain.in_(self.excluded_domains)
+            if self.included_entities:
+                filter_query &= states.entity_id.in_(self.included_entities)
+        # filter if only included domain is configured
+        elif not self.excluded_domains and self.included_domains:
+            filter_query = states.domain.in_(self.included_domains)
+            if self.included_entities:
+                filter_query |= states.entity_id.in_(self.included_entities)
+        # filter if included and excluded domain is configured
+        elif self.excluded_domains and self.included_domains:
+            filter_query = ~states.domain.in_(self.excluded_domains)
+            if self.included_entities:
+                filter_query &= (states.domain.in_(self.included_domains) |
+                                 states.entity_id.in_(self.included_entities))
+            else:
+                filter_query &= (states.domain.in_(self.included_domains) &
+                                 ~states.domain.in_(self.excluded_domains))
+        # no domain filter just included entities
+        elif not self.excluded_domains and not self.included_domains and \
+                self.included_entities:
+            filter_query = states.entity_id.in_(self.included_entities)
+        if filter_query is not None:
+            query = query.filter(filter_query)
+        # finally apply excluded entities filter if configured
+        if self.excluded_entities:
+            query = query.filter(~states.entity_id.in_(self.excluded_entities))
+        return query
 
 
 def _is_significant(state):

tests/components/test_history.py

@@ -43,7 +43,15 @@ class TestComponentHistory(unittest.TestCase):
     def test_setup(self):
         """Test setup method of history."""
         mock_http_component(self.hass)
-        self.assertTrue(setup_component(self.hass, history.DOMAIN, {}))
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_INCLUDE: {
+                    history.CONF_DOMAINS: ['media_player'],
+                    history.CONF_ENTITIES: ['thermostat.test']},
+                history.CONF_EXCLUDE: {
+                    history.CONF_DOMAINS: ['thermostat'],
+                    history.CONF_ENTITIES: ['media_player.test']}}})
+        self.assertTrue(setup_component(self.hass, history.DOMAIN, config))
 
     def test_last_5_states(self):
         """Test retrieving the last 5 states."""
@@ -145,14 +153,236 @@ class TestComponentHistory(unittest.TestCase):
     def test_get_significant_states(self):
         """Test that only significant states are returned.
 
-        We inject a bunch of state updates from media player, zone and
+        We should get back every thermostat change that
-        thermostat. We should get back every thermostat change that
         includes an attribute change, but only the state updates for
         media player (attribute changes are not significant and not returned).
         """
+        zero, four, states = self.record_states()
+        hist = history.get_significant_states(
+            zero, four, filters=history.Filters())
+        assert states == hist
+
+    def test_get_significant_states_entity_id(self):
+        """Test that only significant states are returned for one entity."""
+        zero, four, states = self.record_states()
+        del states['media_player.test2']
+        del states['thermostat.test']
+        del states['thermostat.test2']
+        del states['script.can_cancel_this_one']
+
+        hist = history.get_significant_states(
+            zero, four, 'media_player.test',
+            filters=history.Filters())
+        assert states == hist
+
+    def test_get_significant_states_exclude_domain(self):
+        """Test if significant states are returned when excluding domains.
+
+        We should get back every thermostat change that includes an attribute
+        change, but no media player changes.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test']
+        del states['media_player.test2']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_EXCLUDE: {
+                history.CONF_DOMAINS: ['media_player', ]}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def test_get_significant_states_exclude_entity(self):
+        """Test if significant states are returned when excluding entities.
+
+        We should get back every thermostat and script changes, but no media
+        player changes.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_EXCLUDE: {
+                history.CONF_ENTITIES: ['media_player.test', ]}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def test_get_significant_states_exclude(self):
+        """Test significant states when excluding entities and domains.
+
+        We should not get back every thermostat and media player test changes.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test']
+        del states['thermostat.test']
+        del states['thermostat.test2']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_EXCLUDE: {
+                history.CONF_DOMAINS: ['thermostat', ],
+                history.CONF_ENTITIES: ['media_player.test', ]}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def test_get_significant_states_exclude_include_entity(self):
+        """Test significant states when excluding domains and include entities.
+
+        We should not get back every thermostat and media player test changes.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test2']
+        del states['thermostat.test']
+        del states['thermostat.test2']
+        del states['script.can_cancel_this_one']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {
+                history.CONF_INCLUDE: {
+                    history.CONF_ENTITIES: ['media_player.test',
+                                            'thermostat.test']},
+                history.CONF_EXCLUDE: {
+                    history.CONF_DOMAINS: ['thermostat']}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def test_get_significant_states_include_domain(self):
+        """Test if significant states are returned when including domains.
+
+        We should get back every thermostat and script changes, but no media
+        player changes.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test']
+        del states['media_player.test2']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_INCLUDE: {
+                history.CONF_DOMAINS: ['thermostat', 'script']}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def test_get_significant_states_include_entity(self):
+        """Test if significant states are returned when including entities.
+
+        We should only get back changes of the media_player.test entity.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test2']
+        del states['thermostat.test']
+        del states['thermostat.test2']
+        del states['script.can_cancel_this_one']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_INCLUDE: {
+                history.CONF_ENTITIES: ['media_player.test']}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def test_get_significant_states_include(self):
+        """Test significant states when including domains and entities.
+
+        We should only get back changes of the media_player.test entity and the
+        thermostat domain.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test2']
+        del states['script.can_cancel_this_one']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_INCLUDE: {
+                history.CONF_DOMAINS: ['thermostat'],
+                history.CONF_ENTITIES: ['media_player.test']}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def test_get_significant_states_include_exclude_domain(self):
+        """Test if significant states when excluding and including domains.
+
+        We should not get back any changes since we include only the
+        media_player domain but also exclude it.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test']
+        del states['media_player.test2']
+        del states['thermostat.test']
+        del states['thermostat.test2']
+        del states['script.can_cancel_this_one']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_INCLUDE: {
+                    history.CONF_DOMAINS: ['media_player']},
+                history.CONF_EXCLUDE: {
+                    history.CONF_DOMAINS: ['media_player']}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def test_get_significant_states_include_exclude_entity(self):
+        """Test if significant states when excluding and including domains.
+
+        We should not get back any changes since we include only
+        media_player.test but also exclude it.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test']
+        del states['media_player.test2']
+        del states['thermostat.test']
+        del states['thermostat.test2']
+        del states['script.can_cancel_this_one']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_INCLUDE: {
+                    history.CONF_ENTITIES: ['media_player.test']},
+                history.CONF_EXCLUDE: {
+                    history.CONF_ENTITIES: ['media_player.test']}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def test_get_significant_states_include_exclude(self):
+        """Test if significant states when in/excluding domains and entities.
+
+        We should only get back changes of the media_player.test2 entity.
+        """
+        zero, four, states = self.record_states()
+        del states['media_player.test']
+        del states['thermostat.test']
+        del states['thermostat.test2']
+        del states['script.can_cancel_this_one']
+
+        config = history.CONFIG_SCHEMA({
+            ha.DOMAIN: {},
+            history.DOMAIN: {history.CONF_INCLUDE: {
+                    history.CONF_DOMAINS: ['media_player'],
+                    history.CONF_ENTITIES: ['thermostat.test']},
+                history.CONF_EXCLUDE: {
+                    history.CONF_DOMAINS: ['thermostat'],
+                    history.CONF_ENTITIES: ['media_player.test']}}})
+        self.check_significant_states(zero, four, states, config)
+
+    def check_significant_states(self, zero, four, states, config):
+        """Check if significant states are retrieved."""
+        filters = history.Filters()
+        exclude = config[history.DOMAIN].get(history.CONF_EXCLUDE)
+        if exclude:
+            filters.excluded_entities = exclude[history.CONF_ENTITIES]
+            filters.excluded_domains = exclude[history.CONF_DOMAINS]
+        include = config[history.DOMAIN].get(history.CONF_INCLUDE)
+        if include:
+            filters.included_entities = include[history.CONF_ENTITIES]
+            filters.included_domains = include[history.CONF_DOMAINS]
+
+        hist = history.get_significant_states(zero, four, filters=filters)
+        assert states == hist
+
+    def record_states(self):
+        """Record some test states.
+
+        We inject a bunch of state updates from media player, zone and
+        thermostat.
+        """
         self.init_recorder()
         mp = 'media_player.test'
+        mp2 = 'media_player.test2'
         therm = 'thermostat.test'
+        therm2 = 'thermostat.test2'
         zone = 'zone.home'
         script_nc = 'script.cannot_cancel_this_one'
         script_c = 'script.can_cancel_this_one'
@@ -168,7 +398,7 @@ class TestComponentHistory(unittest.TestCase):
         three = two + timedelta(seconds=1)
         four = three + timedelta(seconds=1)
 
-        states = {therm: [], mp: [], script_c: []}
+        states = {therm: [], therm2: [], mp: [], mp2: [], script_c: []}
         with patch('homeassistant.components.recorder.dt_util.utcnow',
                    return_value=one):
             states[mp].append(
@@ -177,6 +407,9 @@ class TestComponentHistory(unittest.TestCase):
             states[mp].append(
                 set_state(mp, 'YouTube',
                           attributes={'media_title': str(sentinel.mt2)}))
+            states[mp2].append(
+                set_state(mp2, 'YouTube',
+                          attributes={'media_title': str(sentinel.mt2)}))
             states[therm].append(
                 set_state(therm, 20, attributes={'current_temperature': 19.5}))
 
@@ -192,6 +425,8 @@ class TestComponentHistory(unittest.TestCase):
                 set_state(script_c, 'off', attributes={'can_cancel': True}))
             states[therm].append(
                 set_state(therm, 21, attributes={'current_temperature': 19.8}))
+            states[therm2].append(
+                set_state(therm2, 20, attributes={'current_temperature': 19}))
 
         with patch('homeassistant.components.recorder.dt_util.utcnow',
                    return_value=three):
@@ -201,6 +436,7 @@ class TestComponentHistory(unittest.TestCase):
             # Attributes changed even though state is the same
             states[therm].append(
                 set_state(therm, 21, attributes={'current_temperature': 20}))
-
+            # state will be skipped since entity is hidden
-        hist = history.get_significant_states(zero, four)
+            set_state(therm, 22, attributes={'current_temperature': 21,
-        assert states == hist
+                                             'hidden': True})
+        return zero, four, states