Make logbook inherit the recorder filter (#72728)

homeassistant/components/logbook/__init__.py

@@ -7,7 +7,10 @@ from typing import Any
 import voluptuous as vol
 
 from homeassistant.components import frontend
+from homeassistant.components.recorder.const import DOMAIN as RECORDER_DOMAIN
 from homeassistant.components.recorder.filters import (
+    extract_include_exclude_filter_conf,
+    merge_include_exclude_filters,
     sqlalchemy_filter_from_include_exclude_conf,
 )
 from homeassistant.const import (
@@ -115,9 +118,16 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
         hass, "logbook", "logbook", "hass:format-list-bulleted-type"
     )
 
-    if conf := config.get(DOMAIN, {}):
+    recorder_conf = config.get(RECORDER_DOMAIN, {})
-        filters = sqlalchemy_filter_from_include_exclude_conf(conf)
+    logbook_conf = config.get(DOMAIN, {})
-        entities_filter = convert_include_exclude_filter(conf)
+    recorder_filter = extract_include_exclude_filter_conf(recorder_conf)
+    logbook_filter = extract_include_exclude_filter_conf(logbook_conf)
+    merged_filter = merge_include_exclude_filters(recorder_filter, logbook_filter)
+
+    possible_merged_entities_filter = convert_include_exclude_filter(merged_filter)
+    if not possible_merged_entities_filter.empty_filter:
+        filters = sqlalchemy_filter_from_include_exclude_conf(merged_filter)
+        entities_filter = possible_merged_entities_filter
     else:
         filters = None
         entities_filter = None

homeassistant/components/recorder/filters.py

@@ -25,6 +25,40 @@ GLOB_TO_SQL_CHARS = {
     ord("\\"): "\\\\",
 }
 
+FILTER_TYPES = (CONF_EXCLUDE, CONF_INCLUDE)
+FITLER_MATCHERS = (CONF_ENTITIES, CONF_DOMAINS, CONF_ENTITY_GLOBS)
+
+
+def extract_include_exclude_filter_conf(conf: ConfigType) -> dict[str, Any]:
+    """Extract an include exclude filter from configuration.
+
+    This makes a copy so we do not alter the original data.
+    """
+    return {
+        filter_type: {
+            matcher: set(conf.get(filter_type, {}).get(matcher, []))
+            for matcher in FITLER_MATCHERS
+        }
+        for filter_type in FILTER_TYPES
+    }
+
+
+def merge_include_exclude_filters(
+    base_filter: dict[str, Any], add_filter: dict[str, Any]
+) -> dict[str, Any]:
+    """Merge two filters.
+
+    This makes a copy so we do not alter the original data.
+    """
+    return {
+        filter_type: {
+            matcher: base_filter[filter_type][matcher]
+            | add_filter[filter_type][matcher]
+            for matcher in FITLER_MATCHERS
+        }
+        for filter_type in FILTER_TYPES
+    }
+
 
 def sqlalchemy_filter_from_include_exclude_conf(conf: ConfigType) -> Filters | None:
     """Build a sql filter from config."""
@@ -46,13 +80,13 @@ class Filters:
 
     def __init__(self) -> None:
         """Initialise the include and exclude filters."""
-        self.excluded_entities: list[str] = []
+        self.excluded_entities: Iterable[str] = []
-        self.excluded_domains: list[str] = []
+        self.excluded_domains: Iterable[str] = []
-        self.excluded_entity_globs: list[str] = []
+        self.excluded_entity_globs: Iterable[str] = []
 
-        self.included_entities: list[str] = []
+        self.included_entities: Iterable[str] = []
-        self.included_domains: list[str] = []
+        self.included_domains: Iterable[str] = []
-        self.included_entity_globs: list[str] = []
+        self.included_entity_globs: Iterable[str] = []
 
     @property
     def has_config(self) -> bool:

tests/components/logbook/test_websocket_api.py

@@ -483,7 +483,7 @@ async def test_subscribe_unsubscribe_logbook_stream_excluded_entities(
                 CONF_EXCLUDE: {
                     CONF_ENTITIES: ["light.exc"],
                     CONF_DOMAINS: ["switch"],
-                    CONF_ENTITY_GLOBS: "*.excluded",
+                    CONF_ENTITY_GLOBS: ["*.excluded"],
                 }
             },
         },
@@ -672,7 +672,7 @@ async def test_subscribe_unsubscribe_logbook_stream_included_entities(
                 CONF_INCLUDE: {
                     CONF_ENTITIES: ["light.inc"],
                     CONF_DOMAINS: ["switch"],
-                    CONF_ENTITY_GLOBS: "*.included",
+                    CONF_ENTITY_GLOBS: ["*.included"],
                 }
             },
         },
@@ -849,6 +849,194 @@ async def test_subscribe_unsubscribe_logbook_stream_included_entities(
     assert sum(hass.bus.async_listeners().values()) == init_count
 
 
+@patch("homeassistant.components.logbook.websocket_api.EVENT_COALESCE_TIME", 0)
+async def test_logbook_stream_excluded_entities_inherits_filters_from_recorder(
+    hass, recorder_mock, hass_ws_client
+):
+    """Test subscribe/unsubscribe logbook stream inherts filters from recorder."""
+    now = dt_util.utcnow()
+    await asyncio.gather(
+        *[
+            async_setup_component(hass, comp, {})
+            for comp in ("homeassistant", "automation", "script")
+        ]
+    )
+    await async_setup_component(
+        hass,
+        logbook.DOMAIN,
+        {
+            logbook.DOMAIN: {
+                CONF_EXCLUDE: {
+                    CONF_ENTITIES: ["light.additional_excluded"],
+                }
+            },
+            recorder.DOMAIN: {
+                CONF_EXCLUDE: {
+                    CONF_ENTITIES: ["light.exc"],
+                    CONF_DOMAINS: ["switch"],
+                    CONF_ENTITY_GLOBS: ["*.excluded", "*.no_matches"],
+                }
+            },
+        },
+    )
+    await hass.async_block_till_done()
+    init_count = sum(hass.bus.async_listeners().values())
+
+    hass.states.async_set("light.exc", STATE_ON)
+    hass.states.async_set("light.exc", STATE_OFF)
+    hass.states.async_set("switch.any", STATE_ON)
+    hass.states.async_set("switch.any", STATE_OFF)
+    hass.states.async_set("cover.excluded", STATE_ON)
+    hass.states.async_set("cover.excluded", STATE_OFF)
+    hass.states.async_set("light.additional_excluded", STATE_ON)
+    hass.states.async_set("light.additional_excluded", STATE_OFF)
+    hass.states.async_set("binary_sensor.is_light", STATE_ON)
+    hass.states.async_set("binary_sensor.is_light", STATE_OFF)
+    state: State = hass.states.get("binary_sensor.is_light")
+    await hass.async_block_till_done()
+
+    await async_wait_recording_done(hass)
+    websocket_client = await hass_ws_client()
+    await websocket_client.send_json(
+        {"id": 7, "type": "logbook/event_stream", "start_time": now.isoformat()}
+    )
+
+    msg = await asyncio.wait_for(websocket_client.receive_json(), 2)
+    assert msg["id"] == 7
+    assert msg["type"] == TYPE_RESULT
+    assert msg["success"]
+
+    msg = await asyncio.wait_for(websocket_client.receive_json(), 2)
+    assert msg["id"] == 7
+    assert msg["type"] == "event"
+    assert msg["event"]["events"] == [
+        {
+            "entity_id": "binary_sensor.is_light",
+            "state": "off",
+            "when": state.last_updated.timestamp(),
+        }
+    ]
+    assert msg["event"]["start_time"] == now.timestamp()
+    assert msg["event"]["end_time"] > msg["event"]["start_time"]
+    assert msg["event"]["partial"] is True
+
+    hass.states.async_set("light.exc", STATE_ON)
+    hass.states.async_set("light.exc", STATE_OFF)
+    hass.states.async_set("switch.any", STATE_ON)
+    hass.states.async_set("switch.any", STATE_OFF)
+    hass.states.async_set("cover.excluded", STATE_ON)
+    hass.states.async_set("cover.excluded", STATE_OFF)
+    hass.states.async_set("light.additional_excluded", STATE_ON)
+    hass.states.async_set("light.additional_excluded", STATE_OFF)
+    hass.states.async_set("light.alpha", "on")
+    hass.states.async_set("light.alpha", "off")
+    alpha_off_state: State = hass.states.get("light.alpha")
+    hass.states.async_set("light.zulu", "on", {"color": "blue"})
+    hass.states.async_set("light.zulu", "off", {"effect": "help"})
+    zulu_off_state: State = hass.states.get("light.zulu")
+    hass.states.async_set(
+        "light.zulu", "on", {"effect": "help", "color": ["blue", "green"]}
+    )
+    zulu_on_state: State = hass.states.get("light.zulu")
+    await hass.async_block_till_done()
+
+    hass.states.async_remove("light.zulu")
+    await hass.async_block_till_done()
+
+    hass.states.async_set("light.zulu", "on", {"effect": "help", "color": "blue"})
+
+    msg = await asyncio.wait_for(websocket_client.receive_json(), 2)
+    assert msg["id"] == 7
+    assert msg["type"] == "event"
+    assert "partial" not in msg["event"]["events"]
+    assert msg["event"]["events"] == []
+
+    msg = await asyncio.wait_for(websocket_client.receive_json(), 2)
+    assert msg["id"] == 7
+    assert msg["type"] == "event"
+    assert "partial" not in msg["event"]["events"]
+    assert msg["event"]["events"] == [
+        {
+            "entity_id": "light.alpha",
+            "state": "off",
+            "when": alpha_off_state.last_updated.timestamp(),
+        },
+        {
+            "entity_id": "light.zulu",
+            "state": "off",
+            "when": zulu_off_state.last_updated.timestamp(),
+        },
+        {
+            "entity_id": "light.zulu",
+            "state": "on",
+            "when": zulu_on_state.last_updated.timestamp(),
+        },
+    ]
+
+    await async_wait_recording_done(hass)
+    hass.bus.async_fire(
+        EVENT_AUTOMATION_TRIGGERED,
+        {ATTR_NAME: "Mock automation 3", ATTR_ENTITY_ID: "cover.excluded"},
+    )
+    hass.bus.async_fire(
+        EVENT_AUTOMATION_TRIGGERED,
+        {
+            ATTR_NAME: "Mock automation switch matching entity",
+            ATTR_ENTITY_ID: "switch.match_domain",
+        },
+    )
+    hass.bus.async_fire(
+        EVENT_AUTOMATION_TRIGGERED,
+        {ATTR_NAME: "Mock automation switch matching domain", ATTR_DOMAIN: "switch"},
+    )
+    hass.bus.async_fire(
+        EVENT_AUTOMATION_TRIGGERED,
+        {ATTR_NAME: "Mock automation matches nothing"},
+    )
+    hass.bus.async_fire(
+        EVENT_AUTOMATION_TRIGGERED,
+        {ATTR_NAME: "Mock automation 3", ATTR_ENTITY_ID: "light.keep"},
+    )
+    hass.states.async_set("cover.excluded", STATE_ON)
+    hass.states.async_set("cover.excluded", STATE_OFF)
+    await hass.async_block_till_done()
+    msg = await websocket_client.receive_json()
+    assert msg["id"] == 7
+    assert msg["type"] == "event"
+    assert msg["event"]["events"] == [
+        {
+            "context_id": ANY,
+            "domain": "automation",
+            "entity_id": None,
+            "message": "triggered",
+            "name": "Mock automation matches nothing",
+            "source": None,
+            "when": ANY,
+        },
+        {
+            "context_id": ANY,
+            "domain": "automation",
+            "entity_id": "light.keep",
+            "message": "triggered",
+            "name": "Mock automation 3",
+            "source": None,
+            "when": ANY,
+        },
+    ]
+
+    await websocket_client.send_json(
+        {"id": 8, "type": "unsubscribe_events", "subscription": 7}
+    )
+    msg = await asyncio.wait_for(websocket_client.receive_json(), 2)
+
+    assert msg["id"] == 8
+    assert msg["type"] == TYPE_RESULT
+    assert msg["success"]
+
+    # Check our listener got unsubscribed
+    assert sum(hass.bus.async_listeners().values()) == init_count
+
+
 @patch("homeassistant.components.logbook.websocket_api.EVENT_COALESCE_TIME", 0)
 async def test_subscribe_unsubscribe_logbook_stream(
     hass, recorder_mock, hass_ws_client

tests/components/recorder/test_filters.py

@@ -0,0 +1,114 @@
+"""The tests for recorder filters."""
+
+from homeassistant.components.recorder.filters import (
+    extract_include_exclude_filter_conf,
+    merge_include_exclude_filters,
+)
+from homeassistant.helpers.entityfilter import (
+    CONF_DOMAINS,
+    CONF_ENTITIES,
+    CONF_ENTITY_GLOBS,
+    CONF_EXCLUDE,
+    CONF_INCLUDE,
+)
+
+SIMPLE_INCLUDE_FILTER = {
+    CONF_INCLUDE: {
+        CONF_DOMAINS: ["homeassistant"],
+        CONF_ENTITIES: ["sensor.one"],
+        CONF_ENTITY_GLOBS: ["climate.*"],
+    }
+}
+SIMPLE_INCLUDE_FILTER_DIFFERENT_ENTITIES = {
+    CONF_INCLUDE: {
+        CONF_DOMAINS: ["other"],
+        CONF_ENTITIES: ["not_sensor.one"],
+        CONF_ENTITY_GLOBS: ["not_climate.*"],
+    }
+}
+SIMPLE_EXCLUDE_FILTER = {
+    CONF_EXCLUDE: {
+        CONF_DOMAINS: ["homeassistant"],
+        CONF_ENTITIES: ["sensor.one"],
+        CONF_ENTITY_GLOBS: ["climate.*"],
+    }
+}
+SIMPLE_INCLUDE_EXCLUDE_FILTER = {**SIMPLE_INCLUDE_FILTER, **SIMPLE_EXCLUDE_FILTER}
+
+
+def test_extract_include_exclude_filter_conf():
+    """Test we can extract a filter from configuration without altering it."""
+    include_filter = extract_include_exclude_filter_conf(SIMPLE_INCLUDE_FILTER)
+    assert include_filter == {
+        CONF_EXCLUDE: {
+            CONF_DOMAINS: set(),
+            CONF_ENTITIES: set(),
+            CONF_ENTITY_GLOBS: set(),
+        },
+        CONF_INCLUDE: {
+            CONF_DOMAINS: {"homeassistant"},
+            CONF_ENTITIES: {"sensor.one"},
+            CONF_ENTITY_GLOBS: {"climate.*"},
+        },
+    }
+
+    exclude_filter = extract_include_exclude_filter_conf(SIMPLE_EXCLUDE_FILTER)
+    assert exclude_filter == {
+        CONF_INCLUDE: {
+            CONF_DOMAINS: set(),
+            CONF_ENTITIES: set(),
+            CONF_ENTITY_GLOBS: set(),
+        },
+        CONF_EXCLUDE: {
+            CONF_DOMAINS: {"homeassistant"},
+            CONF_ENTITIES: {"sensor.one"},
+            CONF_ENTITY_GLOBS: {"climate.*"},
+        },
+    }
+
+    include_exclude_filter = extract_include_exclude_filter_conf(
+        SIMPLE_INCLUDE_EXCLUDE_FILTER
+    )
+    assert include_exclude_filter == {
+        CONF_INCLUDE: {
+            CONF_DOMAINS: {"homeassistant"},
+            CONF_ENTITIES: {"sensor.one"},
+            CONF_ENTITY_GLOBS: {"climate.*"},
+        },
+        CONF_EXCLUDE: {
+            CONF_DOMAINS: {"homeassistant"},
+            CONF_ENTITIES: {"sensor.one"},
+            CONF_ENTITY_GLOBS: {"climate.*"},
+        },
+    }
+
+    include_exclude_filter[CONF_EXCLUDE][CONF_ENTITIES] = {"cover.altered"}
+    # verify it really is a copy
+    assert SIMPLE_INCLUDE_EXCLUDE_FILTER[CONF_EXCLUDE][CONF_ENTITIES] != {
+        "cover.altered"
+    }
+
+
+def test_merge_include_exclude_filters():
+    """Test we can merge two filters together."""
+    include_exclude_filter_base = extract_include_exclude_filter_conf(
+        SIMPLE_INCLUDE_EXCLUDE_FILTER
+    )
+    include_filter_add = extract_include_exclude_filter_conf(
+        SIMPLE_INCLUDE_FILTER_DIFFERENT_ENTITIES
+    )
+    merged_filter = merge_include_exclude_filters(
+        include_exclude_filter_base, include_filter_add
+    )
+    assert merged_filter == {
+        CONF_EXCLUDE: {
+            CONF_DOMAINS: {"homeassistant"},
+            CONF_ENTITIES: {"sensor.one"},
+            CONF_ENTITY_GLOBS: {"climate.*"},
+        },
+        CONF_INCLUDE: {
+            CONF_DOMAINS: {"other", "homeassistant"},
+            CONF_ENTITIES: {"not_sensor.one", "sensor.one"},
+            CONF_ENTITY_GLOBS: {"climate.*", "not_climate.*"},
+        },
+    }