diff --git a/homeassistant/components/recorder/filters.py b/homeassistant/components/recorder/filters.py index 02c342441a7..45db64e0097 100644 --- a/homeassistant/components/recorder/filters.py +++ b/homeassistant/components/recorder/filters.py @@ -139,49 +139,52 @@ class Filters: have_exclude = self._have_exclude have_include = self._have_include - # Case 1 - no includes or excludes - pass all entities + # Case 1 - No filter + # - All entities included if not have_include and not have_exclude: return None - # Case 2 - includes, no excludes - only include specified entities + # Case 2 - Only includes + # - Entity listed in entities include: include + # - Otherwise, entity matches domain include: include + # - Otherwise, entity matches glob include: include + # - Otherwise: exclude if have_include and not have_exclude: return or_(*includes).self_group() - # Case 3 - excludes, no includes - only exclude specified entities + # Case 3 - Only excludes + # - Entity listed in exclude: exclude + # - Otherwise, entity matches domain exclude: exclude + # - Otherwise, entity matches glob exclude: exclude + # - Otherwise: include if not have_include and have_exclude: return not_(or_(*excludes).self_group()) - # Case 4 - both includes and excludes specified - # Case 4a - include domain or glob specified - # - if domain is included, pass if entity not excluded - # - if glob is included, pass if entity and domain not excluded - # - if domain and glob are not included, pass if entity is included - # note: if both include domain matches then exclude domains ignored. - # If glob matches then exclude domains and glob checked + # Case 4 - Domain and/or glob includes (may also have excludes) + # - Entity listed in entities include: include + # - Otherwise, entity listed in entities exclude: exclude + # - Otherwise, entity matches glob include: include + # - Otherwise, entity matches glob exclude: exclude + # - Otherwise, entity matches domain include: include + # - Otherwise: exclude if self.included_domains or self.included_entity_globs: return or_( - (i_domains & ~(e_entities | e_entity_globs)), - ( - ~i_domains - & or_( - (i_entity_globs & ~(or_(*excludes))), - (~i_entity_globs & i_entities), - ) - ), + i_entities, + (~e_entities & (i_entity_globs | (~e_entity_globs & i_domains))), ).self_group() - # Case 4b - exclude domain or glob specified, include has no domain or glob - # In this one case the traditional include logic is inverted. Even though an - # include is specified since its only a list of entity IDs its used only to - # expose specific entities excluded by domain or glob. Any entities not - # excluded are then presumed included. Logic is as follows - # - if domain or glob is excluded, pass if entity is included - # - if domain is not excluded, pass if entity not excluded by ID + # Case 5 - Domain and/or glob excludes (no domain and/or glob includes) + # - Entity listed in entities include: include + # - Otherwise, entity listed in exclude: exclude + # - Otherwise, entity matches glob exclude: exclude + # - Otherwise, entity matches domain exclude: exclude + # - Otherwise: include if self.excluded_domains or self.excluded_entity_globs: return (not_(or_(*excludes)) | i_entities).self_group() - # Case 4c - neither include or exclude domain specified - # - Only pass if entity is included. Ignore entity excludes. + # Case 6 - No Domain and/or glob includes or excludes + # - Entity listed in entities include: include + # - Otherwise: exclude return i_entities def states_entity_filter(self) -> ClauseList: diff --git a/homeassistant/helpers/entityfilter.py b/homeassistant/helpers/entityfilter.py index d4722eeca44..109c5454cc2 100644 --- a/homeassistant/helpers/entityfilter.py +++ b/homeassistant/helpers/entityfilter.py @@ -145,11 +145,7 @@ def _glob_to_re(glob: str) -> re.Pattern[str]: def _test_against_patterns(patterns: list[re.Pattern[str]], entity_id: str) -> bool: """Test entity against list of patterns, true if any match.""" - for pattern in patterns: - if pattern.match(entity_id): - return True - - return False + return any(pattern.match(entity_id) for pattern in patterns) def _convert_globs_to_pattern_list(globs: list[str] | None) -> list[re.Pattern[str]]: @@ -193,7 +189,7 @@ def _generate_filter_from_sets_and_pattern_lists( return ( entity_id in include_e or domain in include_d - or bool(include_eg and _test_against_patterns(include_eg, entity_id)) + or _test_against_patterns(include_eg, entity_id) ) def entity_excluded(domain: str, entity_id: str) -> bool: @@ -201,14 +197,19 @@ def _generate_filter_from_sets_and_pattern_lists( return ( entity_id in exclude_e or domain in exclude_d - or bool(exclude_eg and _test_against_patterns(exclude_eg, entity_id)) + or _test_against_patterns(exclude_eg, entity_id) ) - # Case 1 - no includes or excludes - pass all entities + # Case 1 - No filter + # - All entities included if not have_include and not have_exclude: return lambda entity_id: True - # Case 2 - includes, no excludes - only include specified entities + # Case 2 - Only includes + # - Entity listed in entities include: include + # - Otherwise, entity matches domain include: include + # - Otherwise, entity matches glob include: include + # - Otherwise: exclude if have_include and not have_exclude: def entity_filter_2(entity_id: str) -> bool: @@ -218,7 +219,11 @@ def _generate_filter_from_sets_and_pattern_lists( return entity_filter_2 - # Case 3 - excludes, no includes - only exclude specified entities + # Case 3 - Only excludes + # - Entity listed in exclude: exclude + # - Otherwise, entity matches domain exclude: exclude + # - Otherwise, entity matches glob exclude: exclude + # - Otherwise: include if not have_include and have_exclude: def entity_filter_3(entity_id: str) -> bool: @@ -228,38 +233,36 @@ def _generate_filter_from_sets_and_pattern_lists( return entity_filter_3 - # Case 4 - both includes and excludes specified - # Case 4a - include domain or glob specified - # - if domain is included, pass if entity not excluded - # - if glob is included, pass if entity and domain not excluded - # - if domain and glob are not included, pass if entity is included - # note: if both include domain matches then exclude domains ignored. - # If glob matches then exclude domains and glob checked + # Case 4 - Domain and/or glob includes (may also have excludes) + # - Entity listed in entities include: include + # - Otherwise, entity listed in entities exclude: exclude + # - Otherwise, entity matches glob include: include + # - Otherwise, entity matches glob exclude: exclude + # - Otherwise, entity matches domain include: include + # - Otherwise: exclude if include_d or include_eg: def entity_filter_4a(entity_id: str) -> bool: """Return filter function for case 4a.""" - domain = split_entity_id(entity_id)[0] - if domain in include_d: - return not ( - entity_id in exclude_e - or bool( - exclude_eg and _test_against_patterns(exclude_eg, entity_id) + return entity_id in include_e or ( + entity_id not in exclude_e + and ( + _test_against_patterns(include_eg, entity_id) + or ( + split_entity_id(entity_id)[0] in include_d + and not _test_against_patterns(exclude_eg, entity_id) ) ) - if _test_against_patterns(include_eg, entity_id): - return not entity_excluded(domain, entity_id) - return entity_id in include_e + ) return entity_filter_4a - # Case 4b - exclude domain or glob specified, include has no domain or glob - # In this one case the traditional include logic is inverted. Even though an - # include is specified since its only a list of entity IDs its used only to - # expose specific entities excluded by domain or glob. Any entities not - # excluded are then presumed included. Logic is as follows - # - if domain or glob is excluded, pass if entity is included - # - if domain is not excluded, pass if entity not excluded by ID + # Case 5 - Domain and/or glob excludes (no domain and/or glob includes) + # - Entity listed in entities include: include + # - Otherwise, entity listed in exclude: exclude + # - Otherwise, entity matches glob exclude: exclude + # - Otherwise, entity matches domain exclude: exclude + # - Otherwise: include if exclude_d or exclude_eg: def entity_filter_4b(entity_id: str) -> bool: @@ -273,6 +276,7 @@ def _generate_filter_from_sets_and_pattern_lists( return entity_filter_4b - # Case 4c - neither include or exclude domain specified - # - Only pass if entity is included. Ignore entity excludes. + # Case 6 - No Domain and/or glob includes or excludes + # - Entity listed in entities include: include + # - Otherwise: exclude return lambda entity_id: entity_id in include_e diff --git a/tests/components/apache_kafka/test_init.py b/tests/components/apache_kafka/test_init.py index 3f594b3fce3..9f5fa2800bc 100644 --- a/tests/components/apache_kafka/test_init.py +++ b/tests/components/apache_kafka/test_init.py @@ -169,7 +169,7 @@ async def test_filtered_allowlist(hass, mock_client): FilterTest("light.excluded_test", False), FilterTest("light.excluded", False), FilterTest("sensor.included_test", True), - FilterTest("climate.included_test", False), + FilterTest("climate.included_test", True), ] await _run_filter_tests(hass, tests, mock_client) diff --git a/tests/components/azure_event_hub/test_init.py b/tests/components/azure_event_hub/test_init.py index cf7226e20b0..c1393483c8c 100644 --- a/tests/components/azure_event_hub/test_init.py +++ b/tests/components/azure_event_hub/test_init.py @@ -176,7 +176,7 @@ async def test_full_batch(hass, entry_with_one_event, mock_create_batch): FilterTest("light.excluded_test", 0), FilterTest("light.excluded", 0), FilterTest("sensor.included_test", 1), - FilterTest("climate.included_test", 0), + FilterTest("climate.included_test", 1), ], ), ( diff --git a/tests/components/google_pubsub/test_init.py b/tests/components/google_pubsub/test_init.py index 1b6d1dbf4b4..71fab923972 100644 --- a/tests/components/google_pubsub/test_init.py +++ b/tests/components/google_pubsub/test_init.py @@ -222,7 +222,7 @@ async def test_filtered_allowlist(hass, mock_client): FilterTest("light.excluded_test", False), FilterTest("light.excluded", False), FilterTest("sensor.included_test", True), - FilterTest("climate.included_test", False), + FilterTest("climate.included_test", True), ] for test in tests: diff --git a/tests/components/history/test_init.py b/tests/components/history/test_init.py index 56ac68d944d..5eb4894c72a 100644 --- a/tests/components/history/test_init.py +++ b/tests/components/history/test_init.py @@ -753,7 +753,7 @@ async def test_fetch_period_api_with_entity_glob_include_and_exclude( { "history": { "exclude": { - "entity_globs": ["light.many*"], + "entity_globs": ["light.many*", "binary_sensor.*"], }, "include": { "entity_globs": ["light.m*"], @@ -769,6 +769,7 @@ async def test_fetch_period_api_with_entity_glob_include_and_exclude( hass.states.async_set("light.many_state_changes", "on") hass.states.async_set("switch.match", "on") hass.states.async_set("media_player.test", "on") + hass.states.async_set("binary_sensor.exclude", "on") await async_wait_recording_done(hass) @@ -778,10 +779,11 @@ async def test_fetch_period_api_with_entity_glob_include_and_exclude( ) assert response.status == HTTPStatus.OK response_json = await response.json() - assert len(response_json) == 3 - assert response_json[0][0]["entity_id"] == "light.match" - assert response_json[1][0]["entity_id"] == "media_player.test" - assert response_json[2][0]["entity_id"] == "switch.match" + assert len(response_json) == 4 + assert response_json[0][0]["entity_id"] == "light.many_state_changes" + assert response_json[1][0]["entity_id"] == "light.match" + assert response_json[2][0]["entity_id"] == "media_player.test" + assert response_json[3][0]["entity_id"] == "switch.match" async def test_entity_ids_limit_via_api(hass, hass_client, recorder_mock): diff --git a/tests/components/influxdb/test_init.py b/tests/components/influxdb/test_init.py index 94071e849c2..27b9ac82ade 100644 --- a/tests/components/influxdb/test_init.py +++ b/tests/components/influxdb/test_init.py @@ -866,7 +866,7 @@ async def test_event_listener_filtered_allowlist( FilterTest("fake.excluded", False), FilterTest("another_fake.denied", False), FilterTest("fake.excluded_entity", False), - FilterTest("another_fake.included_entity", False), + FilterTest("another_fake.included_entity", True), ] execute_filter_test(hass, tests, handler_method, write_api, get_mock_call) diff --git a/tests/components/logbook/test_init.py b/tests/components/logbook/test_init.py index d16b3476d84..31ca1610250 100644 --- a/tests/components/logbook/test_init.py +++ b/tests/components/logbook/test_init.py @@ -2153,7 +2153,7 @@ async def test_include_exclude_events_with_glob_filters( client = await hass_client() entries = await _async_fetch_logbook(client) - assert len(entries) == 6 + assert len(entries) == 7 _assert_entry( entries[0], name="Home Assistant", message="started", domain=ha.DOMAIN ) @@ -2162,6 +2162,7 @@ async def test_include_exclude_events_with_glob_filters( _assert_entry(entries[3], name="bla", entity_id=entity_id, state="20") _assert_entry(entries[4], name="blu", entity_id=entity_id2, state="20") _assert_entry(entries[5], name="included", entity_id=entity_id4, state="30") + _assert_entry(entries[6], name="included", entity_id=entity_id5, state="30") async def test_empty_config(hass, hass_client, recorder_mock): diff --git a/tests/components/recorder/test_filters_with_entityfilter.py b/tests/components/recorder/test_filters_with_entityfilter.py index ed4d4efe066..62bb1b3fa8d 100644 --- a/tests/components/recorder/test_filters_with_entityfilter.py +++ b/tests/components/recorder/test_filters_with_entityfilter.py @@ -514,3 +514,128 @@ async def test_same_entity_included_excluded_include_domain_wins(hass, recorder_ assert filtered_events_entity_ids == filter_accept assert not filtered_events_entity_ids.intersection(filter_reject) + + +async def test_specificly_included_entity_always_wins(hass, recorder_mock): + """Test specificlly included entity always wins.""" + filter_accept = { + "media_player.test2", + "media_player.test3", + "thermostat.test", + "binary_sensor.specific_include", + } + filter_reject = { + "binary_sensor.test2", + "binary_sensor.home", + "binary_sensor.can_cancel_this_one", + } + conf = { + CONF_INCLUDE: { + CONF_ENTITIES: ["binary_sensor.specific_include"], + }, + CONF_EXCLUDE: { + CONF_DOMAINS: ["binary_sensor"], + CONF_ENTITY_GLOBS: ["binary_sensor.*"], + }, + } + + extracted_filter = extract_include_exclude_filter_conf(conf) + entity_filter = convert_include_exclude_filter(extracted_filter) + sqlalchemy_filter = sqlalchemy_filter_from_include_exclude_conf(extracted_filter) + assert sqlalchemy_filter is not None + + for entity_id in filter_accept: + assert entity_filter(entity_id) is True + + for entity_id in filter_reject: + assert entity_filter(entity_id) is False + + ( + filtered_states_entity_ids, + filtered_events_entity_ids, + ) = await _async_get_states_and_events_with_filter( + hass, sqlalchemy_filter, filter_accept | filter_reject + ) + + assert filtered_states_entity_ids == filter_accept + assert not filtered_states_entity_ids.intersection(filter_reject) + + assert filtered_events_entity_ids == filter_accept + assert not filtered_events_entity_ids.intersection(filter_reject) + + +async def test_specificly_included_entity_always_wins_over_glob(hass, recorder_mock): + """Test specificlly included entity always wins over a glob.""" + filter_accept = { + "sensor.apc900va_status", + "sensor.apc900va_battery_charge", + "sensor.apc900va_battery_runtime", + "sensor.apc900va_load", + "sensor.energy_x", + } + filter_reject = { + "sensor.apc900va_not_included", + } + conf = { + CONF_EXCLUDE: { + CONF_DOMAINS: [ + "updater", + "camera", + "group", + "media_player", + "script", + "sun", + "automation", + "zone", + "weblink", + "scene", + "calendar", + "weather", + "remote", + "notify", + "switch", + "shell_command", + "media_player", + ], + CONF_ENTITY_GLOBS: ["sensor.apc900va_*"], + }, + CONF_INCLUDE: { + CONF_DOMAINS: [ + "binary_sensor", + "climate", + "device_tracker", + "input_boolean", + "sensor", + ], + CONF_ENTITY_GLOBS: ["sensor.energy_*"], + CONF_ENTITIES: [ + "sensor.apc900va_status", + "sensor.apc900va_battery_charge", + "sensor.apc900va_battery_runtime", + "sensor.apc900va_load", + ], + }, + } + extracted_filter = extract_include_exclude_filter_conf(conf) + entity_filter = convert_include_exclude_filter(extracted_filter) + sqlalchemy_filter = sqlalchemy_filter_from_include_exclude_conf(extracted_filter) + assert sqlalchemy_filter is not None + + for entity_id in filter_accept: + assert entity_filter(entity_id) is True + + for entity_id in filter_reject: + assert entity_filter(entity_id) is False + + ( + filtered_states_entity_ids, + filtered_events_entity_ids, + ) = await _async_get_states_and_events_with_filter( + hass, sqlalchemy_filter, filter_accept | filter_reject + ) + + assert filtered_states_entity_ids == filter_accept + assert not filtered_states_entity_ids.intersection(filter_reject) + + assert filtered_events_entity_ids == filter_accept + assert not filtered_events_entity_ids.intersection(filter_reject) diff --git a/tests/helpers/test_entityfilter.py b/tests/helpers/test_entityfilter.py index 043fb44a95a..f9d7ca47b4c 100644 --- a/tests/helpers/test_entityfilter.py +++ b/tests/helpers/test_entityfilter.py @@ -91,8 +91,8 @@ def test_excludes_only_with_glob_case_3(): assert testfilter("cover.garage_door") -def test_with_include_domain_case4a(): - """Test case 4a - include and exclude specified, with included domain.""" +def test_with_include_domain_case4(): + """Test case 4 - include and exclude specified, with included domain.""" incl_dom = {"light", "sensor"} incl_ent = {"binary_sensor.working"} excl_dom = {} @@ -108,8 +108,30 @@ def test_with_include_domain_case4a(): assert testfilter("sun.sun") is False -def test_with_include_glob_case4a(): - """Test case 4a - include and exclude specified, with included glob.""" +def test_with_include_domain_exclude_glob_case4(): + """Test case 4 - include and exclude specified, with included domain but excluded by glob.""" + incl_dom = {"light", "sensor"} + incl_ent = {"binary_sensor.working"} + incl_glob = {} + excl_dom = {} + excl_ent = {"light.ignoreme", "sensor.notworking"} + excl_glob = {"sensor.busted"} + testfilter = generate_filter( + incl_dom, incl_ent, excl_dom, excl_ent, incl_glob, excl_glob + ) + + assert testfilter("sensor.test") + assert testfilter("sensor.busted") is False + assert testfilter("sensor.notworking") is False + assert testfilter("light.test") + assert testfilter("light.ignoreme") is False + assert testfilter("binary_sensor.working") + assert testfilter("binary_sensor.another") is False + assert testfilter("sun.sun") is False + + +def test_with_include_glob_case4(): + """Test case 4 - include and exclude specified, with included glob.""" incl_dom = {} incl_glob = {"light.*", "sensor.*"} incl_ent = {"binary_sensor.working"} @@ -129,8 +151,8 @@ def test_with_include_glob_case4a(): assert testfilter("sun.sun") is False -def test_with_include_domain_glob_filtering_case4a(): - """Test case 4a - include and exclude specified, both have domains and globs.""" +def test_with_include_domain_glob_filtering_case4(): + """Test case 4 - include and exclude specified, both have domains and globs.""" incl_dom = {"light"} incl_glob = {"*working"} incl_ent = {} @@ -142,17 +164,64 @@ def test_with_include_domain_glob_filtering_case4a(): ) assert testfilter("sensor.working") - assert testfilter("sensor.notworking") is False + assert testfilter("sensor.notworking") is True # include is stronger assert testfilter("light.test") - assert testfilter("light.notworking") is False + assert testfilter("light.notworking") is True # include is stronger assert testfilter("light.ignoreme") is False - assert testfilter("binary_sensor.not_working") is False + assert testfilter("binary_sensor.not_working") is True # include is stronger assert testfilter("binary_sensor.another") is False assert testfilter("sun.sun") is False -def test_exclude_domain_case4b(): - """Test case 4b - include and exclude specified, with excluded domain.""" +def test_with_include_domain_glob_filtering_case4a_include_strong(): + """Test case 4 - include and exclude specified, both have domains and globs, and a specifically included entity.""" + incl_dom = {"light"} + incl_glob = {"*working"} + incl_ent = {"binary_sensor.specificly_included"} + excl_dom = {"binary_sensor"} + excl_glob = {"*notworking"} + excl_ent = {"light.ignoreme"} + testfilter = generate_filter( + incl_dom, incl_ent, excl_dom, excl_ent, incl_glob, excl_glob + ) + + assert testfilter("sensor.working") + assert testfilter("sensor.notworking") is True # iclude is stronger + assert testfilter("light.test") + assert testfilter("light.notworking") is True # iclude is stronger + assert testfilter("light.ignoreme") is False + assert testfilter("binary_sensor.not_working") is True # iclude is stronger + assert testfilter("binary_sensor.another") is False + assert testfilter("binary_sensor.specificly_included") is True + assert testfilter("sun.sun") is False + + +def test_with_include_glob_filtering_case4a_include_strong(): + """Test case 4 - include and exclude specified, both have globs, and a specifically included entity.""" + incl_dom = {} + incl_glob = {"*working"} + incl_ent = {"binary_sensor.specificly_included"} + excl_dom = {} + excl_glob = {"*broken", "*notworking", "binary_sensor.*"} + excl_ent = {"light.ignoreme"} + testfilter = generate_filter( + incl_dom, incl_ent, excl_dom, excl_ent, incl_glob, excl_glob + ) + + assert testfilter("sensor.working") is True + assert testfilter("sensor.notworking") is True # include is stronger + assert testfilter("sensor.broken") is False + assert testfilter("light.test") is False + assert testfilter("light.notworking") is True # include is stronger + assert testfilter("light.ignoreme") is False + assert testfilter("binary_sensor.not_working") is True # include is stronger + assert testfilter("binary_sensor.another") is False + assert testfilter("binary_sensor.specificly_included") is True + assert testfilter("sun.sun") is False + + +def test_exclude_domain_case5(): + """Test case 5 - include and exclude specified, with excluded domain.""" incl_dom = {} incl_ent = {"binary_sensor.working"} excl_dom = {"binary_sensor"} @@ -168,8 +237,8 @@ def test_exclude_domain_case4b(): assert testfilter("sun.sun") is True -def test_exclude_glob_case4b(): - """Test case 4b - include and exclude specified, with excluded glob.""" +def test_exclude_glob_case5(): + """Test case 5 - include and exclude specified, with excluded glob.""" incl_dom = {} incl_glob = {} incl_ent = {"binary_sensor.working"} @@ -189,8 +258,29 @@ def test_exclude_glob_case4b(): assert testfilter("sun.sun") is True -def test_no_domain_case4c(): - """Test case 4c - include and exclude specified, with no domains.""" +def test_exclude_glob_case5_include_strong(): + """Test case 5 - include and exclude specified, with excluded glob, and a specifically included entity.""" + incl_dom = {} + incl_glob = {} + incl_ent = {"binary_sensor.working"} + excl_dom = {"binary_sensor"} + excl_glob = {"binary_sensor.*"} + excl_ent = {"light.ignoreme", "sensor.notworking"} + testfilter = generate_filter( + incl_dom, incl_ent, excl_dom, excl_ent, incl_glob, excl_glob + ) + + assert testfilter("sensor.test") + assert testfilter("sensor.notworking") is False + assert testfilter("light.test") + assert testfilter("light.ignoreme") is False + assert testfilter("binary_sensor.working") + assert testfilter("binary_sensor.another") is False + assert testfilter("sun.sun") is True + + +def test_no_domain_case6(): + """Test case 6 - include and exclude specified, with no domains.""" incl_dom = {} incl_ent = {"binary_sensor.working"} excl_dom = {}