From 3368e30279c7bfbf7ea621983b85d7b30427dfd9 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Sun, 14 Apr 2019 07:23:01 -0700 Subject: [PATCH] Migrate packages and check config (#23082) * Migrate packages and check config * Fix typing * Fix check config script --- homeassistant/config.py | 128 ++++++++++++---------- homeassistant/helpers/entity_component.py | 8 +- homeassistant/loader.py | 10 +- homeassistant/scripts/check_config.py | 4 +- homeassistant/setup.py | 4 +- tests/common.py | 8 +- tests/test_config.py | 50 +++++---- 7 files changed, 113 insertions(+), 99 deletions(-) diff --git a/homeassistant/config.py b/homeassistant/config.py index fe7f904a4b5..e86a6bf754a 100644 --- a/homeassistant/config.py +++ b/homeassistant/config.py @@ -25,7 +25,9 @@ from homeassistant.const import ( CONF_TYPE, CONF_ID) from homeassistant.core import callback, DOMAIN as CONF_CORE, HomeAssistant from homeassistant.exceptions import HomeAssistantError -from homeassistant.loader import get_component, get_platform +from homeassistant.loader import ( + Integration, async_get_integration, IntegrationNotFound +) from homeassistant.util.yaml import load_yaml, SECRET_YAML import homeassistant.helpers.config_validation as cv from homeassistant.util import dt as date_util, location as loc_util @@ -308,11 +310,14 @@ async def async_hass_config_yaml(hass: HomeAssistant) -> Dict: raise HomeAssistantError( "Config file not found in: {}".format(hass.config.config_dir)) config = load_yaml_config_file(path) - core_config = config.get(CONF_CORE, {}) - merge_packages_config(hass, config, core_config.get(CONF_PACKAGES, {})) return config - return await hass.async_add_executor_job(_load_hass_yaml_config) + config = await hass.async_add_executor_job(_load_hass_yaml_config) + core_config = config.get(CONF_CORE, {}) + await merge_packages_config( + hass, config, core_config.get(CONF_PACKAGES, {}) + ) + return config def find_config_file(config_dir: Optional[str]) -> Optional[str]: @@ -634,8 +639,10 @@ def _recursive_merge( return error -def merge_packages_config(hass: HomeAssistant, config: Dict, packages: Dict, - _log_pkg_error: Callable = _log_pkg_error) -> Dict: +async def merge_packages_config(hass: HomeAssistant, config: Dict, + packages: Dict, + _log_pkg_error: Callable = _log_pkg_error) \ + -> Dict: """Merge packages into the top-level configuration. Mutate config.""" # pylint: disable=too-many-nested-blocks PACKAGES_CONFIG_SCHEMA(packages) @@ -646,12 +653,20 @@ def merge_packages_config(hass: HomeAssistant, config: Dict, packages: Dict, # If component name is given with a trailing description, remove it # when looking for component domain = comp_name.split(' ')[0] - component = get_component(hass, domain) - if component is None: + try: + integration = await async_get_integration(hass, domain) + except IntegrationNotFound: _log_pkg_error(pack_name, comp_name, config, "does not exist") continue + try: + component = integration.get_component() + except ImportError: + _log_pkg_error(pack_name, comp_name, config, + "unable to import") + continue + if hasattr(component, 'PLATFORM_SCHEMA'): if not comp_conf: continue # Ensure we dont add Falsy items to list @@ -701,72 +716,73 @@ def merge_packages_config(hass: HomeAssistant, config: Dict, packages: Dict, return config -@callback -def async_process_component_config( - hass: HomeAssistant, config: Dict, domain: str) -> Optional[Dict]: +async def async_process_component_config( + hass: HomeAssistant, config: Dict, integration: Integration) \ + -> Optional[Dict]: """Check component configuration and return processed configuration. Returns None on error. This method must be run in the event loop. """ - component = get_component(hass, domain) + domain = integration.domain + component = integration.get_component() if hasattr(component, 'CONFIG_SCHEMA'): try: - config = component.CONFIG_SCHEMA(config) # type: ignore + return component.CONFIG_SCHEMA(config) # type: ignore except vol.Invalid as ex: async_log_exception(ex, domain, config, hass) return None - elif (hasattr(component, 'PLATFORM_SCHEMA') or - hasattr(component, 'PLATFORM_SCHEMA_BASE')): - platforms = [] - for p_name, p_config in config_per_platform(config, domain): - # Validate component specific platform schema - try: - if hasattr(component, 'PLATFORM_SCHEMA_BASE'): - p_validated = \ - component.PLATFORM_SCHEMA_BASE( # type: ignore - p_config) - else: - p_validated = component.PLATFORM_SCHEMA( # type: ignore - p_config) - except vol.Invalid as ex: - async_log_exception(ex, domain, p_config, hass) - continue + component_platform_schema = getattr( + component, 'PLATFORM_SCHEMA_BASE', + getattr(component, 'PLATFORM_SCHEMA', None)) - # Not all platform components follow same pattern for platforms - # So if p_name is None we are not going to validate platform - # (the automation component is one of them) - if p_name is None: - platforms.append(p_validated) - continue + if component_platform_schema is None: + return config - platform = get_platform(hass, domain, p_name) - - if platform is None: - continue - - # Validate platform specific schema - if hasattr(platform, 'PLATFORM_SCHEMA'): - # pylint: disable=no-member - try: - p_validated = platform.PLATFORM_SCHEMA( # type: ignore - p_config) - except vol.Invalid as ex: - async_log_exception(ex, '{}.{}'.format(domain, p_name), - p_config, hass) - continue + platforms = [] + for p_name, p_config in config_per_platform(config, domain): + # Validate component specific platform schema + try: + p_validated = component_platform_schema(p_config) + except vol.Invalid as ex: + async_log_exception(ex, domain, p_config, hass) + continue + # Not all platform components follow same pattern for platforms + # So if p_name is None we are not going to validate platform + # (the automation component is one of them) + if p_name is None: platforms.append(p_validated) + continue - # Create a copy of the configuration with all config for current - # component removed and add validated config back in. - filter_keys = extract_domain_configs(config, domain) - config = {key: value for key, value in config.items() - if key not in filter_keys} - config[domain] = platforms + try: + p_integration = await async_get_integration(hass, p_name) + platform = p_integration.get_platform(domain) + except (IntegrationNotFound, ImportError): + continue + + # Validate platform specific schema + if hasattr(platform, 'PLATFORM_SCHEMA'): + # pylint: disable=no-member + try: + p_validated = platform.PLATFORM_SCHEMA( # type: ignore + p_config) + except vol.Invalid as ex: + async_log_exception(ex, '{}.{}'.format(domain, p_name), + p_config, hass) + continue + + platforms.append(p_validated) + + # Create a copy of the configuration with all config for current + # component removed and add validated config back in. + filter_keys = extract_domain_configs(config, domain) + config = {key: value for key, value in config.items() + if key not in filter_keys} + config[domain] = platforms return config diff --git a/homeassistant/helpers/entity_component.py b/homeassistant/helpers/entity_component.py index 7be3d906bfa..5985aa5af32 100644 --- a/homeassistant/helpers/entity_component.py +++ b/homeassistant/helpers/entity_component.py @@ -13,7 +13,7 @@ from homeassistant.core import callback from homeassistant.exceptions import HomeAssistantError from homeassistant.helpers import config_per_platform, discovery from homeassistant.helpers.service import async_extract_entity_ids -from homeassistant.loader import bind_hass +from homeassistant.loader import bind_hass, async_get_integration from homeassistant.util import slugify from .entity_platform import EntityPlatform @@ -276,8 +276,10 @@ class EntityComponent: self.logger.error(err) return None - conf = conf_util.async_process_component_config( - self.hass, conf, self.domain) + integration = await async_get_integration(self.hass, self.domain) + + conf = await conf_util.async_process_component_config( + self.hass, conf, integration) if conf is None: return None diff --git a/homeassistant/loader.py b/homeassistant/loader.py index 44e5ab23d78..243c0790653 100644 --- a/homeassistant/loader.py +++ b/homeassistant/loader.py @@ -1,14 +1,8 @@ """ -The methods for loading Home Assistant components. +The methods for loading Home Assistant integrations. This module has quite some complex parts. I have tried to add as much documentation as possible to keep it understandable. - -Components can be accessed via hass.components.switch from your code. -If you want to retrieve a platform that is part of a component, you should -call get_component(hass, 'switch.your_platform'). In both cases the config -directory is checked to see if it contains a user provided version. If not -available it will check the built-in components and platforms. """ import functools as ft import importlib @@ -100,7 +94,7 @@ class Integration: Will create a stub manifest. """ - comp = get_component(hass, domain) + comp = _load_file(hass, domain, LOOKUP_PATHS) if comp is None: return None diff --git a/homeassistant/scripts/check_config.py b/homeassistant/scripts/check_config.py index 2eb895603dd..04d08d63a82 100644 --- a/homeassistant/scripts/check_config.py +++ b/homeassistant/scripts/check_config.py @@ -320,8 +320,8 @@ def check_ha_config_file(hass): core_config = {} # Merge packages - merge_packages_config( - hass, config, core_config.get(CONF_PACKAGES, {}), _pack_error) + hass.loop.run_until_complete(merge_packages_config( + hass, config, core_config.get(CONF_PACKAGES, {}), _pack_error)) core_config.pop(CONF_PACKAGES, None) # Filter out repeating config sections diff --git a/homeassistant/setup.py b/homeassistant/setup.py index 0160279a859..ab8b471e7d5 100644 --- a/homeassistant/setup.py +++ b/homeassistant/setup.py @@ -126,8 +126,8 @@ async def _async_setup_component(hass: core.HomeAssistant, "%s -> %s", domain, err.from_domain, err.to_domain) return False - processed_config = \ - conf_util.async_process_component_config(hass, config, domain) + processed_config = await conf_util.async_process_component_config( + hass, config, integration) if processed_config is None: log_error("Invalid config.") diff --git a/tests/common.py b/tests/common.py index 255ceacedfb..ce3dc51fcb9 100644 --- a/tests/common.py +++ b/tests/common.py @@ -695,11 +695,11 @@ def assert_setup_component(count, domain=None): """ config = {} - @ha.callback - def mock_psc(hass, config_input, domain_input): + async def mock_psc(hass, config_input, integration): """Mock the prepare_setup_component to capture config.""" - res = async_process_component_config( - hass, config_input, domain_input) + domain_input = integration.domain + res = await async_process_component_config( + hass, config_input, integration) config[domain_input] = None if res is None else res.get(domain_input) _LOGGER.debug("Configuration for %s, Validated: %s, Original %s", domain_input, diff --git a/tests/test_config.py b/tests/test_config.py index 8afad09c946..fa194d17c11 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -14,6 +14,7 @@ import yaml from homeassistant.core import DOMAIN, HomeAssistantError, Config import homeassistant.config as config_util +from homeassistant.loader import async_get_integration from homeassistant.const import ( ATTR_FRIENDLY_NAME, ATTR_HIDDEN, ATTR_ASSUMED_STATE, CONF_LATITUDE, CONF_LONGITUDE, CONF_UNIT_SYSTEM, CONF_NAME, @@ -587,7 +588,7 @@ def merge_log_err(hass): yield logerr -def test_merge(merge_log_err, hass): +async def test_merge(merge_log_err, hass): """Test if we can merge packages.""" packages = { 'pack_dict': {'input_boolean': {'ib1': None}}, @@ -601,7 +602,7 @@ def test_merge(merge_log_err, hass): 'input_boolean': {'ib2': None}, 'light': {'platform': 'test'} } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert merge_log_err.call_count == 0 assert len(config) == 5 @@ -611,7 +612,7 @@ def test_merge(merge_log_err, hass): assert isinstance(config['wake_on_lan'], OrderedDict) -def test_merge_try_falsy(merge_log_err, hass): +async def test_merge_try_falsy(merge_log_err, hass): """Ensure we dont add falsy items like empty OrderedDict() to list.""" packages = { 'pack_falsy_to_lst': {'automation': OrderedDict()}, @@ -622,7 +623,7 @@ def test_merge_try_falsy(merge_log_err, hass): 'automation': {'do': 'something'}, 'light': {'some': 'light'}, } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert merge_log_err.call_count == 0 assert len(config) == 3 @@ -630,7 +631,7 @@ def test_merge_try_falsy(merge_log_err, hass): assert len(config['light']) == 1 -def test_merge_new(merge_log_err, hass): +async def test_merge_new(merge_log_err, hass): """Test adding new components to outer scope.""" packages = { 'pack_1': {'light': [{'platform': 'one'}]}, @@ -643,7 +644,7 @@ def test_merge_new(merge_log_err, hass): config = { config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages}, } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert merge_log_err.call_count == 0 assert 'api' in config @@ -652,7 +653,7 @@ def test_merge_new(merge_log_err, hass): assert len(config['panel_custom']) == 1 -def test_merge_type_mismatch(merge_log_err, hass): +async def test_merge_type_mismatch(merge_log_err, hass): """Test if we have a type mismatch for packages.""" packages = { 'pack_1': {'input_boolean': [{'ib1': None}]}, @@ -665,7 +666,7 @@ def test_merge_type_mismatch(merge_log_err, hass): 'input_select': [{'ib2': None}], 'light': [{'platform': 'two'}] } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert merge_log_err.call_count == 2 assert len(config) == 4 @@ -673,14 +674,14 @@ def test_merge_type_mismatch(merge_log_err, hass): assert len(config['light']) == 2 -def test_merge_once_only_keys(merge_log_err, hass): +async def test_merge_once_only_keys(merge_log_err, hass): """Test if we have a merge for a comp that may occur only once. Keys.""" packages = {'pack_2': {'api': None}} config = { config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages}, 'api': None, } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert config['api'] == OrderedDict() packages = {'pack_2': {'api': { @@ -693,7 +694,7 @@ def test_merge_once_only_keys(merge_log_err, hass): 'key_2': 2, } } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert config['api'] == {'key_1': 1, 'key_2': 2, 'key_3': 3, } # Duplicate keys error @@ -704,11 +705,11 @@ def test_merge_once_only_keys(merge_log_err, hass): config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages}, 'api': {'key': 1, } } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert merge_log_err.call_count == 1 -def test_merge_once_only_lists(hass): +async def test_merge_once_only_lists(hass): """Test if we have a merge for a comp that may occur only once. Lists.""" packages = {'pack_2': {'api': { 'list_1': ['item_2', 'item_3'], @@ -721,14 +722,14 @@ def test_merge_once_only_lists(hass): 'list_1': ['item_1'], } } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert config['api'] == { 'list_1': ['item_1', 'item_2', 'item_3'], 'list_2': ['item_1'], } -def test_merge_once_only_dictionaries(hass): +async def test_merge_once_only_dictionaries(hass): """Test if we have a merge for a comp that may occur only once. Dicts.""" packages = {'pack_2': {'api': { 'dict_1': { @@ -747,7 +748,7 @@ def test_merge_once_only_dictionaries(hass): }, } } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert config['api'] == { 'dict_1': { 'key_1': 1, @@ -758,7 +759,7 @@ def test_merge_once_only_dictionaries(hass): } -def test_merge_id_schema(hass): +async def test_merge_id_schema(hass): """Test if we identify the config schemas correctly.""" types = { 'panel_custom': 'list', @@ -768,14 +769,15 @@ def test_merge_id_schema(hass): 'shell_command': 'dict', 'qwikswitch': 'dict', } - for name, expected_type in types.items(): - module = config_util.get_component(hass, name) + for domain, expected_type in types.items(): + integration = await async_get_integration(hass, domain) + module = integration.get_component() typ, _ = config_util._identify_config_schema(module) assert typ == expected_type, "{} expected {}, got {}".format( - name, expected_type, typ) + domain, expected_type, typ) -def test_merge_duplicate_keys(merge_log_err, hass): +async def test_merge_duplicate_keys(merge_log_err, hass): """Test if keys in dicts are duplicates.""" packages = { 'pack_1': {'input_select': {'ib1': None}}, @@ -784,7 +786,7 @@ def test_merge_duplicate_keys(merge_log_err, hass): config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages}, 'input_select': {'ib1': 1}, } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert merge_log_err.call_count == 1 assert len(config) == 2 @@ -984,7 +986,7 @@ async def test_disallowed_duplicated_auth_mfa_module_config(hass): await config_util.async_process_ha_core_config(hass, core_config) -def test_merge_split_component_definition(hass): +async def test_merge_split_component_definition(hass): """Test components with trailing description in packages are merged.""" packages = { 'pack_1': {'light one': {'l1': None}}, @@ -994,7 +996,7 @@ def test_merge_split_component_definition(hass): config = { config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages}, } - config_util.merge_packages_config(hass, config, packages) + await config_util.merge_packages_config(hass, config, packages) assert len(config) == 4 assert len(config['light one']) == 1