check_config script evolution (#12792)

* Initial async_check_ha_config_file

* check_ha_config_file

* Various fixes

* feedback - return the config

* move_to_check_config

homeassistant/bootstrap.py

@@ -112,18 +112,13 @@ def async_from_config_dict(config: Dict[str, Any],
     if not loader.PREPARED:
         yield from hass.async_add_job(loader.prepare, hass)
 
+    # Make a copy because we are mutating it.
+    config = OrderedDict(config)
+
     # Merge packages
     conf_util.merge_packages_config(
         config, core_config.get(conf_util.CONF_PACKAGES, {}))
 
-    # Make a copy because we are mutating it.
-    # Use OrderedDict in case original one was one.
-    # Convert values to dictionaries if they are None
-    new_config = OrderedDict()
-    for key, value in config.items():
-        new_config[key] = value or {}
-    config = new_config
-
     hass.config_entries = config_entries.ConfigEntries(hass, config)
     yield from hass.config_entries.async_load()
 

homeassistant/config.py

@@ -41,9 +41,9 @@ VERSION_FILE = '.HA_VERSION'
 CONFIG_DIR_NAME = '.homeassistant'
 DATA_CUSTOMIZE = 'hass_customize'
 
-FILE_MIGRATION = [
+FILE_MIGRATION = (
-    ['ios.conf', '.ios.conf'],
+    ('ios.conf', '.ios.conf'),
-]
+)
 
 DEFAULT_CORE_CONFIG = (
     # Tuples (attribute, default, auto detect property, description)
@@ -304,6 +304,9 @@ def load_yaml_config_file(config_path):
         _LOGGER.error(msg)
         raise HomeAssistantError(msg)
 
+    # Convert values to dictionaries if they are None
+    for key, value in conf_dict.items():
+        conf_dict[key] = value or {}
     return conf_dict
 
 
@@ -345,14 +348,22 @@ def process_ha_config_upgrade(hass):
 
 @callback
 def async_log_exception(ex, domain, config, hass):
+    """Log an error for configuration validation.
+
+    This method must be run in the event loop.
+    """
+    if hass is not None:
+        async_notify_setup_error(hass, domain, True)
+    _LOGGER.error(_format_config_error(ex, domain, config))
+
+
+@callback
+def _format_config_error(ex, domain, config):
     """Generate log exception for configuration validation.
 
     This method must be run in the event loop.
     """
     message = "Invalid config for [{}]: ".format(domain)
-    if hass is not None:
-        async_notify_setup_error(hass, domain, True)
-
     if 'extra keys not allowed' in ex.error_message:
         message += '[{}] is an invalid option for [{}]. Check: {}->{}.'\
                    .format(ex.path[-1], domain, domain,
@@ -369,7 +380,7 @@ def async_log_exception(ex, domain, config, hass):
         message += ('Please check the docs at '
                     'https://home-assistant.io/components/{}/'.format(domain))
 
-    _LOGGER.error(message)
+    return message
 
 
 async def async_process_ha_core_config(hass, config):
@@ -497,7 +508,7 @@ async def async_process_ha_core_config(hass, config):
 
 
 def _log_pkg_error(package, component, config, message):
-    """Log an error while merging."""
+    """Log an error while merging packages."""
     message = "Package {} setup failed. Component {} {}".format(
         package, component, message)
 
@@ -523,7 +534,7 @@ def _identify_config_schema(module):
     return '', schema
 
 
-def merge_packages_config(config, packages):
+def merge_packages_config(config, packages, _log_pkg_error=_log_pkg_error):
     """Merge packages into the top-level configuration. Mutate config."""
     # pylint: disable=too-many-nested-blocks
     PACKAGES_CONFIG_SCHEMA(packages)
@@ -589,7 +600,7 @@ def merge_packages_config(config, packages):
 def async_process_component_config(hass, config, domain):
     """Check component configuration and return processed configuration.
 
-    Raise a vol.Invalid exception on error.
+    Returns None on error.
 
     This method must be run in the event loop.
     """

homeassistant/scripts/check_config.py

@@ -1,17 +1,23 @@
-"""Script to ensure a configuration file exists."""
+"""Script to check the configuration file."""
 
 import argparse
 import logging
 import os
-from collections import OrderedDict
+from collections import OrderedDict, namedtuple
 from glob import glob
 from platform import system
 from unittest.mock import patch
 
+import attr
 from typing import Dict, List, Sequence
+import voluptuous as vol
 
-from homeassistant.core import callback
+from homeassistant import bootstrap, core, loader
-from homeassistant import bootstrap, loader, setup, config as config_util
+from homeassistant.config import (
+    get_default_config_dir, CONF_CORE, CORE_CONFIG_SCHEMA,
+    CONF_PACKAGES, merge_packages_config, _format_config_error,
+    find_config_file, load_yaml_config_file, get_component,
+    extract_domain_configs, config_per_platform, get_platform)
 import homeassistant.util.yaml as yaml
 from homeassistant.exceptions import HomeAssistantError
 
@@ -24,35 +30,18 @@ _LOGGER = logging.getLogger(__name__)
 MOCKS = {
     'load': ("homeassistant.util.yaml.load_yaml", yaml.load_yaml),
     'load*': ("homeassistant.config.load_yaml", yaml.load_yaml),
-    'get': ("homeassistant.loader.get_component", loader.get_component),
     'secrets': ("homeassistant.util.yaml._secret_yaml", yaml._secret_yaml),
-    'except': ("homeassistant.config.async_log_exception",
-               config_util.async_log_exception),
-    'package_error': ("homeassistant.config._log_pkg_error",
-                      config_util._log_pkg_error),
-    'logger_exception': ("homeassistant.setup._LOGGER.error",
-                         setup._LOGGER.error),
-    'logger_exception_bootstrap': ("homeassistant.bootstrap._LOGGER.error",
-                                   bootstrap._LOGGER.error),
 }
 SILENCE = (
-    'homeassistant.bootstrap.async_enable_logging',  # callback
+    'homeassistant.scripts.check_config.yaml.clear_secret_cache',
-    'homeassistant.bootstrap.clear_secret_cache',
-    'homeassistant.bootstrap.async_register_signal_handling',  # callback
-    'homeassistant.config.process_ha_config_upgrade',
 )
+
 PATCHES = {}
 
 C_HEAD = 'bold'
 ERROR_STR = 'General Errors'
 
 
-@callback
-def mock_cb(*args):
-    """Callback that returns None."""
-    return None
-
-
 def color(the_color, *args, reset=None):
     """Color helper."""
     from colorlog.escape_codes import escape_codes, parse_colors
@@ -74,11 +63,11 @@ def run(script_args: List) -> int:
         '--script', choices=['check_config'])
     parser.add_argument(
         '-c', '--config',
-        default=config_util.get_default_config_dir(),
+        default=get_default_config_dir(),
         help="Directory that contains the Home Assistant configuration")
     parser.add_argument(
-        '-i', '--info',
+        '-i', '--info', nargs='?',
-        default=None,
+        default=None, const='all',
         help="Show a portion of the config")
     parser.add_argument(
         '-f', '--files',
@@ -89,21 +78,20 @@ def run(script_args: List) -> int:
         action='store_true',
         help="Show secret information")
 
-    args = parser.parse_args()
+    args, unknown = parser.parse_known_args()
+    if unknown:
+        print(color('red', "Unknown arguments:", ', '.join(unknown)))
 
     config_dir = os.path.join(os.getcwd(), args.config)
-    config_path = os.path.join(config_dir, 'configuration.yaml')
-    if not os.path.isfile(config_path):
-        print('Config does not exist:', config_path)
-        return 1
 
     print(color('bold', "Testing configuration at", config_dir))
 
+    res = check(config_dir, args.secrets)
+
     domain_info = []
     if args.info:
         domain_info = args.info.split(',')
 
-    res = check(config_path)
     if args.files:
         print(color(C_HEAD, 'yaml files'), '(used /',
               color('red', 'not used') + ')')
@@ -158,59 +146,23 @@ def run(script_args: List) -> int:
     return len(res['except'])
 
 
-def check(config_path):
+def check(config_dir, secrets=False):
     """Perform a check by mocking hass load functions."""
-    logging.getLogger('homeassistant.core').setLevel(logging.WARNING)
+    logging.getLogger('homeassistant.loader').setLevel(logging.CRITICAL)
-    logging.getLogger('homeassistant.loader').setLevel(logging.WARNING)
-    logging.getLogger('homeassistant.setup').setLevel(logging.WARNING)
-    logging.getLogger('homeassistant.bootstrap').setLevel(logging.ERROR)
-    logging.getLogger('homeassistant.util.yaml').setLevel(logging.INFO)
     res = {
         'yaml_files': OrderedDict(),  # yaml_files loaded
         'secrets': OrderedDict(),  # secret cache and secrets loaded
         'except': OrderedDict(),  # exceptions raised (with config)
-        'components': OrderedDict(),  # successful components
+        'components': None,  # successful components
-        'secret_cache': OrderedDict(),
+        'secret_cache': None,
     }
 
     # pylint: disable=unused-variable
     def mock_load(filename):
-        """Mock hass.util.load_yaml to save config files."""
+        """Mock hass.util.load_yaml to save config file names."""
         res['yaml_files'][filename] = True
         return MOCKS['load'][1](filename)
 
-    # pylint: disable=unused-variable
-    def mock_get(comp_name):
-        """Mock hass.loader.get_component to replace setup & setup_platform."""
-        async def mock_async_setup(*args):
-            """Mock setup, only record the component name & config."""
-            assert comp_name not in res['components'], \
-                "Components should contain a list of platforms"
-            res['components'][comp_name] = args[1].get(comp_name)
-            return True
-        module = MOCKS['get'][1](comp_name)
-
-        if module is None:
-            # Ensure list
-            msg = '{} not found: {}'.format(
-                'Platform' if '.' in comp_name else 'Component', comp_name)
-            res['except'].setdefault(ERROR_STR, []).append(msg)
-            return None
-
-        # Test if platform/component and overwrite setup
-        if '.' in comp_name:
-            module.async_setup_platform = mock_async_setup
-
-            if hasattr(module, 'setup_platform'):
-                del module.setup_platform
-        else:
-            module.async_setup = mock_async_setup
-
-            if hasattr(module, 'setup'):
-                del module.setup
-
-        return module
-
     # pylint: disable=unused-variable
     def mock_secrets(ldr, node):
         """Mock _get_secrets."""
@@ -221,37 +173,14 @@ def check(config_path):
         res['secrets'][node.value] = val
         return val
 
-    def mock_except(ex, domain, config,  # pylint: disable=unused-variable
-                    hass=None):
-        """Mock config.log_exception."""
-        MOCKS['except'][1](ex, domain, config, hass)
-        res['except'][domain] = config.get(domain, config)
-
-    def mock_package_error(  # pylint: disable=unused-variable
-            package, component, config, message):
-        """Mock config_util._log_pkg_error."""
-        MOCKS['package_error'][1](package, component, config, message)
-
-        pkg_key = 'homeassistant.packages.{}'.format(package)
-        res['except'][pkg_key] = config.get('homeassistant', {}) \
-            .get('packages', {}).get(package)
-
-    def mock_logger_exception(msg, *params):
-        """Log logger.exceptions."""
-        res['except'].setdefault(ERROR_STR, []).append(msg % params)
-        MOCKS['logger_exception'][1](msg, *params)
-
-    def mock_logger_exception_bootstrap(msg, *params):
-        """Log logger.exceptions."""
-        res['except'].setdefault(ERROR_STR, []).append(msg % params)
-        MOCKS['logger_exception_bootstrap'][1](msg, *params)
-
     # Patches to skip functions
     for sil in SILENCE:
-        PATCHES[sil] = patch(sil, return_value=mock_cb())
+        PATCHES[sil] = patch(sil)
 
     # Patches with local mock functions
     for key, val in MOCKS.items():
+        if not secrets and key == 'secrets':
+            continue
         # The * in the key is removed to find the mock_function (side_effect)
         # This allows us to use one side_effect to patch multiple locations
         mock_function = locals()['mock_' + key.replace('*', '')]
@@ -260,22 +189,42 @@ def check(config_path):
     # Start all patches
     for pat in PATCHES.values():
         pat.start()
-    # Ensure !secrets point to the patched function
+
-    yaml.yaml.SafeLoader.add_constructor('!secret', yaml._secret_yaml)
+    if secrets:
+        # Ensure !secrets point to the patched function
+        yaml.yaml.SafeLoader.add_constructor('!secret', yaml._secret_yaml)
 
     try:
-        with patch('homeassistant.util.logging.AsyncHandler._process'):
+        class HassConfig():
-            bootstrap.from_config_file(config_path, skip_pip=True)
+            """Hass object with config."""
-        res['secret_cache'] = dict(yaml.__SECRET_CACHE)
+
+            def __init__(self, conf_dir):
+                """Init the config_dir."""
+                self.config = core.Config()
+                self.config.config_dir = conf_dir
+
+        loader.prepare(HassConfig(config_dir))
+
+        res['components'] = check_ha_config_file(config_dir)
+
+        res['secret_cache'] = OrderedDict(yaml.__SECRET_CACHE)
+
+        for err in res['components'].errors:
+            domain = err.domain or ERROR_STR
+            res['except'].setdefault(domain, []).append(err.message)
+            if err.config:
+                res['except'].setdefault(domain, []).append(err.config)
+
     except Exception as err:  # pylint: disable=broad-except
         print(color('red', 'Fatal error while loading config:'), str(err))
-        res['except'].setdefault(ERROR_STR, []).append(err)
+        res['except'].setdefault(ERROR_STR, []).append(str(err))
     finally:
         # Stop all patches
         for pat in PATCHES.values():
             pat.stop()
-        # Ensure !secrets point to the original function
+        if secrets:
-        yaml.yaml.SafeLoader.add_constructor('!secret', yaml._secret_yaml)
+            # Ensure !secrets point to the original function
+            yaml.yaml.SafeLoader.add_constructor('!secret', yaml._secret_yaml)
         bootstrap.clear_secret_cache()
 
     return res
@@ -317,3 +266,125 @@ def dump_dict(layer, indent_count=3, listi=False, **kwargs):
                 dump_dict(i, indent_count + 2, True)
             else:
                 print(' ', indent_str, i)
+
+
+CheckConfigError = namedtuple(  # pylint: disable=invalid-name
+    'CheckConfigError', "message domain config")
+
+
+@attr.s
+class HomeAssistantConfig(OrderedDict):
+    """Configuration result with errors attribute."""
+
+    errors = attr.ib(default=attr.Factory(list))
+
+    def add_error(self, message, domain=None, config=None):
+        """Add a single error."""
+        self.errors.append(CheckConfigError(str(message), domain, config))
+        return self
+
+
+def check_ha_config_file(config_dir):
+    """Check if Home Assistant configuration file is valid."""
+    result = HomeAssistantConfig()
+
+    def _pack_error(package, component, config, message):
+        """Handle errors from packages: _log_pkg_error."""
+        message = "Package {} setup failed. Component {} {}".format(
+            package, component, message)
+        domain = 'homeassistant.packages.{}.{}'.format(package, component)
+        pack_config = core_config[CONF_PACKAGES].get(package, config)
+        result.add_error(message, domain, pack_config)
+
+    def _comp_error(ex, domain, config):
+        """Handle errors from components: async_log_exception."""
+        result.add_error(
+            _format_config_error(ex, domain, config), domain, config)
+
+    # Load configuration.yaml
+    try:
+        config_path = find_config_file(config_dir)
+        if not config_path:
+            return result.add_error("File configuration.yaml not found.")
+        config = load_yaml_config_file(config_path)
+    except HomeAssistantError as err:
+        return result.add_error(err)
+    finally:
+        yaml.clear_secret_cache()
+
+    # Extract and validate core [homeassistant] config
+    try:
+        core_config = config.pop(CONF_CORE, {})
+        core_config = CORE_CONFIG_SCHEMA(core_config)
+        result[CONF_CORE] = core_config
+    except vol.Invalid as err:
+        result.add_error(err, CONF_CORE, core_config)
+        core_config = {}
+
+    # Merge packages
+    merge_packages_config(
+        config, core_config.get(CONF_PACKAGES, {}), _pack_error)
+    del core_config[CONF_PACKAGES]
+
+    # Filter out repeating config sections
+    components = set(key.split(' ')[0] for key in config.keys())
+
+    # Process and validate config
+    for domain in components:
+        component = get_component(domain)
+        if not component:
+            result.add_error("Component not found: {}".format(domain))
+            continue
+
+        if hasattr(component, 'CONFIG_SCHEMA'):
+            try:
+                config = component.CONFIG_SCHEMA(config)
+                result[domain] = config[domain]
+            except vol.Invalid as ex:
+                _comp_error(ex, domain, config)
+                continue
+
+        if not hasattr(component, 'PLATFORM_SCHEMA'):
+            continue
+
+        platforms = []
+        for p_name, p_config in config_per_platform(config, domain):
+            # Validate component specific platform schema
+            try:
+                p_validated = component.PLATFORM_SCHEMA(p_config)
+            except vol.Invalid as ex:
+                _comp_error(ex, domain, config)
+                continue
+
+            # Not all platform components follow same pattern for platforms
+            # So if p_name is None we are not going to validate platform
+            # (the automation component is one of them)
+            if p_name is None:
+                platforms.append(p_validated)
+                continue
+
+            platform = get_platform(domain, p_name)
+
+            if platform is None:
+                result.add_error(
+                    "Platform not found: {}.{}".format(domain, p_name))
+                continue
+
+            # Validate platform specific schema
+            if hasattr(platform, 'PLATFORM_SCHEMA'):
+                # pylint: disable=no-member
+                try:
+                    p_validated = platform.PLATFORM_SCHEMA(p_validated)
+                except vol.Invalid as ex:
+                    _comp_error(
+                        ex, '{}.{}'.format(domain, p_name), p_validated)
+                    continue
+
+            platforms.append(p_validated)
+
+        # Remove config for current component and add validated config back in.
+        for filter_comp in extract_domain_configs(config, domain):
+            del config[filter_comp]
+        result[domain] = platforms
+
+    return result

tests/scripts/test_check_config.py

@@ -1,9 +1,12 @@
 """Test check_config script."""
 import asyncio
 import logging
+import os  # noqa: F401 pylint: disable=unused-import
 import unittest
+from unittest.mock import patch
 
 import homeassistant.scripts.check_config as check_config
+from homeassistant.config import YAML_CONFIG_FILE
 from homeassistant.loader import set_component
 from tests.common import patch_yaml_files, get_test_config_dir
 
@@ -21,21 +24,14 @@ BASE_CONFIG = (
 )
 
 
-def change_yaml_files(check_dict):
+def normalize_yaml_files(check_dict):
-    """Change the ['yaml_files'] property and remove the configuration path.
+    """Remove configuration path from ['yaml_files']."""
-
-    Also removes other files like service.yaml that gets loaded.
-    """
     root = get_test_config_dir()
-    keys = check_dict['yaml_files'].keys()
+    return [key.replace(root, '...')
-    check_dict['yaml_files'] = []
+            for key in sorted(check_dict['yaml_files'].keys())]
-    for key in sorted(keys):
-        if not key.startswith('/'):
-            check_dict['yaml_files'].append(key)
-        if key.startswith(root):
-            check_dict['yaml_files'].append('...' + key[len(root):])
 
 
+# pylint: disable=unsubscriptable-object
 class TestCheckConfig(unittest.TestCase):
     """Tests for the homeassistant.scripts.check_config module."""
 
@@ -51,176 +47,165 @@ class TestCheckConfig(unittest.TestCase):
             asyncio.set_event_loop(asyncio.new_event_loop())
 
         # Will allow seeing full diff
-        self.maxDiff = None
+        self.maxDiff = None  # pylint: disable=invalid-name
 
     # pylint: disable=no-self-use,invalid-name
-    def test_config_platform_valid(self):
+    @patch('os.path.isfile', return_value=True)
+    def test_config_platform_valid(self, isfile_patch):
         """Test a valid platform setup."""
         files = {
-            'light.yaml': BASE_CONFIG + 'light:\n  platform: demo',
+            YAML_CONFIG_FILE: BASE_CONFIG + 'light:\n  platform: demo',
         }
         with patch_yaml_files(files):
-            res = check_config.check(get_test_config_dir('light.yaml'))
+            res = check_config.check(get_test_config_dir())
-            change_yaml_files(res)
+            assert res['components'].keys() == {'homeassistant', 'light'}
-            self.assertDictEqual({
+            assert res['components']['light'] == [{'platform': 'demo'}]
-                'components': {'light': [{'platform': 'demo'}], 'group': None},
+            assert res['except'] == {}
-                'except': {},
+            assert res['secret_cache'] == {}
-                'secret_cache': {},
+            assert res['secrets'] == {}
-                'secrets': {},
+            assert len(res['yaml_files']) == 1
-                'yaml_files': ['.../light.yaml']
-            }, res)
 
-    def test_config_component_platform_fail_validation(self):
+    @patch('os.path.isfile', return_value=True)
+    def test_config_component_platform_fail_validation(self, isfile_patch):
         """Test errors if component & platform not found."""
         files = {
-            'component.yaml': BASE_CONFIG + 'http:\n  password: err123',
+            YAML_CONFIG_FILE: BASE_CONFIG + 'http:\n  password: err123',
         }
         with patch_yaml_files(files):
-            res = check_config.check(get_test_config_dir('component.yaml'))
+            res = check_config.check(get_test_config_dir())
-            change_yaml_files(res)
+            assert res['components'].keys() == {'homeassistant'}
-
+            assert res['except'].keys() == {'http'}
-            self.assertDictEqual({}, res['components'])
+            assert res['except']['http'][1] == {'http': {'password': 'err123'}}
-            res['except'].pop(check_config.ERROR_STR)
+            assert res['secret_cache'] == {}
-            self.assertDictEqual(
+            assert res['secrets'] == {}
-                {'http': {'password': 'err123'}},
+            assert len(res['yaml_files']) == 1
-                res['except']
-            )
-            self.assertDictEqual({}, res['secret_cache'])
-            self.assertDictEqual({}, res['secrets'])
-            self.assertListEqual(['.../component.yaml'], res['yaml_files'])
 
         files = {
-            'platform.yaml': (BASE_CONFIG + 'mqtt:\n\n'
+            YAML_CONFIG_FILE: (BASE_CONFIG + 'mqtt:\n\n'
-                              'light:\n  platform: mqtt_json'),
+                               'light:\n  platform: mqtt_json'),
         }
         with patch_yaml_files(files):
-            res = check_config.check(get_test_config_dir('platform.yaml'))
+            res = check_config.check(get_test_config_dir())
-            change_yaml_files(res)
+            assert res['components'].keys() == {
-            self.assertDictEqual(
+                'homeassistant', 'light', 'mqtt'}
-                {'mqtt': {
+            assert res['components']['light'] == []
-                    'keepalive': 60,
+            assert res['components']['mqtt'] == {
-                    'port': 1883,
+                'keepalive': 60,
-                    'protocol': '3.1.1',
+                'port': 1883,
-                    'discovery': False,
+                'protocol': '3.1.1',
-                    'discovery_prefix': 'homeassistant',
+                'discovery': False,
-                    'tls_version': 'auto',
+                'discovery_prefix': 'homeassistant',
-                },
+                'tls_version': 'auto',
-                 'light': [],
+            }
-                 'group': None},
+            assert res['except'].keys() == {'light.mqtt_json'}
-                res['components']
+            assert res['except']['light.mqtt_json'][1] == {
-            )
+                'platform': 'mqtt_json'}
-            self.assertDictEqual(
+            assert res['secret_cache'] == {}
-                {'light.mqtt_json': {'platform': 'mqtt_json'}},
+            assert res['secrets'] == {}
-                res['except']
+            assert len(res['yaml_files']) == 1
-            )
-            self.assertDictEqual({}, res['secret_cache'])
-            self.assertDictEqual({}, res['secrets'])
-            self.assertListEqual(['.../platform.yaml'], res['yaml_files'])
 
-    def test_component_platform_not_found(self):
+    @patch('os.path.isfile', return_value=True)
+    def test_component_platform_not_found(self, isfile_patch):
         """Test errors if component or platform not found."""
         # Make sure they don't exist
         set_component('beer', None)
-        set_component('light.beer', None)
         files = {
-            'badcomponent.yaml': BASE_CONFIG + 'beer:',
+            YAML_CONFIG_FILE: BASE_CONFIG + 'beer:',
-            'badplatform.yaml': BASE_CONFIG + 'light:\n  platform: beer',
         }
         with patch_yaml_files(files):
-            res = check_config.check(get_test_config_dir('badcomponent.yaml'))
+            res = check_config.check(get_test_config_dir())
-            change_yaml_files(res)
+            assert res['components'].keys() == {'homeassistant'}
-            self.assertDictEqual({}, res['components'])
+            assert res['except'] == {
-            self.assertDictEqual({
+                check_config.ERROR_STR: ['Component not found: beer']}
-                    check_config.ERROR_STR: [
+            assert res['secret_cache'] == {}
-                        'Component not found: beer',
+            assert res['secrets'] == {}
-                        'Setup failed for beer: Component not found.']
+            assert len(res['yaml_files']) == 1
-                }, res['except'])
-            self.assertDictEqual({}, res['secret_cache'])
-            self.assertDictEqual({}, res['secrets'])
-            self.assertListEqual(['.../badcomponent.yaml'], res['yaml_files'])
 
-            res = check_config.check(get_test_config_dir('badplatform.yaml'))
+        set_component('light.beer', None)
-            change_yaml_files(res)
+        files = {
-            assert res['components'] == {'light': [], 'group': None}
+            YAML_CONFIG_FILE: BASE_CONFIG + 'light:\n  platform: beer',
+        }
+        with patch_yaml_files(files):
+            res = check_config.check(get_test_config_dir())
+            assert res['components'].keys() == {'homeassistant', 'light'}
+            assert res['components']['light'] == []
             assert res['except'] == {
                 check_config.ERROR_STR: [
                     'Platform not found: light.beer',
                 ]}
-            self.assertDictEqual({}, res['secret_cache'])
+            assert res['secret_cache'] == {}
-            self.assertDictEqual({}, res['secrets'])
+            assert res['secrets'] == {}
-            self.assertListEqual(['.../badplatform.yaml'], res['yaml_files'])
+            assert len(res['yaml_files']) == 1
 
-    def test_secrets(self):
+    @patch('os.path.isfile', return_value=True)
+    def test_secrets(self, isfile_patch):
         """Test secrets config checking method."""
+        secrets_path = get_test_config_dir('secrets.yaml')
+
         files = {
-            get_test_config_dir('secret.yaml'): (
+            get_test_config_dir(YAML_CONFIG_FILE): BASE_CONFIG + (
-                BASE_CONFIG +
                 'http:\n'
                 '  api_password: !secret http_pw'),
-            'secrets.yaml': ('logger: debug\n'
+            secrets_path: (
-                             'http_pw: abc123'),
+                'logger: debug\n'
+                'http_pw: abc123'),
         }
 
         with patch_yaml_files(files):
-            config_path = get_test_config_dir('secret.yaml')
-            secrets_path = get_test_config_dir('secrets.yaml')
 
-            res = check_config.check(config_path)
+            res = check_config.check(get_test_config_dir(), True)
-            change_yaml_files(res)
 
-            # convert secrets OrderedDict to dict for assertequal
+            assert res['except'] == {}
-            for key, val in res['secret_cache'].items():
+            assert res['components'].keys() == {'homeassistant', 'http'}
-                res['secret_cache'][key] = dict(val)
+            assert res['components']['http'] == {
+                'api_password': 'abc123',
+                'cors_allowed_origins': [],
+                'ip_ban_enabled': True,
+                'login_attempts_threshold': -1,
+                'server_host': '0.0.0.0',
+                'server_port': 8123,
+                'trusted_networks': [],
+                'use_x_forwarded_for': False}
+            assert res['secret_cache'] == {secrets_path: {'http_pw': 'abc123'}}
+            assert res['secrets'] == {'http_pw': 'abc123'}
+            assert normalize_yaml_files(res) == [
+                '.../configuration.yaml', '.../secrets.yaml']
 
-            self.assertDictEqual({
+    @patch('os.path.isfile', return_value=True)
-                'components': {'http': {'api_password': 'abc123',
+    def test_package_invalid(self, isfile_patch): \
-                                        'cors_allowed_origins': [],
-                                        'ip_ban_enabled': True,
-                                        'login_attempts_threshold': -1,
-                                        'server_host': '0.0.0.0',
-                                        'server_port': 8123,
-                                        'trusted_networks': [],
-                                        'use_x_forwarded_for': False}},
-                'except': {},
-                'secret_cache': {secrets_path: {'http_pw': 'abc123'}},
-                'secrets': {'http_pw': 'abc123'},
-                'yaml_files': ['.../secret.yaml', '.../secrets.yaml']
-            }, res)
-
-    def test_package_invalid(self): \
             # pylint: disable=no-self-use,invalid-name
         """Test a valid platform setup."""
         files = {
-            'bad.yaml': BASE_CONFIG + ('  packages:\n'
+            YAML_CONFIG_FILE: BASE_CONFIG + (
-                                       '    p1:\n'
+                '  packages:\n'
-                                       '      group: ["a"]'),
+                '    p1:\n'
+                '      group: ["a"]'),
         }
         with patch_yaml_files(files):
-            res = check_config.check(get_test_config_dir('bad.yaml'))
+            res = check_config.check(get_test_config_dir())
-            change_yaml_files(res)
 
-            err = res['except'].pop('homeassistant.packages.p1')
+            assert res['except'].keys() == {'homeassistant.packages.p1.group'}
-            assert res['except'] == {}
+            assert res['except']['homeassistant.packages.p1.group'][1] == \
-            assert err == {'group': ['a']}
+                {'group': ['a']}
-            assert res['yaml_files'] == ['.../bad.yaml']
+            assert len(res['except']) == 1
-
+            assert res['components'].keys() == {'homeassistant'}
-            assert res['components'] == {}
+            assert len(res['components']) == 1
             assert res['secret_cache'] == {}
             assert res['secrets'] == {}
+            assert len(res['yaml_files']) == 1
 
     def test_bootstrap_error(self): \
             # pylint: disable=no-self-use,invalid-name
         """Test a valid platform setup."""
         files = {
-            'badbootstrap.yaml': BASE_CONFIG + 'automation: !include no.yaml',
+            YAML_CONFIG_FILE: BASE_CONFIG + 'automation: !include no.yaml',
         }
         with patch_yaml_files(files):
-            res = check_config.check(get_test_config_dir('badbootstrap.yaml'))
+            res = check_config.check(get_test_config_dir(YAML_CONFIG_FILE))
-            change_yaml_files(res)
-
             err = res['except'].pop(check_config.ERROR_STR)
             assert len(err) == 1
             assert res['except'] == {}
-            assert res['components'] == {}
+            assert res['components'] == {}  # No components, load failed
             assert res['secret_cache'] == {}
             assert res['secrets'] == {}
+            assert res['yaml_files'] == {}

tests/test_config.py

@@ -158,11 +158,11 @@ class TestConfig(unittest.TestCase):
     def test_load_yaml_config_preserves_key_order(self):
         """Test removal of library."""
         with open(YAML_PATH, 'w') as f:
-            f.write('hello: 0\n')
+            f.write('hello: 2\n')
             f.write('world: 1\n')
 
         self.assertEqual(
-            [('hello', 0), ('world', 1)],
+            [('hello', 2), ('world', 1)],
             list(config_util.load_yaml_config_file(YAML_PATH).items()))
 
     @mock.patch('homeassistant.util.location.detect_location_info',