Attempt fixing flakiness of check config test (#12283)

* Attempt fixing check_config script test flakiness

* Fix logging

* remove cleanup as Python will exit

* Make sure we don't enqueue magicmocks

* Lint

* Reinstate cleanup as it broke secret tests

homeassistant/scripts/check_config.py

@@ -1,4 +1,5 @@
 """Script to ensure a configuration file exists."""
+import asyncio
 import argparse
 import logging
 import os
@@ -34,13 +35,10 @@ MOCKS = {
                                    bootstrap._LOGGER.error),
 }
 SILENCE = (
+    'homeassistant.bootstrap.async_enable_logging',
     'homeassistant.bootstrap.clear_secret_cache',
     'homeassistant.bootstrap.async_register_signal_handling',
-    'homeassistant.core._LOGGER.info',
-    'homeassistant.loader._LOGGER.info',
-    'homeassistant.bootstrap._LOGGER.info',
-    'homeassistant.bootstrap._LOGGER.warning',
-    'homeassistant.util.yaml._LOGGER.debug',
+    'homeassistant.config.process_ha_config_upgrade',
 )
 PATCHES = {}
 
@@ -48,6 +46,12 @@ C_HEAD = 'bold'
 ERROR_STR = 'General Errors'
 
 
+@asyncio.coroutine
+def mock_coro(*args):
+    """Coroutine that returns None."""
+    return None
+
+
 def color(the_color, *args, reset=None):
     """Color helper."""
     from colorlog.escape_codes import escape_codes, parse_colors
@@ -155,6 +159,11 @@ def run(script_args: List) -> int:
 
 def check(config_path):
     """Perform a check by mocking hass load functions."""
+    logging.getLogger('homeassistant.core').setLevel(logging.WARNING)
+    logging.getLogger('homeassistant.loader').setLevel(logging.WARNING)
+    logging.getLogger('homeassistant.setup').setLevel(logging.WARNING)
+    logging.getLogger('homeassistant.bootstrap').setLevel(logging.ERROR)
+    logging.getLogger('homeassistant.util.yaml').setLevel(logging.INFO)
     res = {
         'yaml_files': OrderedDict(),  # yaml_files loaded
         'secrets': OrderedDict(),  # secret cache and secrets loaded
@@ -172,11 +181,12 @@ def check(config_path):
     # pylint: disable=unused-variable
     def mock_get(comp_name):
         """Mock hass.loader.get_component to replace setup & setup_platform."""
-        def mock_setup(*kwargs):
+        @asyncio.coroutine
+        def mock_async_setup(*args):
             """Mock setup, only record the component name & config."""
             assert comp_name not in res['components'], \
                 "Components should contain a list of platforms"
-            res['components'][comp_name] = kwargs[1].get(comp_name)
+            res['components'][comp_name] = args[1].get(comp_name)
             return True
         module = MOCKS['get'][1](comp_name)
 
@@ -189,15 +199,15 @@ def check(config_path):
 
         # Test if platform/component and overwrite setup
         if '.' in comp_name:
-            module.setup_platform = mock_setup
+            module.async_setup_platform = mock_async_setup
 
-            if hasattr(module, 'async_setup_platform'):
-                del module.async_setup_platform
+            if hasattr(module, 'setup_platform'):
+                del module.setup_platform
         else:
-            module.setup = mock_setup
+            module.async_setup = mock_async_setup
 
-            if hasattr(module, 'async_setup'):
-                del module.async_setup
+            if hasattr(module, 'setup'):
+                del module.setup
 
         return module
 
@@ -238,7 +248,7 @@ def check(config_path):
 
     # Patches to skip functions
     for sil in SILENCE:
-        PATCHES[sil] = patch(sil)
+        PATCHES[sil] = patch(sil, return_value=mock_coro())
 
     # Patches with local mock functions
     for key, val in MOCKS.items():

tests/scripts/test_check_config.py

@@ -1,10 +1,10 @@
 """Test check_config script."""
 import asyncio
 import logging
-import os
 import unittest
 
 import homeassistant.scripts.check_config as check_config
+from homeassistant.loader import set_component
 from tests.common import patch_yaml_files, get_test_config_dir
 
 _LOGGER = logging.getLogger(__name__)
@@ -36,14 +36,6 @@ def change_yaml_files(check_dict):
             check_dict['yaml_files'].append('...' + key[len(root):])
 
 
-def tearDownModule(self):  # pylint: disable=invalid-name
-    """Clean files."""
-    # .HA_VERSION created during bootstrap's config update
-    path = get_test_config_dir('.HA_VERSION')
-    if os.path.isfile(path):
-        os.remove(path)
-
-
 class TestCheckConfig(unittest.TestCase):
     """Tests for the homeassistant.scripts.check_config module."""
 
@@ -124,6 +116,9 @@ class TestCheckConfig(unittest.TestCase):
 
     def test_component_platform_not_found(self):
         """Test errors if component or platform not found."""
+        # Make sure they don't exist
+        set_component('beer', None)
+        set_component('light.beer', None)
         files = {
             'badcomponent.yaml': BASE_CONFIG + 'beer:',
             'badplatform.yaml': BASE_CONFIG + 'light:\n  platform: beer',
@@ -162,7 +157,6 @@ class TestCheckConfig(unittest.TestCase):
             'secrets.yaml': ('logger: debug\n'
                              'http_pw: abc123'),
         }
-        self.maxDiff = None
 
         with patch_yaml_files(files):
             config_path = get_test_config_dir('secret.yaml')