From 578d4a9b6ad66ebbb6949904ee956d75f2e597e5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 2 Jun 2020 13:54:11 -0500 Subject: [PATCH] Make the frontend available sooner (Part 1 of 2) (#36263) * Part 1 of 2 (no breaking changes in part 1). When integrations configured via the UI block startup or fail to start, the webserver can remain offline which make it is impossible to recover without manually changing files in .storage since the UI is not available. This change is the foundation that part 2 will build on and enable a listener to start the webserver when the frontend is finished loading. Frontend Changes (home-assistant/frontend#6068) * Address review comments * bump timeout to 1800s, adjust comment * bump timeout to 4h * remove timeout failsafe * and the test --- homeassistant/bootstrap.py | 9 +++ homeassistant/components/http/__init__.py | 49 +++++++++------ homeassistant/components/http/view.py | 6 +- .../components/websocket_api/decorators.py | 5 +- .../components/websocket_api/http.py | 4 +- homeassistant/core.py | 7 +++ tests/components/hassio/test_discovery.py | 5 +- tests/components/http/test_data_validator.py | 2 +- tests/components/http/test_view.py | 16 ++++- tests/components/logbook/test_init.py | 3 +- tests/components/panel_iframe/test_init.py | 63 ++++++++++--------- tests/test_bootstrap.py | 14 +++-- tests/test_core.py | 5 +- 13 files changed, 126 insertions(+), 62 deletions(-) diff --git a/homeassistant/bootstrap.py b/homeassistant/bootstrap.py index 596b0250de6..a4c5fa14fa8 100644 --- a/homeassistant/bootstrap.py +++ b/homeassistant/bootstrap.py @@ -44,6 +44,13 @@ STAGE_1_INTEGRATIONS = { "mqtt_eventstream", # To provide account link implementations "cloud", + # Ensure supervisor is available + "hassio", + # Get the frontend up and running as soon + # as possible so problem integrations can + # be removed + "frontend", + "config", } @@ -399,6 +406,8 @@ async def _async_set_up_integrations( ) if stage_1_domains: + _LOGGER.info("Setting up %s", stage_1_domains) + await async_setup_multi_components(stage_1_domains) # Load all integrations diff --git a/homeassistant/components/http/__init__.py b/homeassistant/components/http/__init__.py index 42d388ffa85..e06ceb087c2 100644 --- a/homeassistant/components/http/__init__.py +++ b/homeassistant/components/http/__init__.py @@ -4,7 +4,7 @@ import logging import os import ssl from traceback import extract_stack -from typing import Optional, cast +from typing import Dict, Optional, cast from aiohttp import web from aiohttp.web_exceptions import HTTPMovedPermanently @@ -15,7 +15,7 @@ from homeassistant.const import ( EVENT_HOMEASSISTANT_STOP, SERVER_PORT, ) -from homeassistant.core import HomeAssistant +from homeassistant.core import Event, HomeAssistant from homeassistant.helpers import storage import homeassistant.helpers.config_validation as cv from homeassistant.loader import bind_hass @@ -216,29 +216,25 @@ async def async_setup(hass, config): ssl_profile=ssl_profile, ) - async def stop_server(event): + startup_listeners = [] + + async def stop_server(event: Event) -> None: """Stop the server.""" await server.stop() - async def start_server(event): + async def start_server(event: Event) -> None: """Start the server.""" + + for listener in startup_listeners: + listener() + hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, stop_server) - await server.start() - # If we are set up successful, we store the HTTP settings for safe mode. - store = storage.Store(hass, STORAGE_VERSION, STORAGE_KEY) + await start_http_server_and_save_config(hass, dict(conf), server) - if CONF_TRUSTED_PROXIES in conf: - conf_to_save = dict(conf) - conf_to_save[CONF_TRUSTED_PROXIES] = [ - str(ip.network_address) for ip in conf_to_save[CONF_TRUSTED_PROXIES] - ] - else: - conf_to_save = conf - - await store.async_save(conf_to_save) - - hass.bus.async_listen_once(EVENT_HOMEASSISTANT_START, start_server) + startup_listeners.append( + hass.bus.async_listen(EVENT_HOMEASSISTANT_START, start_server) + ) hass.http = server @@ -418,3 +414,20 @@ class HomeAssistantHTTP: """Stop the aiohttp server.""" await self.site.stop() await self.runner.cleanup() + + +async def start_http_server_and_save_config( + hass: HomeAssistant, conf: Dict, server: HomeAssistantHTTP +) -> None: + """Startup the http server and save the config.""" + await server.start() # type: ignore + + # If we are set up successful, we store the HTTP settings for safe mode. + store = storage.Store(hass, STORAGE_VERSION, STORAGE_KEY) + + if CONF_TRUSTED_PROXIES in conf: + conf[CONF_TRUSTED_PROXIES] = [ + str(ip.network_address) for ip in conf[CONF_TRUSTED_PROXIES] + ] + + await store.async_save(conf) diff --git a/homeassistant/components/http/view.py b/homeassistant/components/http/view.py index 701c497d88c..eb6c757384e 100644 --- a/homeassistant/components/http/view.py +++ b/homeassistant/components/http/view.py @@ -14,7 +14,7 @@ from aiohttp.web_exceptions import ( import voluptuous as vol from homeassistant import exceptions -from homeassistant.const import CONTENT_TYPE_JSON, HTTP_OK +from homeassistant.const import CONTENT_TYPE_JSON, HTTP_OK, HTTP_SERVICE_UNAVAILABLE from homeassistant.core import Context, is_callback from homeassistant.helpers.json import JSONEncoder @@ -107,8 +107,8 @@ def request_handler_factory(view: HomeAssistantView, handler: Callable) -> Calla async def handle(request: web.Request) -> web.StreamResponse: """Handle incoming request.""" - if not request.app[KEY_HASS].is_running: - return web.Response(status=503) + if request.app[KEY_HASS].is_stopping: + return web.Response(status=HTTP_SERVICE_UNAVAILABLE) authenticated = request.get(KEY_AUTHENTICATED, False) diff --git a/homeassistant/components/websocket_api/decorators.py b/homeassistant/components/websocket_api/decorators.py index 87b5d5baf92..d4a4cff1a8f 100644 --- a/homeassistant/components/websocket_api/decorators.py +++ b/homeassistant/components/websocket_api/decorators.py @@ -1,4 +1,5 @@ """Decorators for the Websocket API.""" +import asyncio from functools import wraps import logging from typing import Awaitable, Callable @@ -31,7 +32,9 @@ def async_response( @wraps(func) def schedule_handler(hass, connection, msg): """Schedule the handler.""" - hass.async_create_task(_handle_async_response(func, hass, connection, msg)) + # As the webserver is now started before the start + # event we do not want to block for websocket responders + asyncio.create_task(_handle_async_response(func, hass, connection, msg)) return schedule_handler diff --git a/homeassistant/components/websocket_api/http.py b/homeassistant/components/websocket_api/http.py index e20e53d139a..ab412e06583 100644 --- a/homeassistant/components/websocket_api/http.py +++ b/homeassistant/components/websocket_api/http.py @@ -165,7 +165,9 @@ class WebSocketHandler: EVENT_HOMEASSISTANT_STOP, handle_hass_stop ) - self._writer_task = self.hass.async_create_task(self._writer()) + # As the webserver is now started before the start + # event we do not want to block for websocket responses + self._writer_task = asyncio.create_task(self._writer()) auth = AuthPhase(self._logger, self.hass, self._send_message, request) connection = None diff --git a/homeassistant/core.py b/homeassistant/core.py index 34df648a4df..eb7457daecb 100644 --- a/homeassistant/core.py +++ b/homeassistant/core.py @@ -209,6 +209,11 @@ class HomeAssistant: """Return if Home Assistant is running.""" return self.state in (CoreState.starting, CoreState.running) + @property + def is_stopping(self) -> bool: + """Return if Home Assistant is stopping.""" + return self.state in (CoreState.stopping, CoreState.final_write) + def start(self) -> int: """Start Home Assistant. @@ -260,6 +265,7 @@ class HomeAssistant: setattr(self.loop, "_thread_ident", threading.get_ident()) self.bus.async_fire(EVENT_HOMEASSISTANT_START) + self.bus.async_fire(EVENT_CORE_CONFIG_UPDATE) try: # Only block for EVENT_HOMEASSISTANT_START listener @@ -1391,6 +1397,7 @@ class Config: "version": __version__, "config_source": self.config_source, "safe_mode": self.safe_mode, + "state": self.hass.state.value, "external_url": self.external_url, "internal_url": self.internal_url, } diff --git a/tests/components/hassio/test_discovery.py b/tests/components/hassio/test_discovery.py index 845c60c2f85..9d148745f18 100644 --- a/tests/components/hassio/test_discovery.py +++ b/tests/components/hassio/test_discovery.py @@ -60,6 +60,9 @@ async def test_hassio_discovery_startup(hass, aioclient_mock, hassio_client): async def test_hassio_discovery_startup_done(hass, aioclient_mock, hassio_client): """Test startup and discovery with hass discovery.""" + aioclient_mock.post( + "http://127.0.0.1/supervisor/options", json={"result": "ok", "data": {}}, + ) aioclient_mock.get( "http://127.0.0.1/discovery", json={ @@ -101,7 +104,7 @@ async def test_hassio_discovery_startup_done(hass, aioclient_mock, hassio_client await async_setup_component(hass, "hassio", {}) await hass.async_block_till_done() - assert aioclient_mock.call_count == 2 + assert aioclient_mock.call_count == 3 assert mock_mqtt.called mock_mqtt.assert_called_with( { diff --git a/tests/components/http/test_data_validator.py b/tests/components/http/test_data_validator.py index b0a14a31bc5..c7b5ed42ccd 100644 --- a/tests/components/http/test_data_validator.py +++ b/tests/components/http/test_data_validator.py @@ -11,7 +11,7 @@ from tests.async_mock import Mock async def get_client(aiohttp_client, validator): """Generate a client that hits a view decorated with validator.""" app = web.Application() - app["hass"] = Mock(is_running=True) + app["hass"] = Mock(is_stopping=False) class TestView(HomeAssistantView): url = "/" diff --git a/tests/components/http/test_view.py b/tests/components/http/test_view.py index a6e4bdc12c8..045f0837983 100644 --- a/tests/components/http/test_view.py +++ b/tests/components/http/test_view.py @@ -19,7 +19,13 @@ from tests.async_mock import AsyncMock, Mock @pytest.fixture def mock_request(): """Mock a request.""" - return Mock(app={"hass": Mock(is_running=True)}, match_info={}) + return Mock(app={"hass": Mock(is_stopping=False)}, match_info={}) + + +@pytest.fixture +def mock_request_with_stopping(): + """Mock a request.""" + return Mock(app={"hass": Mock(is_stopping=True)}, match_info={}) async def test_invalid_json(caplog): @@ -55,3 +61,11 @@ async def test_handling_service_not_found(mock_request): Mock(requires_auth=False), AsyncMock(side_effect=ServiceNotFound("test", "test")), )(mock_request) + + +async def test_not_running(mock_request_with_stopping): + """Test we get a 503 when not running.""" + response = await request_handler_factory( + Mock(requires_auth=False), AsyncMock(side_effect=Unauthorized) + )(mock_request_with_stopping) + assert response.status == 503 diff --git a/tests/components/logbook/test_init.py b/tests/components/logbook/test_init.py index 27d39446fa3..e1341e64e92 100644 --- a/tests/components/logbook/test_init.py +++ b/tests/components/logbook/test_init.py @@ -42,7 +42,8 @@ class TestComponentLogbook(unittest.TestCase): """Set up things to be run when tests are started.""" self.hass = get_test_home_assistant() init_recorder_component(self.hass) # Force an in memory DB - assert setup_component(self.hass, logbook.DOMAIN, self.EMPTY_CONFIG) + with patch("homeassistant.components.http.start_http_server_and_save_config"): + assert setup_component(self.hass, logbook.DOMAIN, self.EMPTY_CONFIG) def tearDown(self): """Stop everything that was started.""" diff --git a/tests/components/panel_iframe/test_init.py b/tests/components/panel_iframe/test_init.py index d586c4c199e..b38f3c4b1fa 100644 --- a/tests/components/panel_iframe/test_init.py +++ b/tests/components/panel_iframe/test_init.py @@ -4,6 +4,7 @@ import unittest from homeassistant import setup from homeassistant.components import frontend +from tests.async_mock import patch from tests.common import get_test_home_assistant @@ -26,38 +27,42 @@ class TestPanelIframe(unittest.TestCase): ] for conf in to_try: - assert not setup.setup_component( - self.hass, "panel_iframe", {"panel_iframe": conf} - ) + with patch( + "homeassistant.components.http.start_http_server_and_save_config" + ): + assert not setup.setup_component( + self.hass, "panel_iframe", {"panel_iframe": conf} + ) def test_correct_config(self): """Test correct config.""" - assert setup.setup_component( - self.hass, - "panel_iframe", - { - "panel_iframe": { - "router": { - "icon": "mdi:network-wireless", - "title": "Router", - "url": "http://192.168.1.1", - "require_admin": True, - }, - "weather": { - "icon": "mdi:weather", - "title": "Weather", - "url": "https://www.wunderground.com/us/ca/san-diego", - "require_admin": True, - }, - "api": {"icon": "mdi:weather", "title": "Api", "url": "/api"}, - "ftp": { - "icon": "mdi:weather", - "title": "FTP", - "url": "ftp://some/ftp", - }, - } - }, - ) + with patch("homeassistant.components.http.start_http_server_and_save_config"): + assert setup.setup_component( + self.hass, + "panel_iframe", + { + "panel_iframe": { + "router": { + "icon": "mdi:network-wireless", + "title": "Router", + "url": "http://192.168.1.1", + "require_admin": True, + }, + "weather": { + "icon": "mdi:weather", + "title": "Weather", + "url": "https://www.wunderground.com/us/ca/san-diego", + "require_admin": True, + }, + "api": {"icon": "mdi:weather", "title": "Api", "url": "/api"}, + "ftp": { + "icon": "mdi:weather", + "title": "FTP", + "url": "ftp://some/ftp", + }, + } + }, + ) panels = self.hass.data[frontend.DATA_PANELS] diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 9597dfa60b8..3ea42d4545b 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -261,7 +261,9 @@ async def test_setup_hass( with patch( "homeassistant.config.async_hass_config_yaml", return_value={"browser": {}, "frontend": {}}, - ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000): + ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000), patch( + "homeassistant.components.http.start_http_server_and_save_config" + ): hass = await bootstrap.async_setup_hass( config_dir=get_test_config_dir(), verbose=verbose, @@ -338,7 +340,7 @@ async def test_setup_hass_invalid_yaml( """Test it works.""" with patch( "homeassistant.config.async_hass_config_yaml", side_effect=HomeAssistantError - ): + ), patch("homeassistant.components.http.start_http_server_and_save_config"): hass = await bootstrap.async_setup_hass( config_dir=get_test_config_dir(), verbose=False, @@ -391,7 +393,9 @@ async def test_setup_hass_safe_mode( hass.config_entries._async_schedule_save() await flush_store(hass.config_entries._store) - with patch("homeassistant.components.browser.setup") as browser_setup: + with patch("homeassistant.components.browser.setup") as browser_setup, patch( + "homeassistant.components.http.start_http_server_and_save_config" + ): hass = await bootstrap.async_setup_hass( config_dir=get_test_config_dir(), verbose=False, @@ -421,7 +425,7 @@ async def test_setup_hass_invalid_core_config( with patch( "homeassistant.config.async_hass_config_yaml", return_value={"homeassistant": {"non-existing": 1}}, - ): + ), patch("homeassistant.components.http.start_http_server_and_save_config"): hass = await bootstrap.async_setup_hass( config_dir=get_test_config_dir(), verbose=False, @@ -451,7 +455,7 @@ async def test_setup_safe_mode_if_no_frontend( with patch( "homeassistant.config.async_hass_config_yaml", return_value={"map": {}, "person": {"invalid": True}}, - ): + ), patch("homeassistant.components.http.start_http_server_and_save_config"): hass = await bootstrap.async_setup_hass( config_dir=get_test_config_dir(), verbose=verbose, diff --git a/tests/test_core.py b/tests/test_core.py index 3bc001b78b6..9fc257eaf2d 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -35,7 +35,7 @@ from homeassistant.exceptions import InvalidEntityFormatError, InvalidStateError import homeassistant.util.dt as dt_util from homeassistant.util.unit_system import METRIC_SYSTEM -from tests.async_mock import MagicMock, Mock, patch +from tests.async_mock import MagicMock, Mock, PropertyMock, patch from tests.common import async_mock_service, get_test_home_assistant PST = pytz.timezone("America/Los_Angeles") @@ -901,6 +901,8 @@ class TestConfig(unittest.TestCase): def test_as_dict(self): """Test as dict.""" self.config.config_dir = "/test/ha-config" + self.config.hass = MagicMock() + type(self.config.hass.state).value = PropertyMock(return_value="RUNNING") expected = { "latitude": 0, "longitude": 0, @@ -914,6 +916,7 @@ class TestConfig(unittest.TestCase): "version": __version__, "config_source": "default", "safe_mode": False, + "state": "RUNNING", "external_url": None, "internal_url": None, }