Remove usage of "run_until_complete" (#16617)

* De-run_forever()-ization

* Use asyncio.run (or our own implementation on Python <3.7)
* hass.start is only used by tests
* setup_and_run_hass() is now async
* Add "main" async hass.run method
* move SIGINT handling to helpers/signal.py
  * add flag to .run to disable hass's signal handlers
* Teach async_start and async_stop to not step on each other
  (more than necessary)

* shorten over-long lines

* restore missing "import asyncio"

* move run_asyncio to homeassistant.util.async_

* LOGGER: warn => warning

* Add "force" flag to async_stop

only useful for testing

* Add 'attrs==18.2.0' to requirements_all.txt

Required for keeping requirements_test_all.txt in sync, where it is in
turn required to prevent auto-downgrading "attrs" during "pip install"

* Fixes for mypy

* Fix "mock_signal" fixture

* Revert mistaken edit

* Flake8 fixes

* mypy fixes

* pylint fix

* Revert adding attrs== to requirements_test*.txt

solved by using "pip -c"

* Rename "run" to "async_run", as per calling conventions

homeassistant/__main__.py

@@ -22,12 +22,12 @@ from homeassistant.const import (
 def attempt_use_uvloop() -> None:
     """Attempt to use uvloop."""
     import asyncio
-
     try:
         import uvloop
-        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
     except ImportError:
         pass
+    else:
+        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
 
 def validate_python() -> None:
@@ -239,10 +239,10 @@ def cmdline() -> List[str]:
     return [arg for arg in sys.argv if arg != '--daemon']
 
 
-def setup_and_run_hass(config_dir: str,
+async def setup_and_run_hass(config_dir: str,
-                       args: argparse.Namespace) -> int:
+                             args: argparse.Namespace) -> int:
     """Set up HASS and run."""
-    from homeassistant import bootstrap
+    from homeassistant import bootstrap, core
 
     # Run a simple daemon runner process on Windows to handle restarts
     if os.name == 'nt' and '--runner' not in sys.argv:
@@ -255,35 +255,34 @@ def setup_and_run_hass(config_dir: str,
                 if exc.returncode != RESTART_EXIT_CODE:
                     sys.exit(exc.returncode)
 
+    hass = core.HomeAssistant()
+
     if args.demo_mode:
         config = {
             'frontend': {},
             'demo': {}
         }  # type: Dict[str, Any]
-        hass = bootstrap.from_config_dict(
+        bootstrap.async_from_config_dict(
-            config, config_dir=config_dir, verbose=args.verbose,
+            config, hass, config_dir=config_dir, verbose=args.verbose,
             skip_pip=args.skip_pip, log_rotate_days=args.log_rotate_days,
             log_file=args.log_file, log_no_color=args.log_no_color)
     else:
         config_file = ensure_config_file(config_dir)
         print('Config directory:', config_dir)
-        hass = bootstrap.from_config_file(
+        await bootstrap.async_from_config_file(
-            config_file, verbose=args.verbose, skip_pip=args.skip_pip,
+            config_file, hass, verbose=args.verbose, skip_pip=args.skip_pip,
             log_rotate_days=args.log_rotate_days, log_file=args.log_file,
             log_no_color=args.log_no_color)
 
-    if hass is None:
-        return -1
-
     if args.open_ui:
         # Imported here to avoid importing asyncio before monkey patch
         from homeassistant.util.async_ import run_callback_threadsafe
 
         def open_browser(_: Any) -> None:
             """Open the web interface in a browser."""
-            if hass.config.api is not None:  # type: ignore
+            if hass.config.api is not None:
                 import webbrowser
-                webbrowser.open(hass.config.api.base_url)  # type: ignore
+                webbrowser.open(hass.config.api.base_url)
 
         run_callback_threadsafe(
             hass.loop,
@@ -291,7 +290,7 @@ def setup_and_run_hass(config_dir: str,
             EVENT_HOMEASSISTANT_START, open_browser
         )
 
-    return hass.start()
+    return await hass.async_run()
 
 
 def try_to_restart() -> None:
@@ -365,11 +364,12 @@ def main() -> int:
     if args.pid_file:
         write_pid(args.pid_file)
 
-    exit_code = setup_and_run_hass(config_dir, args)
+    from homeassistant.util.async_ import asyncio_run
+    exit_code = asyncio_run(setup_and_run_hass(config_dir, args))
     if exit_code == RESTART_EXIT_CODE and not args.runner:
         try_to_restart()
 
-    return exit_code
+    return exit_code  # type: ignore # mypy cannot yet infer it
 
 
 if __name__ == "__main__":

homeassistant/bootstrap.py

@@ -18,7 +18,6 @@ from homeassistant.util.logging import AsyncHandler
 from homeassistant.util.package import async_get_user_site, is_virtual_env
 from homeassistant.util.yaml import clear_secret_cache
 from homeassistant.exceptions import HomeAssistantError
-from homeassistant.helpers.signal import async_register_signal_handling
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -159,7 +158,6 @@ async def async_from_config_dict(config: Dict[str, Any],
     stop = time()
     _LOGGER.info("Home Assistant initialized in %.2fs", stop-start)
 
-    async_register_signal_handling(hass)
     return hass
 
 

homeassistant/core.py

@@ -154,6 +154,8 @@ class HomeAssistant:
         self.state = CoreState.not_running
         self.exit_code = 0  # type: int
         self.config_entries = None  # type: Optional[ConfigEntries]
+        # If not None, use to signal end-of-loop
+        self._stopped = None  # type: Optional[asyncio.Event]
 
     @property
     def is_running(self) -> bool:
@@ -161,23 +163,45 @@ class HomeAssistant:
         return self.state in (CoreState.starting, CoreState.running)
 
     def start(self) -> int:
-        """Start home assistant."""
+        """Start home assistant.
+
+        Note: This function is only used for testing.
+        For regular use, use "await hass.run()".
+        """
         # Register the async start
         fire_coroutine_threadsafe(self.async_start(), self.loop)
 
-        # Run forever and catch keyboard interrupt
+        # Run forever
         try:
             # Block until stopped
             _LOGGER.info("Starting Home Assistant core loop")
             self.loop.run_forever()
-        except KeyboardInterrupt:
-            self.loop.call_soon_threadsafe(
-                self.loop.create_task, self.async_stop())
-            self.loop.run_forever()
         finally:
             self.loop.close()
         return self.exit_code
 
+    async def async_run(self, *, attach_signals: bool = True) -> int:
+        """Home Assistant main entry point.
+
+        Start Home Assistant and block until stopped.
+
+        This method is a coroutine.
+        """
+        if self.state != CoreState.not_running:
+            raise RuntimeError("HASS is already running")
+
+        # _async_stop will set this instead of stopping the loop
+        self._stopped = asyncio.Event()
+
+        await self.async_start()
+        if attach_signals:
+            from homeassistant.helpers.signal \
+                    import async_register_signal_handling
+            async_register_signal_handling(self)
+
+        await self._stopped.wait()
+        return self.exit_code
+
     async def async_start(self) -> None:
         """Finalize startup from inside the event loop.
 
@@ -203,6 +227,13 @@ class HomeAssistant:
 
         # Allow automations to set up the start triggers before changing state
         await asyncio.sleep(0)
+
+        if self.state != CoreState.starting:
+            _LOGGER.warning(
+                'Home Assistant startup has been interrupted. '
+                'Its state may be inconsistent.')
+            return
+
         self.state = CoreState.running
         _async_create_timer(self)
 
@@ -321,13 +352,32 @@ class HomeAssistant:
 
     def stop(self) -> None:
         """Stop Home Assistant and shuts down all threads."""
+        if self.state == CoreState.not_running:  # just ignore
+            return
         fire_coroutine_threadsafe(self.async_stop(), self.loop)
 
-    async def async_stop(self, exit_code: int = 0) -> None:
+    async def async_stop(self, exit_code: int = 0, *,
+                         force: bool = False) -> None:
         """Stop Home Assistant and shuts down all threads.
 
+        The "force" flag commands async_stop to proceed regardless of
+        Home Assistan't current state. You should not set this flag
+        unless you're testing.
+
         This method is a coroutine.
         """
+        if not force:
+            # Some tests require async_stop to run,
+            # regardless of the state of the loop.
+            if self.state == CoreState.not_running:  # just ignore
+                return
+            if self.state == CoreState.stopping:
+                _LOGGER.info("async_stop called twice: ignored")
+                return
+            if self.state == CoreState.starting:
+                # This may not work
+                _LOGGER.warning("async_stop called before startup is complete")
+
         # stage 1
         self.state = CoreState.stopping
         self.async_track_tasks()
@@ -341,7 +391,11 @@ class HomeAssistant:
         self.executor.shutdown()
 
         self.exit_code = exit_code
-        self.loop.stop()
+
+        if self._stopped is not None:
+            self._stopped.set()
+        else:
+            self.loop.stop()
 
 
 @attr.s(slots=True, frozen=True)

homeassistant/helpers/signal.py

@@ -17,7 +17,13 @@ def async_register_signal_handling(hass: HomeAssistant) -> None:
     if sys.platform != 'win32':
         @callback
         def async_signal_handle(exit_code):
-            """Wrap signal handling."""
+            """Wrap signal handling.
+
+            * queue call to shutdown task
+            * re-instate default handler
+            """
+            hass.loop.remove_signal_handler(signal.SIGTERM)
+            hass.loop.remove_signal_handler(signal.SIGINT)
             hass.async_create_task(hass.async_stop(exit_code))
 
         try:
@@ -26,6 +32,12 @@ def async_register_signal_handling(hass: HomeAssistant) -> None:
         except ValueError:
             _LOGGER.warning("Could not bind to SIGTERM")
 
+        try:
+            hass.loop.add_signal_handler(
+                signal.SIGINT, async_signal_handle, 0)
+        except ValueError:
+            _LOGGER.warning("Could not bind to SIGINT")
+
         try:
             hass.loop.add_signal_handler(
                 signal.SIGHUP, async_signal_handle, RESTART_EXIT_CODE)

homeassistant/util/async_.py

@@ -6,12 +6,32 @@ from asyncio import coroutines
 from asyncio.events import AbstractEventLoop
 from asyncio.futures import Future
 
+import asyncio
 from asyncio import ensure_future
-from typing import Any, Union, Coroutine, Callable, Generator
+from typing import Any, Union, Coroutine, Callable, Generator, TypeVar, \
+                   Awaitable
 
 _LOGGER = logging.getLogger(__name__)
 
 
+try:
+    # pylint: disable=invalid-name
+    asyncio_run = asyncio.run  # type: ignore
+except AttributeError:
+    _T = TypeVar('_T')
+
+    def asyncio_run(main: Awaitable[_T], *, debug: bool = False) -> _T:
+        """Minimal re-implementation of asyncio.run (since 3.7)."""
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        loop.set_debug(debug)
+        try:
+            return loop.run_until_complete(main)
+        finally:
+            asyncio.set_event_loop(None)  # type: ignore # not a bug
+            loop.close()
+
+
 def _set_result_unless_cancelled(fut: Future, result: Any) -> None:
     """Set the result only if the Future was not cancelled."""
     if fut.cancelled():

tests/components/alarm_control_panel/test_spc.py

@@ -17,7 +17,7 @@ def hass(loop):
     hass.data['spc_registry'] = SpcRegistry()
     hass.data['spc_api'] = None
     yield hass
-    loop.run_until_complete(hass.async_stop())
+    loop.run_until_complete(hass.async_stop(force=True))
 
 
 @asyncio.coroutine

tests/components/binary_sensor/test_spc.py

@@ -14,7 +14,7 @@ def hass(loop):
     hass = loop.run_until_complete(async_test_home_assistant(loop))
     hass.data['spc_registry'] = SpcRegistry()
     yield hass
-    loop.run_until_complete(hass.async_stop())
+    loop.run_until_complete(hass.async_stop(force=True))
 
 
 @asyncio.coroutine

tests/components/sensor/test_dsmr.py

@@ -182,10 +182,14 @@ def test_reconnect(hass, monkeypatch, mock_connection_factory):
 
     # mock waiting coroutine while connection lasts
     closed = asyncio.Event(loop=hass.loop)
+    # Handshake so that `hass.async_block_till_done()` doesn't cycle forever
+    closed2 = asyncio.Event(loop=hass.loop)
 
     @asyncio.coroutine
     def wait_closed():
         yield from closed.wait()
+        closed2.set()
+        closed.clear()
     protocol.wait_closed = wait_closed
 
     yield from async_setup_component(hass, 'sensor', {'sensor': config})
@@ -195,8 +199,11 @@ def test_reconnect(hass, monkeypatch, mock_connection_factory):
     # indicate disconnect, release wait lock and allow reconnect to happen
     closed.set()
     # wait for lock set to resolve
-    yield from hass.async_block_till_done()
+    yield from closed2.wait()
-    # wait for sleep to resolve
+    closed2.clear()
+    assert not closed.is_set()
+
+    closed.set()
     yield from hass.async_block_till_done()
 
     assert connection_factory.call_count >= 2, \

tests/conftest.py

@@ -73,7 +73,7 @@ def hass(loop, hass_storage):
 
     yield hass
 
-    loop.run_until_complete(hass.async_stop())
+    loop.run_until_complete(hass.async_stop(force=True))
 
 
 @pytest.fixture

tests/helpers/test_discovery.py

@@ -154,7 +154,7 @@ class TestHelpersDiscovery:
         assert 'test_component' in self.hass.config.components
         assert 'switch' in self.hass.config.components
 
-    @patch('homeassistant.bootstrap.async_register_signal_handling')
+    @patch('homeassistant.helpers.signal.async_register_signal_handling')
     def test_1st_discovers_2nd_component(self, mock_signal):
         """Test that we don't break if one component discovers the other.
 

tests/test_bootstrap.py

@@ -22,7 +22,6 @@ _LOGGER = logging.getLogger(__name__)
     'homeassistant.bootstrap.conf_util.process_ha_config_upgrade', Mock())
 @patch('homeassistant.util.location.detect_location_info',
        Mock(return_value=None))
-@patch('homeassistant.bootstrap.async_register_signal_handling', Mock())
 @patch('os.path.isfile', Mock(return_value=True))
 @patch('os.access', Mock(return_value=True))
 @patch('homeassistant.bootstrap.async_enable_logging',
@@ -41,7 +40,6 @@ def test_from_config_file(hass):
 
 
 @patch('homeassistant.bootstrap.async_enable_logging', Mock())
-@patch('homeassistant.bootstrap.async_register_signal_handling', Mock())
 @asyncio.coroutine
 def test_home_assistant_core_config_validation(hass):
     """Test if we pass in wrong information for HA conf."""