Avoid deadlock on shutdown when a task is shielded from cancelation (#56499)

This commit is contained in:
J. Nick Koston 2021-09-21 16:35:14 -05:00 committed by GitHub
parent d494b3539d
commit 9831ff0487
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 149 additions and 1 deletions

View File

@ -26,6 +26,9 @@ from homeassistant.util.thread import deadlock_safe_shutdown
# use case. # use case.
# #
MAX_EXECUTOR_WORKERS = 64 MAX_EXECUTOR_WORKERS = 64
TASK_CANCELATION_TIMEOUT = 5
_LOGGER = logging.getLogger(__name__)
@dataclasses.dataclass @dataclasses.dataclass
@ -105,4 +108,69 @@ async def setup_and_run_hass(runtime_config: RuntimeConfig) -> int:
def run(runtime_config: RuntimeConfig) -> int: def run(runtime_config: RuntimeConfig) -> int:
"""Run Home Assistant.""" """Run Home Assistant."""
asyncio.set_event_loop_policy(HassEventLoopPolicy(runtime_config.debug)) asyncio.set_event_loop_policy(HassEventLoopPolicy(runtime_config.debug))
return asyncio.run(setup_and_run_hass(runtime_config)) # Backport of cpython 3.9 asyncio.run with a _cancel_all_tasks that times out
loop = asyncio.new_event_loop()
try:
asyncio.set_event_loop(loop)
return loop.run_until_complete(setup_and_run_hass(runtime_config))
finally:
try:
_cancel_all_tasks_with_timeout(loop, TASK_CANCELATION_TIMEOUT)
loop.run_until_complete(loop.shutdown_asyncgens())
# Once cpython 3.8 is no longer supported we can use the
# the built-in loop.shutdown_default_executor
loop.run_until_complete(_shutdown_default_executor(loop))
finally:
asyncio.set_event_loop(None)
loop.close()
def _cancel_all_tasks_with_timeout(
loop: asyncio.AbstractEventLoop, timeout: int
) -> None:
"""Adapted _cancel_all_tasks from python 3.9 with a timeout."""
to_cancel = asyncio.all_tasks(loop)
if not to_cancel:
return
for task in to_cancel:
task.cancel()
loop.run_until_complete(asyncio.wait(to_cancel, timeout=timeout))
for task in to_cancel:
if task.cancelled():
continue
if not task.done():
_LOGGER.warning(
"Task could not be canceled and was still running after shutdown: %s",
task,
)
continue
if task.exception() is not None:
loop.call_exception_handler(
{
"message": "unhandled exception during shutdown",
"exception": task.exception(),
"task": task,
}
)
async def _shutdown_default_executor(loop: asyncio.AbstractEventLoop) -> None:
"""Backport of cpython 3.9 schedule the shutdown of the default executor."""
future = loop.create_future()
def _do_shutdown() -> None:
try:
loop._default_executor.shutdown(wait=True) # type: ignore # pylint: disable=protected-access
loop.call_soon_threadsafe(future.set_result, None)
except Exception as ex: # pylint: disable=broad-except
loop.call_soon_threadsafe(future.set_exception, ex)
thread = threading.Thread(target=_do_shutdown)
thread.start()
try:
await future
finally:
thread.join()

View File

@ -1,8 +1,11 @@
"""Test the runner.""" """Test the runner."""
import asyncio
import threading import threading
from unittest.mock import patch from unittest.mock import patch
import pytest
from homeassistant import core, runner from homeassistant import core, runner
from homeassistant.util import executor, thread from homeassistant.util import executor, thread
@ -37,3 +40,80 @@ async def test_setup_and_run_hass(hass, tmpdir):
assert threading._shutdown == thread.deadlock_safe_shutdown assert threading._shutdown == thread.deadlock_safe_shutdown
assert mock_run.called assert mock_run.called
def test_run(hass, tmpdir):
"""Test we can run."""
test_dir = tmpdir.mkdir("config")
default_config = runner.RuntimeConfig(test_dir)
with patch.object(runner, "TASK_CANCELATION_TIMEOUT", 1), patch(
"homeassistant.bootstrap.async_setup_hass", return_value=hass
), patch("threading._shutdown"), patch(
"homeassistant.core.HomeAssistant.async_run"
) as mock_run:
runner.run(default_config)
assert mock_run.called
def test_run_executor_shutdown_throws(hass, tmpdir):
"""Test we can run and we still shutdown if the executor shutdown throws."""
test_dir = tmpdir.mkdir("config")
default_config = runner.RuntimeConfig(test_dir)
with patch.object(runner, "TASK_CANCELATION_TIMEOUT", 1), pytest.raises(
RuntimeError
), patch("homeassistant.bootstrap.async_setup_hass", return_value=hass), patch(
"threading._shutdown"
), patch(
"homeassistant.runner.InterruptibleThreadPoolExecutor.shutdown",
side_effect=RuntimeError,
) as mock_shutdown, patch(
"homeassistant.core.HomeAssistant.async_run"
) as mock_run:
runner.run(default_config)
assert mock_shutdown.called
assert mock_run.called
def test_run_does_not_block_forever_with_shielded_task(hass, tmpdir, caplog):
"""Test we can shutdown and not block forever."""
test_dir = tmpdir.mkdir("config")
default_config = runner.RuntimeConfig(test_dir)
created_tasks = False
async def _async_create_tasks(*_):
nonlocal created_tasks
async def async_raise(*_):
try:
await asyncio.sleep(2)
except asyncio.CancelledError:
raise Exception
async def async_shielded(*_):
try:
await asyncio.sleep(2)
except asyncio.CancelledError:
await asyncio.sleep(2)
asyncio.ensure_future(asyncio.shield(async_shielded()))
asyncio.ensure_future(asyncio.sleep(2))
asyncio.ensure_future(async_raise())
await asyncio.sleep(0.1)
created_tasks = True
return 0
with patch.object(runner, "TASK_CANCELATION_TIMEOUT", 1), patch(
"homeassistant.bootstrap.async_setup_hass", return_value=hass
), patch("threading._shutdown"), patch(
"homeassistant.core.HomeAssistant.async_run", _async_create_tasks
):
runner.run(default_config)
assert created_tasks is True
assert (
"Task could not be canceled and was still running after shutdown" in caplog.text
)