From a6cec21c435d8fda5d9b290426967b6bf9372399 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2020 03:12:18 -0500 Subject: [PATCH] Make executor max_workers consistent between python versions (#38821) The default on python 3.8 is for max_workers is significantly lower than the default on python 3.7 which means we can get starved for workers. To determine a reasonable maximum, the maximum was increased to large number on 5 production instances. The number of worker threads created during startup that were needed to avoid waiting for a thread: HOU 1 - 71 HOU 2 - 48 OGG 1 - 60 OGG 2 - 68 OGG 3 - 64 This lead to a selection of 64 as it was reliable in all cases and did not have a significant memory impact --- homeassistant/runner.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/homeassistant/runner.py b/homeassistant/runner.py index 26e7bab7616..b397f9438f2 100644 --- a/homeassistant/runner.py +++ b/homeassistant/runner.py @@ -11,6 +11,18 @@ from homeassistant import bootstrap from homeassistant.core import callback from homeassistant.helpers.frame import warn_use +# +# Python 3.8 has significantly less workers by default +# than Python 3.7. In order to be consistent between +# supported versions, we need to set max_workers. +# +# In most cases the workers are not I/O bound, as they +# are sleeping/blocking waiting for data from integrations +# updating so this number should be higher than the default +# use case. +# +MAX_EXECUTOR_WORKERS = 64 + @dataclasses.dataclass class RuntimeConfig: @@ -57,7 +69,9 @@ class HassEventLoopPolicy(PolicyBase): # type: ignore if self.debug: loop.set_debug(True) - executor = ThreadPoolExecutor(thread_name_prefix="SyncWorker") + executor = ThreadPoolExecutor( + thread_name_prefix="SyncWorker", max_workers=MAX_EXECUTOR_WORKERS + ) loop.set_default_executor(executor) loop.set_default_executor = warn_use( # type: ignore loop.set_default_executor, "sets default executor on the event loop"