mirror of
https://github.com/home-assistant/core.git
synced 2025-11-08 10:29:27 +00:00
Avoid subprocess memory copy when c library supports posix_spawn (#87958)
* use posix spawn on alpine * Avoid subprocess memory copy when c library supports posix_spawn By default python 3.10 will use the fork() which has to copy all the memory of the parent process (in our case this can be huge since Home Assistant core can use hundreds of megabytes of RAM). By using posix_spawn this is avoided. In python 3.11 vfork will also be available https://github.com/python/cpython/issues/80004#issuecomment-1093810689 https://github.com/python/cpython/pull/11671 but we won't always be able to use it and posix_spawn is considered safer https://bugzilla.kernel.org/show_bug.cgi?id=215813#c14 The subprocess library doesn't know about musl though even though it supports posix_spawn https://git.musl-libc.org/cgit/musl/log/src/process/posix_spawn.c so we have to teach it since it only has checks for glibc1b736838e6/Lib/subprocess.py (L745)The constant is documented as being able to be flipped here: https://docs.python.org/3/library/subprocess.html#disabling-use-of-vfork-or-posix-spawn * Avoid subprocess memory copy when c library supports posix_spawn By default python 3.10 will use the fork() which has to copy memory of the parent process (in our case this can be huge since Home Assistant core can use hundreds of megabytes of RAM). By using posix_spawn this is avoided and subprocess creation does not get discernibly slow the larger the Home Assistant python process grows. In python 3.11 vfork will also be available https://github.com/python/cpython/issues/80004#issuecomment-1093810689 https://github.com/python/cpython/pull/11671 but we won't always be able to use it and posix_spawn is considered safer https://bugzilla.kernel.org/show_bug.cgi?id=215813#c14 The subprocess library doesn't know about musl though even though it supports posix_spawn https://git.musl-libc.org/cgit/musl/log/src/process/posix_spawn.c so we have to teach it since it only has checks for glibc1b736838e6/Lib/subprocess.py (L745)The constant is documented as being able to be flipped here: https://docs.python.org/3/library/subprocess.html#disabling-use-of-vfork-or-posix-spawn * missed some * adjust more tests * coverage
This commit is contained in:
@@ -6,6 +6,7 @@ from asyncio import events
|
||||
import dataclasses
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import traceback
|
||||
from typing import Any
|
||||
@@ -28,6 +29,7 @@ from .util.thread import deadlock_safe_shutdown
|
||||
#
|
||||
MAX_EXECUTOR_WORKERS = 64
|
||||
TASK_CANCELATION_TIMEOUT = 5
|
||||
ALPINE_RELEASE_FILE = "/etc/alpine-release"
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
@@ -153,8 +155,22 @@ async def setup_and_run_hass(runtime_config: RuntimeConfig) -> int:
|
||||
return await hass.async_run()
|
||||
|
||||
|
||||
def _enable_posix_spawn() -> None:
|
||||
"""Enable posix_spawn on Alpine Linux."""
|
||||
# pylint: disable=protected-access
|
||||
if subprocess._USE_POSIX_SPAWN:
|
||||
return
|
||||
|
||||
# The subprocess module does not know about Alpine Linux/musl
|
||||
# and will use fork() instead of posix_spawn() which significantly
|
||||
# less efficient. This is a workaround to force posix_spawn()
|
||||
# on Alpine Linux which is supported by musl.
|
||||
subprocess._USE_POSIX_SPAWN = os.path.exists(ALPINE_RELEASE_FILE)
|
||||
|
||||
|
||||
def run(runtime_config: RuntimeConfig) -> int:
|
||||
"""Run Home Assistant."""
|
||||
_enable_posix_spawn()
|
||||
asyncio.set_event_loop_policy(HassEventLoopPolicy(runtime_config.debug))
|
||||
# Backport of cpython 3.9 asyncio.run with a _cancel_all_tasks that times out
|
||||
loop = asyncio.new_event_loop()
|
||||
|
||||
Reference in New Issue
Block a user