Increase open file descriptor limit on startup (#148940)

Co-authored-by: Jan Čermák <sairon@sairon.cz>
Co-authored-by: Martin Hjelmare <marhje52@gmail.com>
This commit is contained in:
Stefan Agner 2025-07-29 17:42:26 +02:00 committed by GitHub
parent ff7c125334
commit 09e7d8d1a5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 220 additions and 0 deletions

View File

@ -17,6 +17,7 @@ from . import bootstrap
from .core import callback
from .helpers.frame import warn_use
from .util.executor import InterruptibleThreadPoolExecutor
from .util.resource import set_open_file_descriptor_limit
from .util.thread import deadlock_safe_shutdown
#
@ -146,6 +147,7 @@ def _enable_posix_spawn() -> None:
def run(runtime_config: RuntimeConfig) -> int:
"""Run Home Assistant."""
_enable_posix_spawn()
set_open_file_descriptor_limit()
asyncio.set_event_loop_policy(HassEventLoopPolicy(runtime_config.debug))
# Backport of cpython 3.9 asyncio.run with a _cancel_all_tasks that times out
loop = asyncio.new_event_loop()

View File

@ -0,0 +1,65 @@
"""Resource management utilities for Home Assistant."""
from __future__ import annotations
import logging
import os
import resource
from typing import Final
_LOGGER = logging.getLogger(__name__)
# Default soft file descriptor limit to set
DEFAULT_SOFT_FILE_LIMIT: Final = 2048
def set_open_file_descriptor_limit() -> None:
"""Set the maximum open file descriptor soft limit."""
try:
# Check environment variable first, then use default
soft_limit = int(os.environ.get("SOFT_FILE_LIMIT", DEFAULT_SOFT_FILE_LIMIT))
# Get current limits
current_soft, current_hard = resource.getrlimit(resource.RLIMIT_NOFILE)
_LOGGER.debug(
"Current file descriptor limits: soft=%d, hard=%d",
current_soft,
current_hard,
)
# Don't increase if already at or above the desired limit
if current_soft >= soft_limit:
_LOGGER.debug(
"Current soft limit (%d) is already >= desired limit (%d), skipping",
current_soft,
soft_limit,
)
return
# Don't set soft limit higher than hard limit
if soft_limit > current_hard:
_LOGGER.warning(
"Requested soft limit (%d) exceeds hard limit (%d), "
"setting to hard limit",
soft_limit,
current_hard,
)
soft_limit = current_hard
# Set the new soft limit
resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, current_hard))
# Verify the change
new_soft, new_hard = resource.getrlimit(resource.RLIMIT_NOFILE)
_LOGGER.info(
"File descriptor limits updated: soft=%d->%d, hard=%d",
current_soft,
new_soft,
new_hard,
)
except OSError as err:
_LOGGER.error("Failed to set file descriptor limit: %s", err)
except ValueError as err:
_LOGGER.error("Invalid file descriptor limit value: %s", err)

153
tests/util/test_resource.py Normal file
View File

@ -0,0 +1,153 @@
"""Test the resource utility module."""
import os
import resource
from unittest.mock import call, patch
import pytest
from homeassistant.util.resource import (
DEFAULT_SOFT_FILE_LIMIT,
set_open_file_descriptor_limit,
)
@pytest.mark.parametrize(
("original_soft", "expected_calls", "should_log_already_sufficient"),
[
(
1024,
[call(resource.RLIMIT_NOFILE, (DEFAULT_SOFT_FILE_LIMIT, 524288))],
False,
),
(
DEFAULT_SOFT_FILE_LIMIT - 1,
[call(resource.RLIMIT_NOFILE, (DEFAULT_SOFT_FILE_LIMIT, 524288))],
False,
),
(DEFAULT_SOFT_FILE_LIMIT, [], True),
(DEFAULT_SOFT_FILE_LIMIT + 1, [], True),
],
)
def test_set_open_file_descriptor_limit_default(
caplog: pytest.LogCaptureFixture,
original_soft: int,
expected_calls: list,
should_log_already_sufficient: bool,
) -> None:
"""Test setting file limit with default value."""
original_hard = 524288
with (
patch(
"homeassistant.util.resource.resource.getrlimit",
return_value=(original_soft, original_hard),
),
patch("homeassistant.util.resource.resource.setrlimit") as mock_setrlimit,
):
set_open_file_descriptor_limit()
assert mock_setrlimit.call_args_list == expected_calls
assert (
f"Current soft limit ({original_soft}) is already" in caplog.text
) is should_log_already_sufficient
@pytest.mark.parametrize(
(
"original_soft",
"custom_limit",
"expected_calls",
"should_log_already_sufficient",
),
[
(1499, 1500, [call(resource.RLIMIT_NOFILE, (1500, 524288))], False),
(1500, 1500, [], True),
(1501, 1500, [], True),
],
)
def test_set_open_file_descriptor_limit_environment_variable(
caplog: pytest.LogCaptureFixture,
original_soft: int,
custom_limit: int,
expected_calls: list,
should_log_already_sufficient: bool,
) -> None:
"""Test setting file limit from environment variable."""
original_hard = 524288
with (
patch.dict(os.environ, {"SOFT_FILE_LIMIT": str(custom_limit)}),
patch(
"homeassistant.util.resource.resource.getrlimit",
return_value=(original_soft, original_hard),
),
patch("homeassistant.util.resource.resource.setrlimit") as mock_setrlimit,
):
set_open_file_descriptor_limit()
assert mock_setrlimit.call_args_list == expected_calls
assert (
f"Current soft limit ({original_soft}) is already" in caplog.text
) is should_log_already_sufficient
def test_set_open_file_descriptor_limit_exceeds_hard_limit(
caplog: pytest.LogCaptureFixture,
) -> None:
"""Test setting file limit that exceeds hard limit."""
original_soft, original_hard = (1024, 524288)
excessive_limit = original_hard + 1
with (
patch.dict(os.environ, {"SOFT_FILE_LIMIT": str(excessive_limit)}),
patch(
"homeassistant.util.resource.resource.getrlimit",
return_value=(original_soft, original_hard),
),
patch("homeassistant.util.resource.resource.setrlimit") as mock_setrlimit,
):
set_open_file_descriptor_limit()
mock_setrlimit.assert_called_once_with(
resource.RLIMIT_NOFILE, (original_hard, original_hard)
)
assert (
f"Requested soft limit ({excessive_limit}) exceeds hard limit ({original_hard})"
in caplog.text
)
def test_set_open_file_descriptor_limit_os_error(
caplog: pytest.LogCaptureFixture,
) -> None:
"""Test handling OSError when setting file limit."""
with (
patch(
"homeassistant.util.resource.resource.getrlimit",
return_value=(1024, 524288),
),
patch(
"homeassistant.util.resource.resource.setrlimit",
side_effect=OSError("Permission denied"),
),
):
set_open_file_descriptor_limit()
assert "Failed to set file descriptor limit" in caplog.text
assert "Permission denied" in caplog.text
def test_set_open_file_descriptor_limit_value_error(
caplog: pytest.LogCaptureFixture,
) -> None:
"""Test handling ValueError when setting file limit."""
with (
patch.dict(os.environ, {"SOFT_FILE_LIMIT": "invalid_value"}),
patch(
"homeassistant.util.resource.resource.getrlimit",
return_value=(1024, 524288),
),
):
set_open_file_descriptor_limit()
assert "Invalid file descriptor limit value" in caplog.text
assert "'invalid_value'" in caplog.text