From 09e7d8d1a5525cdb4ab51bc58c496d8bc32a6246 Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Tue, 29 Jul 2025 17:42:26 +0200 Subject: [PATCH] Increase open file descriptor limit on startup (#148940) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jan Čermák Co-authored-by: Martin Hjelmare --- homeassistant/runner.py | 2 + homeassistant/util/resource.py | 65 ++++++++++++++ tests/util/test_resource.py | 153 +++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 homeassistant/util/resource.py create mode 100644 tests/util/test_resource.py diff --git a/homeassistant/runner.py b/homeassistant/runner.py index 59775655854..abcf32f2659 100644 --- a/homeassistant/runner.py +++ b/homeassistant/runner.py @@ -17,6 +17,7 @@ from . import bootstrap from .core import callback from .helpers.frame import warn_use from .util.executor import InterruptibleThreadPoolExecutor +from .util.resource import set_open_file_descriptor_limit from .util.thread import deadlock_safe_shutdown # @@ -146,6 +147,7 @@ def _enable_posix_spawn() -> None: def run(runtime_config: RuntimeConfig) -> int: """Run Home Assistant.""" _enable_posix_spawn() + set_open_file_descriptor_limit() asyncio.set_event_loop_policy(HassEventLoopPolicy(runtime_config.debug)) # Backport of cpython 3.9 asyncio.run with a _cancel_all_tasks that times out loop = asyncio.new_event_loop() diff --git a/homeassistant/util/resource.py b/homeassistant/util/resource.py new file mode 100644 index 00000000000..41982df9e50 --- /dev/null +++ b/homeassistant/util/resource.py @@ -0,0 +1,65 @@ +"""Resource management utilities for Home Assistant.""" + +from __future__ import annotations + +import logging +import os +import resource +from typing import Final + +_LOGGER = logging.getLogger(__name__) + +# Default soft file descriptor limit to set +DEFAULT_SOFT_FILE_LIMIT: Final = 2048 + + +def set_open_file_descriptor_limit() -> None: + """Set the maximum open file descriptor soft limit.""" + try: + # Check environment variable first, then use default + soft_limit = int(os.environ.get("SOFT_FILE_LIMIT", DEFAULT_SOFT_FILE_LIMIT)) + + # Get current limits + current_soft, current_hard = resource.getrlimit(resource.RLIMIT_NOFILE) + + _LOGGER.debug( + "Current file descriptor limits: soft=%d, hard=%d", + current_soft, + current_hard, + ) + + # Don't increase if already at or above the desired limit + if current_soft >= soft_limit: + _LOGGER.debug( + "Current soft limit (%d) is already >= desired limit (%d), skipping", + current_soft, + soft_limit, + ) + return + + # Don't set soft limit higher than hard limit + if soft_limit > current_hard: + _LOGGER.warning( + "Requested soft limit (%d) exceeds hard limit (%d), " + "setting to hard limit", + soft_limit, + current_hard, + ) + soft_limit = current_hard + + # Set the new soft limit + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, current_hard)) + + # Verify the change + new_soft, new_hard = resource.getrlimit(resource.RLIMIT_NOFILE) + _LOGGER.info( + "File descriptor limits updated: soft=%d->%d, hard=%d", + current_soft, + new_soft, + new_hard, + ) + + except OSError as err: + _LOGGER.error("Failed to set file descriptor limit: %s", err) + except ValueError as err: + _LOGGER.error("Invalid file descriptor limit value: %s", err) diff --git a/tests/util/test_resource.py b/tests/util/test_resource.py new file mode 100644 index 00000000000..a32ceb1062c --- /dev/null +++ b/tests/util/test_resource.py @@ -0,0 +1,153 @@ +"""Test the resource utility module.""" + +import os +import resource +from unittest.mock import call, patch + +import pytest + +from homeassistant.util.resource import ( + DEFAULT_SOFT_FILE_LIMIT, + set_open_file_descriptor_limit, +) + + +@pytest.mark.parametrize( + ("original_soft", "expected_calls", "should_log_already_sufficient"), + [ + ( + 1024, + [call(resource.RLIMIT_NOFILE, (DEFAULT_SOFT_FILE_LIMIT, 524288))], + False, + ), + ( + DEFAULT_SOFT_FILE_LIMIT - 1, + [call(resource.RLIMIT_NOFILE, (DEFAULT_SOFT_FILE_LIMIT, 524288))], + False, + ), + (DEFAULT_SOFT_FILE_LIMIT, [], True), + (DEFAULT_SOFT_FILE_LIMIT + 1, [], True), + ], +) +def test_set_open_file_descriptor_limit_default( + caplog: pytest.LogCaptureFixture, + original_soft: int, + expected_calls: list, + should_log_already_sufficient: bool, +) -> None: + """Test setting file limit with default value.""" + original_hard = 524288 + with ( + patch( + "homeassistant.util.resource.resource.getrlimit", + return_value=(original_soft, original_hard), + ), + patch("homeassistant.util.resource.resource.setrlimit") as mock_setrlimit, + ): + set_open_file_descriptor_limit() + + assert mock_setrlimit.call_args_list == expected_calls + assert ( + f"Current soft limit ({original_soft}) is already" in caplog.text + ) is should_log_already_sufficient + + +@pytest.mark.parametrize( + ( + "original_soft", + "custom_limit", + "expected_calls", + "should_log_already_sufficient", + ), + [ + (1499, 1500, [call(resource.RLIMIT_NOFILE, (1500, 524288))], False), + (1500, 1500, [], True), + (1501, 1500, [], True), + ], +) +def test_set_open_file_descriptor_limit_environment_variable( + caplog: pytest.LogCaptureFixture, + original_soft: int, + custom_limit: int, + expected_calls: list, + should_log_already_sufficient: bool, +) -> None: + """Test setting file limit from environment variable.""" + original_hard = 524288 + with ( + patch.dict(os.environ, {"SOFT_FILE_LIMIT": str(custom_limit)}), + patch( + "homeassistant.util.resource.resource.getrlimit", + return_value=(original_soft, original_hard), + ), + patch("homeassistant.util.resource.resource.setrlimit") as mock_setrlimit, + ): + set_open_file_descriptor_limit() + + assert mock_setrlimit.call_args_list == expected_calls + assert ( + f"Current soft limit ({original_soft}) is already" in caplog.text + ) is should_log_already_sufficient + + +def test_set_open_file_descriptor_limit_exceeds_hard_limit( + caplog: pytest.LogCaptureFixture, +) -> None: + """Test setting file limit that exceeds hard limit.""" + original_soft, original_hard = (1024, 524288) + excessive_limit = original_hard + 1 + + with ( + patch.dict(os.environ, {"SOFT_FILE_LIMIT": str(excessive_limit)}), + patch( + "homeassistant.util.resource.resource.getrlimit", + return_value=(original_soft, original_hard), + ), + patch("homeassistant.util.resource.resource.setrlimit") as mock_setrlimit, + ): + set_open_file_descriptor_limit() + + mock_setrlimit.assert_called_once_with( + resource.RLIMIT_NOFILE, (original_hard, original_hard) + ) + assert ( + f"Requested soft limit ({excessive_limit}) exceeds hard limit ({original_hard})" + in caplog.text + ) + + +def test_set_open_file_descriptor_limit_os_error( + caplog: pytest.LogCaptureFixture, +) -> None: + """Test handling OSError when setting file limit.""" + with ( + patch( + "homeassistant.util.resource.resource.getrlimit", + return_value=(1024, 524288), + ), + patch( + "homeassistant.util.resource.resource.setrlimit", + side_effect=OSError("Permission denied"), + ), + ): + set_open_file_descriptor_limit() + + assert "Failed to set file descriptor limit" in caplog.text + assert "Permission denied" in caplog.text + + +def test_set_open_file_descriptor_limit_value_error( + caplog: pytest.LogCaptureFixture, +) -> None: + """Test handling ValueError when setting file limit.""" + with ( + patch.dict(os.environ, {"SOFT_FILE_LIMIT": "invalid_value"}), + patch( + "homeassistant.util.resource.resource.getrlimit", + return_value=(1024, 524288), + ), + ): + set_open_file_descriptor_limit() + + assert "Invalid file descriptor limit value" in caplog.text + assert "'invalid_value'" in caplog.text