From 63b507a589063845dc0d2400531160e515b3edad Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Wed, 26 Mar 2025 17:02:58 +0100 Subject: [PATCH] Rate limit D-Bus errors (#5787) It seems that some systems continously run into D-Bus errors overwhelming the system itself but also generating lots of errors on Sentry. Rate limit D-Bus errors to 3 for every message every 30s. --- requirements.txt | 1 + supervisor/utils/dbus.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 877826f50..f30b40c6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ docker==7.1.0 faust-cchardet==2.1.19 gitpython==3.1.44 jinja2==3.1.6 +log-rate-limit==1.4.2 orjson==3.10.16 pulsectl==24.12.0 pyudev==0.24.3 diff --git a/supervisor/utils/dbus.py b/supervisor/utils/dbus.py index a5bc4d102..6284ddd2c 100644 --- a/supervisor/utils/dbus.py +++ b/supervisor/utils/dbus.py @@ -18,6 +18,7 @@ from dbus_fast.aio.message_bus import MessageBus from dbus_fast.aio.proxy_object import ProxyInterface, ProxyObject from dbus_fast.errors import DBusError as DBusFastDBusError from dbus_fast.introspection import Node +from log_rate_limit import RateLimit, StreamRateLimitFilter from ..exceptions import ( DBusError, @@ -38,6 +39,7 @@ from ..exceptions import ( from .sentry import async_capture_exception _LOGGER: logging.Logger = logging.getLogger(__name__) +_LOGGER.addFilter(StreamRateLimitFilter(period_sec=30, allow_next_n=2)) DBUS_INTERFACE_OBJECT_MANAGER: str = "org.freedesktop.DBus.ObjectManager" DBUS_INTERFACE_PROPERTIES: str = "org.freedesktop.DBus.Properties" @@ -151,11 +153,17 @@ class DBus: # The systemd D-Bus activate service has a timeout of 25s, which will raise. We should # not end up here unless the D-Bus broker is majorly overwhelmed. _LOGGER.critical( - "Timeout connecting to %s - %s", self.bus_name, self.object_path + "Timeout connecting to %s - %s", + self.bus_name, + self.object_path, + extra=RateLimit(stream_id=f"dbus_timeout_{self.bus_name}"), ) except EOFError: _LOGGER.warning( - "Busy system at %s - %s", self.bus_name, self.object_path + "Busy system at %s - %s", + self.bus_name, + self.object_path, + extra=RateLimit(stream_id=f"dbus_eof_{self.bus_name}"), ) await asyncio.sleep(3)