Rate limit D-Bus errors (#5787)

It seems that some systems continously run into D-Bus errors
overwhelming the system itself but also generating lots of errors on
Sentry. Rate limit D-Bus errors to 3 for every message every 30s.
This commit is contained in:
Stefan Agner 2025-03-26 17:02:58 +01:00 committed by GitHub
parent af9b1e5b1e
commit 63b507a589
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 11 additions and 2 deletions

View File

@ -16,6 +16,7 @@ docker==7.1.0
faust-cchardet==2.1.19 faust-cchardet==2.1.19
gitpython==3.1.44 gitpython==3.1.44
jinja2==3.1.6 jinja2==3.1.6
log-rate-limit==1.4.2
orjson==3.10.16 orjson==3.10.16
pulsectl==24.12.0 pulsectl==24.12.0
pyudev==0.24.3 pyudev==0.24.3

View File

@ -18,6 +18,7 @@ from dbus_fast.aio.message_bus import MessageBus
from dbus_fast.aio.proxy_object import ProxyInterface, ProxyObject from dbus_fast.aio.proxy_object import ProxyInterface, ProxyObject
from dbus_fast.errors import DBusError as DBusFastDBusError from dbus_fast.errors import DBusError as DBusFastDBusError
from dbus_fast.introspection import Node from dbus_fast.introspection import Node
from log_rate_limit import RateLimit, StreamRateLimitFilter
from ..exceptions import ( from ..exceptions import (
DBusError, DBusError,
@ -38,6 +39,7 @@ from ..exceptions import (
from .sentry import async_capture_exception from .sentry import async_capture_exception
_LOGGER: logging.Logger = logging.getLogger(__name__) _LOGGER: logging.Logger = logging.getLogger(__name__)
_LOGGER.addFilter(StreamRateLimitFilter(period_sec=30, allow_next_n=2))
DBUS_INTERFACE_OBJECT_MANAGER: str = "org.freedesktop.DBus.ObjectManager" DBUS_INTERFACE_OBJECT_MANAGER: str = "org.freedesktop.DBus.ObjectManager"
DBUS_INTERFACE_PROPERTIES: str = "org.freedesktop.DBus.Properties" DBUS_INTERFACE_PROPERTIES: str = "org.freedesktop.DBus.Properties"
@ -151,11 +153,17 @@ class DBus:
# The systemd D-Bus activate service has a timeout of 25s, which will raise. We should # The systemd D-Bus activate service has a timeout of 25s, which will raise. We should
# not end up here unless the D-Bus broker is majorly overwhelmed. # not end up here unless the D-Bus broker is majorly overwhelmed.
_LOGGER.critical( _LOGGER.critical(
"Timeout connecting to %s - %s", self.bus_name, self.object_path "Timeout connecting to %s - %s",
self.bus_name,
self.object_path,
extra=RateLimit(stream_id=f"dbus_timeout_{self.bus_name}"),
) )
except EOFError: except EOFError:
_LOGGER.warning( _LOGGER.warning(
"Busy system at %s - %s", self.bus_name, self.object_path "Busy system at %s - %s",
self.bus_name,
self.object_path,
extra=RateLimit(stream_id=f"dbus_eof_{self.bus_name}"),
) )
await asyncio.sleep(3) await asyncio.sleep(3)