Rate limit D-Bus errors (#5787)

It seems that some systems continously run into D-Bus errors
overwhelming the system itself but also generating lots of errors on
Sentry. Rate limit D-Bus errors to 3 for every message every 30s.
This commit is contained in:
Stefan Agner 2025-03-26 17:02:58 +01:00 committed by GitHub
parent af9b1e5b1e
commit 63b507a589
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 11 additions and 2 deletions

View File

@ -16,6 +16,7 @@ docker==7.1.0
faust-cchardet==2.1.19
gitpython==3.1.44
jinja2==3.1.6
log-rate-limit==1.4.2
orjson==3.10.16
pulsectl==24.12.0
pyudev==0.24.3

View File

@ -18,6 +18,7 @@ from dbus_fast.aio.message_bus import MessageBus
from dbus_fast.aio.proxy_object import ProxyInterface, ProxyObject
from dbus_fast.errors import DBusError as DBusFastDBusError
from dbus_fast.introspection import Node
from log_rate_limit import RateLimit, StreamRateLimitFilter
from ..exceptions import (
DBusError,
@ -38,6 +39,7 @@ from ..exceptions import (
from .sentry import async_capture_exception
_LOGGER: logging.Logger = logging.getLogger(__name__)
_LOGGER.addFilter(StreamRateLimitFilter(period_sec=30, allow_next_n=2))
DBUS_INTERFACE_OBJECT_MANAGER: str = "org.freedesktop.DBus.ObjectManager"
DBUS_INTERFACE_PROPERTIES: str = "org.freedesktop.DBus.Properties"
@ -151,11 +153,17 @@ class DBus:
# The systemd D-Bus activate service has a timeout of 25s, which will raise. We should
# not end up here unless the D-Bus broker is majorly overwhelmed.
_LOGGER.critical(
"Timeout connecting to %s - %s", self.bus_name, self.object_path
"Timeout connecting to %s - %s",
self.bus_name,
self.object_path,
extra=RateLimit(stream_id=f"dbus_timeout_{self.bus_name}"),
)
except EOFError:
_LOGGER.warning(
"Busy system at %s - %s", self.bus_name, self.object_path
"Busy system at %s - %s",
self.bus_name,
self.object_path,
extra=RateLimit(stream_id=f"dbus_eof_{self.bus_name}"),
)
await asyncio.sleep(3)