Sanitize event (#1908)

* Sanitise event

* No need to remove supervisor token

* cleanup

* Typo

* Review comments

* Move and test

* Move and use hdr

supervisor/bootstrap.py

@@ -23,7 +23,6 @@ from .const import (
     MACHINE_ID,
     SOCKET_DOCKER,
     SUPERVISOR_VERSION,
-    CoreStates,
     LogLevel,
     UpdateChannels,
 )
@@ -35,6 +34,7 @@ from .hassos import HassOS
 from .homeassistant import HomeAssistant
 from .host import HostManager
 from .ingress import Ingress
+from .misc.filter import filter_data
 from .misc.hwmon import HwMonitor
 from .misc.scheduler import Scheduler
 from .misc.secrets import SecretsManager
@@ -281,45 +281,9 @@ def supervisor_debugger(coresys: CoreSys) -> None:
 
 def setup_diagnostics(coresys: CoreSys) -> None:
     """Sentry diagnostic backend."""
-    dev_env: bool = bool(os.environ.get(ENV_SUPERVISOR_DEV, 0))
 
-    def filter_data(event, hint):
+    def filter_event(event, _hint):
-        # Ignore issue if system is not supported or diagnostics is disabled
+        return filter_data(coresys, event)
-        if not coresys.config.diagnostics or not coresys.supported or dev_env:
-            return None
-
-        # Not full startup - missing information
-        if coresys.core.state in (CoreStates.INITIALIZE, CoreStates.SETUP):
-            return event
-
-        # Update information
-        event.setdefault("extra", {}).update(
-            {
-                "supervisor": {
-                    "machine": coresys.machine,
-                    "arch": coresys.arch.default,
-                    "docker": coresys.docker.info.version,
-                    "channel": coresys.updater.channel,
-                    "supervisor": coresys.supervisor.version,
-                    "os": coresys.hassos.version,
-                    "host": coresys.host.info.operating_system,
-                    "kernel": coresys.host.info.kernel,
-                    "core": coresys.homeassistant.version,
-                    "audio": coresys.plugins.audio.version,
-                    "dns": coresys.plugins.dns.version,
-                    "multicast": coresys.plugins.multicast.version,
-                    "cli": coresys.plugins.cli.version,
-                }
-            }
-        )
-        event.setdefault("tags", {}).update(
-            {
-                "installation_type": "os" if coresys.hassos.available else "supervised",
-                "machine": coresys.machine,
-            }
-        )
-
-        return event
 
     # Set log level
     sentry_logging = LoggingIntegration(
@@ -329,7 +293,7 @@ def setup_diagnostics(coresys: CoreSys) -> None:
     _LOGGER.info("Initialize Supervisor Sentry")
     sentry_sdk.init(
         dsn="https://9c6ea70f49234442b4746e447b24747e@o427061.ingest.sentry.io/5370612",
-        before_send=filter_data,
+        before_send=filter_event,
         max_breadcrumbs=30,
         integrations=[AioHttpIntegration(), sentry_logging],
         release=SUPERVISOR_VERSION,

supervisor/misc/filter.py

@@ -0,0 +1,82 @@
+"""Filter tools."""
+import os
+import re
+
+from aiohttp import hdrs
+
+from ..const import ENV_SUPERVISOR_DEV, HEADER_TOKEN_OLD, CoreStates
+from ..coresys import CoreSys
+
+RE_URL: re.Pattern = re.compile(r"(\w+:\/\/)(.*\.\w+)(.*)")
+
+
+def sanitize_url(url: str) -> str:
+    """Return a sanitized url."""
+    if not re.match(RE_URL, url):
+        # Not a URL, just return it back
+        return url
+
+    return re.sub(RE_URL, r"\1example.com\3", url)
+
+
+def filter_data(coresys: CoreSys, event: dict) -> dict:
+    """Filter event data before sending to sentry."""
+    dev_env: bool = bool(os.environ.get(ENV_SUPERVISOR_DEV, 0))
+
+    # Ignore issue if system is not supported or diagnostics is disabled
+    if not coresys.config.diagnostics or not coresys.supported or dev_env:
+        return None
+
+    # Not full startup - missing information
+    if coresys.core.state in (CoreStates.INITIALIZE, CoreStates.SETUP):
+        return event
+
+    # Update information
+    event.setdefault("extra", {}).update(
+        {
+            "supervisor": {
+                "machine": coresys.machine,
+                "arch": coresys.arch.default,
+                "docker": coresys.docker.info.version,
+                "channel": coresys.updater.channel,
+                "supervisor": coresys.supervisor.version,
+                "os": coresys.hassos.version,
+                "host": coresys.host.info.operating_system,
+                "kernel": coresys.host.info.kernel,
+                "core": coresys.homeassistant.version,
+                "audio": coresys.plugins.audio.version,
+                "dns": coresys.plugins.dns.version,
+                "multicast": coresys.plugins.multicast.version,
+                "cli": coresys.plugins.cli.version,
+            }
+        }
+    )
+    event.setdefault("tags", []).extend(
+        [
+            ["installation_type", "os" if coresys.hassos.available else "supervised"],
+            ["machine", coresys.machine],
+        ],
+    )
+
+    # Sanitize event
+    for i, tag in enumerate(event.get("tags", [])):
+        key, value = tag
+        if key == "url":
+            event["tags"][i] = [key, sanitize_url(value)]
+
+    if event.get("request"):
+        if event["request"].get("url"):
+            event["request"]["url"] = sanitize_url(event["request"]["url"])
+
+        for i, header in enumerate(event["request"].get("headers", [])):
+            key, value = header
+            if key == hdrs.REFERER:
+                event["request"]["headers"][i] = [key, sanitize_url(value)]
+
+            if key == HEADER_TOKEN_OLD:
+                event["request"]["headers"][i] = [key, "XXXXXXXXXXXXXXXXXXX"]
+
+            if key in [hdrs.HOST, hdrs.X_FORWARDED_HOST]:
+                event["request"]["headers"][i] = [key, "example.com"]
+    print(event)
+    return event

supervisor/utils/__init__.py

@@ -8,7 +8,8 @@ import socket
 from typing import Optional
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
-RE_STRING = re.compile(r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))")
+
+RE_STRING: re.Pattern = re.compile(r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))")
 
 
 def convert_to_ascii(raw: bytes) -> str:

tests/conftest.py

@@ -5,7 +5,7 @@ import pytest
 
 from supervisor.bootstrap import initialize_coresys
 
-# pylint: disable=redefined-outer-name
+# pylint: disable=redefined-outer-name, protected-access
 
 
 @pytest.fixture
@@ -26,6 +26,7 @@ async def coresys(loop, docker):
         coresys_obj = await initialize_coresys()
 
     coresys_obj.ingress.save_data = MagicMock()
+    coresys_obj.arch._default_arch = "amd64"
 
     yield coresys_obj
 

tests/misc/test_filter_data.py

@@ -0,0 +1,85 @@
+"""Test sentry data filter."""
+from unittest.mock import patch
+
+from supervisor.const import CoreStates
+from supervisor.misc.filter import filter_data
+
+SAMPLE_EVENT = {"sample": "event"}
+
+
+def test_diagnostics_disabled(coresys):
+    """Test if diagnostics is disabled."""
+    coresys.config.diagnostics = False
+    coresys.supported = True
+    assert filter_data(coresys, SAMPLE_EVENT) is None
+
+
+def test_not_supported(coresys):
+    """Test if not supported."""
+    coresys.config.diagnostics = True
+    coresys.supported = False
+    assert filter_data(coresys, SAMPLE_EVENT) is None
+
+
+def test_is_dev(coresys):
+    """Test if dev."""
+    coresys.config.diagnostics = True
+    coresys.supported = True
+    with patch("os.environ", return_value=[("ENV_SUPERVISOR_DEV", "1")]):
+        assert filter_data(coresys, SAMPLE_EVENT) is None
+
+
+def test_not_started(coresys):
+    """Test if supervisor not fully started."""
+    coresys.config.diagnostics = True
+    coresys.supported = True
+
+    coresys.core.state = CoreStates.INITIALIZE
+    assert filter_data(coresys, SAMPLE_EVENT) == SAMPLE_EVENT
+
+    coresys.core.state = CoreStates.SETUP
+    assert filter_data(coresys, SAMPLE_EVENT) == SAMPLE_EVENT
+
+
+def test_defaults(coresys):
+    """Test event defaults."""
+    coresys.config.diagnostics = True
+    coresys.supported = True
+
+    coresys.core.state = CoreStates.RUNNING
+    filtered = filter_data(coresys, SAMPLE_EVENT)
+
+    assert ["installation_type", "supervised"] in filtered["tags"]
+    assert filtered["extra"]["supervisor"]["arch"] == "amd64"
+
+
+def test_sanitize(coresys):
+    """Test event sanitation."""
+    event = {
+        "tags": [["url", "https://mydomain.com"]],
+        "request": {
+            "url": "https://mydomain.com",
+            "headers": [
+                ["Host", "mydomain.com"],
+                ["Referer", "https://mydomain.com/api/hassio_ingress/xxx-xxx/"],
+                ["X-Forwarded-Host", "mydomain.com"],
+                ["X-Hassio-Key", "xxx"],
+            ],
+        },
+    }
+    coresys.config.diagnostics = True
+    coresys.supported = True
+
+    coresys.core.state = CoreStates.RUNNING
+    filtered = filter_data(coresys, event)
+
+    assert ["url", "https://example.com"] in filtered["tags"]
+
+    assert filtered["request"]["url"] == "https://example.com"
+
+    assert ["Host", "example.com"] in filtered["request"]["headers"]
+    assert ["Referer", "https://example.com/api/hassio_ingress/xxx-xxx/"] in filtered[
+        "request"
+    ]["headers"]
+    assert ["X-Forwarded-Host", "example.com"] in filtered["request"]["headers"]
+    assert ["X-Hassio-Key", "XXXXXXXXXXXXXXXXXXX"] in filtered["request"]["headers"]

tests/misc/test_sanitise_url.py

@@ -0,0 +1,9 @@
+"""Test supervisor.utils.sanitize_url."""
+from supervisor.misc.filter import sanitize_url
+
+
+def test_sanitize_url():
+    """Test supervisor.utils.sanitize_url."""
+    assert sanitize_url("test") == "test"
+    assert sanitize_url("http://my.duckdns.org") == "http://example.com"
+    assert sanitize_url("http://my.duckdns.org/test") == "http://example.com/test"