Add enhanced logging REST endpoints using systemd-journal-gatewayd (#3291)

* Add enhanced logging REST endpoints using systemd-journal-gatewayd

Add /host/logs/entries and /host/logs/{identifier}/entries to expose log
entries from systemd-journald running on the host. Use
systemd-journal-gatewayd which exposes the logs to the Supervisor via
Unix socket.

Current two query string parameters are allowed: "boot" and "follow".
The first will only return logs since last boot. The second will keep
the HTTP request open and send new log entries as they get added to the
systemd-journal.

* Allow Range header

Forward the Range header to systemd-journal-gatewayd. This allows to
select only a certain amount of log data. The Range header is a standard
header to select only partial amount of data. However, the "entries="
prefix is custom for systemd-journal-gatewayd, denoting that the numbers
following represent log entries (as opposed to bytes or other metrics).

* Avoid connecting if systemd-journal-gatewayd is not available

* Use path for all options

* Add pytests

* Address pylint issues

* Boot ID offsets and slug to identifier

* Fix tests

* API refactor from feedback

* fix tests and add identifiers

* stop isort and pylint fighting

* fix tests

* Update default log identifiers

* Only modify /host/logs endpoints

* Fix bad import

* Load log caches asynchronously at startup

* Allow task to complete in fixture

* Boot IDs and identifiers loaded on demand

* Add suggested identifiers

* Fix tests around boot ids

Co-authored-by: Mike Degatano <michael.degatano@gmail.com>
This commit is contained in:
Stefan Agner 2022-10-13 17:40:11 +02:00 committed by GitHub
parent 1f7c067c90
commit 2ebb405871
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 819 additions and 45 deletions

View File

@ -1,4 +1,5 @@
"""Init file for Supervisor RESTful API."""
from functools import partial
import logging
from pathlib import Path
from typing import Any
@ -103,16 +104,36 @@ class RestAPI(CoreSysAttributes):
self.webapp.add_routes(
[
web.get("/host/info", api_host.info),
web.get("/host/logs", api_host.logs),
web.get("/host/logs", api_host.advanced_logs),
web.get(
"/host/logs/follow",
partial(api_host.advanced_logs, follow=True),
),
web.get("/host/logs/identifiers", api_host.list_identifiers),
web.get("/host/logs/identifiers/{identifier}", api_host.advanced_logs),
web.get(
"/host/logs/identifiers/{identifier}/follow",
partial(api_host.advanced_logs, follow=True),
),
web.get("/host/logs/boots", api_host.list_boots),
web.get("/host/logs/boots/{bootid}", api_host.advanced_logs),
web.get(
"/host/logs/boots/{bootid}/follow",
partial(api_host.advanced_logs, follow=True),
),
web.get(
"/host/logs/boots/{bootid}/identifiers/{identifier}",
api_host.advanced_logs,
),
web.get(
"/host/logs/boots/{bootid}/identifiers/{identifier}/follow",
partial(api_host.advanced_logs, follow=True),
),
web.post("/host/reboot", api_host.reboot),
web.post("/host/shutdown", api_host.shutdown),
web.post("/host/reload", api_host.reload),
web.post("/host/options", api_host.options),
web.get("/host/services", api_host.services),
web.post("/host/services/{service}/stop", api_host.service_stop),
web.post("/host/services/{service}/start", api_host.service_start),
web.post("/host/services/{service}/restart", api_host.service_restart),
web.post("/host/services/{service}/reload", api_host.service_reload),
]
)
@ -514,6 +535,8 @@ class RestAPI(CoreSysAttributes):
"""Register Audio functions."""
api_audio = APIAudio()
api_audio.coresys = self.coresys
api_host = APIHost()
api_host.coresys = self.coresys
self.webapp.add_routes(
[

View File

@ -1,9 +1,12 @@
"""Init file for Supervisor host RESTful API."""
import asyncio
from typing import Awaitable
from contextlib import suppress
import logging
from aiohttp import web
from aiohttp.hdrs import ACCEPT, RANGE
import voluptuous as vol
from voluptuous.error import CoerceInvalid
from ..const import (
ATTR_CHASSIS,
@ -24,6 +27,8 @@ from ..const import (
ATTR_TIMEZONE,
)
from ..coresys import CoreSysAttributes
from ..exceptions import APIError, HostLogError
from ..host.const import PARAM_BOOT_ID, PARAM_FOLLOW, PARAM_SYSLOG_IDENTIFIER
from .const import (
ATTR_AGENT_VERSION,
ATTR_APPARMOR_VERSION,
@ -35,11 +40,15 @@ from .const import (
ATTR_LLMNR_HOSTNAME,
ATTR_STARTUP_TIME,
ATTR_USE_NTP,
CONTENT_TYPE_BINARY,
CONTENT_TYPE_TEXT,
)
from .utils import api_process, api_process_raw, api_validate
from .utils import api_process, api_validate
SERVICE = "service"
_LOGGER: logging.Logger = logging.getLogger(__name__)
IDENTIFIER = "identifier"
BOOTID = "bootid"
DEFAULT_RANGE = 100
SCHEMA_OPTIONS = vol.Schema({vol.Optional(ATTR_HOSTNAME): str})
@ -117,30 +126,72 @@ class APIHost(CoreSysAttributes):
return {ATTR_SERVICES: services}
@api_process
def service_start(self, request):
"""Start a service."""
unit = request.match_info.get(SERVICE)
return [asyncio.shield(self.sys_host.services.start(unit))]
async def list_boots(self, _: web.Request):
"""Return a list of boot IDs."""
boot_ids = await self.sys_host.logs.get_boot_ids()
return {
str(1 + i - len(boot_ids)): boot_id for i, boot_id in enumerate(boot_ids)
}
@api_process
def service_stop(self, request):
"""Stop a service."""
unit = request.match_info.get(SERVICE)
return [asyncio.shield(self.sys_host.services.stop(unit))]
async def list_identifiers(self, _: web.Request):
"""Return a list of syslog identifiers."""
return await self.sys_host.logs.get_identifiers()
async def _get_boot_id(self, possible_offset: str) -> str:
"""Convert offset into boot ID if required."""
with suppress(CoerceInvalid):
offset = vol.Coerce(int)(possible_offset)
try:
return await self.sys_host.logs.get_boot_id(offset)
except (ValueError, HostLogError) as err:
raise APIError() from err
return possible_offset
@api_process
def service_reload(self, request):
"""Reload a service."""
unit = request.match_info.get(SERVICE)
return [asyncio.shield(self.sys_host.services.reload(unit))]
async def advanced_logs(
self, request: web.Request, identifier: str | None = None, follow: bool = False
) -> web.StreamResponse:
"""Return systemd-journald logs."""
params = {}
if identifier:
params[PARAM_SYSLOG_IDENTIFIER] = identifier
elif IDENTIFIER in request.match_info:
params[PARAM_SYSLOG_IDENTIFIER] = request.match_info.get(IDENTIFIER)
else:
params[PARAM_SYSLOG_IDENTIFIER] = self.sys_host.logs.default_identifiers
@api_process
def service_restart(self, request):
"""Restart a service."""
unit = request.match_info.get(SERVICE)
return [asyncio.shield(self.sys_host.services.restart(unit))]
if BOOTID in request.match_info:
params[PARAM_BOOT_ID] = await self._get_boot_id(
request.match_info.get(BOOTID)
)
if follow:
params[PARAM_FOLLOW] = ""
@api_process_raw(CONTENT_TYPE_BINARY)
def logs(self, request: web.Request) -> Awaitable[bytes]:
"""Return host kernel logs."""
return self.sys_host.info.get_dmesg()
if ACCEPT in request.headers and request.headers[ACCEPT] not in [
CONTENT_TYPE_TEXT,
"*/*",
]:
raise APIError(
"Invalid content type requested. Only text/plain supported for now."
)
if RANGE in request.headers:
range_header = request.headers.get(RANGE)
else:
range_header = f"entries=:-{DEFAULT_RANGE}:"
async with self.sys_host.logs.journald_logs(
params=params, range_header=range_header
) as resp:
try:
response = web.StreamResponse()
response.content_type = CONTENT_TYPE_TEXT
await response.prepare(request)
async for data in resp.content:
await response.write(data)
except ConnectionResetError as ex:
raise APIError(
"Connection reset when trying to fetch data from systemd-journald."
) from ex
return response

View File

@ -65,6 +65,8 @@ def api_process(method):
return api_return_ok(data=answer)
if isinstance(answer, web.Response):
return answer
if isinstance(answer, web.StreamResponse):
return answer
elif isinstance(answer, bool) and not answer:
return api_return_error()
return api_return_ok()

View File

@ -0,0 +1,36 @@
[
"NetworkManager",
"bluetoothd",
"bthelper",
"btuart",
"dbus-broker",
"dockerd",
"dropbear",
"fstrim",
"hassos-apparmor",
"hassos-config",
"hassos-expand",
"hassos-overlay",
"hassos-persists",
"hassos-supervisor",
"hassos-zram",
"kernel",
"os-agent",
"rauc",
"systemd",
"systemd-coredump",
"systemd-fsck",
"systemd-growfs",
"systemd-hostnamed",
"systemd-journal-gatewayd",
"systemd-journald",
"systemd-logind",
"systemd-random-seed",
"systemd-resolved",
"systemd-sysctl",
"systemd-time-wait-sync",
"systemd-timesyncd",
"systemd-tmpfiles",
"systemd-udevd",
"wpa_supplicant"
]

View File

@ -72,6 +72,11 @@ class DockerAddon(DockerInterface):
self._hw_listener: EventListener | None = None
@staticmethod
def slug_to_name(slug: str) -> str:
"""Convert slug to container name."""
return f"addon_{slug}"
@property
def image(self) -> str | None:
"""Return name of Docker image."""
@ -111,7 +116,7 @@ class DockerAddon(DockerInterface):
@property
def name(self) -> str:
"""Return name of Docker container."""
return f"addon_{self.addon.slug}"
return DockerAddon.slug_to_name(self.addon.slug)
@property
def environment(self) -> dict[str, str | None]:

View File

@ -274,6 +274,10 @@ class HostNetworkNotFound(HostError):
"""Return if host interface is not found."""
class HostLogError(HostError):
"""Internal error with host log."""
# API

View File

@ -1,6 +1,10 @@
"""Const for host."""
from enum import Enum
PARAM_BOOT_ID = "_BOOT_ID"
PARAM_FOLLOW = "follow"
PARAM_SYSLOG_IDENTIFIER = "SYSLOG_IDENTIFIER"
class InterfaceMethod(str, Enum):
"""Configuration of an interface."""
@ -47,3 +51,12 @@ class HostFeature(str, Enum):
SHUTDOWN = "shutdown"
OS_AGENT = "os_agent"
TIMEDATE = "timedate"
JOURNAL = "journal"
class LogFormat(str, Enum):
"""Log format."""
JOURNAL = "application/vnd.fdo.journal"
JSON = "application/json"
TEXT = "text/plain"

160
supervisor/host/logs.py Normal file
View File

@ -0,0 +1,160 @@
"""Logs control for host."""
from __future__ import annotations
from contextlib import asynccontextmanager
import json
import logging
from pathlib import Path
from aiohttp import ClientError, ClientSession, ClientTimeout
from aiohttp.client_reqrep import ClientResponse
from aiohttp.connector import UnixConnector
from aiohttp.hdrs import ACCEPT, RANGE
from ..coresys import CoreSys, CoreSysAttributes
from ..exceptions import ConfigurationFileError, HostLogError, HostNotSupportedError
from ..utils.json import read_json_file
from .const import PARAM_BOOT_ID, PARAM_SYSLOG_IDENTIFIER, LogFormat
_LOGGER: logging.Logger = logging.getLogger(__name__)
# pylint: disable=no-member
SYSLOG_IDENTIFIERS_JSON: Path = (
Path(__file__).parents[1].joinpath("data/syslog-identifiers.json")
)
# pylint: enable=no-member
SYSTEMD_JOURNAL_GATEWAYD_SOCKET: Path = Path("/run/systemd-journal-gatewayd.sock")
# From systemd catalog for message IDs (`journalctl --dump-catalog``)
# -- b07a249cd024414a82dd00cd181378ff
# Subject: System start-up is now complete
# Defined-By: systemd
BOOT_IDS_QUERY = {"MESSAGE_ID": "b07a249cd024414a82dd00cd181378ff"}
class LogsControl(CoreSysAttributes):
"""Handle systemd-journal logs."""
def __init__(self, coresys: CoreSys):
"""Initialize host power handling."""
self.coresys: CoreSys = coresys
self._profiles: set[str] = set()
self._boot_ids: list[str] = []
self._identifiers: list[str] = []
self._default_identifiers: list[str] = []
@property
def available(self) -> bool:
"""Return True if Unix socket to systemd-journal-gatwayd is available."""
return SYSTEMD_JOURNAL_GATEWAYD_SOCKET.is_socket()
@property
def boot_ids(self) -> list[str]:
"""Get boot IDs from oldest to newest."""
return self._boot_ids
@property
def identifiers(self) -> list[str]:
"""Get syslog identifiers."""
return self._identifiers
@property
def default_identifiers(self) -> list[str]:
"""Get default syslog identifiers."""
return self._default_identifiers
async def load(self) -> None:
"""Load log control."""
try:
self._default_identifiers = read_json_file(SYSLOG_IDENTIFIERS_JSON)
except ConfigurationFileError:
_LOGGER.warning(
"Can't read syslog identifiers json file from %s",
SYSLOG_IDENTIFIERS_JSON,
)
async def get_boot_id(self, offset: int = 0) -> str:
"""
Get ID of a boot by offset.
Current boot is offset = 0, negative numbers go that many in the past.
Positive numbers count up from the oldest boot.
"""
boot_ids = await self.get_boot_ids()
offset -= 1
if offset >= len(boot_ids) or abs(offset) > len(boot_ids):
raise ValueError(f"Logs only contain {len(boot_ids)} boots")
return boot_ids[offset]
async def get_boot_ids(self) -> list[str]:
"""Get boot IDs from oldest to newest."""
if self._boot_ids:
# Doesn't change without a reboot, no reason to query again once cached
return self._boot_ids
try:
async with self.journald_logs(
params=BOOT_IDS_QUERY,
accept=LogFormat.JSON,
timeout=ClientTimeout(total=20),
) as resp:
text = await resp.text()
self._boot_ids = [
json.loads(entry)[PARAM_BOOT_ID]
for entry in text.split("\n")
if entry
]
return self._boot_ids
except (ClientError, TimeoutError) as err:
raise HostLogError(
"Could not get a list of boot IDs from systemd-journal-gatewayd",
_LOGGER.error,
) from err
async def get_identifiers(self) -> list[str]:
"""Get syslog identifiers."""
try:
async with self.journald_logs(
path=f"/fields/{PARAM_SYSLOG_IDENTIFIER}",
timeout=ClientTimeout(total=20),
) as resp:
return (await resp.text()).split("\n")
except (ClientError, TimeoutError) as err:
raise HostLogError(
"Could not get a list of syslog identifiers from systemd-journal-gatewayd",
_LOGGER.error,
) from err
@asynccontextmanager
async def journald_logs(
self,
path: str = "/entries",
params: dict[str, str | list[str]] | None = None,
range_header: str | None = None,
accept: LogFormat = LogFormat.TEXT,
timeout: ClientTimeout | None = None,
) -> ClientResponse:
"""Get logs from systemd-journal-gatewayd.
See https://www.freedesktop.org/software/systemd/man/systemd-journal-gatewayd.service.html for params and more info.
"""
if not self.available:
raise HostNotSupportedError(
"No systemd-journal-gatewayd Unix socket available", _LOGGER.error
)
async with ClientSession(
connector=UnixConnector(path="/run/systemd-journal-gatewayd.sock")
) as session:
headers = {ACCEPT: accept.value}
if range_header:
headers[RANGE] = range_header
async with session.get(
f"http://localhost{path}",
headers=headers,
params=params or {},
timeout=timeout,
) as client_response:
yield client_response

View File

@ -3,9 +3,11 @@ from contextlib import suppress
from functools import lru_cache
import logging
from supervisor.host.logs import LogsControl
from ..const import BusEvent
from ..coresys import CoreSys, CoreSysAttributes
from ..exceptions import HassioError, PulseAudioError
from ..exceptions import HassioError, HostLogError, PulseAudioError
from ..hardware.const import PolicyGroup
from ..hardware.data import Device
from .apparmor import AppArmorControl
@ -32,6 +34,7 @@ class HostManager(CoreSysAttributes):
self._services: ServiceManager = ServiceManager(coresys)
self._network: NetworkManager = NetworkManager(coresys)
self._sound: SoundControl = SoundControl(coresys)
self._logs: LogsControl = LogsControl(coresys)
@property
def apparmor(self) -> AppArmorControl:
@ -63,6 +66,11 @@ class HostManager(CoreSysAttributes):
"""Return host PulseAudio control."""
return self._sound
@property
def logs(self) -> LogsControl:
"""Return host logs handler."""
return self._logs
@property
def features(self) -> list[HostFeature]:
"""Return a list of host features."""
@ -96,6 +104,9 @@ class HostManager(CoreSysAttributes):
if self.sys_dbus.resolved.is_connected:
features.append(HostFeature.RESOLVED)
if self.logs.available:
features.append(HostFeature.JOURNAL)
return features
async def reload(self):
@ -126,6 +137,9 @@ class HostManager(CoreSysAttributes):
with suppress(PulseAudioError):
await self.sound.update()
with suppress(HostLogError):
await self.logs.load()
await self.network.load()
# Register for events

View File

@ -1,4 +1,5 @@
"""Test API security layer."""
from unittest.mock import patch
from aiohttp import web
import pytest
@ -15,7 +16,9 @@ async def api_system(aiohttp_client, run_dir, coresys: CoreSys):
"""Fixture for RestAPI client."""
api = RestAPI(coresys)
api.webapp = web.Application()
await api.load()
with patch("supervisor.docker.supervisor.os") as os:
os.environ = {"SUPERVISOR_NAME": "hassio_supervisor"}
await api.load()
api.webapp.middlewares.append(api.security.system_validation)
yield await aiohttp_client(api.webapp)

View File

@ -1,5 +1,9 @@
"""Test addons api."""
from unittest.mock import MagicMock
from aiohttp.test_utils import TestClient
from supervisor.addons.addon import Addon
from supervisor.const import AddonState
from supervisor.coresys import CoreSys
@ -8,7 +12,9 @@ from supervisor.store.repository import Repository
from ..const import TEST_ADDON_SLUG
async def test_addons_info(api_client, coresys: CoreSys, install_addon_ssh: Addon):
async def test_addons_info(
api_client: TestClient, coresys: CoreSys, install_addon_ssh: Addon
):
"""Test getting addon info."""
install_addon_ssh.state = AddonState.STOPPED
install_addon_ssh.ingress_panel = True
@ -27,7 +33,7 @@ async def test_addons_info(api_client, coresys: CoreSys, install_addon_ssh: Addo
# DEPRECATED - Remove with legacy routing logic on 1/2023
async def test_addons_info_not_installed(
api_client, coresys: CoreSys, repository: Repository
api_client: TestClient, coresys: CoreSys, repository: Repository
):
"""Test getting addon info for not installed addon."""
resp = await api_client.get(f"/addons/{TEST_ADDON_SLUG}/info")
@ -42,3 +48,17 @@ async def test_addons_info_not_installed(
"password": "",
"server": {"tcp_forwarding": False},
}
async def test_api_addon_logs(
api_client: TestClient, docker_logs: MagicMock, install_addon_ssh: Addon
):
"""Test addon logs."""
resp = await api_client.get("/addons/local_ssh/logs")
assert resp.status == 200
assert resp.content_type == "application/octet-stream"
content = await resp.text()
assert content.split("\n")[0:2] == [
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os\x1b[0m",
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/AppArmor\x1b[0m",
]

17
tests/api/test_audio.py Normal file
View File

@ -0,0 +1,17 @@
"""Test audio api."""
from unittest.mock import MagicMock
from aiohttp.test_utils import TestClient
async def test_api_audio_logs(api_client: TestClient, docker_logs: MagicMock):
"""Test audio logs."""
resp = await api_client.get("/audio/logs")
assert resp.status == 200
assert resp.content_type == "application/octet-stream"
content = await resp.text()
assert content.split("\n")[0:2] == [
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os\x1b[0m",
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/AppArmor\x1b[0m",
]

View File

@ -1,11 +1,15 @@
"""Test DNS API."""
from unittest.mock import PropertyMock, patch
from unittest.mock import MagicMock, PropertyMock, patch
from aiohttp.test_utils import TestClient
from supervisor.coresys import CoreSys
from supervisor.dbus.const import MulticastProtocolEnabled
async def test_llmnr_mdns_info(api_client, coresys: CoreSys, dbus_is_connected):
async def test_llmnr_mdns_info(
api_client: TestClient, coresys: CoreSys, dbus_is_connected: PropertyMock
):
"""Test llmnr and mdns in info api."""
coresys.host.sys_dbus.resolved.is_connected = False
@ -38,7 +42,7 @@ async def test_llmnr_mdns_info(api_client, coresys: CoreSys, dbus_is_connected):
assert result["data"]["mdns"] is True
async def test_options(api_client, coresys: CoreSys):
async def test_options(api_client: TestClient, coresys: CoreSys):
"""Test options api."""
assert coresys.plugins.dns.servers == []
assert coresys.plugins.dns.fallback is True
@ -58,3 +62,15 @@ async def test_options(api_client, coresys: CoreSys):
assert coresys.plugins.dns.servers == ["dns://8.8.8.8"]
assert coresys.plugins.dns.fallback is True
restart.assert_called_once()
async def test_api_dns_logs(api_client: TestClient, docker_logs: MagicMock):
"""Test dns logs."""
resp = await api_client.get("/dns/logs")
assert resp.status == 200
assert resp.content_type == "application/octet-stream"
content = await resp.text()
assert content.split("\n")[0:2] == [
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os\x1b[0m",
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/AppArmor\x1b[0m",
]

View File

@ -0,0 +1,21 @@
"""Test homeassistant api."""
from unittest.mock import MagicMock
from aiohttp.test_utils import TestClient
import pytest
@pytest.mark.parametrize("legacy_route", [True, False])
async def test_api_core_logs(
api_client: TestClient, docker_logs: MagicMock, legacy_route: bool
):
"""Test core logs."""
resp = await api_client.get(f"/{'homeassistant' if legacy_route else 'core'}/logs")
assert resp.status == 200
assert resp.content_type == "application/octet-stream"
content = await resp.text()
assert content.split("\n")[0:2] == [
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os\x1b[0m",
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/AppArmor\x1b[0m",
]

View File

@ -1,9 +1,13 @@
"""Test Host API."""
from unittest.mock import MagicMock
from aiohttp.test_utils import TestClient
import pytest
from supervisor.coresys import CoreSys
DEFAULT_RANGE = "entries=:-100:"
# pylint: disable=protected-access
@ -19,7 +23,7 @@ async def fixture_coresys_disk_info(coresys: CoreSys) -> CoreSys:
@pytest.mark.asyncio
async def test_api_host_info(api_client, coresys_disk_info: CoreSys):
async def test_api_host_info(api_client: TestClient, coresys_disk_info: CoreSys):
"""Test host info api."""
coresys = coresys_disk_info
@ -33,7 +37,7 @@ async def test_api_host_info(api_client, coresys_disk_info: CoreSys):
async def test_api_host_features(
api_client, coresys_disk_info: CoreSys, dbus_is_connected
api_client: TestClient, coresys_disk_info: CoreSys, dbus_is_connected
):
"""Test host info features."""
coresys = coresys_disk_info
@ -96,7 +100,7 @@ async def test_api_host_features(
async def test_api_llmnr_mdns_info(
api_client, coresys_disk_info: CoreSys, dbus_is_connected
api_client: TestClient, coresys_disk_info: CoreSys, dbus_is_connected
):
"""Test llmnr and mdns details in info."""
coresys = coresys_disk_info
@ -118,3 +122,132 @@ async def test_api_llmnr_mdns_info(
assert result["data"]["broadcast_llmnr"] is True
assert result["data"]["broadcast_mdns"] is False
assert result["data"]["llmnr_hostname"] == "homeassistant"
async def test_api_boot_ids_info(api_client: TestClient, journald_logs: MagicMock):
"""Test getting boot IDs."""
resp = await api_client.get("/host/logs/boots")
result = await resp.json()
assert result["data"] == {"0": "ccc", "-1": "bbb", "-2": "aaa"}
async def test_api_identifiers_info(api_client: TestClient, journald_logs: MagicMock):
"""Test getting syslog identifiers."""
resp = await api_client.get("/host/logs/identifiers")
result = await resp.json()
assert result["data"] == ["hassio_supervisor", "hassos-config", "kernel"]
async def test_advanced_logs(
api_client: TestClient, coresys: CoreSys, journald_logs: MagicMock
):
"""Test advanced logging API entries with identifier and custom boot."""
await api_client.get("/host/logs")
journald_logs.assert_called_once_with(
params={"SYSLOG_IDENTIFIER": coresys.host.logs.default_identifiers},
range_header=DEFAULT_RANGE,
)
journald_logs.reset_mock()
identifier = "dropbear"
await api_client.get(f"/host/logs/identifiers/{identifier}")
journald_logs.assert_called_once_with(
params={"SYSLOG_IDENTIFIER": identifier}, range_header=DEFAULT_RANGE
)
journald_logs.reset_mock()
bootid = "798cc03bcd77465482b6a1c43dc6a5fc"
await api_client.get(f"/host/logs/boots/{bootid}")
journald_logs.assert_called_once_with(
params={
"_BOOT_ID": bootid,
"SYSLOG_IDENTIFIER": coresys.host.logs.default_identifiers,
},
range_header=DEFAULT_RANGE,
)
journald_logs.reset_mock()
await api_client.get(f"/host/logs/boots/{bootid}/identifiers/{identifier}")
journald_logs.assert_called_once_with(
params={"_BOOT_ID": bootid, "SYSLOG_IDENTIFIER": identifier},
range_header=DEFAULT_RANGE,
)
journald_logs.reset_mock()
headers = {"Range": "entries=:-19:10"}
await api_client.get("/host/logs", headers=headers)
journald_logs.assert_called_once_with(
params={"SYSLOG_IDENTIFIER": coresys.host.logs.default_identifiers},
range_header=headers["Range"],
)
journald_logs.reset_mock()
await api_client.get("/host/logs/follow")
journald_logs.assert_called_once_with(
params={
"SYSLOG_IDENTIFIER": coresys.host.logs.default_identifiers,
"follow": "",
},
range_header=DEFAULT_RANGE,
)
async def test_advanced_logs_boot_id_offset(
api_client: TestClient, coresys: CoreSys, journald_logs: MagicMock
):
"""Test advanced logging API when using an offset as boot ID."""
await api_client.get("/host/logs/boots/0")
journald_logs.assert_called_once_with(
params={
"_BOOT_ID": "ccc",
"SYSLOG_IDENTIFIER": coresys.host.logs.default_identifiers,
},
range_header=DEFAULT_RANGE,
)
journald_logs.reset_mock()
await api_client.get("/host/logs/boots/-2")
journald_logs.assert_called_once_with(
params={
"_BOOT_ID": "aaa",
"SYSLOG_IDENTIFIER": coresys.host.logs.default_identifiers,
},
range_header=DEFAULT_RANGE,
)
journald_logs.reset_mock()
await api_client.get("/host/logs/boots/2")
journald_logs.assert_called_once_with(
params={
"_BOOT_ID": "bbb",
"SYSLOG_IDENTIFIER": coresys.host.logs.default_identifiers,
},
range_header=DEFAULT_RANGE,
)
journald_logs.reset_mock()
async def test_advanced_logs_errors(api_client: TestClient):
"""Test advanced logging API errors."""
# coresys = coresys_logs_control
resp = await api_client.get("/host/logs")
result = await resp.json()
assert result["result"] == "error"
assert result["message"] == "No systemd-journal-gatewayd Unix socket available"
headers = {"Accept": "application/json"}
resp = await api_client.get("/host/logs", headers=headers)
result = await resp.json()
assert result["result"] == "error"
assert (
result["message"]
== "Invalid content type requested. Only text/plain supported for now."
)

View File

@ -0,0 +1,17 @@
"""Test multicast api."""
from unittest.mock import MagicMock
from aiohttp.test_utils import TestClient
async def test_api_multicast_logs(api_client: TestClient, docker_logs: MagicMock):
"""Test multicast logs."""
resp = await api_client.get("/multicast/logs")
assert resp.status == 200
assert resp.content_type == "application/octet-stream"
content = await resp.text()
assert content.split("\n")[0:2] == [
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os\x1b[0m",
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/AppArmor\x1b[0m",
]

View File

@ -1,7 +1,7 @@
"""Test Supervisor API."""
# pylint: disable=protected-access
import asyncio
from unittest.mock import patch
from unittest.mock import MagicMock, patch
from aiohttp.test_utils import TestClient
import pytest
@ -129,3 +129,15 @@ async def test_api_supervisor_options_diagnostics(
assert response.status == 200
assert dbus == ["/io/hass/os-io.hass.os.Diagnostics"]
async def test_api_supervisor_logs(api_client: TestClient, docker_logs: MagicMock):
"""Test supervisor logs."""
resp = await api_client.get("/supervisor/logs")
assert resp.status == 200
assert resp.content_type == "application/octet-stream"
content = await resp.text()
assert content.split("\n")[0:2] == [
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os\x1b[0m",
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/AppArmor\x1b[0m",
]

View File

@ -77,6 +77,12 @@ def load_fixture(filename: str) -> str:
return path.read_text(encoding="utf-8")
def load_binary_fixture(filename: str) -> bytes:
"""Load a fixture without decoding."""
path = Path(Path(__file__).parent.joinpath("fixtures"), filename)
return path.read_bytes()
def exists_fixture(filename: str) -> bool:
"""Check if a fixture exists."""
path = Path(Path(__file__).parent.joinpath("fixtures"), filename)

View File

@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch
from uuid import uuid4
from aiohttp import web
from aiohttp.test_utils import TestClient
from awesomeversion import AwesomeVersion
from dbus_fast.aio.message_bus import MessageBus
from dbus_fast.aio.proxy_object import ProxyInterface, ProxyObject
@ -50,12 +51,19 @@ from supervisor.dbus.systemd import Systemd
from supervisor.dbus.timedate import TimeDate
from supervisor.docker.manager import DockerAPI
from supervisor.docker.monitor import DockerMonitor
from supervisor.host.logs import LogsControl
from supervisor.store.addon import AddonStore
from supervisor.store.repository import Repository
from supervisor.utils.dbus import DBUS_INTERFACE_PROPERTIES, DBus
from supervisor.utils.dt import utcnow
from .common import exists_fixture, get_dbus_name, load_fixture, load_json_fixture
from .common import (
exists_fixture,
get_dbus_name,
load_binary_fixture,
load_fixture,
load_json_fixture,
)
from .const import TEST_ADDON_SLUG
# pylint: disable=redefined-outer-name, protected-access
@ -358,6 +366,18 @@ async def coresys(
await coresys_obj.websession.close()
@pytest.fixture
async def journald_gateway() -> MagicMock:
"""Mock logs control."""
with patch("supervisor.host.logs.Path.is_socket", return_value=True), patch(
"supervisor.host.logs.ClientSession.get"
) as get:
get.return_value.__aenter__.return_value.text = AsyncMock(
return_value=load_fixture("logs_host.txt")
)
yield get
@pytest.fixture
def sys_machine():
"""Mock sys_machine."""
@ -376,7 +396,7 @@ def sys_supervisor():
@pytest.fixture
async def api_client(aiohttp_client, coresys: CoreSys):
async def api_client(aiohttp_client, coresys: CoreSys) -> TestClient:
"""Fixture for RestAPI client."""
@web.middleware
@ -388,7 +408,9 @@ async def api_client(aiohttp_client, coresys: CoreSys):
api = RestAPI(coresys)
api.webapp = web.Application(middlewares=[_security_middleware])
api.start = AsyncMock()
await api.load()
with patch("supervisor.docker.supervisor.os") as os:
os.environ = {"SUPERVISOR_NAME": "hassio_supervisor"}
await api.load()
yield await aiohttp_client(api.webapp)
@ -528,3 +550,34 @@ async def backups(
coresys.backups._backups[backup.slug] = backup
yield coresys.backups.list_backups
@pytest.fixture
async def journald_logs(coresys: CoreSys) -> MagicMock:
"""Mock journald logs and make it available."""
with patch.object(
LogsControl, "available", new=PropertyMock(return_value=True)
), patch.object(
LogsControl, "get_boot_ids", return_value=["aaa", "bbb", "ccc"]
), patch.object(
LogsControl,
"get_identifiers",
return_value=["hassio_supervisor", "hassos-config", "kernel"],
), patch.object(
LogsControl, "journald_logs", new=MagicMock()
) as logs:
await coresys.host.logs.load()
yield logs
@pytest.fixture
async def docker_logs(docker: DockerAPI) -> MagicMock:
"""Mock log output for a container from docker."""
container_mock = MagicMock()
container_mock.logs.return_value = load_binary_fixture("logs_docker_container.txt")
docker.containers.get.return_value = container_mock
with patch("supervisor.docker.supervisor.os") as os:
os.environ = {"SUPERVISOR_NAME": "hassio_supervisor"}
yield container_mock.logs

2
tests/fixtures/logs_boot_ids.txt vendored Normal file
View File

@ -0,0 +1,2 @@
{"_GID":"0","_HOSTNAME":"homeassistant","_UID":"0","__CURSOR":"s=fe3fc256ac7f4d7e9f4abbfa9f5457a6;i=3753c53;b=b2aca10d5ca54fb1b6fb35c85a0efca9;m=5cdaab7;t=5e76f13d3ccbd;x=30603973f0648b80","MESSAGE_ID":"b07a249cd024414a82dd00cd181378ff","SYSLOG_IDENTIFIER":"systemd","CODE_FILE":"src/core/manager.c","_CMDLINE":"/sbin/init","TID":"1","_COMM":"systemd","MESSAGE":"Startup finished in 1.907s (kernel) + 1min 35.456s (userspace) = 1min 37.364s.","_TRANSPORT":"journal","_CAP_EFFECTIVE":"1ffffffffff","CODE_FUNC":"manager_notify_finished","_SYSTEMD_UNIT":"init.scope","KERNEL_USEC":"1907954","PRIORITY":"6","_SYSTEMD_CGROUP":"/init.scope","SYSLOG_FACILITY":"3","USERSPACE_USEC":"95456563","_SOURCE_REALTIME_TIMESTAMP":"1661839143586929","_PID":"1","_EXE":"/usr/lib/systemd/systemd","__REALTIME_TIMESTAMP":"1661839143587005","CODE_LINE":"3450","_SYSTEMD_SLICE":"-.slice","_MACHINE_ID":"41669ad30bf148ad842fdc89597b3c04","_BOOT_ID":"b2aca10d5ca54fb1b6fb35c85a0efca9","__MONOTONIC_TIMESTAMP":"97364663"}
{"_GID":"0","CODE_FILE":"src/core/manager.c","MESSAGE_ID":"b07a249cd024414a82dd00cd181378ff","_SYSTEMD_SLICE":"-.slice","SYSLOG_IDENTIFIER":"systemd","PRIORITY":"6","_SYSTEMD_CGROUP":"/init.scope","TID":"1","_HOSTNAME":"homeassistant","CODE_FUNC":"manager_notify_finished","KERNEL_USEC":"1899908","__MONOTONIC_TIMESTAMP":"97940408","_SOURCE_REALTIME_TIMESTAMP":"1663424928174145","_BOOT_ID":"b1c386a144fd44db8f855d7e907256f8","_TRANSPORT":"journal","__CURSOR":"s=fe3fc256ac7f4d7e9f4abbfa9f5457a6;i=390f660;b=b1c386a144fd44db8f855d7e907256f8;m=5d673b8;t=5e8e04bf4505d;x=e6c39dc6b167e68","SYSLOG_FACILITY":"3","_EXE":"/usr/lib/systemd/systemd","USERSPACE_USEC":"96040432","__REALTIME_TIMESTAMP":"1663424928174173","_MACHINE_ID":"41669ad30bf148ad842fdc89597b3c04","_COMM":"systemd","_SYSTEMD_UNIT":"init.scope","CODE_LINE":"3450","MESSAGE":"Startup finished in 1.899s (kernel) + 1min 36.040s (userspace) = 1min 37.940s.","_PID":"1","_UID":"0","_CAP_EFFECTIVE":"1ffffffffff","_CMDLINE":"/sbin/init"}

View File

@ -0,0 +1,19 @@
22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os
22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/AppArmor
22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/DataDisk
22-10-11 14:04:23 INFO (MainThread) [supervisor.host.sound] Updating PulseAudio information
22-10-11 14:04:23 INFO (MainThread) [supervisor.host.manager] Host information reload completed
22-10-11 14:06:59 DEBUG (MainThread) [supervisor.api.middleware.security] Passthrough /supervisor/ping
22-10-11 14:07:19 DEBUG (MainThread) [supervisor.api.middleware.security] /info access from Home Assistant
22-10-11 14:07:19 DEBUG (MainThread) [supervisor.api.middleware.security] /host/info access from Home Assistant
22-10-11 14:07:19 DEBUG (MainThread) [supervisor.api.middleware.security] /store access from Home Assistant
22-10-11 14:07:19 DEBUG (MainThread) [supervisor.api.middleware.security] /core/info access from Home Assistant
22-10-11 14:07:19 DEBUG (MainThread) [supervisor.api.middleware.security] /supervisor/info access from Home Assistant
22-10-11 14:07:19 DEBUG (MainThread) [supervisor.api.middleware.security] /os/info access from Home Assistant
22-10-11 14:07:45 DEBUG (MainThread) [supervisor.api.middleware.security] /info access from Home Assistant
22-10-11 14:07:45 DEBUG (MainThread) [supervisor.api.middleware.security] /core/info access from Home Assistant
22-10-11 14:07:45 DEBUG (MainThread) [supervisor.api.middleware.security] /supervisor/info access from Home Assistant
22-10-11 14:07:45 DEBUG (MainThread) [supervisor.api.middleware.security] /os/info access from Home Assistant
22-10-11 14:07:45 DEBUG (MainThread) [supervisor.api.middleware.security] /addons/a0d7b954_vscode/stats access from Home Assistant
22-10-11 14:07:46 DEBUG (MainThread) [supervisor.api.middleware.security] /addons/a0d7b954_vscode/changelog access from Home Assistant
22-10-11 14:07:46 DEBUG (MainThread) [supervisor.api.middleware.security] /addons/a0d7b954_vscode/info access from Home Assistant

7
tests/fixtures/logs_host.txt vendored Normal file
View File

@ -0,0 +1,7 @@
Oct 11 20:46:22 odroid-dev systemd[1]: Started Hostname Service.
Oct 11 20:46:22 odroid-dev kernel: audit: type=1334 audit(1665521182.350:257): prog-id=77 op=LOAD
Oct 11 20:46:22 odroid-dev systemd[1]: Started Journal Gateway Service.
Oct 11 20:46:22 odroid-dev systemd-timesyncd[1126]: Network configuration changed, trying to establish connection.
Oct 11 20:46:22 odroid-dev systemd-timesyncd[1126]: Network configuration changed, trying to establish connection.
Oct 11 20:46:22 odroid-dev systemd-timesyncd[1126]: Initial synchronization to time server 192.168.12.5:123 (192.168.12.5).
Oct 11 20:46:22 odroid-dev systemd-journal-gatewayd[4054]: microhttpd: MHD_OPTION_EXTERNAL_LOGGER is not the first option specified for the daemon. Some messages may be printed by the standard MHD logger.

50
tests/fixtures/logs_identifiers.txt vendored Normal file
View File

@ -0,0 +1,50 @@
systemd-timesyncd
dbus-daemon
ghcr.io/home-assistant/aarch64-hassio-dns:2022.04.1/hassio_dns
systemd
systemd-udevd
dockerd
ghcr.io/home-assistant/aarch64-hassio-supervisor:latest/hassio_supervisor
NetworkManager
audit
systemd-journald
kernel
login
systemd-hostnamed
hassos-config
wpa_supplicant
hassos-persists
systemd-tmpfiles
systemd-random-seed
systemd-growfs
hassos-expand
systemd-fsck
dbus-broker
rngd
systemd-logind
ghcr.io/home-assistant/aarch64-hassio-multicast:2022.02.0/hassio_multicast
ghcr.io/home-assistant/aarch64-hassio-cli:2022.08.dev1602/hassio_cli
os-agent
hassos-supervisor
docker
dropbear
systemd-time-wait-sync
systemd-resolved
fstrim
systemd-journal-gatewayd
homeassistant
hassio_multicast
hassio_audio
hassio_dns
hassio_cli
hassio_supervisor
hassio_observer
nm-dispatcher
addon_94cfad5a_example
ghcr.io/home-assistant/odroid-n2-homeassistant:2022.9.0.dev20220830/homeassistant
rauc
ghcr.io/home-assistant/odroid-n2-homeassistant:2022.10.0.dev20220919/homeassistant
ghcr.io/home-assistant/aarch64-hassio-observer:2022.09.dev1101/hassio_observer
ghcr.io/home-assistant/aarch64-addon-example:1.2.0/addon_94cfad5a_example
ghcr.io/home-assistant/aarch64-hassio-audio:2022.05.0/hassio_audio
addon_local_ssh

89
tests/host/test_logs.py Normal file
View File

@ -0,0 +1,89 @@
"""Test host logs control."""
from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch
import pytest
from supervisor.coresys import CoreSys
from supervisor.exceptions import HostNotSupportedError
from supervisor.host.logs import LogsControl
from tests.common import load_fixture
TEST_BOOT_IDS = [
"b2aca10d5ca54fb1b6fb35c85a0efca9",
"b1c386a144fd44db8f855d7e907256f8",
]
async def test_load(coresys: CoreSys):
"""Test load."""
assert coresys.host.logs.default_identifiers == []
await coresys.host.logs.load()
assert coresys.host.logs.boot_ids == []
assert coresys.host.logs.identifiers == []
# File is quite large so just check it loaded
for identifier in ["kernel", "os-agent", "systemd"]:
assert identifier in coresys.host.logs.default_identifiers
async def test_logs(coresys: CoreSys, journald_gateway: MagicMock):
"""Test getting logs and errors."""
assert coresys.host.logs.available is True
async with coresys.host.logs.journald_logs() as resp:
body = await resp.text()
assert (
"Oct 11 20:46:22 odroid-dev systemd[1]: Started Hostname Service." in body
)
with patch.object(
LogsControl, "available", new=PropertyMock(return_value=False)
), pytest.raises(HostNotSupportedError):
async with coresys.host.logs.journald_logs():
pass
async def test_boot_ids(coresys: CoreSys, journald_gateway: MagicMock):
"""Test getting boot ids."""
journald_gateway.return_value.__aenter__.return_value.text = AsyncMock(
return_value=load_fixture("logs_boot_ids.txt")
)
assert TEST_BOOT_IDS == await coresys.host.logs.get_boot_ids()
# Boot ID query should not be run again, mock a failure for it to ensure
journald_gateway.side_effect = TimeoutError()
assert TEST_BOOT_IDS == await coresys.host.logs.get_boot_ids()
assert "b1c386a144fd44db8f855d7e907256f8" == await coresys.host.logs.get_boot_id(0)
# -1 is previous boot. We have 2 boots so -2 is too far
assert "b2aca10d5ca54fb1b6fb35c85a0efca9" == await coresys.host.logs.get_boot_id(-1)
with pytest.raises(ValueError):
await coresys.host.logs.get_boot_id(-2)
# 1 is oldest boot and count up from there. We have 2 boots so 3 is too far
assert "b2aca10d5ca54fb1b6fb35c85a0efca9" == await coresys.host.logs.get_boot_id(1)
assert "b1c386a144fd44db8f855d7e907256f8" == await coresys.host.logs.get_boot_id(2)
with pytest.raises(ValueError):
await coresys.host.logs.get_boot_id(3)
async def test_identifiers(coresys: CoreSys, journald_gateway: MagicMock):
"""Test getting identifiers."""
journald_gateway.return_value.__aenter__.return_value.text = AsyncMock(
return_value=load_fixture("logs_identifiers.txt")
)
# Mock is large so just look for a few different types of identifiers
for identifier in [
"addon_local_ssh",
"hassio_dns",
"hassio_supervisor",
"kernel",
"os-agent",
]:
assert identifier in await coresys.host.logs.get_identifiers()

View File

@ -46,6 +46,7 @@ async def test_load(coresys_dbus: CoreSys, dbus: list[str]):
assert coresys.dbus.network.connectivity_enabled is True
assert coresys.dbus.resolved.multicast_dns == MulticastProtocolEnabled.RESOLVE
assert coresys.dbus.agent.apparmor.version == "2.13.2"
assert len(coresys.host.logs.default_identifiers) > 0
sound_update.assert_called_once()