monitoring: use netdata for internal monitoring

Change-type: patch
Signed-off-by: Matthew McGinn <matthew@balena.io>

docker-compose.yml

@@ -7,6 +7,8 @@ volumes:
     etcher_bash_history:
     etcher_config_electron:
     etcher_config_balena_etcher:
+    netdatalib:
+    netdatacache:
 services:
   influxdb:
     restart: always
@@ -90,3 +92,25 @@ services:
           echo 1 > /sys/devices/system/cpu/cpu3/online &&
           dbus-send --system --dest=org.freedesktop.systemd1 --type=method_call --print-reply /org/freedesktop/systemd1 org.freedesktop.systemd1.Manager.StartUnit string:gov-switch.service string:replace
         "
+  netdata:
+    build: ./etcher-pro-monitoring/netdata
+    # this could be removed if we could simply use env vars in our configuration (only to pass in supervisor API URL)
+    network_mode: 'host'
+    ports:
+      - '19999:19999'
+    environment:
+      - PGID=988
+        # we get this injected as unix:///...sock, but netdata doesn't like that
+      - DOCKER_HOST=/var/run/balena-engine.sock
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - apparmor:unconfined
+    labels:
+      io.balena.features.balena-socket: 1
+      io.balena.features.procfs: 1
+      io.balena.features.supervisor-api: 1
+      io.balena.features.sysfs: 1
+    volumes:
+      - netdatalib:/var/lib/netdata
+      - netdatacache:/var/cache/netdata

etcher-pro-monitoring/netdata/Dockerfile

@@ -0,0 +1,5 @@
+FROM netdata/netdata
+
+WORKDIR /etc/netdata
+COPY httpcheck-supervisor.yml ./go.d/httpcheck.conf
+COPY balenad.conf ./python.d/dockerd.conf

etcher-pro-monitoring/netdata/balenad.conf

@@ -0,0 +1,2 @@
+ update_every : 1
+ priority     : 60000

etcher-pro-monitoring/netdata/httpcheck-supervisor.yml

@@ -0,0 +1,3 @@
+jobs:
+  - name: balena-supervisor
+    url: http://localhost:48484/ping