diff --git a/src/components/NavigationDocs.jsx b/src/components/NavigationDocs.jsx index a7cf7eef7..f5613a27b 100644 --- a/src/components/NavigationDocs.jsx +++ b/src/components/NavigationDocs.jsx @@ -867,6 +867,7 @@ export const docsNavigation = [ { title: 'gRPC Daemon Socket', href: '/client/grpc-socket' }, { title: 'HTTP/JSON Daemon Socket', href: '/client/json-socket' }, { title: 'Environment Variables', href: '/client/environment-variables' }, + { title: 'Local Metrics Endpoint', href: '/client/local-metrics' }, { title: 'MDM Integration', href: '/client/mdm-integration' }, { title: 'Settings', diff --git a/src/pages/client/local-metrics.mdx b/src/pages/client/local-metrics.mdx new file mode 100644 index 000000000..ae4714b66 --- /dev/null +++ b/src/pages/client/local-metrics.mdx @@ -0,0 +1,113 @@ +import {Note} from "@/components/mdx"; + +export const description = 'Scrape NetBird client connection health with Prometheus through the local /metrics endpoint on the client daemon.' + +# Local Metrics Endpoint + +The NetBird client daemon can expose a Prometheus `/metrics` endpoint on the local machine. It reports whether the client is connected to Management and Signal, how many peers it knows and how many of those are connected, the latency to each directly connected peer, and how long connection establishment, sync processing, and logins take. + +The endpoint is opt-in and off by default. When enabled, it binds to `127.0.0.1:9191` unless you set another address. It is independent of [client metrics push](/manage/client-metrics): nothing here is sent anywhere, the data is only served to whatever scrapes the endpoint. + +## Enabling the endpoint + +```bash +netbird up --enable-local-metrics +``` + +To listen on a different port: + +```bash +netbird up --enable-local-metrics --local-metrics-address 127.0.0.1:9300 +``` + +To disable it again: + +```bash +netbird up --enable-local-metrics=false +``` + +The setting is stored in the [profile](/client/profiles) config, so it survives restarts and applies per profile. Switching profiles starts, stops, or rebinds the endpoint to match the profile you switch to. + + + `netbird up` ignores configuration flags when the client is already connected. Run `netbird down` first, then `netbird up` with the flags. + + +Once the client is up, check the endpoint: + +```bash +curl http://127.0.0.1:9191/metrics +``` + +## Exposing it beyond localhost + +The endpoint has no authentication. Anything that can reach it can read your peer names, the latency to each of them, and your connectivity state. Keep it on a loopback address and let the scraper run on the same host. + +If a scraper genuinely has to reach it from elsewhere, binding to a non-loopback address requires root on Linux and macOS, and administrator privileges on Windows. The daemon runs with those privileges itself, so an unprivileged local user must not be able to publish this to the network: + +```bash +sudo netbird down +sudo netbird up --enable-local-metrics --local-metrics-address 0.0.0.0:9191 +``` + +The daemon logs a warning for every non-loopback bind. Put the endpoint behind a firewall rule, or a reverse proxy that adds authentication, if you do this. + +## Scraping with Prometheus + +```yaml +scrape_configs: + - job_name: netbird-client + static_configs: + - targets: ['127.0.0.1:9191'] +``` + +## Metrics reference + +### Connection state + +These are read from the daemon at scrape time and are always present while the daemon runs. + +| Metric | Type | Labels | Description | +| --- | --- | --- | --- | +| `netbird_management_connected` | gauge | — | `1` when connected to the management service, `0` otherwise. | +| `netbird_signal_connected` | gauge | — | `1` when connected to the signal service, `0` otherwise. | +| `netbird_peers` | gauge | — | Number of peers known to this client, matching the total in `netbird status`. Includes peers that are currently offline. | +| `netbird_peers_connected` | gauge | `connection_type` | Number of connected peers, split into `p2p` and `relay`. | +| `netbird_peer_latency_seconds` | gauge | `peer` | Round-trip latency to a directly connected peer, labeled with its FQDN. Relayed connections carry no latency measurement and produce no series. | + +### Connection establishment and management interactions + +These are recorded as the client runs, so they appear only once the engine is up and the corresponding event has happened at least once. They reset when the daemon restarts. + +| Metric | Type | Labels | Description | +| --- | --- | --- | --- | +| `netbird_peer_connection_stage_duration_seconds` | histogram | `stage`, `connection_type`, `attempt_type` | Duration of peer connection establishment stages. | +| `netbird_sync_duration_seconds` | histogram | — | Duration of processing a sync message from the management service. | +| `netbird_sync_phase_duration_seconds` | histogram | `phase` | Duration of an individual sync processing phase, for example `routes_apply`, `filtering`, or `added_peers`. | +| `netbird_login_duration_seconds` | histogram | `success` | Duration of logins to the management service, split by whether the login succeeded. | + +Label values for `netbird_peer_connection_stage_duration_seconds`: + +| Label | Values | +| --- | --- | +| `stage` | `signaling_to_connection`, `connection_to_wg_handshake`, `total` | +| `connection_type` | `ice_p2p`, `ice_turn`, `relay` | +| `attempt_type` | `initial` for the first connection to a peer, `reconnection` for a later one | + + + Every label is a bounded enum except `peer` on `netbird_peer_latency_seconds`, which carries one series per directly connected peer. On a client with many direct connections this is the one metric whose cardinality grows with your network. + + +## Grafana dashboard + +NetBird ships a [client dashboard](/selfhosted/observability/dashboards#client) that graphs everything above. Import [`client.json`](https://github.com/netbirdio/netbird/blob/main/infrastructure_files/observability/grafana/dashboards/client.json) into Grafana and point it at the Prometheus datasource that scrapes your clients. + +## MDM + +Both settings can be enforced through [MDM](/client/mdm-integration): + +| Key | Type | Description | +| --- | --- | --- | +| `enableLocalMetrics` | boolean | Turn the local `/metrics` endpoint on or off. | +| `localMetricsAddress` | string | Listen address for the endpoint, for example `127.0.0.1:9191`. | + +An MDM-supplied address is applied as-is and is not subject to the privilege check, since the policy already comes from an administrator. diff --git a/src/pages/client/mdm-integration.mdx b/src/pages/client/mdm-integration.mdx index ff0e39615..3cd975ecf 100644 --- a/src/pages/client/mdm-integration.mdx +++ b/src/pages/client/mdm-integration.mdx @@ -60,7 +60,7 @@ stale residue on the device. ## Policy keys reference -The same 16 keys apply on every platform. Names are camelCase in the +The same 20 keys apply on every platform. Names are camelCase in the managed-configuration payload; the Windows ADMX template renders the PascalCase variant in the Group Policy Editor — both are recognized. @@ -82,6 +82,8 @@ PascalCase variant in the Group Policy Editor — both are recognized. | `disableUpdateSettings` | boolean | Block every configuration change from UI or CLI on this device (read-only mode). | | `disableProfiles` | boolean | Hide the profile menu in the GUI and reject profile CRUD via CLI. | | `disableNetworks` | boolean | Hide the Networks / Exit Node menus in the GUI and reject the related RPCs. | +| `enableLocalMetrics` | boolean | Expose the client's [local Prometheus `/metrics` endpoint](/client/local-metrics). | +| `localMetricsAddress` | string | Listen address of the local `/metrics` endpoint (default `127.0.0.1:9191`). | | `splitTunnelMode` | string | `allow` or `disallow` — split-tunnel policy mode (Android only at the client level; harmless on desktop). | | `splitTunnelApps` | string | Comma-separated list of package names that the split-tunnel mode applies to (Android only). | @@ -100,6 +102,11 @@ PascalCase variant in the Group Policy Editor — both are recognized. See [Enforcing the Exit Node on Managed Devices](/use-cases/remote-access/exit-nodes#enforcing-the-exit-node-on-managed-devices) for the full recipe, including why the policy must be deployed before users touch the exit node switch. +- `localMetricsAddress` is applied as pushed. Users are held to a + loopback address unless they are root, because the endpoint is + unauthenticated, but a policy already comes from an administrator and + is not subject to that check. A non-loopback address publishes peer + names and connectivity state to anything that can reach the port. - `splitTunnelMode` and `splitTunnelApps` are wired into Android's `VpnService.Builder.addAllowedApplication()` flow; on Windows and macOS the daemon parses the keys but ignores them. They are safe to diff --git a/src/pages/manage/client-metrics.mdx b/src/pages/manage/client-metrics.mdx index 8ac5f7cfc..191e28993 100644 --- a/src/pages/manage/client-metrics.mdx +++ b/src/pages/manage/client-metrics.mdx @@ -52,3 +52,10 @@ The `NB_METRICS_PUSH_ENABLED` environment variable on the client takes precedenc You can additionally set `NB_METRICS_INTERVAL` to a duration value (e.g., `30m`, `1h`) to override how often metrics are pushed. + +## Scraping metrics locally + +Independently of metrics push, a client can expose the same connection stage, sync, and login metrics, plus live +connectivity and per-peer latency gauges, on a local Prometheus endpoint that you scrape yourself. The data never +leaves the machine unless your own scraper collects it. See +[Local Metrics Endpoint](/client/local-metrics). diff --git a/src/pages/selfhosted/observability/dashboards.mdx b/src/pages/selfhosted/observability/dashboards.mdx index 93304c31e..b2c3446ad 100644 --- a/src/pages/selfhosted/observability/dashboards.mdx +++ b/src/pages/selfhosted/observability/dashboards.mdx @@ -1,8 +1,8 @@ -export const description = 'Ready-made Grafana dashboards for NetBird Management, Signal, Relay, and the Enterprise Commercial License management stack.' +export const description = 'Ready-made Grafana dashboards for NetBird Management, Signal, Relay, the client, and the Enterprise Commercial License management stack.' # Grafana dashboards -NetBird ships ready-to-use Grafana dashboards for the Management, Signal, and Relay services, plus an extended Management dashboard for [Enterprise Commercial License](/selfhosted/enterprise/getting-started) deployments. They are maintained in the `netbirdio/netbird` repository under [`infrastructure_files/observability/grafana/dashboards`](https://github.com/netbirdio/netbird/tree/main/infrastructure_files/observability/grafana/dashboards) and import directly into Grafana. +NetBird ships ready-to-use Grafana dashboards for the Management, Signal, and Relay services and for the client, plus an extended Management dashboard for [Enterprise Commercial License](/selfhosted/enterprise/getting-started) deployments. They are maintained in the `netbirdio/netbird` repository under [`infrastructure_files/observability/grafana/dashboards`](https://github.com/netbirdio/netbird/tree/main/infrastructure_files/observability/grafana/dashboards) and import directly into Grafana. ## Available dashboards @@ -12,6 +12,7 @@ NetBird ships ready-to-use Grafana dashboards for the Management, Signal, and Re | Management (Enterprise) | [`management-enterprise.json`](https://github.com/netbirdio/netbird/blob/main/infrastructure_files/observability/grafana/dashboards/management-enterprise.json) | | Signal | [`signal.json`](https://github.com/netbirdio/netbird/blob/main/infrastructure_files/observability/grafana/dashboards/signal.json) | | Relay | [`relay.json`](https://github.com/netbirdio/netbird/blob/main/infrastructure_files/observability/grafana/dashboards/relay.json) | +| Client | [`client.json`](https://github.com/netbirdio/netbird/blob/main/infrastructure_files/observability/grafana/dashboards/client.json) | ### Management @@ -29,6 +30,10 @@ Covers active peers, peer connection durations, message forwarding throughput an Covers connected peers (total / active / idle), peer authentication latency, peer store latency, and inbound/outbound relay traffic bandwidth. +### Client + +Covers management and signal connectivity, known and connected peers split by connection type, per-peer latency, peer connection establishment stages, and sync and login durations. Unlike the service dashboards it reads from the NetBird client itself, which exposes these metrics through an opt-in local endpoint — see [Local Metrics Endpoint](/client/local-metrics) for enabling it and for the metric reference. + ## Importing a dashboard 1. In Grafana, go to **Dashboards → New → Import**. @@ -49,6 +54,8 @@ The Management, Signal, and Relay dashboards expose these template variables: Your deployment may use only a subset of these variables; unused ones can be left at the default `All`. The Enterprise dashboard uses a different variable set — see [its variables table](/selfhosted/enterprise/grafana-dashboard#dashboard-variables). +The Client dashboard uses `datasource`, `job`, and `instance`, where `instance` selects the client to look at. + The Management dashboard expects HTTP request metrics to carry an `exported_endpoint` label rather than `endpoint`. If your Prometheus relabeling drops or renames this label, edit the dashboard panel queries accordingly. @@ -56,3 +63,5 @@ The Management dashboard expects HTTP request metrics to carry an `exported_endp ## Scraping for the dashboards The dashboards assume a Prometheus scrape configuration that reaches the `/metrics` endpoint of each NetBird service. See [Service endpoints](/selfhosted/observability#service-endpoints) for defaults and per-service overrides. + +The Client dashboard scrapes the clients themselves rather than a server component. Its endpoint is off by default and binds to localhost when enabled, so it needs a scraper on each client host — see [Local Metrics Endpoint](/client/local-metrics).